| { |
| "model_repo_id": "kyLELEng/adaptive-retro-gpt-1b", |
| "corpus_repo_id": "kyLELEng/adaptive-retro-gpt-1b-corpus", |
| "datastore_repo_id": "kyLELEng/adaptive-retro-gpt-1b-datastore", |
| "private_repo": true, |
| "push_to_hub": true, |
| "push_datasets": true, |
| "output_dir": "/tmp/adaptive-retro-gpt-1b", |
| "smoke_test": false, |
| "dataset_id": "HuggingFaceFW/fineweb-edu", |
| "dataset_config": "sample-10BT", |
| "train_split": "train", |
| "validation_split": "train", |
| "text_column": "text", |
| "streaming": true, |
| "min_text_chars": 200, |
| "datastore_dataset_id": "wikimedia/wikipedia", |
| "datastore_dataset_config": "20231101.en", |
| "datastore_split": "train", |
| "datastore_text_column": "text", |
| "corpus_train_docs": 80000, |
| "corpus_validation_docs": 4000, |
| "tokenizer_train_docs": 200000, |
| "vocab_size": 50000, |
| "max_train_docs": 120000, |
| "max_eval_docs": 2048, |
| "max_index_docs": 120000, |
| "max_index_chunks": 120000, |
| "chunk_tokens": 96, |
| "min_chunk_tokens": 24, |
| "hash_dim": 1024, |
| "top_k": 2, |
| "retrieval_seq_len": 512, |
| "seq_len": 2048, |
| "d_model": 2048, |
| "n_layers": 20, |
| "n_heads": 16, |
| "dropout": 0.0, |
| "retrieval_layers": "5,11,17", |
| "retrieval_budget_lambda": 0.001, |
| "no_retrieval_prob": 0.1, |
| "random_retrieval_prob": 0.1, |
| "max_steps": 20000, |
| "per_device_batch_size": 2, |
| "gradient_accumulation_steps": 2, |
| "learning_rate": 0.00018, |
| "min_lr_ratio": 0.1, |
| "warmup_steps": 1000, |
| "weight_decay": 0.1, |
| "grad_clip": 1.0, |
| "precision": "bf16", |
| "gradient_checkpointing": false, |
| "log_every": 10, |
| "eval_every": 2000, |
| "save_every": 5000, |
| "max_eval_batches": 32, |
| "seed": 17 |
| } |
