| { |
| "step": 45444, |
| "val_bpb": 0.6811688407173832, |
| "model_config": { |
| "sequence_len": 8192, |
| "vocab_size": 131072, |
| "n_layer": 16, |
| "n_head": 8, |
| "n_kv_head": 8, |
| "n_embd": 1024, |
| "moe_num_experts": 8, |
| "moe_top_k": 2, |
| "moe_layer_interval": 3, |
| "moe_group_size": 4, |
| "moe_expert_intermediate_size": 1792, |
| "moe_adjugate_intermediate_size": 0, |
| "moe_adjugate_scale": 0.05, |
| "moe_router_aux_loss_coef": 0.015, |
| "moe_router_bias_lr": 0.001, |
| "moe_activation_checkpoint": true, |
| "moe_capacity_factor": 0.75, |
| "rotary_scaling_type": "yarn", |
| "rotary_scale_factor": 4.0, |
| "residual_scale": -1.0, |
| "attn_dropout": 0.01, |
| "label_smoothing": 0.0, |
| "z_loss_weight": 0.0, |
| "use_flash_attention": true, |
| "domain_router_dim": 32, |
| "num_domain_tags": 128, |
| "domain_router_features": { |
| "dataset": { |
| "capacity": 128, |
| "mode": "one_hot" |
| }, |
| "quality": { |
| "capacity": 32, |
| "mode": "one_hot" |
| }, |
| "specialty": { |
| "capacity": 64, |
| "mode": "one_hot" |
| }, |
| "modality": { |
| "capacity": 32, |
| "mode": "one_hot" |
| }, |
| "language": { |
| "capacity": 32, |
| "mode": "one_hot" |
| }, |
| "origin": { |
| "capacity": 8, |
| "mode": "one_hot" |
| } |
| } |
| }, |
| "user_config": { |
| "run": "continous_pretraining", |
| "device_type": "", |
| "depth": 16, |
| "max_seq_len": 8192, |
| "moe_num_experts": 8, |
| "moe_top_k": 2, |
| "moe_layer_interval": 3, |
| "moe_group_size": 4, |
| "moe_expert_intermediate_size": 1792, |
| "moe_adjugate_intermediate_size": 0, |
| "moe_adjugate_scale": 0.05, |
| "moe_router_aux_loss_coef": 0.015, |
| "moe_router_bias_lr": 0.001, |
| "moe_activation_checkpoint": true, |
| "moe_capacity_factor": 0.75, |
| "moe_router_bias_reset_interval": 0, |
| "domain_router_dim": 32, |
| "domain_tag_capacity": 128, |
| "quality_tag_capacity": 32, |
| "specialty_tag_capacity": 64, |
| "modality_tag_capacity": 32, |
| "language_tag_capacity": 32, |
| "origin_tag_capacity": 8, |
| "domain_router_feature_modes": "dataset:one_hot,quality:one_hot,specialty:one_hot,modality:one_hot,language:one_hot,origin:one_hot", |
| "rotary_scaling_type": "yarn", |
| "rotary_scale_factor": 4.0, |
| "residual_scale": -1.0, |
| "attn_dropout": 0.01, |
| "label_smoothing": 0.0, |
| "z_loss_weight": 0.0, |
| "use_flash_attention": true, |
| "use_distributed_muon": false, |
| "num_iterations": -1, |
| "target_flops": -1.0, |
| "target_param_data_ratio": 20, |
| "device_batch_size": 1, |
| "total_batch_size": 524288, |
| "max_grad_accum_steps": 0, |
| "embedding_lr": 0.2, |
| "unembedding_lr": 0.004, |
| "weight_decay": 0.0, |
| "freeze_embedding_optimizers": false, |
| "matrix_lr": 0.02, |
| "grad_clip": 1.0, |
| "warmup_ratio": 0.0, |
| "warmdown_ratio": 0.2, |
| "final_lr_frac": 0.0, |
| "eval_every": 500, |
| "eval_tokens": "[redacted]", |
| "core_metric_every": 1000, |
| "core_metric_max_per_task": 2500, |
| "sample_every": 1000, |
| "micro_eval_every": 250, |
| "micro_eval_tokens": "[redacted]", |
| "skip_initial_eval": false, |
| "fast_dev_run": false, |
| "fast_dev_num_iterations": 200, |
| "fast_dev_max_grad_accum_steps": 2, |
| "fast_dev_eval_tokens_multiplier": "[redacted]", |
| "model_tag": "d16_cont", |
| "hf_repo_id": "harshad317/base_Medical_continuous", |
| "hf_repo_type": "model", |
| "hf_path_in_repo": "", |
| "hf_commit_message": "Uploading the base model", |
| "hf_private": false, |
| "hf_token": "[redacted]", |
| "hf_max_shard_size": "2gb", |
| "resume_from_checkpoint": "d16", |
| "resume_from_step": 22722, |
| "resume_load_optimizer": true, |
| "base_dataset_num_shards": 1250, |
| "train_dataset_mix": "", |
| "train_dataset_mix_path": "configs/train_dataset_mix.json", |
| "loader_dedup_window": 300000, |
| "loader_enable_length_bucketing": true, |
| "loader_bucket_bins": "512,1024,2048,4096,8192", |
| "loader_prefetch_batches": 2, |
| "loader_tokenizer_threads": "[redacted]", |
| "loader_tokenizer_batch_size": "[redacted]", |
| "train_single_pass": true, |
| "enable_length_curriculum": false, |
| "sequence_length_schedule": "0.25:512,0.50:1024,0.75:2048,1.0:4096", |
| "length_schedule_round_to": 32, |
| "enable_dataset_reweighting": true, |
| "dataset_reweight_interval": 2000, |
| "dataset_reweight_strength": 0.75, |
| "dataset_reweight_smoothing": 0.2, |
| "dataset_reweight_min_multiplier": 0.25, |
| "dataset_reweight_warmup_steps": 4000, |
| "memory_profile_interval": 500, |
| "dataset_mix_source": "/home/jupyter/nanochat/configs/train_dataset_mix.json" |
| }, |
| "device_batch_size": 1, |
| "max_seq_len": 8192 |
| } |
