| |
| seed_everything: 123 |
| trainer: |
| accelerator: gpu |
| strategy: |
| class_path: lightning.pytorch.strategies.DeepSpeedStrategy |
| init_args: |
| accelerator: null |
| zero_optimization: true |
| stage: 2 |
| remote_device: null |
| offload_optimizer: false |
| offload_parameters: true |
| offload_params_device: cpu |
| nvme_path: /local_nvme |
| params_buffer_count: 5 |
| params_buffer_size: 100000000 |
| max_in_cpu: 1000000000 |
| offload_optimizer_device: cpu |
| optimizer_buffer_count: 4 |
| block_size: 1048576 |
| queue_depth: 8 |
| single_submit: false |
| overlap_events: true |
| thread_count: 1 |
| pin_memory: true |
| sub_group_size: 1000000000000 |
| contiguous_gradients: true |
| overlap_comm: true |
| allgather_partitions: true |
| reduce_scatter: true |
| allgather_bucket_size: 200000000 |
| reduce_bucket_size: 200000000 |
| zero_allow_untested_optimizer: true |
| logging_batch_size_per_gpu: auto |
| config: null |
| logging_level: 30 |
| parallel_devices: null |
| cluster_environment: null |
| loss_scale: 0.0 |
| initial_scale_power: 16 |
| loss_scale_window: 1000 |
| hysteresis: 2 |
| min_loss_scale: 1 |
| partition_activations: false |
| cpu_checkpointing: false |
| contiguous_memory_optimization: false |
| synchronize_checkpoint_boundary: false |
| load_full_weights: false |
| precision_plugin: null |
| process_group_backend: null |
| devices: 8 |
| num_nodes: 1 |
| precision: bf16-true |
| logger: |
| class_path: lightning.pytorch.loggers.TensorBoardLogger |
| init_args: |
| save_dir: /media/logs |
| name: main |
| version: null |
| log_graph: false |
| default_hp_metric: true |
| prefix: '' |
| sub_dir: null |
| comment: '' |
| purge_step: null |
| max_queue: 10 |
| flush_secs: 120 |
| filename_suffix: '' |
| callbacks: null |
| fast_dev_run: false |
| max_epochs: 2 |
| min_epochs: null |
| max_steps: -1 |
| min_steps: null |
| max_time: null |
| limit_train_batches: null |
| limit_val_batches: null |
| limit_test_batches: null |
| limit_predict_batches: null |
| overfit_batches: 0.0 |
| val_check_interval: null |
| check_val_every_n_epoch: 1 |
| num_sanity_val_steps: 0 |
| log_every_n_steps: 1 |
| enable_checkpointing: null |
| enable_progress_bar: null |
| enable_model_summary: null |
| accumulate_grad_batches: 8 |
| gradient_clip_val: null |
| gradient_clip_algorithm: null |
| deterministic: null |
| benchmark: null |
| inference_mode: true |
| use_distributed_sampler: true |
| profiler: null |
| detect_anomaly: false |
| barebones: false |
| plugins: null |
| sync_batchnorm: false |
| reload_dataloaders_every_n_epochs: 0 |
| default_root_dir: null |
| model: |
| config: |
| model_name: Mistral-7B-v0.2 |
| dtype: bfloat16 |
| num_thoughts: 2 |
| thought_length: 8 |
| lookahead_tokens: 4 |
| embedding_grad_weights: 100.0 |
| temperature: 1.0 |
| do_sample: true |
| train_max_length: 120 |
| offload_cache: false |
| top_k: null |
| top_p: null |
| checkpoint_dir: /media/models/unsloth/Mistral-7B-v0.2 |
| weight_decay: 0.001 |
| warmup_steps: 20 |
| policy_weight: 1.0 |
| init_lr: 1.0e-06 |
| optimizer: |
| class_path: torch.optim.AdamW |
| init_args: |
| lr: 1.0e-06 |
| betas: |
| - 0.9 |
| - 0.999 |
| eps: 1.0e-08 |
| weight_decay: 0.001 |
| amsgrad: false |
| maximize: false |
| foreach: null |
| capturable: false |
| differentiable: false |
| fused: null |
| scheduler: null |
| ckpt_path: null |
| data: |
| class_path: src.dataset.OpenWebMathDataModule |
| init_args: |
| data_path: /media/datasets/openwebmath |
| tokenizer: |
| class_path: src.dataset.SpecialTokenizer |
| init_args: |
| checkpoint_dir: /media/models/unsloth/Mistral-7B-v0.2 |
| batch_size: 1 |
| max_seq_length: 120 |
| num_samples: 2048 |
| ignore_index: -100 |
| val_split_fraction: 0.125 |
| seed: 42 |
| num_workers: 1 |
|
|
