| { |
| "model_name": "TinyLlama/TinyLlama-1.1B-Chat-v1.0", |
| "model_type": "llama", |
| "training_params": { |
| "learning_rate": 2e-4, |
| "num_train_epochs": 3, |
| "per_device_train_batch_size": 4, |
| "gradient_accumulation_steps": 4, |
| "warmup_ratio": 0.03, |
| "lr_scheduler_type": "cosine", |
| "max_grad_norm": 0.3, |
| "weight_decay": 0.01 |
| }, |
| "lora_config": { |
| "r": 8, |
| "lora_alpha": 16, |
| "target_modules": ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"], |
| "bias": "none", |
| "task_type": "CAUSAL_LM" |
| }, |
| "generation_config": { |
| "max_length": 200, |
| "temperature": 0.7, |
| "top_p": 0.9, |
| "top_k": 40, |
| "repetition_penalty": 1.1 |
| }, |
| "training_framework": { |
| "name": "unsloth", |
| "quantization": "4bit", |
| "use_flash_attention": true, |
| "use_peft": true |
| } |
| } |
|
|
