| # okto_version: "1.1"
|
| PROJECT "LoRAChatbot"
|
| DESCRIPTION "Fine-tuning a chatbot using LoRA adapters for efficient training"
|
| VERSION "1.0"
|
| AUTHOR "OktoSeek"
|
|
|
| DATASET {
|
| mix_datasets: [
|
| { path: "dataset/base_conversations.jsonl", weight: 60 },
|
| { path: "dataset/specialized_qa.jsonl", weight: 30 },
|
| { path: "dataset/domain_specific.jsonl", weight: 10 }
|
| ]
|
| dataset_percent: 75
|
| sampling: "weighted"
|
| shuffle: true
|
| format: "jsonl"
|
| type: "chat"
|
| language: "en"
|
| }
|
|
|
| MODEL {
|
| base: "oktoseek/base-llm-7b"
|
| architecture: "transformer"
|
| parameters: 7B
|
| context_window: 4096
|
| precision: "fp16"
|
| }
|
|
|
| FT_LORA {
|
| base_model: "oktoseek/base-llm-7b"
|
| train_dataset: "dataset/main.jsonl"
|
| lora_rank: 8
|
| lora_alpha: 32
|
| dataset_percent: 75
|
| mix_datasets: [
|
| { path: "dataset/base_conversations.jsonl", weight: 60 },
|
| { path: "dataset/specialized_qa.jsonl", weight: 30 },
|
| { path: "dataset/domain_specific.jsonl", weight: 10 }
|
| ]
|
| epochs: 5
|
| batch_size: 4
|
| learning_rate: 0.00003
|
| device: "cuda"
|
| target_modules: ["q_proj", "v_proj", "k_proj", "o_proj"]
|
| }
|
|
|
| METRICS {
|
| loss
|
| perplexity
|
| accuracy
|
| f1
|
| rouge_l
|
| }
|
|
|
| VALIDATE {
|
| on_validation: true
|
| frequency: 1
|
| save_best_model: true
|
| metric_to_monitor: "loss"
|
| }
|
|
|
| MONITOR {
|
| level: "full"
|
| log_metrics: [
|
| "loss",
|
| "val_loss",
|
| "accuracy",
|
| "perplexity"
|
| ]
|
| log_system: [
|
| "gpu_memory_used",
|
| "gpu_memory_free",
|
| "cpu_usage",
|
| "ram_used",
|
| "temperature"
|
| ]
|
| log_speed: [
|
| "tokens_per_second",
|
| "samples_per_second"
|
| ]
|
| refresh_interval: 2s
|
| export_to: "runs/lora-chatbot/system.json"
|
| dashboard: true
|
| }
|
|
|
| EXPORT {
|
| format: ["gguf", "okm", "safetensors"]
|
| path: "export/"
|
| quantization: "int8"
|
| }
|
|
|
| LOGGING {
|
| save_logs: true
|
| metrics_file: "runs/lora-chatbot/metrics.json"
|
| training_file: "runs/lora-chatbot/training_logs.json"
|
| log_level: "info"
|
| log_every: 10
|
| }
|
|
|
|
|
|
|
|
|
