release: initial squashed history

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

checkpoints/checkpoint_50.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ae75f2ca515e5776ac7f74af03e7806c3e7611c2aa0976ce4945f08a6db2833
+size 5742376211

config.yaml

@@ -0,0 +1,110 @@
+data:
+  allow_multiple_epochs: true
+  dataloader_in_order: false
+  dataset_cache:
+    cache_dir: null
+    cache_size_gb: null
+    cache_verbose: null
+    enabled: false
+  dataset_manifest:
+  - null
+  dataset_modality:
+  - text_untokenized
+  dataset_weighting:
+  - 1.0
+  hf_fast_tokenizer_rayon_threads: null
+  hf_fast_tokenizers_parallelism: true
+  num_workers: 8
+  prefetch_factor: 4
+  seed: 42
+  seq_len: 2048
+  shuffle: true
+  shuffle_buffer_size: 2000
+  shuffle_initial: 500
+  tokenizer: HuggingFaceTB/SmolVLM2-256M-Video-Instruct
+  type: text_untokenized
+  use_hf_fast_tokenizer: true
+  val_dataset_manifest: []
+  val_dataset_weighting: []
+db_logging: true
+distributed:
+  ddp_static_graph: false
+  device: cuda:0
+  dist_backend: nccl
+  dist_url: env://
+  fsdp: true
+  fsdp_cpu_offload: false
+  fsdp_reshard_after_forward: false
+  local_rank: 0
+  rank: 0
+  use_distributed: true
+  world_size: 128
+ema:
+  alpha: 0.999
+  enabled: false
+  inv_gamma: 1.0
+  max_value: 0.9999
+  min_value: 0.0
+  power: 0.75
+  type: ema
+  update_after_step: 0
+hparams:
+  beta1: 0.9
+  beta2: 0.95
+  decay: '0.2'
+  eps: 1.0e-08
+  force_min_lr: 0.0
+  global_batch_size: 1280
+  grad_checkpointing: false
+  grad_clip_norm: 1.0
+  loss_function: cross_entropy
+  lr: 0.0003
+  lr_cooldown_end: 0.0
+  lr_scheduler: warmup_constant_decay
+  optimizer: adamw
+  per_gpu_batch_size: 10
+  precision: pure_bf16
+  seed: 42
+  torchcompile: true
+  warmup: '1000'
+  wd: 0.01
+  world_size: 128
+  z_loss_coefficient: 0.0001
+log_every_n_steps: 20
+log_level: INFO
+max_checkpoint_limit: null
+model:
+  attn_name: torch_attn
+  cast_output_to_float32: false
+  ffn_type: swiglu
+  freeze: false
+  hidden_dim: 2048
+  is_causal: true
+  max_seq_len: 2048
+  n_heads: 16
+  n_layers: 24
+  norm_eps: 1.0e-05
+  norm_type: lp_layer_norm
+  positional_embedding_type: rotary
+  post_embed_norm: false
+  qk_norm: true
+  resume_from_checkpoint: null
+  resume_weights_only: false
+  type: transformer
+  vocab_size: 49280
+  weight_tying: false
+name: 2026_04_04-21_43_15-model_transformer-lr_0.0003-bsz_1280
+num_checkpoints: 20
+num_epochs: null
+remote_sync: null
+remote_sync_fixed_path: null
+resolve_configs: false
+resolve_configs_path: null
+save_path: /tmp
+total_train_samples: 488294400
+total_val_samples: null
+val_every_n_checkpoints: 1
+wandb: true
+wandb_entity: tri
+wandb_project_name: vla_foundry
+wandb_tags: []

config_model.yaml

@@ -0,0 +1,19 @@
+attn_name: torch_attn
+cast_output_to_float32: false
+ffn_type: swiglu
+freeze: false
+hidden_dim: 2048
+is_causal: true
+max_seq_len: 2048
+n_heads: 16
+n_layers: 24
+norm_eps: 1.0e-05
+norm_type: lp_layer_norm
+positional_embedding_type: rotary
+post_embed_norm: false
+qk_norm: true
+resume_from_checkpoint: null
+resume_weights_only: false
+type: transformer
+vocab_size: 49280
+weight_tying: false