RC_augmentation: false
_dataset_cfg_lookup:
  dlb_cmp_gm12878:
    eval_split: validation
    hf_path: jzshared/dlb_cmp_gm12878
    label_key: label_ut
    mask_key: mask_ut
    num_workers: 0
    path: data/dlb_cmp_gm12878
    pin_memory: true
    reference_id: hg38
    sequence_format: string
    sequence_key: sequence
    shuffle: true
    test_split: test
    train_split: train
    type: cmp_seq
  dlb_cmp_h1hesc:
    eval_split: validation
    hf_path: jzshared/dlb_cmp_h1hesc
    label_key: label_ut
    mask_key: mask_ut
    num_workers: 0
    path: data/dlb_cmp_h1hesc
    pin_memory: true
    reference_id: hg38
    sequence_format: string
    sequence_key: sequence
    shuffle: true
    test_split: test
    train_split: train
    type: cmp_seq
  dlb_cmp_hct116:
    eval_split: validation
    hf_path: jzshared/dlb_cmp_hct116
    label_key: label_ut
    mask_key: mask_ut
    num_workers: 0
    path: data/dlb_cmp_hct116
    pin_memory: true
    reference_id: hg38
    sequence_format: string
    sequence_key: sequence
    shuffle: true
    test_split: test
    train_split: train
    type: cmp_seq
  dlb_cmp_hff:
    eval_split: validation
    hf_path: jzshared/dlb_cmp_hff
    label_key: label_ut
    mask_key: mask_ut
    num_workers: 0
    path: data/dlb_cmp_hff
    pin_memory: true
    reference_id: hg38
    sequence_format: string
    sequence_key: sequence
    shuffle: true
    test_split: test
    train_split: train
    type: cmp_seq
  dlb_cmp_imr90:
    eval_split: validation
    hf_path: jzshared/dlb_cmp_imr90
    label_key: label_ut
    mask_key: mask_ut
    num_workers: 0
    path: data/dlb_cmp_imr90
    pin_memory: true
    reference_id: hg38
    sequence_format: string
    sequence_key: sequence
    shuffle: true
    test_split: test
    train_split: train
    type: cmp_seq
  euks_refseq_region_12.8k:
    hf_path: jzshared/euks_refseq_all_12p8k_merged_10m_20260302
    path: data/euks_refseq_all_12p8k_merged_10m_20260302
    type: refseq
  gencode128k_basic:
    hf_path: jzshared/gencode128k_basic
    path: data/gencode128k_basic
    type: refseq
  gencode128k_debug:
    hf_path: jzshared/gencode128k_debug
    path: data/gencode128k_debug
    type: refseq
  gencode_human_12.8k:
    hf_path: jzshared/gencode_human_12.8k
    path: data/gencode_human_12.8k
    type: refseq
  gencode_human_128k:
    hf_path: jzshared/gencode_human_128k
    path: data/gencode_human_128k
    type: refseq
  hg38_128k:
    hf_path: jzshared/hg38_cds_anchored_128000
    path: data/hg38_cds_anchored_128000
    type: refseq
  hg38_12k:
    hf_path: jzshared/hg38_12800
    path: data/hg38_cds_anchored_len12800_mincds150_1000000samples
    type: refseq
  hg38_cds_4m:
    hf_path: null
    path: data/hg38_cds_dataset_4m_filtered
    type: refseq
  orca32m_cmp_seq:
    eval_split: validation
    hf_path: jzshared/orca32m_cmp
    label_key: label_ut
    mask_key: mask_ut
    num_workers: 0
    path: data/orca32m_cmp_seq
    pin_memory: true
    reference_id: hg38
    sequence_format: string
    sequence_key: sequence
    shuffle: true
    test_split: test
    train_split: train
    type: cmp_seq
_unimportant_cfg:
  fields:
  - gpus
  - debug
  - wandb
  - env
  - uid
  - local_rank
  - is_distributed
  - master_port
  - device_type
  - cluster
  - world_size
  - train_dataset
  - eval_datasets
  - user_cfg
  - rank
  - device
  - hf_access_token
  - hf_private
  - hf_repo
  - hf_user
  - hf_token
  - save_every
  - eval_steps
  - save_steps
  - upload_to_hf
  - logging
  - log_every
  - use_wandb
  - project_root
  - version
  postfix:
  - _path
  - _file
  - _dir
  - _alias
  - _prefix
  prefix:
  - _
add_special_tokens: true
alias: Gencode-MxDNA
arch: hnet
batch_size: 8
bidirectional_strategy: mean
cluster: mila
cmd: python src/scripts/rebuttal/train_mlm.py exp=rebuttal/mlm data=gencode_human_12.8k
  model=hnet/mamba_64m max_len=12800 batch_size=8 eval_batch_size=1 grad_acc_steps=4
  train_steps=7650 eval_steps=125 save_steps=750 log_every=2 num_valid_samples=3000
  upload_to_hf=true wandb.project=DNAFM_v2 tokenizer=mxdna alias=Gencode-MxDNA use_wandb=true
  hf_repo=jzshared/Gencode-MxDNA
config_path: null
data: gencode_human_12.8k
data_alias: ${.data}_${max_len}
dataset: ${_dataset_cfg_lookup[${data}]}
dataset_sequence_key: sequence
device: cuda
device_type: GPU
dirs:
  data_cache: ${project_root}/data_cache/
  data_storage: ${project_root}/data/
  hydra: ${project_root}/temp/hydra/
  output: ${project_root}/output/${data_alias}/${alias}/
  temp: ${project_root}/temp/working_dir/${uid}/
  wandb_cache: ${oc.env:WANDB_CACHE_DIR,${project_root}/temp/wandb_cache/}
epochs: 200
eval_batch_size: 1
eval_steps: 125
grad_acc_steps: 4
hf_private: false
hf_repo: jzshared/Gencode-MxDNA
hf_user: jzshared
is_distributed: true
local_rank: 0
log_every: 2
logging:
  level: info
  log_wandb_metric_to_stdout: true
lr: 0.001
mask_replace_prob: 0.8
master_port: '46807'
max_data_samples: null
max_grad_norm: 2.0
max_len: 12800
max_length: ${max_len}
max_routing_tokens: 0
max_train_steps: ${train_steps}
min_routing_tokens: 8
mixed_precision: bf16
mlm_probability: 0.15
mode: Formal
model:
  arch: hnet
  name: hnet_mamba_64m
model_alias: ${oc.select:model.name,UnknownModel}
model_cfg:
  arch_layout:
  - m4
  - - m15
  - m4
  attn_cfg:
    num_heads:
    - 8
    - 12
    rotary_emb_dim:
    - 16
    - 24
    window_size:
    - 511
    - -1
  d_intermediate:
  - 0
  - 2048
  d_model:
  - 512
  - 768
  max_routing_tokens: ${max_routing_tokens}
  min_routing_tokens: ${min_routing_tokens}
  n_gpt: 1.0
  r_hi: ${r_hi}
  r_low: ${r_low}
  ssm_cfg:
    chunk_size: 256
    d_conv: 4
    d_state: 64
    expand: 2
    head_dim: 64
  tie_embeddings: true
  vocab_size: 9
mxdna_tokenizer_vocab_path: src/scripts/rebuttal/assets/mxdna_1mertokenizer/vocab.txt
name: hnet_base
num_test_samples: 0
num_train_samples: 0
num_valid_samples: 3000
project_root: ${hydra:runtime.cwd}
r_hi: 0.3
r_low: 0.0
random_replace_prob: 0.1
random_truncate: false
rank: 0
reference_loss: null
save_steps: 750
seed: 0
source: ${dataset.type}
tokenizer: mxdna
tokenizer_cache_dir: ${dirs.data_cache}/hf_tokenizers
tokenizer_max_length: null
tokenizer_name_or_path: null
tokenizer_name_or_path_resolved: /gpfs/scratch/guoh/DNAFM/src/scripts/rebuttal/assets/mxdna_1mertokenizer/vocab.txt
tokenizer_pad_to_multiple_of: null
tokenizer_trust_remote_code: false
tokenizer_use_fast: true
tokenizer_vocab_size: 9
train_steps: 7650
training:
  adam_beta1: 0.9
  adam_beta2: 0.95
  bf16: true
  dataloader_drop_last: true
  dataloader_num_workers: 1
  disable_tqdm: false
  do_train: true
  eval_steps: ${eval_steps}
  eval_strategy: steps
  gradient_accumulation_steps: ${grad_acc_steps}
  gradient_checkpointing: false
  group_by_length: false
  hnet_initializer_range: 0.02
  hnet_lr_multiplier: null
  label_names:
  - labels
  learning_rate: ${lr}
  logging_steps: ${log_every}
  lr_scheduler_type: linear
  max_grad_norm: ${max_grad_norm}
  max_train_steps: ${max_train_steps}
  num_train_epochs: ${epochs}
  output_dir: ${dirs.output}
  overrides: {}
  per_device_eval_batch_size: ${eval_batch_size}
  per_device_train_batch_size: ${batch_size}
  remove_unused_columns: false
  report_to: null
  resume_from_checkpoint: null
  save_steps: ${save_steps}
  save_strategy: steps
  use_lr_multiplier: true
  warmup_steps: 500
  weight_decay: 0.1
training_alias: mlm_${tokenizer}_lr${lr}_${train_steps}steps_ms${max_train_steps}_maxlen${max_len}
uid: ywrwxmjk
upload_to_hf: true
use_routing_ceiling: false
use_routing_floor: true
use_wandb: true
valid_test_downsample: null
version: NA
wandb:
  dir: ${dirs.wandb_cache}
  entity: ${oc.select:env.vars.wandb_entity,${oc.env:WANDB_ENTITY,null}}
  id: ywrwxmjk
  mode: online
  name: Gencode-MxDNA
  project: DNAFM_v2
  step_metric: null
  tags: []
  url: https://wandb.ai/jzshared/DNAFM_v2/runs/ywrwxmjk
world_size: 8