{
  "architectures": [
    "NemotronH_Nano_Omni_Reasoning_V3"
  ],
  "auto_map": {
    "AutoConfig": "configuration.NemotronH_Nano_Omni_Reasoning_V3_Config",
    "AutoModel": "modeling.NemotronH_Nano_Omni_Reasoning_V3",
    "AutoModelForCausalLM": "modeling.NemotronH_Nano_Omni_Reasoning_V3"
  },
  "max_sequence_length": 131072,
  "downsample_ratio": 0.5,
  "force_image_size": 512,
  "patch_size": 16,
  "use_thumbnail": true,
  "eos_token_id": 11,
  "model_type": "NemotronH_Nano_Omni_Reasoning_V3",
  "ps_version": "v2",
  "template": "n5h_5p5_nanov2",
  "torch_dtype": "bfloat16",
  "image_tag_type": "internvl",
  "img_context_token_id": 18,
  "video_context_token_id": 131081,
  "img_context_token": "<image>",
  "video_context_token": "<video>",
  "img_start_token": "<img>",
  "img_end_token": "</img>",
  "vit_hidden_size": 1280,
  "projector_hidden_size": 20480,
  "norm_mean": [
    0.48145466,
    0.4578275,
    0.40821073
  ],
  "norm_std": [
    0.26862954,
    0.26130258,
    0.27577711
  ],
  "video_pruning_rate": 0.7,
  "sound_context_token_id": 27,
  "sound_context_token": "<so_embedding>",
  "sound_config": {
    "model_type": "parakeet",
    "hidden_size": 1024,
    "num_attention_heads": 8,
    "num_hidden_layers": 24,
    "intermediate_size": 4096,
    "conv_kernel_size": 9,
    "convolution_bias": false,
    "subsampling_conv_channels": 256,
    "subsampling_conv_kernel_size": 3,
    "subsampling_conv_stride": 2,
    "subsampling_factor": 8,
    "num_mel_bins": 128,
    "projection_hidden_size": 4096,
    "projection_bias": false,
    "sampling_rate": 16000
  },
  "llm_config": {
    "architectures": [
      "NemotronHForCausalLM"
    ],
    "auto_map": {
      "AutoConfig": "configuration_nemotron_h.NemotronHConfig",
      "AutoModelForCausalLM": "modeling_nemotron_h.NemotronHForCausalLM"
    },
    "model_type": "nemotron_h",
    "bos_token_id": 1,
    "chunk_size": 128,
    "conv_kernel": 4,
    "expand": 2,
    "eos_token_id": 11,
    "pad_token_id": 0,
    "torch_dtype": "bfloat16",
    "transformers_version": "4.55.4",
    "attention_bias": false,
    "attention_dropout": 0.0,
    "head_dim": 128,
    "hidden_dropout": 0.0,
    "hidden_size": 2688,
    "hybrid_override_pattern": "MEMEM*EMEMEM*EMEMEM*EMEMEM*EMEMEM*EMEMEMEM*EMEMEMEME",
    "initializer_range": 0.02,
    "intermediate_size": 1856,
    "layer_norm_epsilon": 1e-05,
    "mamba_head_dim": 64,
    "mamba_hidden_act": "silu",
    "mamba_num_heads": 64,
    "mamba_proj_bias": false,
    "max_position_embeddings": 262144,
    "mlp_bias": false,
    "mlp_hidden_act": "relu2",
    "moe_intermediate_size": 1856,
    "moe_shared_expert_intermediate_size": 3712,
    "n_group": 1,
    "n_groups": 8,
    "n_routed_experts": 128,
    "n_shared_experts": 1,
    "norm_eps": 1e-05,
    "norm_topk_prob": true,
    "num_attention_heads": 32,
    "num_experts_per_tok": 6,
    "num_hidden_layers": 52,
    "num_key_value_heads": 2,
    "num_logits_to_keep": 1,
    "partial_rotary_factor": 1.0,
    "rescale_prenorm_residual": true,
    "residual_in_fp32": false,
    "rope_theta": 10000,
    "routed_scaling_factor": 2.5,
    "sliding_window": null,
    "ssm_state_size": 128,
    "tie_word_embeddings": false,
    "time_step_floor": 0.0001,
    "time_step_limit": [
      0.0,
      1e+30
    ],
    "time_step_max": 0.1,
    "time_step_min": 0.001,
    "topk_group": 1,
    "use_bias": false,
    "use_cache": true,
    "use_conv_bias": true,
    "use_mamba_kernels": true,
    "vocab_size": 131072
  },
  "vision_config": {
    "auto_map": {
      "AutoConfig": "nvidia/C-RADIOv2-H--hf_model.RADIOConfig",
      "AutoModel": "nvidia/C-RADIOv2-H--hf_model.RADIOModel"
    },
    "adaptor_configs": {},
    "adaptor_names": null,
    "architectures": [
      "RADIOModel"
    ],
    "args": {
      "aa": null,
      "amp": true,
      "amp_dtype": "bfloat16",
      "amp_impl": "native",
      "aug_repeats": 0,
      "aug_splits": 0,
      "bn_eps": null,
      "bn_momentum": null,
      "cache_dir": null,
      "channels_last": false,
      "checkpoint_hist": 10,
      "chk_keep_forever": 100,
      "class_map": "",
      "clip_grad": null,
      "clip_mode": "norm",
      "cls_token_per_teacher": true,
      "coco_annotations_file": "/datasets/coco2017-adlsa/annotations/captions_val2017.json",
      "coco_image_dir": "/datasets/coco2017-adlsa/val2017",
      "color_jitter": 0.4,
      "cooldown_epochs": 0,
      "cpe_max_size": 2048,
      "crd_loss": false,
      "crd_loss_weight": 0.8,
      "crop_pct": null,
      "cutmix": 0.0,
      "cutmix_minmax": null,
      "dataset_download": false,
      "debug_full_knn": false,
      "decay_epochs": 90,
      "decay_milestones": [
        90,
        180,
        270
      ],
      "decay_rate": 0.1,
      "depchain": true,
      "dist_bn": "reduce",
      "dist_norm_weight": 0.0,
      "distributed": true,
      "drop": 0.0,
      "drop_block": null,
      "drop_connect": null,
      "drop_path": null,
      "dtype": "bfloat16",
      "epoch_repeats": 0.0,
      "eval": false,
      "eval_metric": "knn_top1",
      "eval_teacher": false,
      "eval_teacher_only": false,
      "eval_throughput": false,
      "fast_norm": false,
      "fd_loss_fn": "MSE",
      "feature_normalization": "SHIP_NORM",
      "feature_summarizer": "cls_token",
      "feature_upscale_factor": null,
      "force_new_wandb_id": false,
      "force_spectral_reparam": true,
      "freeze_bn": false,
      "fsdp": false,
      "fuser": "",
      "gp": null,
      "grad_accum_steps": 1,
      "grad_checkpointing": false,
      "head_init_bias": null,
      "head_init_scale": null,
      "head_warmup": 5,
      "head_weight_decay": 0.001,
      "hflip": 0.5,
      "img_size": null,
      "in_chans": null,
      "initial_checkpoint": null,
      "input_size": null,
      "interpolation": "",
      "layer_decay": null,
      "local_rank": 0,
      "log_interval": 50,
      "log_mlflow": false,
      "log_wandb": true,
      "loss_auto_balance": false,
      "lr_base": 0.1,
      "lr_base_scale": "",
      "lr_base_size": 256,
      "lr_cycle_decay": 0.5,
      "lr_cycle_limit": 1,
      "lr_cycle_mul": 1.0,
      "lr_k_decay": 1.0,
      "lr_noise": null,
      "lr_noise_pct": 0.67,
      "lr_noise_std": 1.0,
      "mean": null,
      "mesa": false,
      "min_lr": 0,
      "mixup": 0.0,
      "mixup_mode": "batch",
      "mixup_off_epoch": 0,
      "mixup_prob": 1.0,
      "mixup_switch_prob": 0.5,
      "mlp_hidden_size": 1520,
      "mlp_num_inner": 3,
      "mlp_version": "v2",
      "model": "vit_huge_patch16_224",
      "model_kwargs": {},
      "model_norm": false,
      "momentum": 0.9,
      "no_aug": false,
      "no_ddp_bb": true,
      "no_prefetcher": false,
      "no_resume_opt": false,
      "num_classes": null,
      "opt_betas": null,
      "opt_eps": null,
      "patience_epochs": 10,
      "pin_mem": false,
      "prefetcher": true,
      "pretrained": false,
      "rank": 0,
      "ratio": [
        0.75,
        1.3333333333333333
      ],
      "recount": 1,
      "recovery_interval": 0,
      "register_multiple": 10,
      "remode": "pixel",
      "reprob": 0.0,
      "reset_loss_state": false,
      "resplit": false,
      "save_images": false,
      "scale": [
        0.5,
        1.0
      ],
      "sched": "cosine",
      "seed": 42,
      "smoothing": 0.1,
      "spectral_heads": false,
      "spectral_reparam": false,
      "split_bn": false,
      "start_epoch": null,
      "std": null,
      "stream_teachers": true,
      "sync_bn": false,
      "synchronize_step": false,
      "teachers": [
        {
          "fd_normalize": false,
          "feature_distillation": true,
          "input_size": 378,
          "model": "ViT-H-14-378-quickgelu",
          "name": "clip",
          "pretrained": "dfn5b",
          "type": "open_clip",
          "use_summary": true
        },
        {
          "fd_normalize": false,
          "feature_distillation": true,
          "input_size": 378,
          "model": "ViT-SO400M-14-SigLIP-384",
          "name": "siglip",
          "pretrained": "webli",
          "type": "open_clip",
          "use_summary": true
        },
        {
          "fd_normalize": false,
          "feature_distillation": true,
          "input_size": 378,
          "model": "dinov2_vitg14_reg",
          "name": "dino_v2",
          "type": "dino_v2",
          "use_summary": true
        },
        {
          "fd_normalize": false,
          "feature_distillation": true,
          "input_size": 1024,
          "model": "vit-h",
          "name": "sam",
          "type": "sam",
          "use_summary": false
        }
      ],
      "torchcompile": null,
      "torchscript": false,
      "train_interpolation": "random",
      "train_split": "train",
      "tta": 0,
      "use_coco": false,
      "use_multi_epochs_loader": false,
      "val_ema_only": false,
      "val_split": "val",
      "vflip": 0.0,
      "vitdet_version": 1,
      "wandb_entity": "",
      "wandb_job_type": "",
      "wandb_name": "",
      "wandb_project": "",
      "warmup_lr": 1e-05,
      "warmup_prefix": false,
      "worker_seeding": "all",
      "workers": 8,
      "world_size": 256,
      "min_num_patches": 1024,
      "max_num_patches": 13312
    },
    "feature_normalizer_config": null,
    "inter_feature_normalizer_config": null,
    "max_resolution": 2048,
    "patch_size": 16,
    "preferred_resolution": [
      768,
      768
    ],
    "torch_dtype": "bfloat16",
    "version": "radio_v2.5-h",
    "vitdet_window_size": null,
    "min_num_patches": 1024,
    "max_num_patches": 13312,
    "video_target_num_patches": 1024,
    "video_maintain_aspect_ratio": true,
    "video_temporal_patch_size": 2,
    "video_prompt_version": 2,
    "separate_video_embedder": true
  }
}