| { |
| "activation_func": "swiglu", |
| "activation_func_fp8_input_store": false, |
| "add_bias_linear": false, |
| "architectures": [ |
| "ZayaForCausalLM" |
| ], |
| "attention_bias": false, |
| "attention_dropout": 0.0, |
| "bias_activation_fusion": true, |
| "bos_token_id": 2, |
| "cca": true, |
| "cca_num_q_heads": 8, |
| "dtype": "bfloat16", |
| "eos_token_id": 106, |
| "ffn_hidden_size": 4096, |
| "gated_linear_unit": true, |
| "hidden_size": 2048, |
| "kv_channels": 128, |
| "lm_head_bias": false, |
| "mamba_cache_dtype": "float32", |
| "max_position_embeddings": 131072, |
| "model_type": "zaya", |
| "moe_router_topk": 1, |
| "norm_epsilon": 1e-05, |
| "normalization": "RMSNorm", |
| "num_attention_heads": 16, |
| "num_experts": 16, |
| "num_hidden_layers": 80, |
| "num_key_value_heads": 2, |
| "num_query_groups": 2, |
| "pad_token_id": 0, |
| "partial_rotary_factor": 0.5, |
| "residual_in_fp32": true, |
| "rope_scaling": false, |
| "rope_theta": 5000000, |
| "scale_residual_merge": true, |
| "sliding_window": null, |
| "transformers_version": "4.57.1", |
| "use_cache": true, |
| "vocab_size": 262272, |
| "zaya_mlp_expansion": 256, |
| "zaya_use_eda": true, |
| "zaya_use_mod": true, |
| "capabilities": { |
| "reasoning_parser": "qwen3", |
| "tool_parser": "zaya_xml", |
| "think_in_template": false, |
| "supports_tools": true, |
| "supports_thinking": true, |
| "family": "zaya", |
| "modality": "text", |
| "cache_type": "hybrid" |
| }, |
| "weight_format": "mxtq", |
| "zaya_expert_layout": "split_switch_mlp", |
| "tie_word_embeddings": true, |
| "mxtq_bits": { |
| "routed_expert": { |
| "gate_proj": 2, |
| "up_proj": 2, |
| "down_proj": 4 |
| }, |
| "attention": 8, |
| "router": 16, |
| "embed_tokens": 8, |
| "lm_head": 8, |
| "cca_conv": 16, |
| "norms_residual": 16 |
| }, |
| "mxtq_seed": 42, |
| "quantization": { |
| "bits": 8, |
| "group_size": 32, |
| "mode": "affine", |
| "routed_expert_bits": { |
| "gate_proj": 2, |
| "up_proj": 2, |
| "down_proj": 4 |
| }, |
| "mxtq_bits": { |
| "routed_expert": { |
| "gate_proj": 2, |
| "up_proj": 2, |
| "down_proj": 4 |
| }, |
| "attention": 8, |
| "router": 16, |
| "embed_tokens": 8, |
| "lm_head": 8, |
| "cca_conv": 16, |
| "norms_residual": 16 |
| }, |
| "expert_layout": "split_switch_mlp" |
| } |
| } |
