Upload Vortex model

vortex_config.py

@@ -1,38 +1,36 @@
 """
-Vortex-7B model configuration.
-Optimized for 8GB VRAM (4060 laptop) and MacBook Pro M2/M3.
+Vortex-13B model configuration.
+Optimized for 16GB VRAM (4060 Ti laptop) and MacBook Pro M3 Max.
 """
 
-VORTEX_7B_CONFIG = {
+VORTEX_13B_CONFIG = {
     # Model dimensions
-    "d_model": 4096,
-    "num_layers": 32,
-    "num_heads": 32,
+    "d_model": 5120,
+    "num_layers": 40,
+    "num_heads": 40,
     "head_dim": 128,  # d_model // num_heads
 
     # State-space layer parameters
-    "d_state": 16,          # SSM state dimension
+    "d_state": 32,          # SSM state dimension (larger for bigger model)
     "d_conv": 4,            # SSM convolution width
 
     # Attention parameters
     "window_size": 512,     # Local attention window
-    "use_flash_attention": True,  # CUDA only
+    "use_flash_attention": True,
 
     # Feed-forward parameters
-    "ffn_expansion": 4,     # Hidden dim = d_model * expansion
-    "num_domains": 7,       # Physics, Math, Chemistry, Biology, Earth, Space, Zoology
-
-    # Tokenizer parameters
+    "ffn_expansion": 4,
+    "num_domains": 7,
     "vocab_size": 50000,
     "max_seq_len": 16384,
 
-    # Layer ratio: 60% SSM, 40% attention
-    "ssm_ratio": 0.6,
+    # Layer ratio: 50% SSM, 50% attention (more memory for attention)
+    "ssm_ratio": 0.5,
 
     # Data types
     "dtype": "bfloat16",
 
-    # Special tokens
+    # Special tokens (same as 7B)
     "special_tokens": {
         "[PAD]": 0,
         "[UNK]": 1,
@@ -55,10 +53,9 @@ VORTEX_7B_CONFIG = {
         "[ZOO]": 18,
     },
 
-    # Domain tags
     "domain_tags": ["[MATH]", "[CHEM]", "[BIO]", "[PHYS]", "[EARTH]", "[SPACE]", "[ZOO]"],
 
-    # Science module flags (enable/disable for ablation)
+    # Science module flags
     "enable_equation_module": True,
     "enable_numerical_module": True,
     "enable_citation_module": True,
@@ -67,5 +64,5 @@ VORTEX_7B_CONFIG = {
 
 
 def get_config():
-    """Return the 7B configuration dictionary."""
-    return VORTEX_7B_CONFIG
+    """Return the 13B configuration dictionary."""
+    return VORTEX_13B_CONFIG