fix: 14B + bnb int4 (AWQ build failed; bnb proven, no compile)

app.py

@@ -13,7 +13,7 @@ import gradio as gr
 import spaces
 import torch
 
-BASE_MODEL = os.environ.get("BASE_MODEL", "Qwen/Qwen2.5-Coder-32B-Instruct-AWQ")
+BASE_MODEL = os.environ.get("BASE_MODEL", "Qwen/Qwen2.5-Coder-14B-Instruct")
 HF_TOKEN   = os.environ.get("HF_TOKEN", "")
 
 SYSTEM = (
@@ -36,10 +36,17 @@ def _load_lazy():
         BASE_MODEL, token=HF_TOKEN or None, trust_remote_code=True)
     if _tok.pad_token_id is None:
         _tok.pad_token_id = _tok.eos_token_id
-    # AWQ pre-quantized — no quant config needed at load time
+    # bitsandbytes 4-bit — proven, no compile (AWQ build was failing)
+    from transformers import BitsAndBytesConfig
+    bnb = BitsAndBytesConfig(
+        load_in_4bit=True,
+        bnb_4bit_compute_dtype=torch.bfloat16,
+        bnb_4bit_quant_type="nf4",
+        bnb_4bit_use_double_quant=True,
+    )
     _model = AutoModelForCausalLM.from_pretrained(
         BASE_MODEL, token=HF_TOKEN or None, trust_remote_code=True,
-        device_map="cuda", torch_dtype=torch.bfloat16)
+        device_map="cuda", quantization_config=bnb)
     return _model, _tok