fix: 14B + bnb int4

app.py

@@ -8,7 +8,7 @@ import gradio as gr
 import spaces
 import torch
 
-BASE_MODEL = os.environ.get("BASE_MODEL", "Qwen/Qwen2.5-Coder-32B-Instruct-AWQ")
+BASE_MODEL = os.environ.get("BASE_MODEL", "Qwen/Qwen2.5-Coder-14B-Instruct")
 HF_TOKEN   = os.environ.get("HF_TOKEN", "")
 
 DOMAIN_SEEDS = {
@@ -50,9 +50,16 @@ def _load_lazy():
         BASE_MODEL, token=HF_TOKEN or None, trust_remote_code=True)
     if _tok.pad_token_id is None:
         _tok.pad_token_id = _tok.eos_token_id
+    from transformers import BitsAndBytesConfig
+    bnb = BitsAndBytesConfig(
+        load_in_4bit=True,
+        bnb_4bit_compute_dtype=torch.bfloat16,
+        bnb_4bit_quant_type="nf4",
+        bnb_4bit_use_double_quant=True,
+    )
     _model = AutoModelForCausalLM.from_pretrained(
         BASE_MODEL, token=HF_TOKEN or None, trust_remote_code=True,
-        device_map="cuda", torch_dtype=torch.bfloat16)
+        device_map="cuda", quantization_config=bnb)
     return _model, _tok