init

app.py

@@ -0,0 +1,381 @@
+"""
+HuggingFace Space — Gemma 4 26B A4B Coding API
+Model : unsloth/gemma-4-26B-A4B-it-GGUF  →  UD-IQ3_XXS (11.2 GB)
+RAM   : fits in 16 GB with ~4 GB left for KV cache at ctx=4096
+Params: temp=0.3, top_p=0.9, min_p=0.1, top_k=20 (tuned for coding per reddit)
+
+Endpoints
+  GET  /           → landing page
+  GET  /health     → status (also used by self-ping)
+  GET  /v1/models  → OpenAI model list
+  POST /v1/chat/completions → OpenAI-compatible
+  POST /v1/messages         → Anthropic-compatible  ← Claude Code uses this
+"""
+
+import os, json, time, uuid, asyncio, threading
+from contextlib import asynccontextmanager
+from typing import Optional, List, Union, Any, Dict
+
+import httpx
+from fastapi import FastAPI, HTTPException
+from fastapi.responses import HTMLResponse, StreamingResponse, JSONResponse
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+
+# ── Config ────────────────────────────────────────────────────────────────────
+MODEL_REPO   = os.getenv("MODEL_REPO", "unsloth/gemma-4-26B-A4B-it-GGUF")
+MODEL_FILE   = os.getenv("MODEL_FILE", "gemma-4-26B-A4B-it-UD-IQ3_XXS.gguf")
+MODEL_DIR    = "/app/models"
+MODEL_PATH   = f"{MODEL_DIR}/{MODEL_FILE}"
+SPACE_URL    = os.getenv("SPACE_URL", "")
+
+# Context 4096 keeps KV cache ≤2 GB — safe with 11.2 GB model on 16 GB RAM
+N_CTX        = int(os.getenv("N_CTX",     "4096"))
+N_THREADS    = int(os.getenv("N_THREADS", "2"))
+
+# Coding-optimised defaults (OP's settings from reddit thread)
+DEFAULT_TEMP  = float(os.getenv("DEFAULT_TEMP",   "0.3"))
+DEFAULT_TOP_P = float(os.getenv("DEFAULT_TOP_P",  "0.9"))
+DEFAULT_MIN_P = float(os.getenv("DEFAULT_MIN_P",  "0.1"))
+DEFAULT_TOP_K = int(os.getenv("DEFAULT_TOP_K",    "20"))
+
+MODEL_ALIAS  = "gemma-4-26b"
+llm          = None
+
+# ── Model download + load ─────────────────────────────────────────────────────
+def download_model():
+    from huggingface_hub import hf_hub_download
+    os.makedirs(MODEL_DIR, exist_ok=True)
+    if not os.path.exists(MODEL_PATH):
+        print(f"[model] Downloading {MODEL_FILE} (~11.2 GB)...")
+        hf_hub_download(
+            repo_id=MODEL_REPO,
+            filename=MODEL_FILE,
+            local_dir=MODEL_DIR,
+        )
+        print("[model] Download complete.")
+
+def load_model():
+    global llm
+    from llama_cpp import Llama
+    download_model()
+    print("[model] Loading Gemma 4 26B IQ3_XXS into RAM...")
+    llm = Llama(
+        model_path   = MODEL_PATH,
+        n_ctx        = N_CTX,
+        n_threads    = N_THREADS,
+        n_batch      = 512,
+        n_gpu_layers = 0,       # HF free tier is CPU-only
+        verbose      = False,
+        chat_format  = None,    # auto-detect from GGUF metadata (Gemma 4 template)
+    )
+    print(f"[model] Gemma 4 26B ready — ctx={N_CTX}, threads={N_THREADS}")
+
+# ── Self-ping ─────────────────────────────────────────────────────────────────
+async def self_ping_loop():
+    while True:
+        await asyncio.sleep(25 * 60)
+        if SPACE_URL:
+            try:
+                async with httpx.AsyncClient(timeout=15) as c:
+                    r = await c.get(f"{SPACE_URL}/health")
+                    print(f"[ping] {r.status_code}")
+            except Exception as e:
+                print(f"[ping] failed: {e}")
+
+# ── App ───────────────────────────────────────────────────────────────────────
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    threading.Thread(target=load_model, daemon=True).start()
+    asyncio.create_task(self_ping_loop())
+    yield
+
+app = FastAPI(title="Gemma 4 Coding API", lifespan=lifespan)
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+    allow_credentials=True,
+)
+
+# ── Helpers ───────────────────────────────────────────────────────────────────
+def _check_model():
+    if llm is None:
+        raise HTTPException(
+            503,
+            detail="Model still loading — first boot downloads ~11 GB, wait ~5-10 min"
+        )
+
+def _extract_text(content) -> str:
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        parts = []
+        for block in content:
+            if isinstance(block, dict):
+                if block.get("type") == "text":
+                    parts.append(block.get("text", ""))
+                elif block.get("type") == "tool_result":
+                    parts.append(_extract_text(block.get("content", "")))
+            else:
+                parts.append(str(block))
+        return "".join(parts)
+    return str(content)
+
+# ── Health ────────────────────────────────────────────────────────────────────
+@app.get("/health")
+async def health():
+    return {
+        "status": "ok",
+        "model_loaded": llm is not None,
+        "model": MODEL_FILE,
+        "ctx": N_CTX,
+    }
+
+# ══ OpenAI-compatible  /v1/chat/completions ══════════════════════════════════
+class OAIMessage(BaseModel):
+    role: str
+    content: Union[str, List[Any]]
+
+class OAIRequest(BaseModel):
+    model: str = MODEL_ALIAS
+    messages: List[OAIMessage]
+    temperature: float = DEFAULT_TEMP
+    top_p: float = DEFAULT_TOP_P
+    min_p: float = DEFAULT_MIN_P
+    top_k: int = DEFAULT_TOP_K
+    max_tokens: int = 2048
+    stream: bool = False
+    stop: Optional[List[str]] = None
+
+@app.get("/v1/models")
+async def oai_models():
+    return {
+        "object": "list",
+        "data": [{
+            "id": MODEL_ALIAS,
+            "object": "model",
+            "created": int(time.time()),
+            "owned_by": "google-deepmind",
+        }],
+    }
+
+@app.post("/v1/chat/completions")
+async def oai_chat(req: OAIRequest):
+    _check_model()
+    msgs = [
+        {"role": m.role, "content": _extract_text(m.content)}
+        for m in req.messages
+    ]
+    kwargs = dict(
+        messages    = msgs,
+        temperature = req.temperature,
+        top_p       = req.top_p,
+        min_p       = req.min_p,
+        top_k       = req.top_k,
+        max_tokens  = req.max_tokens,
+        stop        = req.stop,
+    )
+
+    if req.stream:
+        async def gen():
+            rid = f"chatcmpl-{uuid.uuid4().hex[:8]}"
+            ts  = int(time.time())
+            for chunk in llm.create_chat_completion(**kwargs, stream=True):
+                data = {
+                    "id": rid,
+                    "object": "chat.completion.chunk",
+                    "created": ts,
+                    "model": req.model,
+                    "choices": [{
+                        "index": 0,
+                        "delta": chunk["choices"][0]["delta"],
+                        "finish_reason": chunk["choices"][0]["finish_reason"],
+                    }],
+                }
+                yield f"data: {json.dumps(data)}\n\n"
+            yield "data: [DONE]\n\n"
+        return StreamingResponse(gen(), media_type="text/event-stream")
+
+    result = llm.create_chat_completion(**kwargs, stream=False)
+    return JSONResponse(result)
+
+# ══ Anthropic-compatible  /v1/messages  (Claude Code) ═══════════════════════
+class AnthropicMessage(BaseModel):
+    role: str
+    content: Union[str, List[Dict]]
+
+class AnthropicRequest(BaseModel):
+    model: str = MODEL_ALIAS
+    messages: List[AnthropicMessage]
+    system: Optional[str] = None
+    max_tokens: int = 2048
+    temperature: float = DEFAULT_TEMP
+    top_p: float = DEFAULT_TOP_P
+    top_k: int = DEFAULT_TOP_K
+    stream: bool = False
+    stop_sequences: Optional[List[str]] = None
+
+@app.post("/v1/messages")
+async def anthropic_messages(req: AnthropicRequest):
+    _check_model()
+    msgs = []
+    if req.system:
+        msgs.append({"role": "system", "content": req.system})
+    for m in req.messages:
+        msgs.append({"role": m.role, "content": _extract_text(m.content)})
+
+    kwargs = dict(
+        messages    = msgs,
+        temperature = req.temperature,
+        top_p       = req.top_p,
+        min_p       = DEFAULT_MIN_P,  # always apply min_p for coding accuracy
+        top_k       = req.top_k,
+        max_tokens  = req.max_tokens,
+        stop        = req.stop_sequences,
+    )
+
+    if req.stream:
+        async def gen():
+            msg_id = f"msg_{uuid.uuid4().hex[:20]}"
+            yield f"data: {json.dumps({'type':'message_start','message':{'id':msg_id,'type':'message','role':'assistant','content':[],'model':req.model,'stop_reason':None,'usage':{'input_tokens':0,'output_tokens':0}}})}\n\n"
+            yield f"data: {json.dumps({'type':'content_block_start','index':0,'content_block':{'type':'text','text':''}})}\n\n"
+            full = ""
+            for chunk in llm.create_chat_completion(**kwargs, stream=True):
+                dt = chunk["choices"][0]["delta"].get("content", "")
+                if dt:
+                    full += dt
+                    yield f"data: {json.dumps({'type':'content_block_delta','index':0,'delta':{'type':'text_delta','text':dt}})}\n\n"
+            yield f"data: {json.dumps({'type':'content_block_stop','index':0})}\n\n"
+            yield f"data: {json.dumps({'type':'message_delta','delta':{'stop_reason':'end_turn','stop_sequence':None},'usage':{'output_tokens':len(full.split())}})}\n\n"
+            yield f"data: {json.dumps({'type':'message_stop'})}\n\n"
+        return StreamingResponse(
+            gen(),
+            media_type="text/event-stream",
+            headers={"anthropic-version": "2023-06-01"},
+        )
+
+    result = llm.create_chat_completion(**kwargs, stream=False)
+    text   = result["choices"][0]["message"]["content"]
+    usage  = result.get("usage", {})
+    return JSONResponse({
+        "id":           f"msg_{uuid.uuid4().hex[:20]}",
+        "type":         "message",
+        "role":         "assistant",
+        "content":      [{"type": "text", "text": text}],
+        "model":        req.model,
+        "stop_reason":  "end_turn",
+        "stop_sequence": None,
+        "usage": {
+            "input_tokens":  usage.get("prompt_tokens", 0),
+            "output_tokens": usage.get("completion_tokens", 0),
+        },
+    })
+
+# ══ Landing page ══════════════════════════════════════════════════════════════
+@app.get("/", response_class=HTMLResponse)
+async def landing():
+    sc = "#22c55e" if llm is not None else "#f59e0b"
+    st = "Model ready" if llm is not None else "Loading model... (~5-10 min on first boot)"
+    return LANDING_HTML.replace("{{SC}}", sc).replace("{{ST}}", st)
+
+LANDING_HTML = r"""<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1">
+<title>Gemma 4 26B Coding API</title>
+<style>
+*{box-sizing:border-box;margin:0;padding:0}
+body{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;background:#0d0d12;color:#e2e2ed;min-height:100vh;display:flex;flex-direction:column;align-items:center;padding:3.5rem 1.5rem 4rem}
+h1{font-size:2.1rem;font-weight:700;background:linear-gradient(130deg,#818cf8 20%,#34d399 80%);-webkit-background-clip:text;-webkit-text-fill-color:transparent;margin-bottom:.35rem;letter-spacing:-.5px}
+.tagline{color:#6b7280;font-size:.93rem;margin-bottom:2.5rem;text-align:center;line-height:1.5}
+.badge{display:inline-flex;align-items:center;gap:.45rem;background:#151520;border:1px solid #2a2a3a;border-radius:999px;padding:.3rem .9rem;font-size:.8rem;margin:.25rem}
+.dot{width:7px;height:7px;border-radius:50%;background:{{SC}};flex-shrink:0}
+.badges{display:flex;flex-wrap:wrap;justify-content:center;margin-bottom:2.8rem}
+.cards{display:grid;grid-template-columns:repeat(auto-fit,minmax(290px,1fr));gap:1.1rem;width:100%;max-width:920px;margin-bottom:2.8rem}
+.card{background:#13131c;border:1px solid #252535;border-radius:14px;padding:1.3rem 1.5rem}
+.card-title{font-size:.72rem;font-weight:600;text-transform:uppercase;letter-spacing:.1em;color:#6b7280;margin-bottom:.75rem}
+pre{background:#090910;border:1px solid #1e1e2e;border-radius:9px;padding:.85rem 1rem;font-family:'JetBrains Mono','Fira Code',monospace;font-size:.78rem;color:#a5b4fc;line-height:1.65;overflow-x:auto;white-space:pre-wrap;word-break:break-all}
+.ep-table{width:100%;max-width:920px;border-collapse:collapse;margin-bottom:2rem}
+.ep-table thead th{font-size:.72rem;text-transform:uppercase;letter-spacing:.08em;color:#4b5563;padding:.5rem .8rem;border-bottom:1px solid #1e1e2e;text-align:left}
+.ep-table tbody tr{border-bottom:1px solid #161622}
+.ep-table tbody td{padding:.7rem .8rem;font-size:.84rem}
+.method{display:inline-block;font-size:.68rem;font-weight:700;padding:.18rem .5rem;border-radius:5px;min-width:42px;text-align:center}
+.get{background:#064e3b;color:#34d399}.post{background:#1e3a5f;color:#60a5fa}
+.path{font-family:monospace;color:#e2e8f0;font-size:.85rem}
+.note{font-size:.78rem;color:#4b5563}
+.tip{background:#131a1f;border:1px solid #1d3040;border-radius:10px;padding:1rem 1.25rem;width:100%;max-width:920px;font-size:.82rem;color:#7dd3fc;line-height:1.6;margin-bottom:1.2rem}
+footer{margin-top:2.5rem;font-size:.75rem;color:#374151;text-align:center;line-height:1.8}
+</style>
+</head>
+<body>
+<h1>Gemma 4 26B A4B</h1>
+<p class="tagline">Coding-tuned · Anthropic &amp; OpenAI compatible · HuggingFace Spaces</p>
+
+<div class="badges">
+  <span class="badge"><span class="dot"></span>{{ST}}</span>
+  <span class="badge" style="color:#9ca3af">IQ3_XXS · 11.2 GB</span>
+  <span class="badge" style="color:#9ca3af">ctx 4096 · 2 vCPU · 16 GB RAM</span>
+  <span class="badge" style="color:#9ca3af">temp 0.3 · top-k 20 · min-p 0.1</span>
+</div>
+
+<div class="cards">
+  <div class="card">
+    <div class="card-title">Claude Code setup</div>
+    <pre>export ANTHROPIC_BASE_URL=\
+  https://YOUR-USER-space-name.hf.space
+export ANTHROPIC_API_KEY=gemma4-local
+
+claude --model gemma-4-26b</pre>
+  </div>
+  <div class="card">
+    <div class="card-title">OpenAI Python client</div>
+    <pre>from openai import OpenAI
+client = OpenAI(
+  base_url="https://YOUR-SPACE.hf.space/v1",
+  api_key="gemma4-local",
+)
+r = client.chat.completions.create(
+  model="gemma-4-26b",
+  messages=[{"role":"user",
+    "content":"write binary search"}],
+)</pre>
+  </div>
+  <div class="card">
+    <div class="card-title">curl quick test</div>
+    <pre>curl YOUR-SPACE.hf.space/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "gemma-4-26b",
+    "messages": [
+      {"role":"user",
+       "content":"hello"}
+    ]
+  }'</pre>
+  </div>
+</div>
+
+<div class="tip">
+  <strong>First boot:</strong> The model (~11.2 GB) downloads from HuggingFace on first start — allow 5–10 min.
+  <code style="background:#0d1b26;padding:1px 5px;border-radius:4px">/health</code> returns
+  <code style="background:#0d1b26;padding:1px 5px;border-radius:4px">model_loaded: false</code>
+  until ready. Subsequent restarts load from disk in ~60 s. Self-pings every 25 min to prevent sleep.
+</div>
+
+<table class="ep-table">
+  <thead><tr><th>Method</th><th>Path</th><th>Notes</th></tr></thead>
+  <tbody>
+    <tr><td><span class="method get">GET</span></td><td class="path">/health</td><td class="note">Status + model_loaded</td></tr>
+    <tr><td><span class="method get">GET</span></td><td class="path">/v1/models</td><td class="note">Model list (OpenAI)</td></tr>
+    <tr><td><span class="method post">POST</span></td><td class="path">/v1/chat/completions</td><td class="note">OpenAI-compatible · streaming supported</td></tr>
+    <tr><td><span class="method post">POST</span></td><td class="path">/v1/messages</td><td class="note">Anthropic-compatible · used by Claude Code</td></tr>
+  </tbody>
+</table>
+
+<footer>
+  Gemma 4 26B A4B · unsloth UD-IQ3_XXS · llama-cpp-python + OpenBLAS<br>
+  Self-pings /health every 25 min · April 2026
+</footer>
+</body>
+</html>"""