New application file

Dockerfile

@@ -1,16 +1,14 @@
-# Read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker
-# you will also find guides on how best to write your Dockerfile
+FROM python:3.10-slim
 
-FROM python:3.9
+WORKDIR /app
 
-RUN useradd -m -u 1000 user
-USER user
-ENV PATH="/home/user/.local/bin:$PATH"
+ENV HF_HOME=/tmp/hf
+ENV TRANSFORMERS_CACHE=/tmp/hf
+ENV HF_HUB_DISABLE_TELEMETRY=1
 
-WORKDIR /app
+COPY requirements.txt .
+RUN pip install --upgrade pip && pip install --no-cache-dir -r requirements.txt
 
-COPY --chown=user ./requirements.txt requirements.txt
-RUN pip install --no-cache-dir --upgrade -r requirements.txt
+COPY app.py .
 
-COPY --chown=user . /app
-CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
+CMD uvicorn app:app --host 0.0.0.0 --port ${PORT:-7860}

app.py

@@ -1,7 +1,61 @@
-from fastapi import FastAPI
+import os
+import threading
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+from transformers import pipeline
 
 app = FastAPI()
 
+## you can change model. On hugging face go to Models and then you have the ID. For example:
+## Nanbeige/Nanbeige4.1-3B
+
+## Careful about how big the model is, as HF free resources are limited
+
+MODEL_ID = "Qwen/Qwen2.5-1.5B-Instruct"
+
+_pipe = None
+_pipe_lock = threading.Lock()
+
+class Request(BaseModel):
+    prompt: str
+    temperature: float = 0.0
+    max_tokens: int = 50 ## you can pass the parameter in the request
+
+
 @app.get("/")
-def greet_json():
-    return {"Hello": "World!"}
+def health():
+    return {"status": "running", "model_loaded": _pipe is not None}
+
+
+def get_pipe():
+    global _pipe
+    if _pipe is None:
+        with _pipe_lock:
+            if _pipe is None:
+                _pipe = pipeline(
+                    "text-generation",
+                    model=MODEL_ID,
+                    device=-1
+                )
+    return _pipe
+
+
+@app.post("/generate") ## this is the endpoint that you call in the notebook
+def generate(req: Request):
+    try:
+        pipe = get_pipe()
+
+        do_sample = req.temperature > 0
+
+        out = pipe(
+            req.prompt,
+            max_new_tokens=int(req.max_tokens),
+            temperature=float(req.temperature),
+            do_sample=do_sample,
+            return_full_text=False
+        )
+
+        return {"response": out[0]["generated_text"].strip()}
+
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))

requirements.txt

@@ -1,2 +1,5 @@
 fastapi
 uvicorn[standard]
+transformers
+accelerate
+torch --index-url https://download.pytorch.org/whl/cpu