Create app.py

app.py

@@ -0,0 +1,42 @@
+import gradio as gr
+import spaces
+import torch
+from threading import Thread
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+from fastapi.responses import HTMLResponse
+from pathlib import Path
+
+app = gr.Server()
+HOME = Path(__file__).parent
+
+MODEL_ID = "HuggingFaceTB/SmolLM2-1.7B-Instruct"
+tok = AutoTokenizer.from_pretrained(MODEL_ID)
+model = AutoModelForCausalLM.from_pretrained(MODEL_ID, torch_dtype=torch.bfloat16).to("cuda")
+
+@spaces.GPU
+def _generate(text: str):
+    inputs = tok.apply_chat_template(
+        [{"role": "user", "content": f"Summarize in 3 bullets:\n\n{text}"}],
+        return_tensors="pt", add_generation_prompt=True,
+    ).to("cuda")
+    streamer = TextIteratorStreamer(tok, skip_prompt=True, skip_special_tokens=True)
+    Thread(target=model.generate, kwargs=dict(
+        inputs=inputs, streamer=streamer, max_new_tokens=300, do_sample=False,
+    )).start()
+    return streamer
+
+@app.mcp.tool(name="summarize")
+@app.api(name="summarize", concurrency_limit=1, stream_every=0.2)
+def summarize(text: str):
+    """Summarize the input text into 3 bullet points."""
+    out = ""
+    for chunk in _generate(text):
+        out += chunk
+        yield out
+
+@app.get("/", response_class=HTMLResponse)
+async def index():
+    return (HOME / "index.html").read_text(encoding="utf-8")
+
+if __name__ == "__main__":
+    app.launch(mcp_server=True)