Update script_gen.py

script_gen.py

@@ -2,10 +2,11 @@
 VoiceVerse AI — Script Generation Module.
 
 Generates spoken-style scripts from retrieved document chunks
-using Mistral-7B-Instruct via the Hugging Face Inference API.
+using SmolLM3-3B via the Hugging Face Inference API.
 
 Design decisions:
-  - Serverless HF Inference API avoids loading a 14 GB model locally
+  - Serverless HF Inference API avoids loading a large model locally
+  - SmolLM3-3B is deployed on the free hf-inference provider
   - Prompt template enforces podcast/narration structure
   - Max 1024 new tokens keeps scripts a reasonable length for TTS
   - Temperature 0.7 balances creativity with factual grounding
@@ -16,7 +17,7 @@ from huggingface_hub import InferenceClient
 from utils import logger
 
 # — Configuration ————————————————————————————————————
-MODEL_ID = "HuggingFaceH4/zephyr-7b-beta"
+MODEL_ID = "HuggingFaceTB/SmolLM3-3B"
 MAX_NEW_TOKENS = 1024
 TEMPERATURE = 0.7
 
@@ -80,7 +81,7 @@ def generate_script(
     # Combine chunks into a single context block
     context = "\n\n".join(context_chunks)
 
-    # Truncate if too long (Mistral context window is 32k, but we want focused input)
+    # Truncate if too long
     max_context_chars = 6000
     if len(context) > max_context_chars:
         context = context[:max_context_chars]
@@ -94,22 +95,28 @@ def generate_script(
 
     client = _get_client()
 
-    # Call the model using chat completion (conversational task)
-    response = client.chat_completion(
+    # Call the model using text_generation
+    response = client.text_generation(
+        prompt=_format_prompt(SYSTEM_PROMPT, user_message),
         model=MODEL_ID,
-        messages=[
-            {"role": "system", "content": SYSTEM_PROMPT},
-            {"role": "user", "content": user_message},
-        ],
-        max_tokens=MAX_NEW_TOKENS,
+        max_new_tokens=MAX_NEW_TOKENS,
         temperature=TEMPERATURE,
         top_p=0.9,
+        do_sample=True,
     )
 
-    script = response.choices[0].message.content.strip()
+    script = response.strip()
 
     if not script:
         raise RuntimeError("The model returned an empty script. Please try again.")
 
     logger.info("Script generated: %d characters", len(script))
-    return script
+    return script
+
+
+def _format_prompt(system: str, user: str) -> str:
+    """
+    Format a prompt for SmolLM3 instruction following.
+    Uses a simple system + user format.
+    """
+    return f"<|im_start|>system\n{system}<|im_end|>\n<|im_start|>user\n{user}<|im_end|>\n<|im_start|>assistant\n"