Remove handler.py

handler.py

@@ -1,102 +0,0 @@
-import json
-from typing import Any
-
-import torch
-from peft import PeftModel
-from transformers import AutoModelForCausalLM, AutoTokenizer
-
-
-class EndpointHandler:
-    """
-    Custom Inference Endpoint handler for adapter-only LoRA repos.
-    """
-
-    def __init__(self, model_dir: str, **kwargs: Any):
-        adapter_cfg_path = f"{model_dir}/adapter_config.json"
-        with open(adapter_cfg_path, "r", encoding="utf-8") as f:
-            adapter_cfg = json.load(f)
-
-        base_model_id = adapter_cfg.get("base_model_name_or_path", "Qwen/Qwen3-4B")
-
-        # Endpoints are usually more stable with the canonical base model id.
-        if "unsloth" in base_model_id and "bnb-4bit" in base_model_id:
-            # Try to infer the base model if it's an unsloth bnb-4bit one
-            # For Qwen3-4B-unsloth-bnb-4bit, the base is likely Qwen/Qwen3-4B
-            if "Qwen3-4B" in base_model_id:
-                base_model_id = "Qwen/Qwen3-4B"
-            elif "Qwen2.5" in base_model_id:
-                base_model_id = "Qwen/Qwen2.5-7B" # Or whatever the base is
-
-        self.tokenizer = AutoTokenizer.from_pretrained(model_dir, use_fast=True)
-        dtype = torch.float16 if torch.cuda.is_available() else torch.float32
-
-        base_model = AutoModelForCausalLM.from_pretrained(
-            base_model_id,
-            torch_dtype=dtype,
-            device_map="auto" if torch.cuda.is_available() else None,
-            low_cpu_mem_usage=True,
-        )
-        self.model = PeftModel.from_pretrained(base_model, model_dir)
-        self.model.eval()
-        if not torch.cuda.is_available():
-            self.model.to("cpu")
-
-    def _format_prompt(self, inputs: Any) -> str:
-        if isinstance(inputs, str):
-            return inputs
-
-        # Support chat-style inputs:
-        # [{"role":"system","content":"..."},{"role":"user","content":"..."}]
-        if isinstance(inputs, list) and inputs and isinstance(inputs[0], dict) and "role" in inputs[0]:
-            try:
-                return self.tokenizer.apply_chat_template(
-                    inputs,
-                    add_generation_prompt=True,
-                    tokenize=False,
-                    enable_thinking=False,
-                )
-            except TypeError:
-                return self.tokenizer.apply_chat_template(
-                    inputs,
-                    add_generation_prompt=True,
-                    tokenize=False,
-                )
-
-        if isinstance(inputs, dict):
-            return inputs.get("prompt") or inputs.get("text") or json.dumps(inputs)
-
-        return str(inputs)
-
-    def __call__(self, data: Any) -> dict[str, str]:
-        payload = data if isinstance(data, dict) else {"inputs": data}
-        params = payload.get("parameters", {}) or {}
-
-        prompt = self._format_prompt(payload.get("inputs", ""))
-        max_new_tokens = int(params.get("max_new_tokens", 128))
-        temperature = float(params.get("temperature", 0.2))
-        top_p = float(params.get("top_p", 0.95))
-        top_k = int(params.get("top_k", 0))
-        if top_k < 0:
-            top_k = 0
-
-        enc = self.tokenizer([prompt], return_tensors="pt")
-        device = next(self.model.parameters()).device
-        enc = {k: v.to(device) for k, v in enc.items()}
-
-        with torch.no_grad():
-            out = self.model.generate(
-                **enc,
-                max_new_tokens=max_new_tokens,
-                do_sample=temperature > 0,
-                temperature=max(temperature, 1e-5),
-                top_p=top_p,
-                top_k=top_k,
-                eos_token_id=self.tokenizer.eos_token_id,
-                pad_token_id=self.tokenizer.pad_token_id,
-            )
-
-        generated_text = self.tokenizer.decode(
-            out[0][enc["input_ids"].shape[1]:],
-            skip_special_tokens=True,
-        )
-        return {"generated_text": generated_text}