File size: 1,389 Bytes
42c3583 a9f105b 42c3583 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 | import gradio as gr
from transformers import AutoProcessor, AutoModelForCausalLM
TARGET_MODEL_ID = "google/gemma-4-E2B-it"
ASSISTANT_MODEL_ID = "google/gemma-4-E2B-it-assistant"
# Target Model
processor = AutoProcessor.from_pretrained(TARGET_MODEL_ID)
target_model = AutoModelForCausalLM.from_pretrained(
TARGET_MODEL_ID,
dtype="auto",
device_map="auto",
)
# Assistant Model (the drafter)
assistant_model = AutoModelForCausalLM.from_pretrained(
ASSISTANT_MODEL_ID,
dtype="auto",
device_map="auto",
)
def greet(name):
# Prompt
messages = [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": name},
]
# Process input
text = processor.apply_chat_template(
messages,
tokenize=False,
add_generation_prompt=True,
)
inputs = processor(text=text, return_tensors="pt").to(target_model.device)
input_len = inputs["input_ids"].shape[-1]
# Generate output
outputs = target_model.generate(
**inputs,
assistant_model=assistant_model,
max_new_tokens=256,
)
response = processor.decode(outputs[0][input_len:], skip_special_tokens=False)
# Parse output
textofinal =processor.parse_response(response)
return textofinal
demo = gr.Interface(fn=greet, inputs="text", outputs="text")
demo.launch()
|