app.py

import gradio as gr
from llama_cpp import Llama


llm = Llama.from_pretrained(
    repo_id="maddes8cht/smallcloudai-Refact-1_6B-fim-gguf",
    filename="smallcloudai-Refact-1_6B-fim-Q4_K_M.gguf",
    numa=True,
    seed=-1,
    n_batch=1024,
    n_ctx=4095,
)

def respond(prompt: str):
    print(llm(prompt, max_tokens=64))
    return prompt

prompt = "<fim_prefix>import socket\n\ndef ping_exponential_backoff():\n    <fim_suffix>\n\nif __name__ == \"main\":\n   ping_exponential_backoff()<fim_middle>"
demo = gr.Interface(fn=respond, inputs=[gr.TextArea(prompt)], outputs=[gr.TextArea()])
demo.launch(server_name="0.0.0.0", server_port=7860)