app.py

"""Token counter — count tokens for text across multiple model families.

Uses agentfit's pluggable counter. Falls back to char-based estimates when
exact tokenizers aren't available.
"""

import gradio as gr
from agentfit import count


MODELS = [
    "claude-sonnet-4-6",
    "claude-haiku-4-5",
    "gpt-5",
    "gpt-4.1",
    "default",
]


def count_tokens(text: str, comparison: str):
    """Count tokens for the given text across selected models."""
    if not text.strip():
        return "_Enter some text to count tokens._"

    models = [m.strip() for m in comparison.split(",") if m.strip()] if comparison else MODELS
    rows = ["| Model | Tokens | Chars/token |", "|---|---:|---:|"]
    char_count = len(text)
    for m in models:
        try:
            n = count([{"role": "user", "content": text}], model=m)
            ratio = f"{char_count / n:.2f}" if n else "—"
            rows.append(f"| `{m}` | {n} | {ratio} |")
        except Exception as e:
            rows.append(f"| `{m}` | — | error: {e} |")
    return "\n".join(rows) + f"\n\n**Input:** {char_count} chars · {len(text.split())} words"


with gr.Blocks(title="Token Counter — across model families", theme=gr.themes.Soft()) as demo:
    gr.Markdown(
        """
        # Token Counter

        Paste any text and see how it tokenizes across Claude, GPT, and other model families.
        Powered by [`agentfit`](https://pypi.org/project/agentfit-py/) — pure Python, no API calls.

        💡 Useful for: budgeting context windows, comparing tokenizer efficiency for non-English text, sanity-checking your own counter.
        """
    )

    with gr.Row():
        with gr.Column():
            txt = gr.Textbox(
                value="The quick brown fox jumps over the lazy dog.",
                label="Text",
                lines=10,
                placeholder="Paste text here...",
            )
            models_in = gr.Textbox(
                value=", ".join(MODELS),
                label="Models (comma-separated)",
            )
            btn = gr.Button("Count", variant="primary")
        out = gr.Markdown()
    btn.click(count_tokens, inputs=[txt, models_in], outputs=out)

    gr.Examples(
        examples=[
            ["Hello, world!", "claude-sonnet-4-6, gpt-5, default"],
            ["你好世界 こんにちは 안녕하세요", "claude-sonnet-4-6, gpt-5, default"],
            ["🚀🎉🤖✨", "claude-sonnet-4-6, gpt-5, default"],
            ["function add(a, b) {\n  return a + b;\n}", "claude-sonnet-4-6, gpt-5"],
        ],
        inputs=[txt, models_in],
    )

    gr.Markdown(
        """
        ---
        Part of [The Agent Reliability Stack](https://mukundakatta.github.io/agent-stack/) · MIT licensed
        """
    )


if __name__ == "__main__":
    demo.launch(server_name="0.0.0.0", server_port=7860)