add 4o tokenizer

app.py

@@ -4,19 +4,24 @@ import tiktoken
 
 anthropic = Anthropic()
 gpt_35_encoder = tiktoken.get_encoding("cl100k_base")
+gpt_4o_encoder = tiktoken.get_encoding("o200k_base")
 
 def count_anthropic_tokens(text: str) -> int:
     return anthropic.count_tokens(text)
 
-def count_gpt_tokens(text: str) -> int:
+def count_gpt_tokens_35(text: str) -> int:
     return len(gpt_35_encoder.encode(text))
 
+def count_gpt_tokens_4o(text: str) -> int:
+    return len(gpt_4o_encoder.encode(text))
+
 def tokenize(text: str) -> tuple:
     anthropic_count = count_anthropic_tokens(text)
-    gpt_tokens_count = count_gpt_tokens(text)
+    gpt_tokens_count_35 = count_gpt_tokens_35(text)
+    gpt_tokens_count_4o = count_gpt_tokens_4o(text)
     char_count = len(text)
 
-    return anthropic_count, gpt_tokens_count, char_count
+    return anthropic_count, gpt_tokens_count_35, gpt_tokens_count_4o, char_count
 
 counter = gr.Interface(
     title="Token and Character Counter",
@@ -25,6 +30,7 @@ counter = gr.Interface(
     outputs=[
         gr.Number(label="Anthropic API (Claude) Token Count", interactive=False),
         gr.Number(label="OpenAI API (GPT-3.5 / GPT-4) Token Count", interactive=False),
+        gr.Number(label="OpenAI API (GPT-4o) Token Count", interactive=False),
         gr.Number(label="Character Count", interactive=False)
     ],
     live=True,