add

README.md

@@ -10,4 +10,17 @@ pinned: false
 short_description: 'Language Extractor: Polyglot Tagger'
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+## Offline FLEURS cache
+
+The demo can now pull examples from a local, text-only FLEURS parquet cache instead of relying on Tatoeba.
+
+Build the cache once with:
+
+```bash
+./.venv/bin/python fleurs_cache.py
+```
+
+That downloads the FLEURS TSV metadata, dedupes repeated sentences, drops unused columns, and writes a reusable lean parquet file at `data/fleurs/fleurs_text_only.parquet`.
+Run it once while online; after that, the app reads only the local parquet and does not need the network.

app.py

@@ -12,6 +12,7 @@ import pandas as pd
 import gradio as gr
 from transformers import AutoModelForTokenClassification, AutoTokenizer, pipeline
 
+from fleurs_cache import fetch_random_fleurs_sentence, fetch_random_fleurs_sentence_mix
 from language import ALL_LANGS, LANG_ISO2_TO_ISO3
 from tatoeba import fetch_random_tatoeba_sentence, fetch_random_tatoeba_sentence_mix
 
@@ -152,11 +153,11 @@ def build_ui_state(
     }
 
 
-def build_tatoeba_validation(
+def build_example_validation(
     classifier_scores: dict[str, float],
     expected_langs: list[str],
 ) -> dict[str, Any]:
-    """Compare derived scores against known Tatoeba source languages."""
+    """Compare derived scores against known source languages."""
     expected_langs = [lang for lang in expected_langs if lang]
     expected_set = set(expected_langs)
     top_lang = next(iter(classifier_scores), None)
@@ -181,8 +182,8 @@ def build_tatoeba_validation(
     }
 
 
-def render_tatoeba_validation_html(validation: dict[str, Any]) -> str:
-    """Render a compact validation card for Tatoeba examples."""
+def render_validation_html(validation: dict[str, Any], *, source_label: str) -> str:
+    """Render a compact validation card for a labeled example source."""
     if not validation:
         return ""
 
@@ -199,7 +200,7 @@ def render_tatoeba_validation_html(validation: dict[str, Any]) -> str:
 
     return f"""
     <div class="validation-strip">
-      <div class="validation-kicker">Tatoeba validation</div>
+      <div class="validation-kicker">{source_label} validation</div>
       <div class="validation-main">{validation_score:.1%}</div>
       <div class="validation-status {status_class}">{status_label}</div>
       <div class="validation-subtitle">
@@ -214,6 +215,19 @@ def render_tatoeba_validation_html(validation: dict[str, Any]) -> str:
     """
 
 
+def build_tatoeba_validation(
+    classifier_scores: dict[str, float],
+    expected_langs: list[str],
+) -> dict[str, Any]:
+    """Backward-compatible wrapper for existing Tatoeba callers."""
+    return build_example_validation(classifier_scores, expected_langs)
+
+
+def render_tatoeba_validation_html(validation: dict[str, Any]) -> str:
+    """Backward-compatible wrapper for existing Tatoeba callers."""
+    return render_validation_html(validation, source_label="Tatoeba")
+
+
 def render_prediction_summary(
     *,
     text: str,
@@ -453,7 +467,7 @@ def load_random_tatoeba_example() -> tuple[str, str, pd.DataFrame, dict[str, Any
     sentence = fetch_random_tatoeba_sentence()
     text = sentence["text"]
     summary, spans, raw, ui_state, _, *chip_updates = predict(text)
-    validation = build_tatoeba_validation(
+    validation = build_example_validation(
         raw.get("classifier_scores", {}),
         [sentence.get("lang_iso2", "")],
     )
@@ -466,7 +480,7 @@ def load_random_tatoeba_example() -> tuple[str, str, pd.DataFrame, dict[str, Any
         "sentence_lang_iso3": sentence.get("lang_iso3"),
         "tatoeba_validation": validation,
     }
-    validation_html = render_tatoeba_validation_html(validation)
+    validation_html = render_validation_html(validation, source_label="Tatoeba")
     summary = render_prediction_summary(
         text=text,
         selected_lang=ui_state.get("selected_lang", raw.get("selected_lang", "")),
@@ -483,7 +497,7 @@ def load_random_tatoeba_mix_example() -> tuple[str, str, pd.DataFrame, dict[str,
     mix = fetch_random_tatoeba_sentence_mix()
     text = mix["text"]
     summary, spans, raw, ui_state, _, *chip_updates = predict(text)
-    validation = build_tatoeba_validation(
+    validation = build_example_validation(
         raw.get("classifier_scores", {}),
         mix.get("langs", []),
     )
@@ -496,7 +510,87 @@ def load_random_tatoeba_mix_example() -> tuple[str, str, pd.DataFrame, dict[str,
         "sentences": mix["sentences"],
         "tatoeba_validation": validation,
     }
-    validation_html = render_tatoeba_validation_html(validation)
+    validation_html = render_validation_html(validation, source_label="Tatoeba")
+    summary = render_prediction_summary(
+        text=text,
+        selected_lang=ui_state.get("selected_lang", raw.get("selected_lang", "")),
+        dominant_lang=ui_state.get("dominant_lang", raw.get("selected_lang", "")),
+        lang_stats=ui_state.get("lang_stats", {}),
+        classifier_scores=ui_state.get("classifier_scores", {}),
+        overall_confidence=float(ui_state.get("overall_confidence", 0.0)),
+        ignored_artifacts=int(ui_state.get("ignored_artifacts", 0)),
+    )
+    return text, summary, spans, raw, ui_state, validation_html, *chip_updates
+
+
+def load_random_fleurs_example() -> tuple[str, str, pd.DataFrame, dict[str, Any], dict[str, Any], str]:
+    try:
+        sentence = fetch_random_fleurs_sentence()
+    except FileNotFoundError as exc:
+        empty = pd.DataFrame(columns=["token", "language", "score", "start", "end"])
+        message = (
+            "<div class='empty-state'>"
+            f"{exc}"
+            "</div>"
+        )
+        return "", message, empty, {}, {}, "", *[gr.update(value="", visible=False) for _ in range(6)]
+    text = sentence["text"]
+    summary, spans, raw, ui_state, _, *chip_updates = predict(text)
+    validation = build_example_validation(
+        raw.get("classifier_scores", {}),
+        [sentence.get("lang_iso2", "")],
+    )
+    raw = {
+        **raw,
+        "source": "fleurs",
+        "fleurs_sentence_id": sentence.get("fleurs_id"),
+        "fleurs_split": sentence.get("split"),
+        "fleurs_source_lang": sentence.get("source_lang"),
+        "fleurs_model_lang": sentence.get("model_lang"),
+        "fleurs_language": sentence.get("language"),
+        "fleurs_lang_group": sentence.get("lang_group"),
+        "fleurs_validation": validation,
+    }
+    validation_html = render_validation_html(validation, source_label="FLEURS")
+    summary = render_prediction_summary(
+        text=text,
+        selected_lang=ui_state.get("selected_lang", raw.get("selected_lang", "")),
+        dominant_lang=ui_state.get("dominant_lang", raw.get("selected_lang", "")),
+        lang_stats=ui_state.get("lang_stats", {}),
+        classifier_scores=ui_state.get("classifier_scores", {}),
+        overall_confidence=float(ui_state.get("overall_confidence", 0.0)),
+        ignored_artifacts=int(ui_state.get("ignored_artifacts", 0)),
+    )
+    return text, summary, spans, raw, ui_state, validation_html, *chip_updates
+
+
+def load_random_fleurs_mix_example() -> tuple[str, str, pd.DataFrame, dict[str, Any], dict[str, Any], str]:
+    try:
+        mix = fetch_random_fleurs_sentence_mix()
+    except FileNotFoundError as exc:
+        empty = pd.DataFrame(columns=["token", "language", "score", "start", "end"])
+        message = (
+            "<div class='empty-state'>"
+            f"{exc}"
+            "</div>"
+        )
+        return "", message, empty, {}, {}, "", *[gr.update(value="", visible=False) for _ in range(6)]
+    text = mix["text"]
+    summary, spans, raw, ui_state, _, *chip_updates = predict(text)
+    validation = build_example_validation(
+        raw.get("classifier_scores", {}),
+        mix.get("langs", []),
+    )
+    raw = {
+        **raw,
+        "source": "fleurs-mix",
+        "lang_count": mix["lang_count"],
+        "sentence_langs": mix["langs"],
+        "sentence_lang_iso3s": mix["lang_iso3s"],
+        "sentences": mix["sentences"],
+        "fleurs_validation": validation,
+    }
+    validation_html = render_validation_html(validation, source_label="FLEURS")
     summary = render_prediction_summary(
         text=text,
         selected_lang=ui_state.get("selected_lang", raw.get("selected_lang", "")),
@@ -887,7 +981,7 @@ with gr.Blocks(title="Polyglot Tagger Studio") as demo:
             )
             validation_strip = gr.HTML()
             gr.Markdown(
-                "Use the Tatoeba buttons for fresh examples, or paste your own text."
+                "Use the FLEURS buttons for fresh examples, or paste your own text."
             )
             with gr.Row(elem_classes=["action-strip"]):
                 with gr.Column(scale=1, min_width=0):
@@ -896,9 +990,9 @@ with gr.Blocks(title="Polyglot Tagger Studio") as demo:
                     clear_btn = gr.Button("Clear", elem_classes=["action-btn", "action-clear"])
             with gr.Row(elem_classes=["action-strip", "action-stack"]):
                 with gr.Column(scale=1, min_width=0):
-                    random_btn = gr.Button("Random sentence", elem_classes=["action-btn", "action-secondary"])
+                    random_btn = gr.Button("Random FLEURS sentence", elem_classes=["action-btn", "action-secondary"])
                 with gr.Column(scale=1, min_width=0):
-                    random_mix_btn = gr.Button("Random mix", elem_classes=["action-btn", "action-secondary"])
+                    random_mix_btn = gr.Button("Random FLEURS mix", elem_classes=["action-btn", "action-secondary"])
         with gr.Column(scale=7):
             summary = gr.HTML()
             prediction_state = gr.State({})
@@ -929,16 +1023,16 @@ with gr.Blocks(title="Polyglot Tagger Studio") as demo:
         api_name="analyze",
     )
     random_btn.click(
-        fn=load_random_tatoeba_example,
+        fn=load_random_fleurs_example,
         inputs=None,
         outputs=[input_text, summary, spans, raw, prediction_state, validation_strip, chip_0, chip_1, chip_2, chip_3, chip_4, chip_5],
-        api_name="random_tatoeba_sentence",
+        api_name="random_fleurs_sentence",
     )
     random_mix_btn.click(
-        fn=load_random_tatoeba_mix_example,
+        fn=load_random_fleurs_mix_example,
         inputs=None,
         outputs=[input_text, summary, spans, raw, prediction_state, validation_strip, chip_0, chip_1, chip_2, chip_3, chip_4, chip_5],
-        api_name="random_tatoeba_mix",
+        api_name="random_fleurs_mix",
     )
     input_text.submit(
         fn=predict,

data/fleurs/fleurs_text_only.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f348bba789b3a7f051f586b6424ba445eb122be69d91f2b8a23db4c3bafae02
+size 20278131

requirements.txt

@@ -2,3 +2,5 @@ torch
 transformers
 gradio
 pandas
+datasets
+pyarrow