Create app.py

app.py

@@ -0,0 +1,241 @@
+#!/usr/bin/env python3
+"""
+Kabyle-letter checker – Hugging Face Space edition
+Run locally:  python app.py
+On Spaces the container starts this file automatically.
+"""
+
+import io, re, unicodedata, uuid
+from collections import Counter
+from pathlib import Path
+
+from flask import (Flask, render_template_string, request, send_file,
+                   url_for, flash, redirect)
+from tqdm import tqdm
+
+# ------------------------------------------------------------------ constants
+ALLOWED = set(
+    "a b c d e f g h i j k l m n o p q r s t u v w x y z "
+    "č ḍ ǧ ḥ ɣ ṛ ṣ ṭ ɛ ẓ".split()
+)
+UPLOAD_DIR = Path("uploads")
+CSV_DIR    = Path("csv")
+UPLOAD_DIR.mkdir(exist_ok=True)
+CSV_DIR.mkdir(exist_ok=True)
+
+# ------------------------------------------------------------------ Flask app
+app = Flask(__name__)
+app.secret_key = uuid.uuid4().hex       # random on every restart
+
+# ------------------------------------------------------------------ helpers
+def _is_ok(ch):  # quick reusable test
+    return ch.isalpha() and (ch.lower() in ALLOWED)
+
+def analyse(path: Path):
+    """
+    Returns:
+        counter   – Counter of illegal chars
+        sentences – list[dict] with keys
+                     line_nr, sentence, bad_chars (set), highlights (HTML)
+    """
+    counter, sentences = Counter(), []
+    with path.open(encoding="utf-8") as fh:
+        for line_nr, sentence in enumerate(tqdm(fh, unit=" lines", desc="Analysing"), 1):
+            bad = {ch for ch in sentence if not _is_ok(ch) if ch.isalpha()}
+            if bad:
+                # build red-span version
+                highlighted = re.sub(
+                    r"([^\s])",
+                    lambda m: (f'<span style="color:red">{m.group(0)}</span>'
+                               if not _is_ok(m.group(0)) and m.group(0).isalpha()
+                               else m.group(0)),
+                    sentence,
+                )
+                sentences.append({
+                    "line_nr": line_nr,
+                    "sentence": sentence.rstrip(),
+                    "bad_chars": bad,
+                    "highlights": highlighted.rstrip(),
+                })
+                for ch in bad:
+                    counter[ch] += 1
+    return counter, sentences
+
+# ------------------------------------------------------------------ routes
+@app.route("/", methods=["GET", "POST"])
+def index():
+    if request.method == "POST":
+        file = request.files.get("file")
+        if not file or file.filename == "":
+            flash("No file selected")
+            return redirect(request.url)
+
+        uid   = uuid.uuid4().hex
+        fpath = UPLOAD_DIR / f"{uid}.txt"
+        file.save(fpath)
+
+        counter, sentences = analyse(fpath)
+
+        # CSV
+        csv_io = io.StringIO()
+        csv_io.write("char,codepoint,count,unicode_name\n")
+        for ch, freq in counter.most_common():
+            csv_io.write(f"{ch},U+{ord(ch):04X},{freq},{unicodedata.name(ch,'')}\n")
+        (CSV_DIR / f"{uid}.csv").write_text(csv_io.getvalue(), encoding="utf-8")
+
+        # store sentences list for /show/<uid>
+        (CSV_DIR / f"{uid}_sentences.txt").write_text(
+            "\n".join(f'{s["line_nr"]:>6}  {s["sentence"]}' for s in sentences),
+            encoding="utf-8",
+        )
+
+        table_rows = [
+            {"char": ch, "code": f"U+{ord(ch):04X}", "count": freq,
+             "name": unicodedata.name(ch, "")}
+            for ch, freq in counter.most_common()
+        ]
+        total_lines = sum(1 for _ in fpath.open(encoding="utf-8"))
+        return render_template_string(
+            TEMPLATE_RESULT,
+            table=table_rows,
+            total_lines=total_lines,
+            csv_url=url_for("download", uid=uid),
+            show_url=url_for("show_sentences", uid=uid),
+            sent_count=len(sentences),
+        )
+    return render_template_string(TEMPLATE_INDEX)
+
+@app.route("/csv/<uid>")
+def download(uid):
+    return send_file(CSV_DIR / f"{uid}.csv", as_attachment=True,
+                     download_name="unsupported_stats.csv")
+
+@app.route("/show/<uid>")
+def show_sentences(uid):
+    """Display affected sentences with red highlights."""
+    sentences = []
+    for raw in (CSV_DIR / f"{uid}_sentences.txt").read_text(encoding="utf-8").splitlines():
+        # recover line_nr + sentence
+        line_nr, sentence = raw[:6].strip(), raw[8:]
+        bad = {ch for ch in sentence if not _is_ok(ch) if ch.isalpha()}
+        highlighted = re.sub(
+            r"([^\s])",
+            lambda m: (f'<span style="color:red;font-weight:bold">{m.group(0)}</span>'
+                       if not _is_ok(m.group(0)) and m.group(0).isalpha()
+                       else m.group(0)),
+            sentence,
+        )
+        sentences.append({
+            "line_nr": line_nr,
+            "highlights": highlighted,
+            "sentence": sentence,
+        })
+    return render_template_string(
+        TEMPLATE_SHOW,
+        sentences=sentences,
+        back_url=url_for("index"),
+    )
+
+# ------------------------------------------------------------------ templates
+TEMPLATE_INDEX = """
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <title>Tatoeba Kabyle Corpus Standardisation Checker</title>
+  <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css" rel="stylesheet">
+</head>
+<body class="bg-light">
+<div class="container py-5">
+  <h1 class="mb-3">Tatoeba Kabyle Corpus Standardisation Checker</h1>
+  <p class="lead mb-4">Upload Kabyle sentences from Tatoeba and spot letters/symbols that are <strong>not</strong> part of the official Kabyle alphabet (CLDR). Download a CSV or view highlighted sentences ready for correction.</p>
+
+  <form method="post" enctype="multipart/form-data" class="mb-4">
+    <div class="input-group">
+      <input type="file" name="file" class="form-control" required>
+      <button class="btn btn-primary" type="submit">Upload & analyse</button>
+    </div>
+  </form>
+
+  {% with msgs = get_flashed_messages() %}
+    {% if msgs %}<div class="alert alert-warning">{{ msgs[0] }}</div>{% endif %}
+  {% endwith %}
+</div>
+</body>
+</html>
+"""
+
+TEMPLATE_RESULT = """
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <title>Tatoeba Kabyle Corpus Standardisation Checker</title>
+  <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css" rel="stylesheet">
+</head>
+<body class="bg-light">
+<div class="container py-5">
+  <h1 class="mb-3">Tatoeba Kabyle Corpus Standardisation Checker</h1>
+
+  <div class="alert alert-info">Analysed {{ total_lines }} lines.
+   Found {{ sent_count }} sentence(s) with non-standard letters.</div>
+
+  <h2>Result table</h2>
+  <div class="table-responsive">
+    <table class="table table-sm table-striped align-middle">
+      <thead class="table-light">
+        <tr><th>Char</th><th>Codepoint</th><th>Count</th><th>Unicode name</th></tr>
+      </thead>
+      <tbody>
+      {% for row in table %}
+        <tr>
+          <td style="font-size:1.5rem">{{ row.char }}</td>
+          <td><code>{{ row.code }}</code></td>
+          <td>{{ row.count }}</td>
+          <td><small>{{ row.name }}</small></td>
+        </tr>
+      {% endfor %}
+      </tbody>
+    </table>
+  </div>
+
+  <a class="btn btn-outline-success" href="{{ csv_url }}">⬇ Download CSV</a>
+  <a class="btn btn-outline-primary ms-2" href="{{ show_url }}">👁 Show affected sentences</a>
+</div>
+</body>
+</html>
+"""
+
+TEMPLATE_SHOW = """
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <title>Tatoeba Kabyle Corpus Standardisation Checker</title>
+  <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css" rel="stylesheet">
+</head>
+<body class="bg-light">
+<div class="container py-5">
+  <h1 class="mb-3">Tatoeba Kabyle Corpus Standardisation Checker</h1>
+  <a class="btn btn-outline-secondary mb-3" href="{{ back_url }}">← Back</a>
+
+  <ol>
+  {% for s in sentences %}
+    <li class="mb-2">
+      <pre style="display:inline; margin:0; padding:0; border:none; background:none;">{{ s.highlights|safe }}</pre>
+      <small class="text-muted">(line {{ s.line_nr }})</small>
+      <a class="btn btn-sm btn-outline-primary ms-2"
+         href="https://tatoeba.org/en/sentences/search?from=kab&query={{ s.sentence|urlencode }}&to="
+         target="_blank" title="Search on Tatoeba" aria-label="Search">🔍</a>
+    </li>
+  {% endfor %}
+  </ol>
+</div>
+</body>
+</html>
+"""
+
+# ------------------------------------------------------------------ entrypoint
+if __name__ == "__main__":
+    # 0.0.0.0:7860 is mandatory for Hugging Face Spaces
+    app.run(host="0.0.0.0", port=7860, debug=False)