Update upload_suggestions.py

upload_suggestions.py

@@ -10,13 +10,13 @@ from huggingface_hub import HfApi, hf_hub_download
 HF_TOKEN = os.environ.get("HF_TOKEN")
 REPO_ID = "axxam/libretranslate-suggestions"
 DEST_PATH_IN_REPO = "suggestions.json"
-DEST_CSV_PATH_IN_REPO = "suggestions.csv"
+DEST_CSV_IN_REPO = "suggestions.csv"
 REPO_TYPE = "dataset"
+
 JSON_OUTPUT_PATH = "/tmp/suggestions.json"
 CSV_OUTPUT_PATH = "/tmp/suggestions.csv"
 CHECKSUM_FILE = "/tmp/.last_suggestions_checksum"
 
-# Known or expected locations for the DB
 possible_paths = [
     "/app/db/suggestions.db",
     "/app/suggestions.db",
@@ -90,39 +90,36 @@ def merge_with_existing(suggestions, existing_json_path):
 
     changed = False
     for s in suggestions:
-        s_clean = {
-            "source_text": s["source_text"],
-            "suggested_text": s["suggested_text"],
-            "source_lang": s["source_lang"],
-            "target_lang": s["target_lang"],
-        }
-
-        existing_entry = existing.get(s["id"])
-        if not existing_entry:
+        if s["id"] not in existing:
             changed = True
-            existing[s["id"]] = {**s_clean, "timestamp": datetime.now().isoformat()}
+            existing[s["id"]] = {
+                "source_text": s["source_text"],
+                "suggested_text": s["suggested_text"],
+                "source_lang": s["source_lang"],
+                "target_lang": s["target_lang"],
+                "timestamp": datetime.now().isoformat()
+            }
 
     if not changed:
         print("No new suggestions — skipping write/upload.")
         return None
 
-    # Write merged JSON
+    # Create final list
     final = []
     for id_, data in existing.items():
         final.append({**data, "id": id_})
 
+    # Write JSON
     with open(JSON_OUTPUT_PATH, "w", encoding="utf-8") as f:
         json.dump(final, f, indent=2, ensure_ascii=False)
 
-    # Write merged CSV
-    with open(CSV_OUTPUT_PATH, "w", encoding="utf-8", newline='') as csvfile:
-        fieldnames = ["id", "source_text", "suggested_text", "source_lang", "target_lang", "timestamp"]
-        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+    # Write CSV
+    with open(CSV_OUTPUT_PATH, "w", encoding="utf-8", newline="") as f:
+        writer = csv.DictWriter(f, fieldnames=["id", "source_text", "suggested_text", "source_lang", "target_lang", "timestamp"])
         writer.writeheader()
-        for row in final:
-            writer.writerow(row)
+        writer.writerows(final)
 
-    return JSON_OUTPUT_PATH, CSV_OUTPUT_PATH
+    return JSON_OUTPUT_PATH  # Return one of them for checksum comparison
 
 def get_checksum(filepath):
     if not os.path.exists(filepath):
@@ -142,11 +139,10 @@ def upload_if_updated(json_path, csv_path):
             old_checksum = f.read().strip()
 
     if new_checksum != old_checksum:
-        print("Uploading updated files to Hugging Face...")
+        print("Uploading updated suggestions.json and suggestions.csv to Hugging Face...")
         try:
             api = HfApi()
 
-            # Upload JSON
             api.upload_file(
                 path_or_fileobj=json_path,
                 path_in_repo=DEST_PATH_IN_REPO,
@@ -154,16 +150,13 @@ def upload_if_updated(json_path, csv_path):
                 repo_type=REPO_TYPE,
                 token=HF_TOKEN
             )
-
-            # Upload CSV
-            if csv_path and os.path.exists(csv_path):
-                api.upload_file(
-                    path_or_fileobj=csv_path,
-                    path_in_repo=DEST_CSV_PATH_IN_REPO,
-                    repo_id=REPO_ID,
-                    repo_type=REPO_TYPE,
-                    token=HF_TOKEN
-                )
+            api.upload_file(
+                path_or_fileobj=csv_path,
+                path_in_repo=DEST_CSV_IN_REPO,
+                repo_id=REPO_ID,
+                repo_type=REPO_TYPE,
+                token=HF_TOKEN
+            )
 
             with open(CHECKSUM_FILE, "w") as f:
                 f.write(new_checksum)
@@ -191,10 +184,10 @@ def main():
         return
 
     existing_path = download_existing_suggestions()
-    result = merge_with_existing(suggestions, existing_path)
-    if result:
-        merged_json, merged_csv = result
-        upload_if_updated(merged_json, merged_csv)
+    merged_json = merge_with_existing(suggestions, existing_path)
+
+    if merged_json:
+        upload_if_updated(JSON_OUTPUT_PATH, CSV_OUTPUT_PATH)
 
 if __name__ == "__main__":
     main()