Update upload_suggestions.py

upload_suggestions.py

@@ -9,13 +9,14 @@ from huggingface_hub import HfApi, hf_hub_download
 # Settings
 HF_TOKEN = os.environ.get("HF_TOKEN")
 REPO_ID = "axxam/libretranslate-suggestions"
-DEST_PATH_IN_REPO = "suggestions.json"
-DEST_CSV_IN_REPO = "suggestions.csv"
+DEST_JSON_PATH_IN_REPO = "suggestions.json"
+DEST_CSV_PATH_IN_REPO = "suggestions.csv"
 REPO_TYPE = "dataset"
 
 JSON_OUTPUT_PATH = "/tmp/suggestions.json"
 CSV_OUTPUT_PATH = "/tmp/suggestions.csv"
-CHECKSUM_FILE = "/tmp/.last_suggestions_checksum"
+CHECKSUM_FILE_JSON = "/tmp/.last_suggestions_json_checksum"
+CHECKSUM_FILE_CSV = "/tmp/.last_suggestions_csv_checksum"
 
 possible_paths = [
     "/app/db/suggestions.db",
@@ -56,12 +57,12 @@ def extract_suggestions(db_path):
         print(f"SQLite error: {e}")
     return suggestions
 
-def download_existing_suggestions():
+def download_existing_json():
     try:
         path = hf_hub_download(
             repo_id=REPO_ID,
             repo_type=REPO_TYPE,
-            filename=DEST_PATH_IN_REPO,
+            filename=DEST_JSON_PATH_IN_REPO,
             token=HF_TOKEN,
             local_dir="/tmp"
         )
@@ -90,36 +91,43 @@ def merge_with_existing(suggestions, existing_json_path):
 
     changed = False
     for s in suggestions:
-        if s["id"] not in existing:
+        s_clean = {
+            "source_text": s["source_text"],
+            "suggested_text": s["suggested_text"],
+            "source_lang": s["source_lang"],
+            "target_lang": s["target_lang"],
+        }
+
+        existing_entry = existing.get(s["id"])
+        if not existing_entry:
             changed = True
-            existing[s["id"]] = {
-                "source_text": s["source_text"],
-                "suggested_text": s["suggested_text"],
-                "source_lang": s["source_lang"],
-                "target_lang": s["target_lang"],
-                "timestamp": datetime.now().isoformat()
-            }
+            existing[s["id"]] = {**s_clean, "timestamp": datetime.now().isoformat()}
 
     if not changed:
         print("No new suggestions — skipping write/upload.")
         return None
 
-    # Create final list
+    # Write merged JSON
     final = []
     for id_, data in existing.items():
         final.append({**data, "id": id_})
 
-    # Write JSON
     with open(JSON_OUTPUT_PATH, "w", encoding="utf-8") as f:
         json.dump(final, f, indent=2, ensure_ascii=False)
 
-    # Write CSV
-    with open(CSV_OUTPUT_PATH, "w", encoding="utf-8", newline="") as f:
-        writer = csv.DictWriter(f, fieldnames=["id", "source_text", "suggested_text", "source_lang", "target_lang", "timestamp"])
-        writer.writeheader()
-        writer.writerows(final)
+    # Also write CSV
+    write_csv(final, CSV_OUTPUT_PATH)
+
+    return JSON_OUTPUT_PATH
 
-    return JSON_OUTPUT_PATH  # Return one of them for checksum comparison
+def write_csv(suggestions, csv_path):
+    with open(csv_path, "w", newline="", encoding="utf-8") as csvfile:
+        writer = csv.DictWriter(csvfile, fieldnames=[
+            "id", "source_text", "suggested_text", "source_lang", "target_lang", "timestamp"
+        ])
+        writer.writeheader()
+        for item in suggestions:
+            writer.writerow(item)
 
 def get_checksum(filepath):
     if not os.path.exists(filepath):
@@ -127,45 +135,35 @@ def get_checksum(filepath):
     with open(filepath, "rb") as f:
         return hashlib.md5(f.read()).hexdigest()
 
-def upload_if_updated(json_path, csv_path):
-    if not json_path or not os.path.exists(json_path):
+def upload_if_updated(filepath, dest_path, checksum_file):
+    if not filepath or not os.path.exists(filepath):
         return
 
-    new_checksum = get_checksum(json_path)
+    new_checksum = get_checksum(filepath)
     old_checksum = None
 
-    if os.path.exists(CHECKSUM_FILE):
-        with open(CHECKSUM_FILE, "r") as f:
+    if os.path.exists(checksum_file):
+        with open(checksum_file, "r") as f:
             old_checksum = f.read().strip()
 
     if new_checksum != old_checksum:
-        print("Uploading updated suggestions.json and suggestions.csv to Hugging Face...")
+        print(f"Uploading updated {os.path.basename(dest_path)} to Hugging Face...")
         try:
             api = HfApi()
-
-            api.upload_file(
-                path_or_fileobj=json_path,
-                path_in_repo=DEST_PATH_IN_REPO,
-                repo_id=REPO_ID,
-                repo_type=REPO_TYPE,
-                token=HF_TOKEN
-            )
             api.upload_file(
-                path_or_fileobj=csv_path,
-                path_in_repo=DEST_CSV_IN_REPO,
+                path_or_fileobj=filepath,
+                path_in_repo=dest_path,
                 repo_id=REPO_ID,
                 repo_type=REPO_TYPE,
                 token=HF_TOKEN
             )
-
-            with open(CHECKSUM_FILE, "w") as f:
+            with open(checksum_file, "w") as f:
                 f.write(new_checksum)
-
-            print(f"Upload successful at {datetime.now().isoformat()}")
+            print(f"Upload successful: {dest_path} at {datetime.now().isoformat()}")
         except Exception as e:
-            print("Upload failed:", e)
+            print(f"Upload failed for {dest_path}:", e)
     else:
-        print("No changes in file — skipping upload.")
+        print(f"No changes in {os.path.basename(dest_path)} — skipping upload.")
 
 def main():
     print(f"===== Application Startup at {datetime.now().isoformat()} =====")
@@ -183,11 +181,12 @@ def main():
         print("No suggestions found — skipping.")
         return
 
-    existing_path = download_existing_suggestions()
+    existing_path = download_existing_json()
     merged_json = merge_with_existing(suggestions, existing_path)
 
     if merged_json:
-        upload_if_updated(JSON_OUTPUT_PATH, CSV_OUTPUT_PATH)
+        upload_if_updated(JSON_OUTPUT_PATH, DEST_JSON_PATH_IN_REPO, CHECKSUM_FILE_JSON)
+        upload_if_updated(CSV_OUTPUT_PATH, DEST_CSV_PATH_IN_REPO, CHECKSUM_FILE_CSV)
 
 if __name__ == "__main__":
     main()