Adding RAI checks

app.py

@@ -1,22 +1,10 @@
-import mlcroissant._src.operation_graph.operations.download as dl_mod
 import requests
 import os
-
-# Make sure the HF token is loaded
-HF_TOKEN = os.environ.get("HF_TOKEN")
-
-# Set the environment variables Croissant expects
-os.environ["CROISSANT_BASIC_AUTH_USERNAME"] = "hf_user"
-os.environ["CROISSANT_BASIC_AUTH_PASSWORD"] = HF_TOKEN or ""
-
-print("[DEBUG] HF_TOKEN is", "set" if HF_TOKEN else "missing")
-print("[DEBUG] Basic auth env set for Croissant")
-
 import gradio as gr
 import json
 import time
 import traceback
-from validation import validate_json, validate_croissant, validate_records, generate_validation_report
+from validation import validate_json, validate_croissant, validate_records, validate_rai, generate_validation_report
 
 def process_file(file):
     results = []
@@ -45,6 +33,10 @@ def process_file(file):
     records_message = records_message.replace("\n✓\n", "\n")
     results.append(("Records Generation Test", records_valid, records_message, records_status))
 
+    # Check 4: Responsible AI metadata
+    rai_valid, rai_message = validate_rai(json_data)
+    results.append(("Responsible AI Metadata", rai_valid, rai_message, "pass" if rai_valid else "error"))
+
     # Generate final report
     report = generate_validation_report(filename, json_data, results)
 
@@ -64,6 +56,7 @@ def create_ui():
         1. If the file is valid JSON
         2. If it passes Croissant schema validation
         3. If records can be generated within a reasonable time
+        4. If all required Responsible AI metadata fields are present
         """)
         
         # Track the active tab for conditional UI updates
@@ -107,7 +100,8 @@ def create_ui():
                         label="Report Content", 
                         visible=True,
                         show_copy_button=True,
-                        lines=10
+                        lines=10,
+                        elem_id="report-text-box"
                     )
         
         # Define CSS for the validation UI
@@ -287,6 +281,12 @@ def create_ui():
         .gr-accordion[data-open=true] > .label-wrap::after {
             transform: rotate(90deg);
         }
+
+        /* Prevent report textbox from bubbling scroll to the page */
+        #report-text-box textarea {
+            overflow-y: auto !important;
+            overscroll-behavior: contain;
+        }
         </style>
         """)
         
@@ -380,6 +380,10 @@ def create_ui():
                 records_valid, records_message, records_status = validate_records(json_data)
                 results.append(("Records Generation Test (Optional)", records_valid, records_message, records_status))
 
+                # Check 4: Responsible AI metadata
+                rai_valid, rai_message = validate_rai(json_data)
+                results.append(("Responsible AI Metadata", rai_valid, rai_message, "pass" if rai_valid else "error"))
+
                 # Generate report
                 report = generate_validation_report(url.split("/")[-1], json_data, results)
                 report_filename = f"report_croissant-validation_{json_data.get('name', 'unnamed')}.md"
@@ -438,11 +442,11 @@ def create_ui():
                     status_class = "status-success"
                     status_icon = "✓"
                     message_with_emoji = "✅ " + message
-                elif status == "warning":
+                elif status == "warning" and "Records" in test_name:
                     status_class = "status-warning"
                     status_icon = "?"
                     message_with_emoji = "⚠️ Could not automatically generate records. This is oftentimes not an issue (e.g. datasets could be too large or too complex), and it's not required to pass this test to submit to NeurIPS.\n\n" + message
-                else:  # error
+                else:  # error or non-records warning
                     status_class = "status-error"
                     status_icon = "✗"
                     message_with_emoji = "❌ " + message

validation.py

@@ -2,21 +2,40 @@ import mlcroissant._src.operation_graph.operations.download as dl_mod
 import requests
 import os
 
-# Make sure the HF token is loaded
 HF_TOKEN = os.environ.get("HF_TOKEN")
+print("[DEBUG] HF_TOKEN is", "set" if HF_TOKEN else "missing")
 
-# Set the environment variables Croissant expects
-os.environ["CROISSANT_BASIC_AUTH_USERNAME"] = "hf_user"
-os.environ["CROISSANT_BASIC_AUTH_PASSWORD"] = HF_TOKEN or ""
+# Only send HF credentials when downloading from huggingface.co.
+# The default get_basic_auth_from_env() applies auth to ALL URLs, which
+# causes non-HF hosts (e.g. OpenML) to return 400 Bad Request.
+_orig_download_from_http = dl_mod.Download._download_from_http
 
-print("[DEBUG] HF_TOKEN is", "set" if HF_TOKEN else "missing")
-print("[DEBUG] Basic auth env set for Croissant")
+def _hf_aware_download(self, filepath):
+    url = self.node.content_url or ""
+    if HF_TOKEN and "huggingface.co" in url:
+        os.environ["CROISSANT_BASIC_AUTH_USERNAME"] = "hf_user"
+        os.environ["CROISSANT_BASIC_AUTH_PASSWORD"] = HF_TOKEN
+    else:
+        os.environ.pop("CROISSANT_BASIC_AUTH_USERNAME", None)
+        os.environ.pop("CROISSANT_BASIC_AUTH_PASSWORD", None)
+    return _orig_download_from_http(self, filepath)
+
+dl_mod.Download._download_from_http = _hf_aware_download
 
+import logging
 import mlcroissant as mlc
 import func_timeout
 import json
 import traceback
 
+# Suppress noisy mlcroissant pattern-matching warnings
+logging.getLogger("root").addFilter(
+    lambda r: "Could not match" not in r.getMessage()
+)
+logging.getLogger().addFilter(
+    lambda r: "Could not match" not in r.getMessage()
+)
+
 WAIT_TIME = 10 * 60  # seconds
 
 def validate_json(file_path):
@@ -96,6 +115,30 @@ def validate_records(json_data):
         error_message = f"Unexpected error during records validation: {str(e)}\n\n{error_details}"
         return False, error_message, "error"
     
+RAI_FIELDS = [
+    "rai:dataLimitations",
+    "rai:dataBiases",
+    "rai:personalSensitiveInformation",
+    "rai:dataUseCases",
+    "rai:dataSocialImpact",
+    "rai:hasSyntheticData",
+    "prov:wasGeneratedBy",
+]
+
+RAI_GUIDELINES_URL = "https://neurips.cc/Conferences/2026/EvaluationsDatasetsHosting"
+
+def validate_rai(json_data):
+    """Check that all required Responsible AI metadata fields are present."""
+    missing = [field for field in RAI_FIELDS if field not in json_data]
+    if not missing:
+        return True, "All required Responsible AI metadata fields are present."
+    missing_list = "\n".join(f"- `{f}`" for f in missing)
+    message = (
+        f"The following required Responsible AI metadata fields are missing:\n{missing_list}\n\n"
+        f"Please refer to the <a href='{RAI_GUIDELINES_URL}' target='_blank'>NeurIPS guidelines for instructions</a> on how to add them."
+    )
+    return False, message
+
 def generate_validation_report(filename, json_data, results):
     """Generate a detailed validation report in markdown format."""
     report = []