diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..fa8debcaf4c6c24650a584d5abb523c4e1e2be13 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +__pycache__/ +*.py[cod] +.DS_Store +.ipynb_checkpoints/ +*.log +tmp/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..9663b6210060f0b72e444d3e11a4460912268e9c --- /dev/null +++ b/LICENSE @@ -0,0 +1,9 @@ +This release is provided for scholarly reproducibility of the associated paper. + +Code files are released under the MIT License. +Paper-output tables, figures, and bundled summary artifacts are released for +non-commercial research and review use with attribution. + +Raw data from NOAA, NASA, LANDFIRE, Wildfire Risk to Communities, LandScan, +WFIGS, MTBS, and model providers are not redistributed here. Users must obtain +those data from the original providers and comply with their terms. diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3f05c384c9f16973a8bab5bb5ffeeb81de06d84a --- /dev/null +++ b/README.md @@ -0,0 +1,146 @@ +--- +license: mit +tags: + - wildfire + - geospatial + - earth-observation + - foundation-models + - evaluation + - reproducibility + - paper-artifact +pretty_name: FireWx-FM and Wildfire Evaluation Contracts +--- + +# FireWx-FM and Wildfire Evaluation Contracts + +This repository contains the public code and lightweight artifacts for the +paper *Does Your Wildfire Prediction Model Actually Work, or Just Score Well?* + +The release has two parts: + +- **FireWx-FM reference backbone artifacts.** FireWx-FM is a wildfire-specialized + reference model used as an in-domain comparator in the paper. +- **Fixed-contract evaluation artifacts.** The paper evaluates wildfire transfer + under fixed task, metric, matching-rule, scope, and head-family choices. + +This is a paper-artifact repository. It includes scripts, compact summary files, +paper tables, and paper figures needed to inspect and reproduce reported +outputs. It does **not** redistribute raw weather, fire, fuel, exposure, +incident, perimeter, feature-cache, or private model files. + +## Key Results in the Release + +The bundled paper outputs reproduce the main results reported in the manuscript. + +| Check | Paper artifact | What it shows | +|---|---|---| +| Matching-rule sensitivity (RQ1) | `paper_outputs/figures/fig_fireprone_contract_progression_compact.pdf` | The same occupancy outputs can move sharply from exact to tolerated to union \(F_1\), especially under fire-prone scopes. | +| Fixed-feature head selection (RQ2) | `paper_outputs/figures/fig_selection_regret_rq2.tikz` | Selecting a head by ranking evidence can lose decision performance relative to direct decision selection. | +| Same-contract transfer matrix (RQ3) | `paper_outputs/tables/tab_primary_results.tex` | FireWx-FM, Prithvi-WxC, Aurora, ClimaX, StormCast, DLWP, FCN, FengWu, FuXi, Pangu-Weather, and AlphaEarth are compared under fixed occupancy and spread contracts. | +| Supporting task forms (RQ4) | `paper_outputs/tables/tab_supporting_results.tex` and `paper_outputs/figures/fig_task_rank_map.pdf` | Backbone ranking changes across burned area, analog retrieval, smoke PM2.5, and extreme heat task forms. | + +Selected displayed values from the paper: + +- FireWx-FM reference occupancy union \(F_1\): `59.0656 ± 2.7372`. +- ClimaX occupancy union \(F_1\): `60.1506 ± 7.5865`. +- FireWx-FM fire-spread AP: `30.0900 ± 1.2500`. +- FireWx-FM smoke PM2.5 RMSE: `4.4646 ± 0.0060`. +- AlphaEarth smoke PM2.5 RMSE: `4.4403 ± 0.0488`. + +Values are stored as TeX table cells under `paper_outputs/tables/`. The +corresponding compact CSV/JSON summaries are under `artifacts/results/`. + +## Repository Layout + +```text +artifacts/ + manifests/ table/figure provenance metadata and SHA-256 hashes + results/ compact CSV/JSON summaries used by paper outputs +data_sources/ source list and download notes for raw data +docs/ artifact map and Hugging Face release notes +experiments/ sanitized raw-rerun scripts and Slurm templates +paper_outputs/ + figures/ PDF/TikZ figures used by the manuscript + tables/ TeX table blocks used by the manuscript +scripts/ release rebuild and audit scripts +``` + +## Quick Reproduction + +The paper-output path uses only the Python standard library. + +```bash +python3 scripts/reproduce_paper_outputs.py +``` + +This command: + +- rebuilds the RQ1 fire-prone progression figure from summary JSON; +- rebuilds the RQ2 selection-regret TikZ figure from CSV; +- rebuilds the RQ4 rank-map PDF from the released main tables; +- checks SHA-256 hashes for all final paper outputs; +- audits that stale labels, local paths, and incomplete placeholders are absent. + +Expected terminal tail: + +```text +Paper-output checksum check passed. +Release audit passed. +Rebuilt reproducible outputs and passed release audit. +``` + +## What Is Included + +- Final paper table TeX files under `paper_outputs/tables/`. +- Final paper figures under `paper_outputs/figures/`. +- Small released CSV/JSON summary artifacts under `artifacts/results/`. +- Builder scripts for reproducible paper-output figures under `scripts/`. +- Sanitized raw-rerun reference scripts under `experiments/raw_reference/`. +- Data-source documentation under `data_sources/DATA_SOURCES.md`. +- A table/figure provenance map under `docs/artifact_map.md`. + +## What Is Not Included + +Raw data are not bundled. The paper uses public or provider-hosted resources, +including NOAA HRRR, NASA FIRMS, LANDFIRE, Wildfire Risk to Communities, +LandScan, WFIGS, MTBS, and external Earth-FM/backbone sources. + +See `data_sources/DATA_SOURCES.md` for the role of each source and public access +entry points. Full raw-data reruns require users to obtain those sources +independently and rebuild local feature caches. + +## Reproducibility Scope + +There are two levels of reproducibility: + +1. **Paper-output reproduction from bundled artifacts.** This is lightweight and + does not require raw data, GPUs, or Slurm. It verifies the exact files used by + the manuscript. +2. **Raw-data reruns.** These require separately downloaded source data, local + preprocessing, model dependencies, and compute resources. The repository + provides sanitized scripts and Slurm templates, but not the raw inputs. + +## Intended Use + +Use this repository to inspect paper values, reproduce released figures from +summary artifacts, audit table/figure provenance, or adapt the fixed-contract +evaluation workflow for wildfire transfer studies. + +Do not use this repository as a raw dataset mirror. Do not treat the included +summary artifacts as a substitute for the original data sources. + +## Citation + +If you use this release, please cite: + +```bibtex +@misc{wildfire_fm_evaluation_contracts_2026, + title = {Does Your Wildfire Prediction Model Actually Work, or Just Score Well?}, + author = {Anonymous}, + year = {2026}, + note = {FireWx-FM and fixed-contract wildfire evaluation code and artifacts} +} +``` + +The BibTeX entry will be updated with arXiv metadata after the preprint is +public. diff --git a/artifacts/manifests/paper_outputs.sha256 b/artifacts/manifests/paper_outputs.sha256 new file mode 100644 index 0000000000000000000000000000000000000000..a3813a18a1c4d96a811bd2663bcbdc61381cc95e --- /dev/null +++ b/artifacts/manifests/paper_outputs.sha256 @@ -0,0 +1,20 @@ +b369d13e0419fa8272ccdc994b6642f3b141248a879c030218e387c583537eb2 paper_outputs/figures/fig_fireprone_contract_progression_compact.pdf +b2e56403e2774c457dd12c4685e2dc7492e22e32df46fcc5c37b3087110f2439 paper_outputs/figures/fig_selection_regret_rq2.tikz +bc4d35ad9cb4c1f9ba8f31c7c340d9684c9dd2d55f5a2e60604a2b58b90cbe40 paper_outputs/figures/fig_task_contract_tiles.pdf +c382f5d69f25cc2f5db174601a33d0fd0928b44910a2a4b1c131954bd42113d9 paper_outputs/figures/fig_task_rank_map.pdf +015ab951b0af5c130e4894092a5dd0bb0fd62e710467163a9df8246d8cf369f4 paper_outputs/figures/matching.pdf +e8abbd2668517f5cae14933ed943fe103e74132886b0ff48ecd1685978549504 paper_outputs/tables/tab_app_analog_rank_depth.tex +81db28aace3366625f1cfd5935892eb5af672d5ecd8327e6dcba00b7b04e2b3c paper_outputs/tables/tab_app_burned_area_median_acre.tex +4a93401ef355c02eb0cc6b2e9a1506f9ed9d912301ec6829581247e40991bdfb paper_outputs/tables/tab_app_contract_params_full.tex +3c5398c28e6243b1784b27d2e9eab1a5c60e6e6d2cfd14a79aa6fd1e0499b871 paper_outputs/tables/tab_app_head_architectures.tex +f740b8f076490e852efa88fa8180ca08bb6b12901ff3ec3687c7e5c0b236da4e paper_outputs/tables/tab_app_heat_event_pr.tex +86e97a394ceae8cc6eafd6d1021b44d13a117378ead87bfee662cc90a1e0e54b paper_outputs/tables/tab_app_matching_rule_params.tex +0b1ad4587dd440fdabf771000b1c971daa9222e946a3404c9beae10dd7ea67c6 paper_outputs/tables/tab_app_occupancy_ppr_scope.tex +4e79672c28a938cd9ba1bc0e423e7169eca389251a22357aff6fe84d3cbfa889 paper_outputs/tables/tab_app_scope_params.tex +6850ee131e203f66392c79f17f59214672b362274f42285b252b83ac0ede1eb3 paper_outputs/tables/tab_app_seed_robustness.tex +1ca91ca451f846e59cb62ea64a616780c698b9dee80918a05467bd6c40df2dd5 paper_outputs/tables/tab_app_smoke_high_event.tex +cd65372622e8dd388adb1122a3e93b22d2090fba836405b08a078d5159b182de paper_outputs/tables/tab_app_spread_ap_by_scope.tex +a31d4a4e0f2f1c7f90a5610acea77aef5a48e63c754ab2159a42473dce2c3b94 paper_outputs/tables/tab_appendix_selection_regret_tolerance.tex +22614e90568cc562c023c540bdfdec14c0923ecf55d432fffa2619625b856092 paper_outputs/tables/tab_fireprone_contract_progression.tex +6672c62a150d83a351f4fa23ac04537d9aaae01af6056f689437d9b7d8bcee40 paper_outputs/tables/tab_primary_results.tex +717555b2584658c936aa8fc27b63f1068dc5f796a297bcef0576cf020b3ddaf8 paper_outputs/tables/tab_supporting_results.tex diff --git a/artifacts/manifests/paper_outputs.yml b/artifacts/manifests/paper_outputs.yml new file mode 100644 index 0000000000000000000000000000000000000000..7d0b714b1185be6bbc975982a22ef18b017a124c --- /dev/null +++ b/artifacts/manifests/paper_outputs.yml @@ -0,0 +1,60 @@ +figures: + fig:toy_occupancy_contract: + output: paper_outputs/figures/matching.pdf + type: static_vector + fig:task_contract_tiles: + output: paper_outputs/figures/fig_task_contract_tiles.pdf + type: static_vector + fig:selection_regret_diagnostic: + output: paper_outputs/figures/fig_selection_regret_rq2.tikz + builder: scripts/build_selection_regret_rq2_figure.py + inputs: + - artifacts/results/selection_regret_scope_sweep_20260505.csv + fig:fireprone_contract_progression: + output: paper_outputs/figures/fig_fireprone_contract_progression_compact.pdf + builder: scripts/build_fireprone_contract_progression_figure.py + inputs: + - artifacts/results/fireprone_contract_progression_summary.json + fig:task_comparator_normalized_map: + output: paper_outputs/figures/fig_task_rank_map.pdf + builder: scripts/build_task_rank_map.py + inputs: + - paper_outputs/tables/tab_primary_results.tex + - paper_outputs/tables/tab_supporting_results.tex +tables: + tab:primary_results: + output: paper_outputs/tables/tab_primary_results.tex + tab:supporting_results: + output: paper_outputs/tables/tab_supporting_results.tex + tab:app_matching_rule_params: + output: paper_outputs/tables/tab_app_matching_rule_params.tex + tab:app_contract_params_full: + output: paper_outputs/tables/tab_app_contract_params_full.tex + tab:app_scope_params: + output: paper_outputs/tables/tab_app_scope_params.tex + tab:fireprone_contract_progression: + output: paper_outputs/tables/tab_fireprone_contract_progression.tex + inputs: + - artifacts/results/fireprone_contract_progression_summary.json + tab:appendix_selection_regret_tolerance: + output: paper_outputs/tables/tab_appendix_selection_regret_tolerance.tex + inputs: + - artifacts/results/selection_regret_all_backbones_20260504.csv + tab:app_occupancy_ppr_scope: + output: paper_outputs/tables/tab_app_occupancy_ppr_scope.tex + inputs: + - artifacts/results/fireprone_contract_progression_summary.json + tab:app_spread_ap_by_scope: + output: paper_outputs/tables/tab_app_spread_ap_by_scope.tex + tab:app_burned_area_median_acre: + output: paper_outputs/tables/tab_app_burned_area_median_acre.tex + tab:app_analog_rank_depth: + output: paper_outputs/tables/tab_app_analog_rank_depth.tex + tab:app_smoke_high_event: + output: paper_outputs/tables/tab_app_smoke_high_event.tex + tab:app_heat_event_pr: + output: paper_outputs/tables/tab_app_heat_event_pr.tex + tab:app_seed_robustness: + output: paper_outputs/tables/tab_app_seed_robustness.tex + tab:app_head_architectures: + output: paper_outputs/tables/tab_app_head_architectures.tex diff --git a/artifacts/results/fireprone_contract_progression_summary.json b/artifacts/results/fireprone_contract_progression_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..c02b01cf0b9ef58124777217fc6d8d8190cd3c0a --- /dev/null +++ b/artifacts/results/fireprone_contract_progression_summary.json @@ -0,0 +1,16540 @@ +{ + "min_runs": 5, + "missing_or_incomplete": [], + "nondegenerate_prediction_issues": [], + "repair_run_roots": [], + "repair_scopes": [ + "train_fire_top02pct" + ], + "repair_sources": [ + "REDACTED_LOCAL_RESULT_PATH" + ], + "repair_threshold_max_ppr": 0.999999, + "repair_threshold_min_ppr": 1e-06, + "repair_threshold_selector": "strict_f1_nondegenerate", + "rows": [ + { + "difference": 0.38916402243638726, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0006305504019789734, + "scope": "full_domain", + "seed": 1, + "source_priority": 0, + "strict_f1": 0.01841620626151013, + "threshold": 0.005, + "threshold_key": "0.0050", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.3233671389763082, + "union_combo": "t3_s8", + "union_f1": 0.4075802286978974 + }, + { + "difference": 0.4907581884518499, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0024215889977243494, + "scope": "train_fire_top05pct", + "seed": 1, + "source_priority": 0, + "strict_f1": 0.05801376597836775, + "threshold": 0.005, + "threshold_key": "0.0050", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.45453465315678554, + "union_combo": "t3_s8", + "union_f1": 0.5487719544302176 + }, + { + "difference": 0.48639654658611775, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0012293459978232908, + "scope": "train_fire_top10pct", + "seed": 1, + "source_priority": 0, + "strict_f1": 0.05730937348227295, + "threshold": 0.005, + "threshold_key": "0.0050", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.4503813713224908, + "union_combo": "t3_s8", + "union_f1": 0.5437059200683907 + }, + { + "difference": 0.4576561443425643, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0006233766233766234, + "scope": "train_fire_top20pct", + "seed": 1, + "source_priority": 0, + "strict_f1": 0.0544280442804428, + "threshold": 0.005, + "threshold_key": "0.0050", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.42554776574916503, + "union_combo": "t3_s8", + "union_f1": 0.5120841886230071 + }, + { + "difference": 0.4724120147971174, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0015512677798392085, + "scope": "full_domain", + "seed": 123, + "source_priority": 0, + "strict_f1": 0.015582982831138383, + "threshold": 0.005, + "threshold_key": "0.0050", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.3412019506417814, + "union_combo": "t3_s8", + "union_f1": 0.4879949976282558 + }, + { + "difference": 0.572037609102215, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.005397249431087365, + "scope": "train_fire_top05pct", + "seed": 123, + "source_priority": 0, + "strict_f1": 0.06425702811244981, + "threshold": 0.005, + "threshold_key": "0.0050", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.44689699140777084, + "union_combo": "t3_s8", + "union_f1": 0.6362946372146648 + }, + { + "difference": 0.5674882177936724, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.002728307113881468, + "scope": "train_fire_top10pct", + "seed": 123, + "source_priority": 0, + "strict_f1": 0.06358911647814125, + "threshold": 0.005, + "threshold_key": "0.0050", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.4435187312985661, + "union_combo": "t3_s8", + "union_f1": 0.6310773342718137 + }, + { + "difference": 0.5375844952591261, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0013673469387755102, + "scope": "train_fire_top20pct", + "seed": 123, + "source_priority": 0, + "strict_f1": 0.06170275882527441, + "threshold": 0.005, + "threshold_key": "0.0050", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.4259185655133553, + "union_combo": "t3_s8", + "union_f1": 0.5992872540844005 + }, + { + "difference": 0.3681553641584426, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0005820655534941249, + "scope": "full_domain", + "seed": 42, + "source_priority": 0, + "strict_f1": 0.024598770061496925, + "threshold": 0.005, + "threshold_key": "0.0050", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.3224566312114399, + "union_combo": "t3_s8", + "union_f1": 0.3927541342199395 + }, + { + "difference": 0.42865099938480467, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.002226179875333927, + "scope": "train_fire_top05pct", + "seed": 42, + "source_priority": 0, + "strict_f1": 0.07774936061381074, + "threshold": 0.005, + "threshold_key": "0.0050", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.46103410735073025, + "union_combo": "t3_s8", + "union_f1": 0.5064003599986154 + }, + { + "difference": 0.42494003459095464, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.00112051053725141, + "scope": "train_fire_top10pct", + "seed": 42, + "source_priority": 0, + "strict_f1": 0.07711821410451548, + "threshold": 0.005, + "threshold_key": "0.0050", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.4570756272994282, + "union_combo": "t3_s8", + "union_f1": 0.5020582486954701 + }, + { + "difference": 0.39876530350195205, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0005602968460111318, + "scope": "train_fire_top20pct", + "seed": 42, + "source_priority": 0, + "strict_f1": 0.07357212003872217, + "threshold": 0.005, + "threshold_key": "0.0050", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.42986611450839224, + "union_combo": "t3_s8", + "union_f1": 0.4723374235406742 + }, + { + "difference": 0.1971458843464264, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.00030327767470624613, + "scope": "full_domain", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.018581463856786766, + "threshold": 0.02, + "threshold_key": "0.0200", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.19137568834619276, + "union_combo": "t3_s8", + "union_f1": 0.21572734820321318 + }, + { + "difference": 0.32645966955454164, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0025873157217769865, + "scope": "train_fire_top05pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.06853879105188007, + "threshold": 0.01, + "threshold_key": "0.0100", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.348489928787613, + "union_combo": "t3_s8", + "union_f1": 0.39499846060642174 + }, + { + "difference": 0.29747106554895125, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.001793311566241219, + "scope": "train_fire_top10pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.05725646123260437, + "threshold": 0.01, + "threshold_key": "0.0100", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.31417824074074074, + "union_combo": "t3_s8", + "union_f1": 0.3547275267815556 + }, + { + "difference": 0.22027935343108077, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.00035250463821892396, + "scope": "train_fire_top20pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.047398843930635835, + "threshold": 0.02, + "threshold_key": "0.0200", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.25536195152859176, + "union_combo": "t3_s8", + "union_f1": 0.2676781973617166 + }, + { + "difference": 0.3415199178431181, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.00038899196042053183, + "scope": "full_domain", + "seed": 99, + "source_priority": 0, + "strict_f1": 0.02585193889541716, + "threshold": 0.01, + "threshold_key": "0.0100", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.29397699288019324, + "union_combo": "t3_s8", + "union_f1": 0.36737185673853523 + }, + { + "difference": 0.42366919991467084, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.001496487582863362, + "scope": "train_fire_top05pct", + "seed": 99, + "source_priority": 0, + "strict_f1": 0.07710843373493977, + "threshold": 0.01, + "threshold_key": "0.0100", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.4329934518241347, + "union_combo": "t3_s8", + "union_f1": 0.5007776336496106 + }, + { + "difference": 0.42043572680093133, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0007507173246264965, + "scope": "train_fire_top10pct", + "seed": 99, + "source_priority": 0, + "strict_f1": 0.07655502392344497, + "threshold": 0.01, + "threshold_key": "0.0100", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.4297532170909699, + "union_combo": "t3_s8", + "union_f1": 0.4969907507243763 + }, + { + "difference": 0.3953377323997304, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0003753865182436611, + "scope": "train_fire_top20pct", + "seed": 99, + "source_priority": 0, + "strict_f1": 0.0724391624221845, + "threshold": 0.01, + "threshold_key": "0.0100", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.40493051697817745, + "union_combo": "t3_s8", + "union_f1": 0.4677768948219149 + }, + { + "difference": 0.1955214345352177, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9846233766233766, + "scope": "full_domain", + "seed": 1, + "source_priority": 0, + "strict_f1": 0.0004925501476206198, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.07032155974511028, + "union_combo": "t3_s8", + "union_f1": 0.19601398468283832 + }, + { + "difference": 0.38844131515497954, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9234194122885129, + "scope": "train_fire_top05pct", + "seed": 1, + "source_priority": 0, + "strict_f1": 0.005636060100166944, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.15977956000729543, + "union_combo": "t3_s8", + "union_f1": 0.39407737525514647 + }, + { + "difference": 0.2282163144290469, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9575541703769664, + "scope": "train_fire_top10pct", + "seed": 1, + "source_priority": 0, + "strict_f1": 0.002747305898968793, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08713204456539597, + "union_combo": "t3_s8", + "union_f1": 0.2309636203280157 + }, + { + "difference": 0.16866850620027352, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9746196660482375, + "scope": "train_fire_top20pct", + "seed": 1, + "source_priority": 0, + "strict_f1": 0.001471035812113219, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06246138809231819, + "union_combo": "t3_s8", + "union_f1": 0.17013954201238674 + }, + { + "difference": 0.1955214345352177, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9846233766233766, + "scope": "full_domain", + "seed": 123, + "source_priority": 0, + "strict_f1": 0.0004925501476206198, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.07032155974511028, + "union_combo": "t3_s8", + "union_f1": 0.19601398468283832 + }, + { + "difference": 0.38844131515497954, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9234194122885129, + "scope": "train_fire_top05pct", + "seed": 123, + "source_priority": 0, + "strict_f1": 0.005636060100166944, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.15977956000729543, + "union_combo": "t3_s8", + "union_f1": 0.39407737525514647 + }, + { + "difference": 0.2282163144290469, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9575541703769664, + "scope": "train_fire_top10pct", + "seed": 123, + "source_priority": 0, + "strict_f1": 0.002747305898968793, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08713204456539597, + "union_combo": "t3_s8", + "union_f1": 0.2309636203280157 + }, + { + "difference": 0.16866850620027352, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9746196660482375, + "scope": "train_fire_top20pct", + "seed": 123, + "source_priority": 0, + "strict_f1": 0.001471035812113219, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06246138809231819, + "union_combo": "t3_s8", + "union_f1": 0.17013954201238674 + }, + { + "difference": 0.1941569792865045, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9922968460111318, + "scope": "full_domain", + "seed": 42, + "source_priority": 0, + "strict_f1": 0.0004887421693148924, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06979673814819128, + "union_combo": "t3_s8", + "union_f1": 0.19464572145581938 + }, + { + "difference": 0.38784279977334685, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.92520035618878, + "scope": "train_fire_top05pct", + "seed": 42, + "source_priority": 0, + "strict_f1": 0.00562524160545995, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.15949652359184557, + "union_combo": "t3_s8", + "union_f1": 0.3934680413788068 + }, + { + "difference": 0.2275932584637248, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.960522410210745, + "scope": "train_fire_top10pct", + "seed": 42, + "source_priority": 0, + "strict_f1": 0.0027388277045923585, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.0868744819295806, + "union_combo": "t3_s8", + "union_f1": 0.23033208616831716 + }, + { + "difference": 0.16810681009431644, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9781818181818182, + "scope": "train_fire_top20pct", + "seed": 42, + "source_priority": 0, + "strict_f1": 0.0014656828060244616, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06224100702898884, + "union_combo": "t3_s8", + "union_f1": 0.1695724929003409 + }, + { + "difference": 0.1941569792865045, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9922968460111318, + "scope": "full_domain", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.0004887421693148924, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06979673814819128, + "union_combo": "t3_s8", + "union_f1": 0.19464572145581938 + }, + { + "difference": 0.38784279977334685, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.92520035618878, + "scope": "train_fire_top05pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.00562524160545995, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.15949652359184557, + "union_combo": "t3_s8", + "union_f1": 0.3934680413788068 + }, + { + "difference": 0.2275932584637248, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.960522410210745, + "scope": "train_fire_top10pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.0027388277045923585, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.0868744819295806, + "union_combo": "t3_s8", + "union_f1": 0.23033208616831716 + }, + { + "difference": 0.16810681009431644, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9781818181818182, + "scope": "train_fire_top20pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.0014656828060244616, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06224100702898884, + "union_combo": "t3_s8", + "union_f1": 0.1695724929003409 + }, + { + "difference": 0.1955214345352177, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9846233766233766, + "scope": "full_domain", + "seed": 99, + "source_priority": 0, + "strict_f1": 0.0004925501476206198, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.07032155974511028, + "union_combo": "t3_s8", + "union_f1": 0.19601398468283832 + }, + { + "difference": 0.38844131515497954, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9234194122885129, + "scope": "train_fire_top05pct", + "seed": 99, + "source_priority": 0, + "strict_f1": 0.005636060100166944, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.15977956000729543, + "union_combo": "t3_s8", + "union_f1": 0.39407737525514647 + }, + { + "difference": 0.2282163144290469, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9575541703769664, + "scope": "train_fire_top10pct", + "seed": 99, + "source_priority": 0, + "strict_f1": 0.002747305898968793, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08713204456539597, + "union_combo": "t3_s8", + "union_f1": 0.2309636203280157 + }, + { + "difference": 0.16866850620027352, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9746196660482375, + "scope": "train_fire_top20pct", + "seed": 99, + "source_priority": 0, + "strict_f1": 0.001471035812113219, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06246138809231819, + "union_combo": "t3_s8", + "union_f1": 0.17013954201238674 + }, + { + "difference": 0.1928199932355324, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 1.0, + "scope": "full_domain", + "seed": 1, + "source_priority": 0, + "strict_f1": 0.00048497822606044473, + "threshold": 2e-05, + "threshold_key": "0.0000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06928162590128548, + "union_combo": "t3_s8", + "union_f1": 0.19330497146159284 + }, + { + "difference": 0.3644918022569579, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 1.0, + "scope": "train_fire_top05pct", + "seed": 1, + "source_priority": 0, + "strict_f1": 0.005205570700778368, + "threshold": 2e-05, + "threshold_key": "0.0000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.14852385215852132, + "union_combo": "t3_s8", + "union_f1": 0.36969737295773625 + }, + { + "difference": 0.21975162068302262, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 1.0, + "scope": "train_fire_top10pct", + "seed": 1, + "source_priority": 0, + "strict_f1": 0.0026308476146364055, + "threshold": 2e-05, + "threshold_key": "0.0000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08362683149966108, + "union_combo": "t3_s8", + "union_f1": 0.22238246829765904 + }, + { + "difference": 0.16481712333667892, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 1.0, + "scope": "train_fire_top20pct", + "seed": 1, + "source_priority": 0, + "strict_f1": 0.0014337271963217482, + "threshold": 2e-05, + "threshold_key": "0.0000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06094415351278495, + "union_combo": "t3_s8", + "union_f1": 0.16625085053300068 + }, + { + "difference": 0.19494871620879223, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9878311688311688, + "scope": "full_domain", + "seed": 123, + "source_priority": 0, + "strict_f1": 0.0004909510781395833, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.07010120797866884, + "union_combo": "t3_s8", + "union_f1": 0.19543966728693182 + }, + { + "difference": 0.3879673384615635, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9248293262095577, + "scope": "train_fire_top05pct", + "seed": 123, + "source_priority": 0, + "strict_f1": 0.0056274920321647175, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.15955540678900526, + "union_combo": "t3_s8", + "union_f1": 0.39359483049372823 + }, + { + "difference": 0.22772278113809574, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9599040269120411, + "scope": "train_fire_top10pct", + "seed": 123, + "source_priority": 0, + "strict_f1": 0.002740589677112216, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08692801516378341, + "union_combo": "t3_s8", + "union_f1": 0.23046337081520796 + }, + { + "difference": 0.16822352141468047, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9774397031539889, + "scope": "train_fire_top20pct", + "seed": 123, + "source_priority": 0, + "strict_f1": 0.001466794800465328, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06228679141188911, + "union_combo": "t3_s8", + "union_f1": 0.1696903162151458 + }, + { + "difference": 0.19417262666147908, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9922864564007421, + "scope": "full_domain", + "seed": 42, + "source_priority": 0, + "strict_f1": 0.0004887472853776956, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06980139080211295, + "union_combo": "t3_s8", + "union_f1": 0.19466137394685679 + }, + { + "difference": 0.36525822235524846, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9974176313446127, + "scope": "train_fire_top05pct", + "seed": 42, + "source_priority": 0, + "strict_f1": 0.0052190130376387305, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.14887976367271333, + "union_combo": "t3_s8", + "union_f1": 0.3704772353928872 + }, + { + "difference": 0.22041063105557585, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9966310477886613, + "scope": "train_fire_top10pct", + "seed": 42, + "source_priority": 0, + "strict_f1": 0.0026397290373912042, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08389766026250713, + "union_combo": "t3_s8", + "union_f1": 0.22305036009296705 + }, + { + "difference": 0.1653872236463206, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9962374768089054, + "scope": "train_fire_top20pct", + "seed": 42, + "source_priority": 0, + "strict_f1": 0.0014391381051513698, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.061167283891925196, + "union_combo": "t3_s8", + "union_f1": 0.16682636175147197 + }, + { + "difference": 0.19283033214709877, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9999406307977736, + "scope": "full_domain", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.00048500701355756607, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06928559683931526, + "union_combo": "t3_s8", + "union_f1": 0.19331533916065632 + }, + { + "difference": 0.3646677061475394, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9994063520332442, + "scope": "train_fire_top05pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.005208654759994569, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.14860552018347153, + "union_combo": "t3_s8", + "union_f1": 0.36987636090753395 + }, + { + "difference": 0.2198095249514603, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9997031760166222, + "scope": "train_fire_top10pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.0026316277173409446, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08365062280406448, + "union_combo": "t3_s8", + "union_f1": 0.22244115266880124 + }, + { + "difference": 0.16483953803239848, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9998515769944342, + "scope": "train_fire_top20pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.0014339398734177216, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06095292465378015, + "union_combo": "t3_s8", + "union_f1": 0.1662734779058162 + }, + { + "difference": 0.1955214345352177, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9846233766233766, + "scope": "full_domain", + "seed": 99, + "source_priority": 0, + "strict_f1": 0.0004925501476206198, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.07032155974511028, + "union_combo": "t3_s8", + "union_f1": 0.19601398468283832 + }, + { + "difference": 0.38844131515497954, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9234194122885129, + "scope": "train_fire_top05pct", + "seed": 99, + "source_priority": 0, + "strict_f1": 0.005636060100166944, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.15977956000729543, + "union_combo": "t3_s8", + "union_f1": 0.39407737525514647 + }, + { + "difference": 0.2282163144290469, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9575541703769664, + "scope": "train_fire_top10pct", + "seed": 99, + "source_priority": 0, + "strict_f1": 0.002747305898968793, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08713204456539597, + "union_combo": "t3_s8", + "union_f1": 0.2309636203280157 + }, + { + "difference": 0.16866850620027352, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9746196660482375, + "scope": "train_fire_top20pct", + "seed": 99, + "source_priority": 0, + "strict_f1": 0.001471035812113219, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06246138809231819, + "union_combo": "t3_s8", + "union_f1": 0.17013954201238674 + }, + { + "difference": 0.1934553039285264, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9963636363636363, + "scope": "full_domain", + "seed": 1, + "source_priority": 0, + "strict_f1": 0.0004867477888159953, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06952568867327341, + "union_combo": "t3_s8", + "union_f1": 0.19394205171734238 + }, + { + "difference": 0.3644918022569579, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 1.0, + "scope": "train_fire_top05pct", + "seed": 1, + "source_priority": 0, + "strict_f1": 0.005205570700778368, + "threshold": 2e-05, + "threshold_key": "0.0000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.14852385215852132, + "union_combo": "t3_s8", + "union_f1": 0.36969737295773625 + }, + { + "difference": 0.21975162068302262, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 1.0, + "scope": "train_fire_top10pct", + "seed": 1, + "source_priority": 0, + "strict_f1": 0.0026308476146364055, + "threshold": 2e-05, + "threshold_key": "0.0000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08362683149966108, + "union_combo": "t3_s8", + "union_f1": 0.22238246829765904 + }, + { + "difference": 0.16481712333667892, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 1.0, + "scope": "train_fire_top20pct", + "seed": 1, + "source_priority": 0, + "strict_f1": 0.0014337271963217482, + "threshold": 2e-05, + "threshold_key": "0.0000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06094415351278495, + "union_combo": "t3_s8", + "union_f1": 0.16625085053300068 + }, + { + "difference": 0.1955214345352177, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9846233766233766, + "scope": "full_domain", + "seed": 123, + "source_priority": 0, + "strict_f1": 0.0004925501476206198, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.07032155974511028, + "union_combo": "t3_s8", + "union_f1": 0.19601398468283832 + }, + { + "difference": 0.38844131515497954, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9234194122885129, + "scope": "train_fire_top05pct", + "seed": 123, + "source_priority": 0, + "strict_f1": 0.005636060100166944, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.15977956000729543, + "union_combo": "t3_s8", + "union_f1": 0.39407737525514647 + }, + { + "difference": 0.2282163144290469, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9575541703769664, + "scope": "train_fire_top10pct", + "seed": 123, + "source_priority": 0, + "strict_f1": 0.002747305898968793, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08713204456539597, + "union_combo": "t3_s8", + "union_f1": 0.2309636203280157 + }, + { + "difference": 0.16866850620027352, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9746196660482375, + "scope": "train_fire_top20pct", + "seed": 123, + "source_priority": 0, + "strict_f1": 0.001471035812113219, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06246138809231819, + "union_combo": "t3_s8", + "union_f1": 0.17013954201238674 + }, + { + "difference": 0.1955214345352177, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9846233766233766, + "scope": "full_domain", + "seed": 42, + "source_priority": 0, + "strict_f1": 0.0004925501476206198, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.07032155974511028, + "union_combo": "t3_s8", + "union_f1": 0.19601398468283832 + }, + { + "difference": 0.38844131515497954, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9234194122885129, + "scope": "train_fire_top05pct", + "seed": 42, + "source_priority": 0, + "strict_f1": 0.005636060100166944, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.15977956000729543, + "union_combo": "t3_s8", + "union_f1": 0.39407737525514647 + }, + { + "difference": 0.2282163144290469, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9575541703769664, + "scope": "train_fire_top10pct", + "seed": 42, + "source_priority": 0, + "strict_f1": 0.002747305898968793, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08713204456539597, + "union_combo": "t3_s8", + "union_f1": 0.2309636203280157 + }, + { + "difference": 0.16866850620027352, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9746196660482375, + "scope": "train_fire_top20pct", + "seed": 42, + "source_priority": 0, + "strict_f1": 0.001471035812113219, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06246138809231819, + "union_combo": "t3_s8", + "union_f1": 0.17013954201238674 + }, + { + "difference": 0.19345790561199683, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9963487940630797, + "scope": "full_domain", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.000486755037982903, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06952668837151453, + "union_combo": "t3_s8", + "union_f1": 0.19394466064997973 + }, + { + "difference": 0.36457973300001806, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9997031760166222, + "scope": "train_fire_top05pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.005207112273731229, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.1485646749474909, + "union_combo": "t3_s8", + "union_f1": 0.3697868452737493 + }, + { + "difference": 0.21978056900402743, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9998515880083111, + "scope": "train_fire_top10pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.0026312376081679546, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08363872545998453, + "union_combo": "t3_s8", + "union_f1": 0.22241180661219537 + }, + { + "difference": 0.1648283299225597, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.999925788497217, + "scope": "train_fire_top20pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.0014338335269832637, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06094853876771749, + "union_combo": "t3_s8", + "union_f1": 0.16626216344954295 + }, + { + "difference": 0.1955214345352177, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9846233766233766, + "scope": "full_domain", + "seed": 99, + "source_priority": 0, + "strict_f1": 0.0004925501476206198, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.07032155974511028, + "union_combo": "t3_s8", + "union_f1": 0.19601398468283832 + }, + { + "difference": 0.38844131515497954, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9234194122885129, + "scope": "train_fire_top05pct", + "seed": 99, + "source_priority": 0, + "strict_f1": 0.005636060100166944, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.15977956000729543, + "union_combo": "t3_s8", + "union_f1": 0.39407737525514647 + }, + { + "difference": 0.2282163144290469, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9575541703769664, + "scope": "train_fire_top10pct", + "seed": 99, + "source_priority": 0, + "strict_f1": 0.002747305898968793, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08713204456539597, + "union_combo": "t3_s8", + "union_f1": 0.2309636203280157 + }, + { + "difference": 0.16866850620027352, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9746196660482375, + "scope": "train_fire_top20pct", + "seed": 99, + "source_priority": 0, + "strict_f1": 0.001471035812113219, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06246138809231819, + "union_combo": "t3_s8", + "union_f1": 0.17013954201238674 + }, + { + "difference": 0.1928199932355324, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 1.0, + "scope": "full_domain", + "seed": 1, + "source_priority": 0, + "strict_f1": 0.00048497822606044473, + "threshold": 2e-05, + "threshold_key": "0.0000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06928162590128548, + "union_combo": "t3_s8", + "union_f1": 0.19330497146159284 + }, + { + "difference": 0.3644918022569579, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 1.0, + "scope": "train_fire_top05pct", + "seed": 1, + "source_priority": 0, + "strict_f1": 0.005205570700778368, + "threshold": 2e-05, + "threshold_key": "0.0000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.14852385215852132, + "union_combo": "t3_s8", + "union_f1": 0.36969737295773625 + }, + { + "difference": 0.21975162068302262, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 1.0, + "scope": "train_fire_top10pct", + "seed": 1, + "source_priority": 0, + "strict_f1": 0.0026308476146364055, + "threshold": 2e-05, + "threshold_key": "0.0000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08362683149966108, + "union_combo": "t3_s8", + "union_f1": 0.22238246829765904 + }, + { + "difference": 0.16481712333667892, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 1.0, + "scope": "train_fire_top20pct", + "seed": 1, + "source_priority": 0, + "strict_f1": 0.0014337271963217482, + "threshold": 2e-05, + "threshold_key": "0.0000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06094415351278495, + "union_combo": "t3_s8", + "union_f1": 0.16625085053300068 + }, + { + "difference": 0.1955214345352177, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9846233766233766, + "scope": "full_domain", + "seed": 123, + "source_priority": 0, + "strict_f1": 0.0004925501476206198, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.07032155974511028, + "union_combo": "t3_s8", + "union_f1": 0.19601398468283832 + }, + { + "difference": 0.38844131515497954, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9234194122885129, + "scope": "train_fire_top05pct", + "seed": 123, + "source_priority": 0, + "strict_f1": 0.005636060100166944, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.15977956000729543, + "union_combo": "t3_s8", + "union_f1": 0.39407737525514647 + }, + { + "difference": 0.2282163144290469, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9575541703769664, + "scope": "train_fire_top10pct", + "seed": 123, + "source_priority": 0, + "strict_f1": 0.002747305898968793, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08713204456539597, + "union_combo": "t3_s8", + "union_f1": 0.2309636203280157 + }, + { + "difference": 0.16866850620027352, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9746196660482375, + "scope": "train_fire_top20pct", + "seed": 123, + "source_priority": 0, + "strict_f1": 0.001471035812113219, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06246138809231819, + "union_combo": "t3_s8", + "union_f1": 0.17013954201238674 + }, + { + "difference": 0.19415173790531431, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9923265306122449, + "scope": "full_domain", + "seed": 42, + "source_priority": 0, + "strict_f1": 0.0004887275525827606, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06979472310089276, + "union_combo": "t3_s8", + "union_f1": 0.19464046545789707 + }, + { + "difference": 0.38764370394898157, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9257940041555358, + "scope": "train_fire_top05pct", + "seed": 42, + "source_priority": 0, + "strict_f1": 0.005621644664100071, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.15940240082223983, + "union_combo": "t3_s8", + "union_f1": 0.39326534861308166 + }, + { + "difference": 0.22753113988967633, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9608192341941229, + "scope": "train_fire_top10pct", + "seed": 42, + "source_priority": 0, + "strict_f1": 0.002737982762277539, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08684880939041824, + "union_combo": "t3_s8", + "union_f1": 0.23026912265195387 + }, + { + "difference": 0.16808348725958006, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9783302411873841, + "scope": "train_fire_top20pct", + "seed": 42, + "source_priority": 0, + "strict_f1": 0.0014654606094294809, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.062231858228220235, + "union_combo": "t3_s8", + "union_f1": 0.16954894786900954 + }, + { + "difference": 0.19479328198743862, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9887050092764379, + "scope": "full_domain", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.000490517270597817, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.07004142068407385, + "union_combo": "t3_s8", + "union_f1": 0.19528379925803643 + }, + { + "difference": 0.38764370394898157, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9257940041555358, + "scope": "train_fire_top05pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.005621644664100071, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.15940240082223983, + "union_combo": "t3_s8", + "union_f1": 0.39326534861308166 + }, + { + "difference": 0.22753113988967633, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9608192341941229, + "scope": "train_fire_top10pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.002737982762277539, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08684880939041824, + "union_combo": "t3_s8", + "union_f1": 0.23026912265195387 + }, + { + "difference": 0.16808348725958006, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9783302411873841, + "scope": "train_fire_top20pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.0014654606094294809, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.062231858228220235, + "union_combo": "t3_s8", + "union_f1": 0.16954894786900954 + }, + { + "difference": 0.1955214345352177, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9846233766233766, + "scope": "full_domain", + "seed": 99, + "source_priority": 0, + "strict_f1": 0.0004925501476206198, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.07032155974511028, + "union_combo": "t3_s8", + "union_f1": 0.19601398468283832 + }, + { + "difference": 0.38844131515497954, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9234194122885129, + "scope": "train_fire_top05pct", + "seed": 99, + "source_priority": 0, + "strict_f1": 0.005636060100166944, + "threshold": 0.0001, + "threshold_key": "0.0001", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.15977956000729543, + "union_combo": "t3_s8", + "union_f1": 0.39407737525514647 + }, + { + "difference": 0.2282163144290469, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9575541703769664, + "scope": "train_fire_top10pct", + "seed": 99, + "source_priority": 0, + "strict_f1": 0.002747305898968793, + "threshold": 0.0001, + "threshold_key": "0.0001", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08713204456539597, + "union_combo": "t3_s8", + "union_f1": 0.2309636203280157 + }, + { + "difference": 0.16866850620027352, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9746196660482375, + "scope": "train_fire_top20pct", + "seed": 99, + "source_priority": 0, + "strict_f1": 0.001471035812113219, + "threshold": 0.0001, + "threshold_key": "0.0001", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06246138809231819, + "union_combo": "t3_s8", + "union_f1": 0.17013954201238674 + }, + { + "difference": 0.601348973225457, + "label": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.014596289424860853, + "scope": "full_domain", + "seed": 0, + "source_priority": 0, + "strict_f1": 0.006568199246490847, + "threshold": 0.95, + "threshold_key": "0.9500", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.29937890424270136, + "union_combo": "t3_s8", + "union_f1": 0.6079171724719479 + }, + { + "difference": 0.7136230759527924, + "label": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.02449292569506283, + "scope": "train_fire_top05pct", + "seed": 0, + "source_priority": 0, + "strict_f1": 0.04873596787441818, + "threshold": 0.95, + "threshold_key": "0.9500", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.40145293653054454, + "union_combo": "t3_s8", + "union_f1": 0.7623590438272105 + }, + { + "difference": 0.7104053704635315, + "label": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.012246462847531414, + "scope": "train_fire_top10pct", + "seed": 0, + "source_priority": 0, + "strict_f1": 0.0486915291328531, + "threshold": 0.95, + "threshold_key": "0.9500", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.4004617740449798, + "union_combo": "t3_s8", + "union_f1": 0.7590968995963846 + }, + { + "difference": 0.6811711568125168, + "label": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.006123685837971553, + "scope": "train_fire_top20pct", + "seed": 0, + "source_priority": 0, + "strict_f1": 0.04827336828783222, + "threshold": 0.95, + "threshold_key": "0.9500", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.39128423640663973, + "union_combo": "t3_s8", + "union_f1": 0.7294445251003491 + }, + { + "difference": 0.1941734129578403, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9922820037105752, + "scope": "full_domain", + "seed": 1, + "source_priority": 0, + "strict_f1": 0.0004887494780088282, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06980169309182878, + "union_combo": "t3_s8", + "union_f1": 0.19466216243584913 + }, + { + "difference": 0.36528470808376384, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9973285841495992, + "scope": "train_fire_top05pct", + "seed": 1, + "source_priority": 0, + "strict_f1": 0.005219477804850899, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.14889206690983883, + "union_combo": "t3_s8", + "union_f1": 0.3705041858886147 + }, + { + "difference": 0.2204193669035129, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9965865241911547, + "scope": "train_fire_top10pct", + "seed": 1, + "source_priority": 0, + "strict_f1": 0.0026398468145228757, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08390125124244428, + "union_combo": "t3_s8", + "union_f1": 0.22305921371803578 + }, + { + "difference": 0.16539060875783795, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9962152133580705, + "scope": "train_fire_top20pct", + "seed": 1, + "source_priority": 0, + "strict_f1": 0.001439170243914543, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06116860905264603, + "union_combo": "t3_s8", + "union_f1": 0.1668297790017525 + }, + { + "difference": 0.19479855806395832, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9886753246753247, + "scope": "full_domain", + "seed": 123, + "source_priority": 0, + "strict_f1": 0.0004905319945803844, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.07004345000159352, + "union_combo": "t3_s8", + "union_f1": 0.1952890900585387 + }, + { + "difference": 0.3877432263256208, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9254971801721579, + "scope": "train_fire_top05pct", + "seed": 123, + "source_priority": 0, + "strict_f1": 0.005623442559599164, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.15944944831691155, + "union_combo": "t3_s8", + "union_f1": 0.39336666888522 + }, + { + "difference": 0.22756219493836724, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9606708222024339, + "scope": "train_fire_top10pct", + "seed": 123, + "source_priority": 0, + "strict_f1": 0.002738405168257641, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08686164376307684, + "union_combo": "t3_s8", + "union_f1": 0.23030060010662487 + }, + { + "difference": 0.16809514786801025, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9782560296846011, + "scope": "train_fire_top20pct", + "seed": 123, + "source_priority": 0, + "strict_f1": 0.0014655716993051168, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06223643229238444, + "union_combo": "t3_s8", + "union_f1": 0.16956071956731536 + }, + { + "difference": 0.1955214345352177, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9846233766233766, + "scope": "full_domain", + "seed": 42, + "source_priority": 0, + "strict_f1": 0.0004925501476206198, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.07032155974511028, + "union_combo": "t3_s8", + "union_f1": 0.19601398468283832 + }, + { + "difference": 0.38844131515497954, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9234194122885129, + "scope": "train_fire_top05pct", + "seed": 42, + "source_priority": 0, + "strict_f1": 0.005636060100166944, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.15977956000729543, + "union_combo": "t3_s8", + "union_f1": 0.39407737525514647 + }, + { + "difference": 0.2282163144290469, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9575541703769664, + "scope": "train_fire_top10pct", + "seed": 42, + "source_priority": 0, + "strict_f1": 0.002747305898968793, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08713204456539597, + "union_combo": "t3_s8", + "union_f1": 0.2309636203280157 + }, + { + "difference": 0.16866850620027352, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9746196660482375, + "scope": "train_fire_top20pct", + "seed": 42, + "source_priority": 0, + "strict_f1": 0.001471035812113219, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06246138809231819, + "union_combo": "t3_s8", + "union_f1": 0.17013954201238674 + }, + { + "difference": 0.19479855806395832, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9886753246753247, + "scope": "full_domain", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.0004905319945803844, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.07004345000159352, + "union_combo": "t3_s8", + "union_f1": 0.1952890900585387 + }, + { + "difference": 0.38784279977334685, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.92520035618878, + "scope": "train_fire_top05pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.00562524160545995, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.15949652359184557, + "union_combo": "t3_s8", + "union_f1": 0.3934680413788068 + }, + { + "difference": 0.2275932584637248, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.960522410210745, + "scope": "train_fire_top10pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.0027388277045923585, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.0868744819295806, + "union_combo": "t3_s8", + "union_f1": 0.23033208616831716 + }, + { + "difference": 0.16810681009431644, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9781818181818182, + "scope": "train_fire_top20pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.0014656828060244616, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06224100702898884, + "union_combo": "t3_s8", + "union_f1": 0.1695724929003409 + }, + { + "difference": 0.19415960008322136, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9922820037105752, + "scope": "full_domain", + "seed": 99, + "source_priority": 0, + "strict_f1": 0.0004887494780088282, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06979774571547358, + "union_combo": "t3_s8", + "union_f1": 0.1946483495612302 + }, + { + "difference": 0.3877432263256208, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9254971801721579, + "scope": "train_fire_top05pct", + "seed": 99, + "source_priority": 0, + "strict_f1": 0.005623442559599164, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.15944944831691155, + "union_combo": "t3_s8", + "union_f1": 0.39336666888522 + }, + { + "difference": 0.22756219493836724, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9606708222024339, + "scope": "train_fire_top10pct", + "seed": 99, + "source_priority": 0, + "strict_f1": 0.002738405168257641, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08686164376307684, + "union_combo": "t3_s8", + "union_f1": 0.23030060010662487 + }, + { + "difference": 0.16809514786801025, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9782560296846011, + "scope": "train_fire_top20pct", + "seed": 99, + "source_priority": 0, + "strict_f1": 0.0014655716993051168, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06223643229238444, + "union_combo": "t3_s8", + "union_f1": 0.16956071956731536 + }, + { + "difference": 0.3044358568583716, + "label": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.057648732220160794, + "scope": "full_domain", + "seed": 17, + "source_priority": 0, + "strict_f1": 0.001786130143937306, + "threshold": 0.08, + "threshold_key": "0.0800", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.16383974771641738, + "union_combo": "t3_s8", + "union_f1": 0.3062219870023089 + }, + { + "difference": 0.5668021940668209, + "label": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.05733155238943306, + "scope": "train_fire_top05pct", + "seed": 17, + "source_priority": 0, + "strict_f1": 0.01972516815912186, + "threshold": 0.08, + "threshold_key": "0.0800", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.3500819147736832, + "union_combo": "t3_s8", + "union_f1": 0.5865273622259427 + }, + { + "difference": 0.5537110636994692, + "label": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.02979989116453943, + "scope": "train_fire_top10pct", + "seed": 17, + "source_priority": 0, + "strict_f1": 0.018998410174880763, + "threshold": 0.08, + "threshold_key": "0.0800", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.3409571337826908, + "union_combo": "t3_s8", + "union_f1": 0.57270947387435 + }, + { + "difference": 0.44734760227295295, + "label": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.010166975881261595, + "scope": "train_fire_top20pct", + "seed": 17, + "source_priority": 0, + "strict_f1": 0.01931818181818182, + "threshold": 0.1, + "threshold_key": "0.1000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.28953047944835925, + "union_combo": "t3_s8", + "union_f1": 0.4666657840911348 + }, + { + "difference": 0.2924864697969017, + "label": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.04519196042053185, + "scope": "full_domain", + "seed": 37, + "source_priority": 0, + "strict_f1": 0.0019600476944939, + "threshold": 0.1, + "threshold_key": "0.1000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.17299721297318826, + "union_combo": "t3_s8", + "union_f1": 0.2944465174913956 + }, + { + "difference": 0.5053802602935323, + "label": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.04792965271593945, + "scope": "train_fire_top05pct", + "seed": 37, + "source_priority": 0, + "strict_f1": 0.024960845732184808, + "threshold": 0.1, + "threshold_key": "0.1000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.35323298220195054, + "union_combo": "t3_s8", + "union_f1": 0.5303411060257172 + }, + { + "difference": 0.486215877804802, + "label": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.025880577817354308, + "scope": "train_fire_top10pct", + "seed": 37, + "source_priority": 0, + "strict_f1": 0.023191305534082125, + "threshold": 0.1, + "threshold_key": "0.1000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.33689092573275314, + "union_combo": "t3_s8", + "union_f1": 0.5094071833388841 + }, + { + "difference": 0.26320327785336994, + "label": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0027860235003092145, + "scope": "train_fire_top20pct", + "seed": 37, + "source_priority": 0, + "strict_f1": 0.028949691085613415, + "threshold": 0.2, + "threshold_key": "0.2000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.20397124965652888, + "union_combo": "t3_s8", + "union_f1": 0.29215296893898335 + }, + { + "difference": 0.17556822794728488, + "label": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.02625751391465677, + "scope": "full_domain", + "seed": 57, + "source_priority": 0, + "strict_f1": 0.001008153911497155, + "threshold": 0.1, + "threshold_key": "0.1000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.09460750931776095, + "union_combo": "t3_s8", + "union_f1": 0.17657638185878205 + }, + { + "difference": 0.5636022889410879, + "label": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.1306866528148808, + "scope": "train_fire_top05pct", + "seed": 57, + "source_priority": 0, + "strict_f1": 0.017035016422646553, + "threshold": 0.05, + "threshold_key": "0.0500", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.3101459291213085, + "union_combo": "t3_s8", + "union_f1": 0.5806373053637345 + }, + { + "difference": 0.3946007462885974, + "label": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.02746487582863362, + "scope": "train_fire_top10pct", + "seed": 57, + "source_priority": 0, + "strict_f1": 0.018477139910622206, + "threshold": 0.08, + "threshold_key": "0.0800", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.2606560686639405, + "union_combo": "t3_s8", + "union_f1": 0.41307788619921965 + }, + { + "difference": 0.30932859219900066, + "label": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.021262213976499692, + "scope": "train_fire_top20pct", + "seed": 57, + "source_priority": 0, + "strict_f1": 0.01226752201682564, + "threshold": 0.08, + "threshold_key": "0.0800", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.19661692001462533, + "union_combo": "t3_s8", + "union_f1": 0.3215961142158263 + }, + { + "difference": 0.26339192421292845, + "label": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0589282622139765, + "scope": "full_domain", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.0016304484151205278, + "threshold": 0.08, + "threshold_key": "0.0800", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.14442504878529014, + "union_combo": "t3_s8", + "union_f1": 0.265022372628049 + }, + { + "difference": 0.45626799482364494, + "label": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.06984763035519936, + "scope": "train_fire_top05pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.016864097224592905, + "threshold": 0.08, + "threshold_key": "0.0800", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.28520211660517514, + "union_combo": "t3_s8", + "union_f1": 0.47313209204823786 + }, + { + "difference": 0.3591438068453628, + "label": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.052691204115959234, + "scope": "train_fire_top10pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.011312372621310312, + "threshold": 0.08, + "threshold_key": "0.0800", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.2160234001333052, + "union_combo": "t3_s8", + "union_f1": 0.3704561794666731 + }, + { + "difference": 0.2695667418377301, + "label": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.04168893011750155, + "scope": "train_fire_top20pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.007379212786746584, + "threshold": 0.08, + "threshold_key": "0.0800", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.1600767886426284, + "union_combo": "t3_s8", + "union_f1": 0.2769459546244767 + }, + { + "difference": 0.36515720594022893, + "label": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.028077427334570193, + "scope": "full_domain", + "seed": 97, + "source_priority": 0, + "strict_f1": 0.002078902199880332, + "threshold": 0.1, + "threshold_key": "0.1000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.16987085515333694, + "union_combo": "t3_s8", + "union_f1": 0.36723610814010926 + }, + { + "difference": 0.5906566149800531, + "label": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.16420797467102008, + "scope": "train_fire_top05pct", + "seed": 97, + "source_priority": 0, + "strict_f1": 0.011684287006420426, + "threshold": 0.05, + "threshold_key": "0.0500", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.2874938355126855, + "union_combo": "t3_s8", + "union_f1": 0.6023409019864735 + }, + { + "difference": 0.48212021492719026, + "label": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.11266572672405263, + "scope": "train_fire_top10pct", + "seed": 97, + "source_priority": 0, + "strict_f1": 0.008571862589787548, + "threshold": 0.05, + "threshold_key": "0.0500", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.22837595071441363, + "union_combo": "t3_s8", + "union_f1": 0.4906920775169778 + }, + { + "difference": 0.38082140073143794, + "label": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.02008596165739023, + "scope": "train_fire_top20pct", + "seed": 97, + "source_priority": 0, + "strict_f1": 0.008323671928416423, + "threshold": 0.1, + "threshold_key": "0.1000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.19681923783547378, + "union_combo": "t3_s8", + "union_f1": 0.3891450726598544 + }, + { + "difference": 0.3840989936961055, + "label": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.03192430426716141, + "scope": "full_domain", + "seed": 17, + "source_priority": 0, + "strict_f1": 0.002068681772914111, + "threshold": 0.15, + "threshold_key": "0.1500", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.19094441931639336, + "union_combo": "t3_s8", + "union_f1": 0.38616767546901964 + }, + { + "difference": 0.46051807014713964, + "label": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.06528643514395963, + "scope": "train_fire_top05pct", + "seed": 17, + "source_priority": 0, + "strict_f1": 0.012095158293562608, + "threshold": 0.15, + "threshold_key": "0.1500", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.25572856051368525, + "union_combo": "t3_s8", + "union_f1": 0.47261322844070225 + }, + { + "difference": 0.30442339402679647, + "label": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.03344711586029484, + "scope": "train_fire_top10pct", + "seed": 17, + "source_priority": 0, + "strict_f1": 0.006403642961329112, + "threshold": 0.2, + "threshold_key": "0.2000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.16176542623012913, + "union_combo": "t3_s8", + "union_f1": 0.3108270369881256 + }, + { + "difference": 0.2121721179401987, + "label": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.01035930735930736, + "scope": "train_fire_top20pct", + "seed": 17, + "source_priority": 0, + "strict_f1": 0.004578192172408018, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.10554731176319206, + "union_combo": "t3_s8", + "union_f1": 0.2167503101126067 + }, + { + "difference": 0.33775307067336774, + "label": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.009495980210265924, + "scope": "full_domain", + "seed": 37, + "source_priority": 0, + "strict_f1": 0.00208290997764682, + "threshold": 0.2, + "threshold_key": "0.2000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.16485748333864078, + "union_combo": "t3_s8", + "union_f1": 0.33983598065101456 + }, + { + "difference": 0.48713449736377723, + "label": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.01251113089937667, + "scope": "train_fire_top05pct", + "seed": 37, + "source_priority": 0, + "strict_f1": 0.024537870112874206, + "threshold": 0.2, + "threshold_key": "0.2000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.27999632336313063, + "union_combo": "t3_s8", + "union_f1": 0.5116723674766515 + }, + { + "difference": 0.4238672329193606, + "label": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.00802908875037103, + "scope": "train_fire_top10pct", + "seed": 37, + "source_priority": 0, + "strict_f1": 0.019849146486701073, + "threshold": 0.2, + "threshold_key": "0.2000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.23420382391023928, + "union_combo": "t3_s8", + "union_f1": 0.4437163794060617 + }, + { + "difference": 0.3647698181559645, + "label": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.005085961657390229, + "scope": "train_fire_top20pct", + "seed": 37, + "source_priority": 0, + "strict_f1": 0.016197783461210574, + "threshold": 0.2, + "threshold_key": "0.2000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.19512358967789506, + "union_combo": "t3_s8", + "union_f1": 0.3809676016171751 + }, + { + "difference": 0.2830040701913329, + "label": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0016064316635745208, + "scope": "full_domain", + "seed": 57, + "source_priority": 0, + "strict_f1": 0.0033447053314602988, + "threshold": 0.5, + "threshold_key": "0.5000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.16083569292964747, + "union_combo": "t3_s8", + "union_f1": 0.2863487755227932 + }, + { + "difference": 0.4820839133891601, + "label": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0197734243593549, + "scope": "train_fire_top05pct", + "seed": 57, + "source_priority": 0, + "strict_f1": 0.01569234169521494, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.300679830289768, + "union_combo": "t3_s8", + "union_f1": 0.49777625508437506 + }, + { + "difference": 0.387627119265477, + "label": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.015055159790244386, + "scope": "train_fire_top10pct", + "seed": 57, + "source_priority": 0, + "strict_f1": 0.010726695875509895, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.2256264421482596, + "union_combo": "t3_s8", + "union_f1": 0.3983538151409869 + }, + { + "difference": 0.3250188323968997, + "label": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.010080395794681509, + "scope": "train_fire_top20pct", + "seed": 57, + "source_priority": 0, + "strict_f1": 0.00859106529209622, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.18647162974553239, + "union_combo": "t3_s8", + "union_f1": 0.33360989768899596 + }, + { + "difference": 0.4822930267328773, + "label": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.027048113790970933, + "scope": "full_domain", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.0026467855605157606, + "threshold": 0.15, + "threshold_key": "0.1500", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.23211789308126832, + "union_combo": "t3_s8", + "union_f1": 0.48493981229339306 + }, + { + "difference": 0.5645980289087711, + "label": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.06537300880577818, + "scope": "train_fire_top05pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.01222529471692621, + "threshold": 0.15, + "threshold_key": "0.1500", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.3028035236186743, + "union_combo": "t3_s8", + "union_f1": 0.5768233236256973 + }, + { + "difference": 0.4452328295737776, + "label": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.030024982685267636, + "scope": "train_fire_top10pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.008918001736248123, + "threshold": 0.2, + "threshold_key": "0.2000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.2270292295163525, + "union_combo": "t3_s8", + "union_f1": 0.4541508313100257 + }, + { + "difference": 0.3226698082083355, + "label": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.009654916512059368, + "scope": "train_fire_top20pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.008704984497972811, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.1596269732904185, + "union_combo": "t3_s8", + "union_f1": 0.3313747927063083 + }, + { + "difference": 0.5017259192233794, + "label": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.005936920222634508, + "scope": "full_domain", + "seed": 97, + "source_priority": 0, + "strict_f1": 0.00400312243549969, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.22655113161542761, + "union_combo": "t3_s8", + "union_f1": 0.5057290416588791 + }, + { + "difference": 0.6396713821034217, + "label": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.07633570792520035, + "scope": "train_fire_top05pct", + "seed": 97, + "source_priority": 0, + "strict_f1": 0.016606090988845718, + "threshold": 0.15, + "threshold_key": "0.1500", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.3296389749733903, + "union_combo": "t3_s8", + "union_f1": 0.6562774730922674 + }, + { + "difference": 0.5525178187961205, + "label": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.049730384881765115, + "scope": "train_fire_top10pct", + "seed": 97, + "source_priority": 0, + "strict_f1": 0.012986069049061175, + "threshold": 0.15, + "threshold_key": "0.1500", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.2724598040570218, + "union_combo": "t3_s8", + "union_f1": 0.5655038878451817 + }, + { + "difference": 0.4298567373898511, + "label": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.006658627087198516, + "scope": "train_fire_top20pct", + "seed": 97, + "source_priority": 0, + "strict_f1": 0.011738073279114613, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.20219147661108805, + "union_combo": "t3_s8", + "union_f1": 0.4415948106689657 + }, + { + "difference": 0.2798111779709097, + "label": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.004433395176252319, + "scope": "full_domain", + "seed": 17, + "source_priority": 0, + "strict_f1": 0.003438698584843275, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.1569315132056949, + "union_combo": "t3_s8", + "union_f1": 0.283249876555753 + }, + { + "difference": 0.31898912374476807, + "label": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.008098347679825863, + "scope": "train_fire_top05pct", + "seed": 17, + "source_priority": 0, + "strict_f1": 0.01848001848001848, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.21207047330588913, + "union_combo": "t3_s8", + "union_f1": 0.3374691422247865 + }, + { + "difference": 0.28149494747245746, + "label": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.005720045513010785, + "scope": "train_fire_top10pct", + "seed": 17, + "source_priority": 0, + "strict_f1": 0.014059753954305799, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.17969557482138315, + "union_combo": "t3_s8", + "union_f1": 0.29555470142676327 + }, + { + "difference": 0.2536326799641239, + "label": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0035126777983920843, + "scope": "train_fire_top20pct", + "seed": 17, + "source_priority": 0, + "strict_f1": 0.011695906432748539, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.1579992352547558, + "union_combo": "t3_s8", + "union_f1": 0.26532858639687246 + }, + { + "difference": 0.2236424973966951, + "label": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0023098330241187385, + "scope": "full_domain", + "seed": 37, + "source_priority": 0, + "strict_f1": 0.0018414421399496025, + "threshold": 0.2, + "threshold_key": "0.2000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.10327893388535589, + "union_combo": "t3_s8", + "union_f1": 0.2254839395366447 + }, + { + "difference": 0.26699212025153246, + "label": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0032452755515979025, + "scope": "train_fire_top05pct", + "seed": 37, + "source_priority": 0, + "strict_f1": 0.01267427122940431, + "threshold": 0.2, + "threshold_key": "0.2000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.13288915578187116, + "union_combo": "t3_s8", + "union_f1": 0.27966639148093675 + }, + { + "difference": 0.24813584863600618, + "label": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0019738794894627485, + "scope": "train_fire_top10pct", + "seed": 37, + "source_priority": 0, + "strict_f1": 0.011273957158962795, + "threshold": 0.2, + "threshold_key": "0.2000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.1191832198916516, + "union_combo": "t3_s8", + "union_f1": 0.259409805794969 + }, + { + "difference": 0.2344501305987902, + "label": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0010810142238713667, + "scope": "train_fire_top20pct", + "seed": 37, + "source_priority": 0, + "strict_f1": 0.010316368638239341, + "threshold": 0.2, + "threshold_key": "0.2000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.11041562343515275, + "union_combo": "t3_s8", + "union_f1": 0.24476649923702953 + }, + { + "difference": 0.06735555938074993, + "label": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.00042881880024737165, + "scope": "full_domain", + "seed": 57, + "source_priority": 0, + "strict_f1": 0.001842299189388357, + "threshold": 0.5, + "threshold_key": "0.5000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.042906126190853267, + "union_combo": "t3_s8", + "union_f1": 0.06919785857013828 + }, + { + "difference": 0.3168732218355133, + "label": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.005567923221529633, + "scope": "train_fire_top05pct", + "seed": 57, + "source_priority": 0, + "strict_f1": 0.0205686630369026, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.1866312847963942, + "union_combo": "t3_s8", + "union_f1": 0.3374418848724159 + }, + { + "difference": 0.3141659158267793, + "label": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0028556940734144652, + "scope": "train_fire_top10pct", + "seed": 57, + "source_priority": 0, + "strict_f1": 0.02015411973918198, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.1840127014362662, + "union_combo": "t3_s8", + "union_f1": 0.33432003556596124 + }, + { + "difference": 0.2991917676543021, + "label": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0015157699443413728, + "scope": "train_fire_top20pct", + "seed": 57, + "source_priority": 0, + "strict_f1": 0.018831348656881748, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.17388386719173746, + "union_combo": "t3_s8", + "union_f1": 0.31802311631118385 + }, + { + "difference": 0.18397565646547948, + "label": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0016931354359925788, + "scope": "full_domain", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.00268370607028754, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.09710793321083327, + "union_combo": "t3_s8", + "union_f1": 0.186659362535767 + }, + { + "difference": 0.20932175095916816, + "label": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0030746017611556348, + "scope": "train_fire_top05pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.013054830287206266, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.12305221580797777, + "union_combo": "t3_s8", + "union_f1": 0.22237658124637444 + }, + { + "difference": 0.19895488431057093, + "label": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0019998515880083113, + "scope": "train_fire_top10pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.011185682326621925, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.11778287307219655, + "union_combo": "t3_s8", + "union_f1": 0.21014056663719285 + }, + { + "difference": 0.19412591498868878, + "label": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0010822510822510823, + "scope": "train_fire_top20pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.010309278350515464, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.11130870584431331, + "union_combo": "t3_s8", + "union_f1": 0.20443519333920424 + }, + { + "difference": 0.43726313068468925, + "label": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.012518243661100804, + "scope": "full_domain", + "seed": 97, + "source_priority": 0, + "strict_f1": 0.0032567291196169464, + "threshold": 0.2, + "threshold_key": "0.2000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.20002472134774485, + "union_combo": "t3_s8", + "union_f1": 0.4405198598043062 + }, + { + "difference": 0.3146235231202003, + "label": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.010032650638171565, + "scope": "train_fire_top05pct", + "seed": 97, + "source_priority": 0, + "strict_f1": 0.013695949911954608, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.15917085028122405, + "union_combo": "t3_s8", + "union_f1": 0.3283194730321549 + }, + { + "difference": 0.10438312130672354, + "label": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0004947066389630949, + "scope": "train_fire_top10pct", + "seed": 97, + "source_priority": 0, + "strict_f1": 0.005460750853242321, + "threshold": 0.5, + "threshold_key": "0.5000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.04684227229450968, + "union_combo": "t3_s8", + "union_f1": 0.10984387215996587 + }, + { + "difference": 0.1019380452757028, + "label": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0003116883116883117, + "scope": "train_fire_top20pct", + "seed": 97, + "source_priority": 0, + "strict_f1": 0.004807692307692307, + "threshold": 0.5, + "threshold_key": "0.5000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.04393081279657125, + "union_combo": "t3_s8", + "union_f1": 0.1067457375833951 + }, + { + "difference": 0.3742624367300788, + "label": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.004230179344465059, + "scope": "full_domain", + "seed": 17, + "source_priority": 0, + "strict_f1": 0.003097173828881146, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.20441214417386547, + "union_combo": "t3_s8", + "union_f1": 0.37735961055895995 + }, + { + "difference": 0.530575061359281, + "label": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.024220837043633127, + "scope": "train_fire_top05pct", + "seed": 17, + "source_priority": 0, + "strict_f1": 0.02138840232322301, + "threshold": 0.2, + "threshold_key": "0.2000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.30180702036802187, + "union_combo": "t3_s8", + "union_f1": 0.551963463682504 + }, + { + "difference": 0.43990346446961154, + "label": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0040170179083803305, + "scope": "train_fire_top10pct", + "seed": 17, + "source_priority": 0, + "strict_f1": 0.019476002782286114, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.26166119072060445, + "union_combo": "t3_s8", + "union_f1": 0.45937946725189766 + }, + { + "difference": 0.40169955480281794, + "label": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0031719233147804574, + "scope": "train_fire_top20pct", + "seed": 17, + "source_priority": 0, + "strict_f1": 0.013674670058832883, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.25519274478475146, + "union_combo": "t3_s8", + "union_f1": 0.4153742248616508 + }, + { + "difference": 0.217453250561222, + "label": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.001522943722943723, + "scope": "full_domain", + "seed": 37, + "source_priority": 0, + "strict_f1": 0.0029424127784783522, + "threshold": 0.2, + "threshold_key": "0.2000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.13539446199130012, + "union_combo": "t3_s8", + "union_f1": 0.22039566333970034 + }, + { + "difference": 0.3862603850081952, + "label": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.007007519540912239, + "scope": "train_fire_top05pct", + "seed": 37, + "source_priority": 0, + "strict_f1": 0.02109053497942387, + "threshold": 0.15, + "threshold_key": "0.1500", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.26086649919553234, + "union_combo": "t3_s8", + "union_f1": 0.4073509199876191 + }, + { + "difference": 0.23030507895417707, + "label": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0016448995745522905, + "scope": "train_fire_top10pct", + "seed": 37, + "source_priority": 0, + "strict_f1": 0.016701461377870565, + "threshold": 0.2, + "threshold_key": "0.2000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.1656142112861716, + "union_combo": "t3_s8", + "union_f1": 0.24700654033204764 + }, + { + "difference": 0.21606998197866278, + "label": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0009029066171923315, + "scope": "train_fire_top20pct", + "seed": 37, + "source_priority": 0, + "strict_f1": 0.015267175572519083, + "threshold": 0.2, + "threshold_key": "0.2000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.15340459969134643, + "union_combo": "t3_s8", + "union_f1": 0.23133715755118187 + }, + { + "difference": 0.35798567074450705, + "label": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0026514533085961658, + "scope": "full_domain", + "seed": 57, + "source_priority": 0, + "strict_f1": 0.005897939994871356, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.22566599998043468, + "union_combo": "t3_s8", + "union_f1": 0.3638836107393784 + }, + { + "difference": 0.49281035455029043, + "label": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.004068962105471455, + "scope": "train_fire_top05pct", + "seed": 57, + "source_priority": 0, + "strict_f1": 0.030370370370370364, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.3499638832422191, + "union_combo": "t3_s8", + "union_f1": 0.5231807249206608 + }, + { + "difference": 0.45700386933894555, + "label": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.002442614029880281, + "scope": "train_fire_top10pct", + "seed": 57, + "source_priority": 0, + "strict_f1": 0.026973684210526316, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.3231296638237823, + "union_combo": "t3_s8", + "union_f1": 0.48397755354947186 + }, + { + "difference": 0.4314537012527686, + "label": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0013982683982683983, + "scope": "train_fire_top20pct", + "seed": 57, + "source_priority": 0, + "strict_f1": 0.0239695995323005, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.30046102485294895, + "union_combo": "t3_s8", + "union_f1": 0.4554233007850691 + }, + { + "difference": 0.44532955766011967, + "label": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.005430797773654916, + "scope": "full_domain", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.0033573873422136956, + "threshold": 0.2, + "threshold_key": "0.2000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.2682493391647262, + "union_combo": "t3_s8", + "union_f1": 0.44868694500233336 + }, + { + "difference": 0.6138219771674631, + "label": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0665751459384585, + "scope": "train_fire_top05pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.016660707901322847, + "threshold": 0.1, + "threshold_key": "0.1000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.38200179460101663, + "union_combo": "t3_s8", + "union_f1": 0.6304826850687859 + }, + { + "difference": 0.34740284065122157, + "label": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0019404867913327397, + "scope": "train_fire_top10pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.011389521640091117, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.2251054852320675, + "union_combo": "t3_s8", + "union_f1": 0.35879236229131267 + }, + { + "difference": 0.2820322009484037, + "label": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0013902288188002473, + "scope": "train_fire_top20pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.008802816901408451, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.18265891116039673, + "union_combo": "t3_s8", + "union_f1": 0.29083501784981214 + }, + { + "difference": 0.45054053460571325, + "label": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.008687322201607915, + "scope": "full_domain", + "seed": 97, + "source_priority": 0, + "strict_f1": 0.003573506191307238, + "threshold": 0.2, + "threshold_key": "0.2000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.21789178988640112, + "union_combo": "t3_s8", + "union_f1": 0.4541140407970205 + }, + { + "difference": 0.5715381697222122, + "label": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.04470416542990007, + "scope": "train_fire_top05pct", + "seed": 97, + "source_priority": 0, + "strict_f1": 0.012024257632789628, + "threshold": 0.15, + "threshold_key": "0.1500", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.30008183753413636, + "union_combo": "t3_s8", + "union_f1": 0.5835624273550017 + }, + { + "difference": 0.45337080969477445, + "label": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.01593821114079351, + "scope": "train_fire_top10pct", + "seed": 97, + "source_priority": 0, + "strict_f1": 0.008170871559633027, + "threshold": 0.2, + "threshold_key": "0.2000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.22512986809908905, + "union_combo": "t3_s8", + "union_f1": 0.4615416812544075 + }, + { + "difference": 0.43708558281057147, + "label": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.011240568954854669, + "scope": "train_fire_top20pct", + "seed": 97, + "source_priority": 0, + "strict_f1": 0.006516342573438146, + "threshold": 0.2, + "threshold_key": "0.2000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.20602494796886814, + "union_combo": "t3_s8", + "union_f1": 0.4436019253840096 + }, + { + "difference": 0.2698875125828892, + "label": "Pangu-Weather", + "model_tag": "pangu_weather_h24", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0034029684601113172, + "scope": "full_domain", + "seed": 17, + "source_priority": 0, + "strict_f1": 0.0018999796430752525, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.11199020227536667, + "union_combo": "t3_s8", + "union_f1": 0.27178749222596443 + }, + { + "difference": 0.2627312378871571, + "label": "Pangu-Weather", + "model_tag": "pangu_weather_h24", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.00599584446423271, + "scope": "train_fire_top05pct", + "seed": 17, + "source_priority": 0, + "strict_f1": 0.006323656223052602, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.1236361475439318, + "union_combo": "t3_s8", + "union_f1": 0.26905489411020966 + }, + { + "difference": 0.2112720683625158, + "label": "Pangu-Weather", + "model_tag": "pangu_weather_h24", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.004856782428020184, + "scope": "train_fire_top10pct", + "seed": 17, + "source_priority": 0, + "strict_f1": 0.004407051282051282, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.09804613935969869, + "union_combo": "t3_s8", + "union_f1": 0.2156791196445671 + }, + { + "difference": 0.19289957921827447, + "label": "Pangu-Weather", + "model_tag": "pangu_weather_h24", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0036982065553494126, + "scope": "train_fire_top20pct", + "seed": 17, + "source_priority": 0, + "strict_f1": 0.003081232492997199, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.0918614020950846, + "union_combo": "t3_s8", + "union_f1": 0.19598081171127166 + }, + { + "difference": 0.44278198559528875, + "label": "Pangu-Weather", + "model_tag": "pangu_weather_h24", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.015876932591218307, + "scope": "full_domain", + "seed": 37, + "source_priority": 0, + "strict_f1": 0.002025689424980434, + "threshold": 0.1, + "threshold_key": "0.1000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.17658327178527805, + "union_combo": "t3_s8", + "union_f1": 0.4448076750202692 + }, + { + "difference": 0.5419792852745725, + "label": "Pangu-Weather", + "model_tag": "pangu_weather_h24", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.02707034728406055, + "scope": "train_fire_top05pct", + "seed": 37, + "source_priority": 0, + "strict_f1": 0.014334527877323108, + "threshold": 0.1, + "threshold_key": "0.1000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.23786880092608761, + "union_combo": "t3_s8", + "union_f1": 0.5563138131518957 + }, + { + "difference": 0.467280448823381, + "label": "Pangu-Weather", + "model_tag": "pangu_weather_h24", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.018105026219451866, + "scope": "train_fire_top10pct", + "seed": 37, + "source_priority": 0, + "strict_f1": 0.010952623535404991, + "threshold": 0.1, + "threshold_key": "0.1000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.19299696406724362, + "union_combo": "t3_s8", + "union_f1": 0.478233072358786 + }, + { + "difference": 0.31356600176953114, + "label": "Pangu-Weather", + "model_tag": "pangu_weather_h24", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.004520098948670378, + "scope": "train_fire_top20pct", + "seed": 37, + "source_priority": 0, + "strict_f1": 0.003306175463454953, + "threshold": 0.15, + "threshold_key": "0.1500", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.10046960132658891, + "union_combo": "t3_s8", + "union_f1": 0.3168721772329861 + }, + { + "difference": 0.3065577101854246, + "label": "Pangu-Weather", + "model_tag": "pangu_weather_h24", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0028299319727891157, + "scope": "full_domain", + "seed": 57, + "source_priority": 0, + "strict_f1": 0.00458918723078781, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.14940756501428915, + "union_combo": "t3_s8", + "union_f1": 0.31114689741621243 + }, + { + "difference": 0.3107866198278387, + "label": "Pangu-Weather", + "model_tag": "pangu_weather_h24", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.006668645493222519, + "scope": "train_fire_top05pct", + "seed": 57, + "source_priority": 0, + "strict_f1": 0.012796587576646227, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.15839502058327895, + "union_combo": "t3_s8", + "union_f1": 0.3235832074044849 + }, + { + "difference": 0.285638850087505, + "label": "Pangu-Weather", + "model_tag": "pangu_weather_h24", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.004374443455031167, + "scope": "train_fire_top10pct", + "seed": 57, + "source_priority": 0, + "strict_f1": 0.010430247718383311, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.14381976498868418, + "union_combo": "t3_s8", + "union_f1": 0.2960690978058883 + }, + { + "difference": 0.2659461938342217, + "label": "Pangu-Weather", + "model_tag": "pangu_weather_h24", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.002423005565862709, + "scope": "train_fire_top20pct", + "seed": 57, + "source_priority": 0, + "strict_f1": 0.0094525403702245, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.1330139941503748, + "union_combo": "t3_s8", + "union_f1": 0.2753987342044462 + }, + { + "difference": 0.1610876335708797, + "label": "Pangu-Weather", + "model_tag": "pangu_weather_h24", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0013768707482993197, + "scope": "full_domain", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.00412434125105018, + "threshold": 0.5, + "threshold_key": "0.5000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08503531719948608, + "union_combo": "t3_s8", + "union_f1": 0.1652119748219299 + }, + { + "difference": 0.27422409940357173, + "label": "Pangu-Weather", + "model_tag": "pangu_weather_h24", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.019518650440288908, + "scope": "train_fire_top05pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.00961323496534764, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.13752896018113672, + "union_combo": "t3_s8", + "union_f1": 0.2838373343689194 + }, + { + "difference": 0.14003318233886966, + "label": "Pangu-Weather", + "model_tag": "pangu_weather_h24", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0035594142673394677, + "scope": "train_fire_top10pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.005579507988840983, + "threshold": 0.5, + "threshold_key": "0.5000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.07891517659613233, + "union_combo": "t3_s8", + "union_f1": 0.14561269032771065 + }, + { + "difference": 0.13031562548755435, + "label": "Pangu-Weather", + "model_tag": "pangu_weather_h24", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0019332096474953618, + "scope": "train_fire_top20pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.0051329911339244045, + "threshold": 0.5, + "threshold_key": "0.5000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.07335569213391166, + "union_combo": "t3_s8", + "union_f1": 0.13544861662147875 + }, + { + "difference": 0.13447416377324056, + "label": "Pangu-Weather", + "model_tag": "pangu_weather_h24", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0013461966604823748, + "scope": "full_domain", + "seed": 97, + "source_priority": 0, + "strict_f1": 0.0015570260801868433, + "threshold": 0.5, + "threshold_key": "0.5000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.04422040571422221, + "union_combo": "t3_s8", + "union_f1": 0.1360311898534274 + }, + { + "difference": 0.35676759703557515, + "label": "Pangu-Weather", + "model_tag": "pangu_weather_h24", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.014237657069357871, + "scope": "train_fire_top05pct", + "seed": 97, + "source_priority": 0, + "strict_f1": 0.010571134928791661, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.16460271440438795, + "union_combo": "t3_s8", + "union_f1": 0.3673387319643668 + }, + { + "difference": 0.11744148944724705, + "label": "Pangu-Weather", + "model_tag": "pangu_weather_h24", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.002389433066191748, + "scope": "train_fire_top10pct", + "seed": 97, + "source_priority": 0, + "strict_f1": 0.002669336002669336, + "threshold": 0.5, + "threshold_key": "0.5000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.03347640061525648, + "union_combo": "t3_s8", + "union_f1": 0.12011082544991639 + }, + { + "difference": 0.11926427213402073, + "label": "Pangu-Weather", + "model_tag": "pangu_weather_h24", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0014211502782931354, + "scope": "train_fire_top20pct", + "seed": 97, + "source_priority": 0, + "strict_f1": 0.002313475997686524, + "threshold": 0.5, + "threshold_key": "0.5000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.036969834429575875, + "union_combo": "t3_s8", + "union_f1": 0.12157774813170726 + }, + { + "difference": 0.3981469865854522, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.01413716759431045, + "scope": "full_domain", + "seed": 17, + "source_priority": 0, + "strict_f1": 0.0019783244452090142, + "threshold": 0.2, + "threshold_key": "0.2000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.1809729686535962, + "union_combo": "t3_s8", + "union_f1": 0.40012531103066123 + }, + { + "difference": 0.4579044245182327, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.027451271396062134, + "scope": "train_fire_top05pct", + "seed": 17, + "source_priority": 0, + "strict_f1": 0.011848926191063935, + "threshold": 0.2, + "threshold_key": "0.2000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.23514322359982898, + "union_combo": "t3_s8", + "union_f1": 0.46975335070929664 + }, + { + "difference": 0.37162889777394226, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.021221678044919364, + "scope": "train_fire_top10pct", + "seed": 17, + "source_priority": 0, + "strict_f1": 0.008011413520632135, + "threshold": 0.2, + "threshold_key": "0.2000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.18336249012390426, + "union_combo": "t3_s8", + "union_f1": 0.3796403112945744 + }, + { + "difference": 0.3180949408491859, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.017168212739641312, + "scope": "train_fire_top20pct", + "seed": 17, + "source_priority": 0, + "strict_f1": 0.005117388748660143, + "threshold": 0.2, + "threshold_key": "0.2000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.14938342628742876, + "union_combo": "t3_s8", + "union_f1": 0.323212329597846 + }, + { + "difference": 0.3350135537819443, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.00479208410636982, + "scope": "full_domain", + "seed": 37, + "source_priority": 0, + "strict_f1": 0.0014740203906154037, + "threshold": 0.15, + "threshold_key": "0.1500", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.14188194761227685, + "union_combo": "t3_s8", + "union_f1": 0.3364875741725597 + }, + { + "difference": 0.42661251656646776, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.006297615514000198, + "scope": "train_fire_top05pct", + "seed": 37, + "source_priority": 0, + "strict_f1": 0.012218828103304637, + "threshold": 0.15, + "threshold_key": "0.1500", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.1893747086942956, + "union_combo": "t3_s8", + "union_f1": 0.4388313446697724 + }, + { + "difference": 0.4013551577246312, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.004044226773523301, + "scope": "train_fire_top10pct", + "seed": 37, + "source_priority": 0, + "strict_f1": 0.010149942329873126, + "threshold": 0.15, + "threshold_key": "0.1500", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.1623425297604306, + "union_combo": "t3_s8", + "union_f1": 0.41150510005450436 + }, + { + "difference": 0.36502027762951006, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0025015460729746444, + "scope": "train_fire_top20pct", + "seed": 37, + "source_priority": 0, + "strict_f1": 0.008453410182516812, + "threshold": 0.15, + "threshold_key": "0.1500", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.15039592221466835, + "union_combo": "t3_s8", + "union_f1": 0.3734736878120269 + }, + { + "difference": 0.35070363810152716, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.004669140383426098, + "scope": "full_domain", + "seed": 57, + "source_priority": 0, + "strict_f1": 0.003978746443051044, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.18301623358471802, + "union_combo": "t3_s8", + "union_f1": 0.3546823845445782 + }, + { + "difference": 0.4222173656123986, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.008479271791827446, + "scope": "train_fire_top05pct", + "seed": 57, + "source_priority": 0, + "strict_f1": 0.017845192951148784, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.2501388218123639, + "union_combo": "t3_s8", + "union_f1": 0.44006255856354737 + }, + { + "difference": 0.40500926707190377, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.004714554269318294, + "scope": "train_fire_top10pct", + "seed": 57, + "source_priority": 0, + "strict_f1": 0.016403526758253024, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.23804674197160172, + "union_combo": "t3_s8", + "union_f1": 0.4214127938301568 + }, + { + "difference": 0.3810054677452299, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0026456400742115027, + "scope": "train_fire_top20pct", + "seed": 57, + "source_priority": 0, + "strict_f1": 0.01471129091577786, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.2249788440916092, + "union_combo": "t3_s8", + "union_f1": 0.3957167586610078 + }, + { + "difference": 0.21893843540252053, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0012771799628942485, + "scope": "full_domain", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.0037437942540896885, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.12218430629860097, + "union_combo": "t3_s8", + "union_f1": 0.22268222965661022 + }, + { + "difference": 0.21672193002919604, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.004447412684278223, + "scope": "train_fire_top05pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.010515247108307044, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.12664654352782623, + "union_combo": "t3_s8", + "union_f1": 0.22723717713750308 + }, + { + "difference": 0.18892752891362113, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0033887404768972, + "scope": "train_fire_top10pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.00788436268068331, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.1079162350962175, + "union_combo": "t3_s8", + "union_f1": 0.19681189159430446 + }, + { + "difference": 0.17487748330642924, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.001932591218305504, + "scope": "train_fire_top20pct", + "seed": 7, + "source_priority": 0, + "strict_f1": 0.0070011668611435224, + "threshold": 0.3, + "threshold_key": "0.3000", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.09937369519832985, + "union_combo": "t3_s8", + "union_f1": 0.18187865016757276 + }, + { + "difference": 0.4653493031095141, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.029129251700680273, + "scope": "full_domain", + "seed": 97, + "source_priority": 0, + "strict_f1": 0.0026024230327071516, + "threshold": 0.15, + "threshold_key": "0.1500", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.2264875300093167, + "union_combo": "t3_s8", + "union_f1": 0.46795172614222125 + }, + { + "difference": 0.5794332814358789, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.056067576926882357, + "scope": "train_fire_top05pct", + "seed": 97, + "source_priority": 0, + "strict_f1": 0.015850265576258328, + "threshold": 0.15, + "threshold_key": "0.1500", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.3098064830268362, + "union_combo": "t3_s8", + "union_f1": 0.5952835470121373 + }, + { + "difference": 0.5050483949122122, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.03677772830711388, + "scope": "train_fire_top10pct", + "seed": 97, + "source_priority": 0, + "strict_f1": 0.012206999545484059, + "threshold": 0.15, + "threshold_key": "0.1500", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.25501453744070673, + "union_combo": "t3_s8", + "union_f1": 0.5172553944576963 + }, + { + "difference": 0.44522188799641144, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.026268398268398267, + "scope": "train_fire_top20pct", + "seed": 97, + "source_priority": 0, + "strict_f1": 0.008937574479787332, + "threshold": 0.15, + "threshold_key": "0.1500", + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.226729727407505, + "union_combo": "t3_s8", + "union_f1": 0.45415946247619876 + }, + { + "difference": 0.5480197595988799, + "label": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.016084972170686457, + "scope": "full_domain", + "seed": 1, + "source_priority": 1, + "strict_f1": 0.004136112962850736, + "threshold": 0.95, + "threshold_key": "0.95", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.2985122125101588, + "union_combo": "t3_s8", + "union_f1": 0.5521558725617307 + }, + { + "difference": 0.6696707231715443, + "label": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.023750865736618186, + "scope": "train_fire_top05pct", + "seed": 1, + "source_priority": 1, + "strict_f1": 0.03321760345312939, + "threshold": 0.95, + "threshold_key": "0.95", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.39555976109165375, + "union_combo": "t3_s8", + "union_f1": 0.7028883266246737 + }, + { + "difference": 0.6666121634162013, + "label": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.011875432868309093, + "scope": "train_fire_top10pct", + "seed": 1, + "source_priority": 1, + "strict_f1": 0.033186462923033656, + "threshold": 0.95, + "threshold_key": "0.95", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.3945515652074689, + "union_combo": "t3_s8", + "union_f1": 0.699798626339235 + }, + { + "difference": 0.6388534040739275, + "label": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0059381570810142234, + "scope": "train_fire_top20pct", + "seed": 1, + "source_priority": 1, + "strict_f1": 0.032893514216688347, + "threshold": 0.95, + "threshold_key": "0.95", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.3852239693382854, + "union_combo": "t3_s8", + "union_f1": 0.6717469182906158 + }, + { + "difference": 0.5862595758183573, + "label": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.021380705009276437, + "scope": "full_domain", + "seed": 123, + "source_priority": 1, + "strict_f1": 0.0026426577586601383, + "threshold": 0.95, + "threshold_key": "0.95", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.2756667293290642, + "union_combo": "t3_s8", + "union_f1": 0.5889022335770174 + }, + { + "difference": 0.6960690170722506, + "label": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.04860987434451371, + "scope": "train_fire_top05pct", + "seed": 123, + "source_priority": 1, + "strict_f1": 0.02462935239291061, + "threshold": 0.9, + "threshold_key": "0.9", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.3702273155255325, + "union_combo": "t3_s8", + "union_f1": 0.7206983694651612 + }, + { + "difference": 0.6932811848453125, + "label": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.024304937172256853, + "scope": "train_fire_top10pct", + "seed": 123, + "source_priority": 1, + "strict_f1": 0.02461746391852102, + "threshold": 0.9, + "threshold_key": "0.9", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.369479820868587, + "union_combo": "t3_s8", + "union_f1": 0.7178986487638336 + }, + { + "difference": 0.6678424934752141, + "label": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.012153370439084725, + "scope": "train_fire_top20pct", + "seed": 123, + "source_priority": 1, + "strict_f1": 0.024505093215452623, + "threshold": 0.9, + "threshold_key": "0.9", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.36252633147981417, + "union_combo": "t3_s8", + "union_f1": 0.6923475866906666 + }, + { + "difference": 0.6194077454910121, + "label": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.019636239950525665, + "scope": "full_domain", + "seed": 42, + "source_priority": 1, + "strict_f1": 0.0045296167247386755, + "threshold": 0.95, + "threshold_key": "0.95", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.3056545098251043, + "union_combo": "t3_s8", + "union_f1": 0.6239373622157508 + }, + { + "difference": 0.7095179230119556, + "label": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.033442168793905214, + "scope": "train_fire_top05pct", + "seed": 42, + "source_priority": 1, + "strict_f1": 0.038147512864494, + "threshold": 0.95, + "threshold_key": "0.95", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.4063154181985604, + "union_combo": "t3_s8", + "union_f1": 0.7476654358764496 + }, + { + "difference": 0.7063341380484894, + "label": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.016721084396952607, + "scope": "train_fire_top10pct", + "seed": 42, + "source_priority": 1, + "strict_f1": 0.0381213575591361, + "threshold": 0.95, + "threshold_key": "0.95", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.4052951105632299, + "union_combo": "t3_s8", + "union_f1": 0.7444554956076256 + }, + { + "difference": 0.677407206441351, + "label": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.008361162646876932, + "scope": "train_fire_top20pct", + "seed": 42, + "source_priority": 1, + "strict_f1": 0.03787465940054495, + "threshold": 0.95, + "threshold_key": "0.95", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.39585180444145995, + "union_combo": "t3_s8", + "union_f1": 0.7152818658418959 + }, + { + "difference": 0.601348973225457, + "label": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.014596289424860853, + "scope": "full_domain", + "seed": 7, + "source_priority": 1, + "strict_f1": 0.006568199246490847, + "threshold": 0.95, + "threshold_key": "0.95", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.29937890424270136, + "union_combo": "t3_s8", + "union_f1": 0.6079171724719479 + }, + { + "difference": 0.7136230759527924, + "label": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.02449292569506283, + "scope": "train_fire_top05pct", + "seed": 7, + "source_priority": 1, + "strict_f1": 0.04873596787441818, + "threshold": 0.95, + "threshold_key": "0.95", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.40145293653054454, + "union_combo": "t3_s8", + "union_f1": 0.7623590438272105 + }, + { + "difference": 0.7104053704635315, + "label": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.012246462847531414, + "scope": "train_fire_top10pct", + "seed": 7, + "source_priority": 1, + "strict_f1": 0.0486915291328531, + "threshold": 0.95, + "threshold_key": "0.95", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.4004617740449798, + "union_combo": "t3_s8", + "union_f1": 0.7590968995963846 + }, + { + "difference": 0.6811711568125168, + "label": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.006123685837971553, + "scope": "train_fire_top20pct", + "seed": 7, + "source_priority": 1, + "strict_f1": 0.04827336828783222, + "threshold": 0.95, + "threshold_key": "0.95", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.39128423640663973, + "union_combo": "t3_s8", + "union_f1": 0.7294445251003491 + }, + { + "difference": 0.5755105867935725, + "label": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.012341125541125542, + "scope": "full_domain", + "seed": 99, + "source_priority": 1, + "strict_f1": 0.004855561780634761, + "threshold": 0.95, + "threshold_key": "0.95", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.30820726968158313, + "union_combo": "t3_s8", + "union_f1": 0.5803661485742072 + }, + { + "difference": 0.6744974001959292, + "label": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.022801028989809045, + "scope": "train_fire_top05pct", + "seed": 99, + "source_priority": 1, + "strict_f1": 0.0332911515623479, + "threshold": 0.95, + "threshold_key": "0.95", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.3895272012200133, + "union_combo": "t3_s8", + "union_f1": 0.707788551758277 + }, + { + "difference": 0.6715102385713686, + "label": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.011400514494904522, + "scope": "train_fire_top10pct", + "seed": 99, + "source_priority": 1, + "strict_f1": 0.033258776621608484, + "threshold": 0.95, + "threshold_key": "0.95", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.38853542457185924, + "union_combo": "t3_s8", + "union_f1": 0.7047690151929771 + }, + { + "difference": 0.6443640032843302, + "label": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.005700680272108844, + "scope": "train_fire_top20pct", + "seed": 99, + "source_priority": 1, + "strict_f1": 0.03295432645981885, + "threshold": 0.95, + "threshold_key": "0.95", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.37935948303093225, + "union_combo": "t3_s8", + "union_f1": 0.6773183297441491 + }, + { + "difference": 0.14280219707769823, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.21716512059369203, + "scope": "full_domain", + "seed": 1, + "source_priority": 2, + "strict_f1": 0.0004938156417811271, + "threshold": 0.0002848515287041664, + "threshold_key": "0.00028485153", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.05024333401294363, + "union_combo": "t3_s8", + "union_f1": 0.14329601271947937 + }, + { + "difference": 0.4699697200479976, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.08275452656574651, + "scope": "train_fire_top05pct", + "seed": 1, + "source_priority": 2, + "strict_f1": 0.026484309350641822, + "threshold": 0.0002847809228114784, + "threshold_key": "0.00028478092", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.24351172066402033, + "union_combo": "t3_s8", + "union_f1": 0.4964540293986394 + }, + { + "difference": 0.46744943745251477, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.04137726328287326, + "scope": "train_fire_top10pct", + "seed": 1, + "source_priority": 2, + "strict_f1": 0.026476637409113293, + "threshold": 0.0002847809228114784, + "threshold_key": "0.00028478092", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.24290193233931348, + "union_combo": "t3_s8", + "union_f1": 0.49392607486162804 + }, + { + "difference": 0.24417949078081141, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.7199758812615955, + "scope": "full_domain", + "seed": 123, + "source_priority": 2, + "strict_f1": 0.0006488102455003172, + "threshold": 0.00028207036666572094, + "threshold_key": "0.00028207037", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.09040502340162183, + "union_combo": "t3_s8", + "union_f1": 0.24482830102631173 + }, + { + "difference": 0.3890802257636093, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.921086870485802, + "scope": "train_fire_top05pct", + "seed": 123, + "source_priority": 2, + "strict_f1": 0.005618158058227469, + "threshold": 0.0002819437358994037, + "threshold_key": "0.00028194374", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.16005632828519903, + "union_combo": "t3_s8", + "union_f1": 0.39469838382183675 + }, + { + "difference": 0.2284351905540453, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9565152864351439, + "scope": "train_fire_top10pct", + "seed": 123, + "source_priority": 2, + "strict_f1": 0.00275028568108307, + "threshold": 0.00028190662851557136, + "threshold_key": "0.00028190663", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08722255267001126, + "union_combo": "t3_s8", + "union_f1": 0.23118547623512836 + }, + { + "difference": 0.2520341870433926, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.5915943104514533, + "scope": "full_domain", + "seed": 42, + "source_priority": 2, + "strict_f1": 0.0007076278562951189, + "threshold": 0.00028167077107355, + "threshold_key": "0.00028167077", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.09343380358557288, + "union_combo": "t3_s8", + "union_f1": 0.2527418148996877 + }, + { + "difference": 0.25643683344302254, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.48403581676066093, + "scope": "train_fire_top05pct", + "seed": 42, + "source_priority": 2, + "strict_f1": 0.00618071474679909, + "threshold": 0.00028234132332727313, + "threshold_key": "0.00028234132", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.10264712599745741, + "union_combo": "t3_s8", + "union_f1": 0.26261754818982164 + }, + { + "difference": 0.28330440732569195, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.735743791431681, + "scope": "train_fire_top10pct", + "seed": 42, + "source_priority": 2, + "strict_f1": 0.0035136704264394467, + "threshold": 0.00028129236306995153, + "threshold_key": "0.00028129236", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.11052705132032803, + "union_combo": "t3_s8", + "union_f1": 0.2868180777521314 + }, + { + "difference": 0.2611249045411472, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.6671256648113791, + "scope": "full_domain", + "seed": 7, + "source_priority": 2, + "strict_f1": 0.0007124228455695601, + "threshold": 0.0002701166733984173, + "threshold_key": "0.00027011667", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.09742777619989017, + "union_combo": "t3_s8", + "union_f1": 0.2618373273867167 + }, + { + "difference": 0.38950157948139946, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9150341347580885, + "scope": "train_fire_top05pct", + "seed": 7, + "source_priority": 2, + "strict_f1": 0.00565521517042468, + "threshold": 0.00026978866662830114, + "threshold_key": "0.00026978867", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.16052688500868942, + "union_combo": "t3_s8", + "union_f1": 0.39515679465182413 + }, + { + "difference": 0.2284351905540453, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9565152864351439, + "scope": "train_fire_top10pct", + "seed": 7, + "source_priority": 2, + "strict_f1": 0.00275028568108307, + "threshold": 0.00026914896443486214, + "threshold_key": "0.00026914896", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08722255267001126, + "union_combo": "t3_s8", + "union_f1": 0.23118547623512836 + }, + { + "difference": 0.25176783806212244, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.5833835497835498, + "scope": "full_domain", + "seed": 99, + "source_priority": 2, + "strict_f1": 0.0007150400212604667, + "threshold": 0.0002832056488841772, + "threshold_key": "0.00028320565", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.09353511619384626, + "union_combo": "t3_s8", + "union_f1": 0.25248287808338293 + }, + { + "difference": 0.21988543507354893, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.4582789156030474, + "scope": "train_fire_top05pct", + "seed": 99, + "source_priority": 2, + "strict_f1": 0.0053561461508737284, + "threshold": 0.00028413085965439677, + "threshold_key": "0.00028413086", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08994268846280176, + "union_combo": "t3_s8", + "union_f1": 0.22524158122442267 + }, + { + "difference": 0.2799506337602843, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.749557237558128, + "scope": "train_fire_top10pct", + "seed": 99, + "source_priority": 2, + "strict_f1": 0.0034589138681033788, + "threshold": 0.000282647815765813, + "threshold_key": "0.00028264782", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.10903157063807417, + "union_combo": "t3_s8", + "union_f1": 0.28340954762838766 + }, + { + "difference": 0.6362769893231447, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.041071861471861475, + "scope": "full_domain", + "seed": 1, + "source_priority": 2, + "strict_f1": 0.003394935139973715, + "threshold": 0.001442534703828549, + "threshold_key": "0.0014425347", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.3112148878371163, + "union_combo": "t3_s8", + "union_f1": 0.6396719244631184 + }, + { + "difference": 0.6341272298959009, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.13341248639556744, + "scope": "train_fire_top05pct", + "seed": 1, + "source_priority": 2, + "strict_f1": 0.011492789729228419, + "threshold": 0.0014882655838293205, + "threshold_key": "0.0014882656", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.3186326966868806, + "union_combo": "t3_s8", + "union_f1": 0.6456200196251293 + }, + { + "difference": 0.6312315933404747, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.06670624319778372, + "scope": "train_fire_top10pct", + "seed": 1, + "source_priority": 2, + "strict_f1": 0.011490700169087834, + "threshold": 0.0014882655838293205, + "threshold_key": "0.0014882656", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.3179193199949169, + "union_combo": "t3_s8", + "union_f1": 0.6427222935095626 + }, + { + "difference": 0.604731231099908, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.12421632653061225, + "scope": "full_domain", + "seed": 123, + "source_priority": 2, + "strict_f1": 0.002897888198757764, + "threshold": 0.0005393968409013734, + "threshold_key": "0.00053939684", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.2855276491036767, + "union_combo": "t3_s8", + "union_f1": 0.6076291192986658 + }, + { + "difference": 0.7052352926158846, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.3074774908479272, + "scope": "train_fire_top05pct", + "seed": 123, + "source_priority": 2, + "strict_f1": 0.012252516711603196, + "threshold": 0.0005436268660297483, + "threshold_key": "0.00054362687", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.3566174282966785, + "union_combo": "t3_s8", + "union_f1": 0.7174878093274878 + }, + { + "difference": 0.6886984637018222, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.14598421885821708, + "scope": "train_fire_top10pct", + "seed": 123, + "source_priority": 2, + "strict_f1": 0.010293697838827224, + "threshold": 0.0006776005131691771, + "threshold_key": "0.00067760051", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.34872837593596695, + "union_combo": "t3_s8", + "union_f1": 0.6989921615406495 + }, + { + "difference": 0.615867982755411, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.037147804576376, + "scope": "full_domain", + "seed": 42, + "source_priority": 2, + "strict_f1": 0.004637761701086003, + "threshold": 0.0014632086704014654, + "threshold_key": "0.0014632087", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.31289270592627744, + "union_combo": "t3_s8", + "union_f1": 0.620505744456497 + }, + { + "difference": 0.6028322641867834, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.14358118135945386, + "scope": "train_fire_top05pct", + "seed": 42, + "source_priority": 2, + "strict_f1": 0.014246556800108287, + "threshold": 0.0014632082671483923, + "threshold_key": "0.0014632083", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.32161191611096424, + "union_combo": "t3_s8", + "union_f1": 0.6170788209868917 + }, + { + "difference": 0.5999371340636104, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.07179059067972693, + "scope": "train_fire_top10pct", + "seed": 42, + "source_priority": 2, + "strict_f1": 0.014244146704560834, + "threshold": 0.0014632082671483923, + "threshold_key": "0.0014632083", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.3208182317552862, + "union_combo": "t3_s8", + "union_f1": 0.6141812807681712 + }, + { + "difference": 0.46922312530335947, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.025392455163883734, + "scope": "full_domain", + "seed": 7, + "source_priority": 2, + "strict_f1": 0.0027501821392557145, + "threshold": 0.0013572092986540852, + "threshold_key": "0.0013572093", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.24162386260287075, + "union_combo": "t3_s8", + "union_f1": 0.4719733074426152 + }, + { + "difference": 0.7267635176891648, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.3068887899475611, + "scope": "train_fire_top05pct", + "seed": 7, + "source_priority": 2, + "strict_f1": 0.013106997858124742, + "threshold": 0.0007465386633598005, + "threshold_key": "0.00074653866", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.3655334236435691, + "union_combo": "t3_s8", + "union_f1": 0.7398705155472896 + }, + { + "difference": 0.7230353684802844, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.15368556446027506, + "scope": "train_fire_top10pct", + "seed": 7, + "source_priority": 2, + "strict_f1": 0.01308555880921415, + "threshold": 0.0007519771209341887, + "threshold_key": "0.00075197712", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.36399388919405107, + "union_combo": "t3_s8", + "union_f1": 0.7361209272894985 + }, + { + "difference": 0.664031416861268, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.055986765615337046, + "scope": "full_domain", + "seed": 99, + "source_priority": 2, + "strict_f1": 0.003717439410136951, + "threshold": 0.001021229882830197, + "threshold_key": "0.0010212299", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.33641526956922785, + "union_combo": "t3_s8", + "union_f1": 0.6677488562714049 + }, + { + "difference": 0.7272887839717904, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.3090951815573365, + "scope": "train_fire_top05pct", + "seed": 99, + "source_priority": 2, + "strict_f1": 0.013585576434738445, + "threshold": 0.0006166165592580956, + "threshold_key": "0.00061661656", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.3665570668664193, + "union_combo": "t3_s8", + "union_f1": 0.7408743604065289 + }, + { + "difference": 0.7230514919507499, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.15408504007123774, + "scope": "train_fire_top10pct", + "seed": 99, + "source_priority": 2, + "strict_f1": 0.013497596536465794, + "threshold": 0.0006294738670532621, + "threshold_key": "0.00062947387", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.36524448333846904, + "union_combo": "t3_s8", + "union_f1": 0.7365490884872157 + }, + { + "difference": 0.1624675064035813, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.21222844774273344, + "scope": "full_domain", + "seed": 1, + "source_priority": 2, + "strict_f1": 0.0005518596739603732, + "threshold": 0.00027441306156106293, + "threshold_key": "0.00027441306", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06046181002685303, + "union_combo": "t3_s8", + "union_f1": 0.16301936607754167 + }, + { + "difference": 0.4598793069893796, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.08085485307212822, + "scope": "train_fire_top05pct", + "seed": 1, + "source_priority": 2, + "strict_f1": 0.026612927125626056, + "threshold": 0.00027438640245236456, + "threshold_key": "0.0002743864", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.24179585238367046, + "union_combo": "t3_s8", + "union_f1": 0.4864922341150057 + }, + { + "difference": 0.4574594224967701, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.04042742653606411, + "scope": "train_fire_top10pct", + "seed": 1, + "source_priority": 2, + "strict_f1": 0.02660504251473943, + "threshold": 0.00027438640245236456, + "threshold_key": "0.0002743864", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.2411933776075557, + "union_combo": "t3_s8", + "union_f1": 0.4840644650115095 + }, + { + "difference": 0.24509851810436645, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.7109064935064935, + "scope": "full_domain", + "seed": 123, + "source_priority": 2, + "strict_f1": 0.0006508233558970649, + "threshold": 0.0002703598001971841, + "threshold_key": "0.0002703598", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.09112517773927671, + "union_combo": "t3_s8", + "union_f1": 0.2457493414602635 + }, + { + "difference": 0.38912034744679525, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9211709706144257, + "scope": "train_fire_top05pct", + "seed": 123, + "source_priority": 2, + "strict_f1": 0.005639067545640032, + "threshold": 0.0002697413437999785, + "threshold_key": "0.00026974134", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.16012924166858108, + "union_combo": "t3_s8", + "union_f1": 0.3947594149924353 + }, + { + "difference": 0.2284057583930383, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9564299495399228, + "scope": "train_fire_top10pct", + "seed": 123, + "source_priority": 2, + "strict_f1": 0.0027453654202759313, + "threshold": 0.0002697413437999785, + "threshold_key": "0.00026974134", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08722526453259725, + "union_combo": "t3_s8", + "union_f1": 0.23115112381331424 + }, + { + "difference": 0.2016630331978699, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.8827594310451453, + "scope": "full_domain", + "seed": 42, + "source_priority": 2, + "strict_f1": 0.0005395660023552084, + "threshold": 0.00024381681578233838, + "threshold_key": "0.00024381682", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.07610963330033767, + "union_combo": "t3_s8", + "union_f1": 0.20220259920022512 + }, + { + "difference": 0.3868339716949791, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9162387454239636, + "scope": "train_fire_top05pct", + "seed": 42, + "source_priority": 2, + "strict_f1": 0.00568010509540423, + "threshold": 0.0002431461471132934, + "threshold_key": "0.00024314615", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.16063421440737302, + "union_combo": "t3_s8", + "union_f1": 0.3925140767903833 + }, + { + "difference": 0.2276584353760274, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9562827743148313, + "scope": "train_fire_top10pct", + "seed": 42, + "source_priority": 2, + "strict_f1": 0.002750953470140531, + "threshold": 0.00024276715703308582, + "threshold_key": "0.00024276716", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08724283500251707, + "union_combo": "t3_s8", + "union_f1": 0.23040938884616793 + }, + { + "difference": 0.24742507585853188, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.7071789734075449, + "scope": "full_domain", + "seed": 7, + "source_priority": 2, + "strict_f1": 0.0006577494583008349, + "threshold": 0.0002784867538139224, + "threshold_key": "0.00027848675", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.09170559714407109, + "union_combo": "t3_s8", + "union_f1": 0.2480828253168327 + }, + { + "difference": 0.38914191998978065, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9213416444048679, + "scope": "train_fire_top05pct", + "seed": 7, + "source_priority": 2, + "strict_f1": 0.005648734389013078, + "threshold": 0.0002780304348561913, + "threshold_key": "0.00027803043", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.1601110414083686, + "union_combo": "t3_s8", + "union_f1": 0.39479065437879374 + }, + { + "difference": 0.2284351905540453, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9565152864351439, + "scope": "train_fire_top10pct", + "seed": 7, + "source_priority": 2, + "strict_f1": 0.00275028568108307, + "threshold": 0.0002780304348561913, + "threshold_key": "0.00027803043", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08722255267001126, + "union_combo": "t3_s8", + "union_f1": 0.23118547623512836 + }, + { + "difference": 0.24307011894354136, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.655947804576376, + "scope": "full_domain", + "seed": 99, + "source_priority": 2, + "strict_f1": 0.000687991383709005, + "threshold": 0.00027882601716555655, + "threshold_key": "0.00027882602", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08961425169301492, + "union_combo": "t3_s8", + "union_f1": 0.24375811032725037 + }, + { + "difference": 0.27868986084470576, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.28134213911150685, + "scope": "train_fire_top05pct", + "seed": 99, + "source_priority": 2, + "strict_f1": 0.009651904247534756, + "threshold": 0.00028155450127087533, + "threshold_key": "0.0002815545", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.1110438397786047, + "union_combo": "t3_s8", + "union_f1": 0.2883417650922405 + }, + { + "difference": 0.02818447450809833, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0007742158899772435, + "scope": "train_fire_top10pct", + "seed": 99, + "source_priority": 2, + "strict_f1": 0.0011827321111768183, + "threshold": 0.000285720539977774, + "threshold_key": "0.00028572054", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.019439944457301548, + "union_combo": "t3_s8", + "union_f1": 0.029367206619275145 + }, + { + "difference": 0.2215775413347868, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.5006159554730983, + "scope": "full_domain", + "seed": 1, + "source_priority": 2, + "strict_f1": 0.0006040339888888368, + "threshold": 0.00028278783429414034, + "threshold_key": "0.00028278783", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.07892160427759505, + "union_combo": "t3_s8", + "union_f1": 0.22218157532367563 + }, + { + "difference": 0.38844131515497954, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9234194122885129, + "scope": "train_fire_top05pct", + "seed": 1, + "source_priority": 2, + "strict_f1": 0.005636060100166944, + "threshold": 0.00018521470337873325, + "threshold_key": "0.0001852147", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.15977956000729543, + "union_combo": "t3_s8", + "union_f1": 0.39407737525514647 + }, + { + "difference": 0.2282163144290469, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9575541703769664, + "scope": "train_fire_top10pct", + "seed": 1, + "source_priority": 2, + "strict_f1": 0.002747305898968793, + "threshold": 0.00018521470337873325, + "threshold_key": "0.0001852147", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08713204456539597, + "union_combo": "t3_s8", + "union_f1": 0.2309636203280157 + }, + { + "difference": 0.20020081119628272, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9588571428571429, + "scope": "full_domain", + "seed": 123, + "source_priority": 2, + "strict_f1": 0.0005057825179019543, + "threshold": 0.00028123805532231927, + "threshold_key": "0.00028123806", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.07212988048730175, + "union_combo": "t3_s8", + "union_f1": 0.20070659371418467 + }, + { + "difference": 0.38844131515497954, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9234194122885129, + "scope": "train_fire_top05pct", + "seed": 123, + "source_priority": 2, + "strict_f1": 0.005636060100166944, + "threshold": 0.0002438416049699299, + "threshold_key": "0.0002438416", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.15977956000729543, + "union_combo": "t3_s8", + "union_f1": 0.39407737525514647 + }, + { + "difference": 0.2282163144290469, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9575541703769664, + "scope": "train_fire_top10pct", + "seed": 123, + "source_priority": 2, + "strict_f1": 0.002747305898968793, + "threshold": 0.0002438416049699299, + "threshold_key": "0.0002438416", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08713204456539597, + "union_combo": "t3_s8", + "union_f1": 0.2309636203280157 + }, + { + "difference": 0.17211673370575886, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.22581076066790354, + "scope": "full_domain", + "seed": 42, + "source_priority": 2, + "strict_f1": 0.0005777939978365554, + "threshold": 0.00028831177041865885, + "threshold_key": "0.00028831177", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06182533987250242, + "union_combo": "t3_s8", + "union_f1": 0.17269452770359542 + }, + { + "difference": 0.46663155596188977, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.08637577916295637, + "scope": "train_fire_top05pct", + "seed": 42, + "source_priority": 2, + "strict_f1": 0.02590687977762335, + "threshold": 0.0002883114793803543, + "threshold_key": "0.00028831148", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.23967146506700618, + "union_combo": "t3_s8", + "union_f1": 0.4925384357395131 + }, + { + "difference": 0.46415551438013897, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.043187889581478185, + "scope": "train_fire_top10pct", + "seed": 42, + "source_priority": 2, + "strict_f1": 0.02589968042239822, + "threshold": 0.0002883114793803543, + "threshold_key": "0.00028831148", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.23908194736539978, + "union_combo": "t3_s8", + "union_f1": 0.4900551948025372 + }, + { + "difference": 0.20983886056574044, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9099814471243043, + "scope": "full_domain", + "seed": 7, + "source_priority": 2, + "strict_f1": 0.000532941187181528, + "threshold": 0.00028816089616157115, + "threshold_key": "0.0002881609", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.07586161173658511, + "union_combo": "t3_s8", + "union_f1": 0.21037180175292197 + }, + { + "difference": 0.38844131515497954, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9234194122885129, + "scope": "train_fire_top05pct", + "seed": 7, + "source_priority": 2, + "strict_f1": 0.005636060100166944, + "threshold": 0.0002606456328066997, + "threshold_key": "0.00026064563", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.15977956000729543, + "union_combo": "t3_s8", + "union_f1": 0.39407737525514647 + }, + { + "difference": 0.2282163144290469, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9575541703769664, + "scope": "train_fire_top10pct", + "seed": 7, + "source_priority": 2, + "strict_f1": 0.002747305898968793, + "threshold": 0.0002606456328066997, + "threshold_key": "0.00026064563", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08713204456539597, + "union_combo": "t3_s8", + "union_f1": 0.2309636203280157 + }, + { + "difference": 0.20276874034790468, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.5032875695732839, + "scope": "full_domain", + "seed": 99, + "source_priority": 2, + "strict_f1": 0.0005418761442097979, + "threshold": 0.0002606518683023751, + "threshold_key": "0.00026065187", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06950848282026648, + "union_combo": "t3_s8", + "union_f1": 0.2033106164921145 + }, + { + "difference": 0.4264130583920869, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.013950727218759276, + "scope": "train_fire_top05pct", + "seed": 99, + "source_priority": 2, + "strict_f1": 0.027781926811053025, + "threshold": 0.0002606526104500517, + "threshold_key": "0.00026065261", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.24416957191197788, + "union_combo": "t3_s8", + "union_f1": 0.4541949852031399 + }, + { + "difference": 0.4238775751761356, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.006975363609379638, + "scope": "train_fire_top10pct", + "seed": 99, + "source_priority": 2, + "strict_f1": 0.02774049217002237, + "threshold": 0.0002606526104500517, + "threshold_key": "0.00026065261", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.24342288270264262, + "union_combo": "t3_s8", + "union_f1": 0.451618067346158 + }, + { + "difference": 0.12941486750349504, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.3248534322820037, + "scope": "full_domain", + "seed": 1, + "source_priority": 2, + "strict_f1": 0.00043296285460247506, + "threshold": 0.000301085616229102, + "threshold_key": "0.00030108562", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.04396525865017935, + "union_combo": "t3_s8", + "union_f1": 0.12984783035809752 + }, + { + "difference": 0.46737645889869145, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.09290590679726922, + "scope": "train_fire_top05pct", + "seed": 1, + "source_priority": 2, + "strict_f1": 0.023876731839958566, + "threshold": 0.000301085616229102, + "threshold_key": "0.00030108562", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.23852766585861737, + "union_combo": "t3_s8", + "union_f1": 0.49125319073865004 + }, + { + "difference": 0.4648919131947094, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.04645295339863461, + "scope": "train_fire_top10pct", + "seed": 1, + "source_priority": 2, + "strict_f1": 0.023870550161812297, + "threshold": 0.000301085616229102, + "threshold_key": "0.00030108562", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.23793892088542526, + "union_combo": "t3_s8", + "union_f1": 0.4887624633565217 + }, + { + "difference": 0.22393621200451744, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.7873692022263451, + "scope": "full_domain", + "seed": 123, + "source_priority": 2, + "strict_f1": 0.0005917233172122231, + "threshold": 0.0002731947461143136, + "threshold_key": "0.00027319475", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08255863031622387, + "union_combo": "t3_s8", + "union_f1": 0.22452793532172965 + }, + { + "difference": 0.38844131515497954, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9234194122885129, + "scope": "train_fire_top05pct", + "seed": 123, + "source_priority": 2, + "strict_f1": 0.005636060100166944, + "threshold": 0.0002716928720474243, + "threshold_key": "0.00027169287", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.15977956000729543, + "union_combo": "t3_s8", + "union_f1": 0.39407737525514647 + }, + { + "difference": 0.2282163144290469, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9575541703769664, + "scope": "train_fire_top10pct", + "seed": 123, + "source_priority": 2, + "strict_f1": 0.002747305898968793, + "threshold": 0.0002716928720474243, + "threshold_key": "0.00027169287", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08713204456539597, + "union_combo": "t3_s8", + "union_f1": 0.2309636203280157 + }, + { + "difference": 0.26027328981051695, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.62891280148423, + "scope": "full_domain", + "seed": 42, + "source_priority": 2, + "strict_f1": 0.0007179477702826634, + "threshold": 0.0002782046503853053, + "threshold_key": "0.00027820465", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.09668169596405224, + "union_combo": "t3_s8", + "union_f1": 0.2609912375807996 + }, + { + "difference": 0.2505963038813504, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.43306619174829325, + "scope": "train_fire_top05pct", + "seed": 42, + "source_priority": 2, + "strict_f1": 0.006835665824509611, + "threshold": 0.00027877578395418823, + "threshold_key": "0.00027877578", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.10250537509480662, + "union_combo": "t3_s8", + "union_f1": 0.25743196970586 + }, + { + "difference": 0.2840988865146931, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.7009498367468091, + "scope": "train_fire_top10pct", + "seed": 42, + "source_priority": 2, + "strict_f1": 0.00356800070444239, + "threshold": 0.00027818421949632466, + "threshold_key": "0.00027818422", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.1112312688602769, + "union_combo": "t3_s8", + "union_f1": 0.2876668872191355 + }, + { + "difference": 0.24941796232451194, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.7037476808905381, + "scope": "full_domain", + "seed": 7, + "source_priority": 2, + "strict_f1": 0.0006714969233599232, + "threshold": 0.0002892915508709848, + "threshold_key": "0.00028929155", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.09264506128556889, + "union_combo": "t3_s8", + "union_f1": 0.25008945924787185 + }, + { + "difference": 0.38914191998978065, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9213416444048679, + "scope": "train_fire_top05pct", + "seed": 7, + "source_priority": 2, + "strict_f1": 0.005648734389013078, + "threshold": 0.0002883674460463226, + "threshold_key": "0.00028836745", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.1601110414083686, + "union_combo": "t3_s8", + "union_f1": 0.39479065437879374 + }, + { + "difference": 0.2284351905540453, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9565152864351439, + "scope": "train_fire_top10pct", + "seed": 7, + "source_priority": 2, + "strict_f1": 0.00275028568108307, + "threshold": 0.0002883674460463226, + "threshold_key": "0.00028836745", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08722255267001126, + "union_combo": "t3_s8", + "union_f1": 0.23118547623512836 + }, + { + "difference": 0.2529130183198567, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.5876512059369202, + "scope": "full_domain", + "seed": 99, + "source_priority": 2, + "strict_f1": 0.0007136363665052921, + "threshold": 0.0002819171058945358, + "threshold_key": "0.00028191711", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.09390595738710937, + "union_combo": "t3_s8", + "union_f1": 0.253626654686362 + }, + { + "difference": 0.2658654982272957, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.5093499554764025, + "scope": "train_fire_top05pct", + "seed": 99, + "source_priority": 2, + "strict_f1": 0.005865442686314773, + "threshold": 0.000282936030998826, + "threshold_key": "0.00028293603", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.10517133941552075, + "union_combo": "t3_s8", + "union_f1": 0.2717309409136105 + }, + { + "difference": 0.2816775567740636, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.7423567824280202, + "scope": "train_fire_top10pct", + "seed": 99, + "source_priority": 2, + "strict_f1": 0.003485751823118051, + "threshold": 0.00028128057601861656, + "threshold_key": "0.00028128058", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.10982263054722373, + "union_combo": "t3_s8", + "union_f1": 0.28516330859718164 + }, + { + "difference": 0.44473130599773897, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.02069016697588126, + "scope": "train_fire_top20pct", + "seed": 1, + "source_priority": 3, + "strict_f1": 0.026403975040443728, + "threshold": 0.0002847809228114784, + "threshold_key": "0.00028478092", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.23725772191057362, + "union_combo": "t3_s8", + "union_f1": 0.4711352810381827 + }, + { + "difference": 0.16866850620027352, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9746196660482375, + "scope": "train_fire_top20pct", + "seed": 123, + "source_priority": 3, + "strict_f1": 0.001471035812113219, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06246138809231819, + "union_combo": "t3_s8", + "union_f1": 0.17013954201238674 + }, + { + "difference": 0.21405981286832887, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.7109764996907854, + "scope": "train_fire_top20pct", + "seed": 42, + "source_priority": 3, + "strict_f1": 0.00196209796760366, + "threshold": 0.00028130068676546216, + "threshold_key": "0.00028130069", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08141778117479781, + "union_combo": "t3_s8", + "union_f1": 0.21602191083593253 + }, + { + "difference": 0.17276769792528876, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9366672850958565, + "scope": "train_fire_top20pct", + "seed": 7, + "source_priority": 3, + "strict_f1": 0.0014870516331508276, + "threshold": 0.0002701335761230439, + "threshold_key": "0.00027013358", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06425881062262738, + "union_combo": "t3_s8", + "union_f1": 0.1742547495584396 + }, + { + "difference": 0.2137212534415811, + "label": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.7215015460729747, + "scope": "train_fire_top20pct", + "seed": 99, + "source_priority": 3, + "strict_f1": 0.0019489171350575599, + "threshold": 0.0002826936251949519, + "threshold_key": "0.00028269363", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08112237765113199, + "union_combo": "t3_s8", + "union_f1": 0.21567017057663868 + }, + { + "difference": 0.5984324485861181, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.030718614718614718, + "scope": "train_fire_top20pct", + "seed": 1, + "source_priority": 3, + "strict_f1": 0.011488825936418005, + "threshold": 0.0015083511573617753, + "threshold_key": "0.0015083512", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.30575189275627285, + "union_combo": "t3_s8", + "union_f1": 0.6099212745225361 + }, + { + "difference": 0.5828418325957657, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.12288311688311689, + "scope": "train_fire_top20pct", + "seed": 123, + "source_priority": 3, + "strict_f1": 0.006734646906365392, + "threshold": 0.0006820998141378855, + "threshold_key": "0.00068209981", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.2833418180729231, + "union_combo": "t3_s8", + "union_f1": 0.5895764795021311 + }, + { + "difference": 0.6150773048223223, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.041087198515769946, + "scope": "train_fire_top20pct", + "seed": 42, + "source_priority": 3, + "strict_f1": 0.013757803485310216, + "threshold": 0.0013755693906898965, + "threshold_key": "0.0013755694", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.32901058451573606, + "union_combo": "t3_s8", + "union_f1": 0.6288351083076325 + }, + { + "difference": 0.47288261881572224, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.020547928262213976, + "scope": "train_fire_top20pct", + "seed": 7, + "source_priority": 3, + "strict_f1": 0.010760193101843772, + "threshold": 0.0013572138352511622, + "threshold_key": "0.0013572138", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.24066482674710968, + "union_combo": "t3_s8", + "union_f1": 0.48364281191756603 + }, + { + "difference": 0.6825790409197952, + "label": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.07197278911564625, + "scope": "train_fire_top20pct", + "seed": 99, + "source_priority": 3, + "strict_f1": 0.008694912370256934, + "threshold": 0.0009633156281329466, + "threshold_key": "0.00096331563", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.35193063179186285, + "union_combo": "t3_s8", + "union_f1": 0.691273953290052 + }, + { + "difference": 0.43562410120810985, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0202152133580705, + "scope": "train_fire_top20pct", + "seed": 1, + "source_priority": 3, + "strict_f1": 0.026530371070668874, + "threshold": 0.00027438640245236456, + "threshold_key": "0.0002743864", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.23561614605975847, + "union_combo": "t3_s8", + "union_f1": 0.4621544722787787 + }, + { + "difference": 0.16803346217809073, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9786487322201608, + "scope": "train_fire_top20pct", + "seed": 123, + "source_priority": 3, + "strict_f1": 0.0014649840398829276, + "threshold": 0.0002300136402482167, + "threshold_key": "0.00023001364", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.062212235501567396, + "union_combo": "t3_s8", + "union_f1": 0.16949844621797366 + }, + { + "difference": 0.16835043410148648, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9766338899196042, + "scope": "train_fire_top20pct", + "seed": 42, + "source_priority": 3, + "strict_f1": 0.0014680041534393378, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.062336581960243484, + "union_combo": "t3_s8", + "union_f1": 0.16981843825492582 + }, + { + "difference": 0.16739547174950506, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9827272727272728, + "scope": "train_fire_top20pct", + "seed": 7, + "source_priority": 3, + "strict_f1": 0.0014589084597825474, + "threshold": 0.00023887513088993728, + "threshold_key": "0.00023887513", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06196203978961812, + "union_combo": "t3_s8", + "union_f1": 0.1688543802092876 + }, + { + "difference": 0.08847200309630879, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.34681447124304265, + "scope": "train_fire_top20pct", + "seed": 99, + "source_priority": 3, + "strict_f1": 0.0021531819937041672, + "threshold": 0.00028135377215221524, + "threshold_key": "0.00028135377", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.031681254572138745, + "union_combo": "t3_s8", + "union_f1": 0.09062518509001295 + }, + { + "difference": 0.16866850620027352, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9746196660482375, + "scope": "train_fire_top20pct", + "seed": 1, + "source_priority": 3, + "strict_f1": 0.001471035812113219, + "threshold": 0.0002, + "threshold_key": "0.0002", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06246138809231819, + "union_combo": "t3_s8", + "union_f1": 0.17013954201238674 + }, + { + "difference": 0.16866850620027352, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9746196660482375, + "scope": "train_fire_top20pct", + "seed": 123, + "source_priority": 3, + "strict_f1": 0.001471035812113219, + "threshold": 0.0002438417504890822, + "threshold_key": "0.00024384175", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06246138809231819, + "union_combo": "t3_s8", + "union_f1": 0.17013954201238674 + }, + { + "difference": 0.44182468955354215, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.021595547309833023, + "scope": "train_fire_top20pct", + "seed": 42, + "source_priority": 3, + "strict_f1": 0.025831485587583147, + "threshold": 0.0002883114793803543, + "threshold_key": "0.00028831148", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.23362286716686329, + "union_combo": "t3_s8", + "union_f1": 0.4676561751411253 + }, + { + "difference": 0.16866850620027352, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.9746196660482375, + "scope": "train_fire_top20pct", + "seed": 7, + "source_priority": 3, + "strict_f1": 0.001471035812113219, + "threshold": 0.000260645778325852, + "threshold_key": "0.00026064578", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.06246138809231819, + "union_combo": "t3_s8", + "union_f1": 0.17013954201238674 + }, + { + "difference": 0.4011682044338078, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.0034879406307977735, + "scope": "train_fire_top20pct", + "seed": 99, + "source_priority": 3, + "strict_f1": 0.027352941176470587, + "threshold": 0.0002606526104500517, + "threshold_key": "0.00026065261", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.2365506772536198, + "union_combo": "t3_s8", + "union_f1": 0.4285211456102784 + }, + { + "difference": 0.4425322203556718, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.020630797773654917, + "scope": "train_fire_top20pct", + "seed": 1, + "source_priority": 3, + "strict_f1": 0.022074159907300116, + "threshold": 0.00030111230444163084, + "threshold_key": "0.0003011123", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.23443811051591282, + "union_combo": "t3_s8", + "union_f1": 0.4646063802629719 + }, + { + "difference": 0.16790877204956894, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.979443413729128, + "scope": "train_fire_top20pct", + "seed": 123, + "source_priority": 3, + "strict_f1": 0.0014637962799384197, + "threshold": 0.0002497032401151955, + "threshold_key": "0.00024970324", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.062163327857870446, + "union_combo": "t3_s8", + "union_f1": 0.16937256832950737 + }, + { + "difference": 0.21164756227103257, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.6907606679035251, + "scope": "train_fire_top20pct", + "seed": 42, + "source_priority": 3, + "strict_f1": 0.0020248273888312522, + "threshold": 0.0002781348885037005, + "threshold_key": "0.00027813489", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08079366172039935, + "union_combo": "t3_s8", + "union_f1": 0.21367238965986382 + }, + { + "difference": 0.16790877204956894, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.979443413729128, + "scope": "train_fire_top20pct", + "seed": 7, + "source_priority": 3, + "strict_f1": 0.0014637962799384197, + "threshold": 0.00021927931811660528, + "threshold_key": "0.00021927932", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.062163327857870446, + "union_combo": "t3_s8", + "union_f1": 0.16937256832950737 + }, + { + "difference": 0.21401759488544256, + "label": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "predicted_positive_rate": 0.7217068645640075, + "scope": "train_fire_top20pct", + "seed": 99, + "source_priority": 3, + "strict_f1": 0.001948363237912615, + "threshold": 0.0002812395105138421, + "threshold_key": "0.00028123951", + "threshold_selector": "strict_f1_nondegenerate", + "tolerance_combo": "t0_s8", + "tolerance_f1": 0.08122801407099456, + "union_combo": "t3_s8", + "union_f1": 0.21596595812335517 + } + ], + "run_roots": [ + "REDACTED_LOCAL_RESULT_PATH" + ], + "seed_variation_issues": [], + "summary": [ + { + "difference": { + "max": 0.4724120147971174, + "mean": 0.3536794407162983, + "min": 0.1971458843464264, + "n": 5, + "std": 0.10027055938000065 + }, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.0015512677798392085, + "mean": 0.000691230674087817, + "min": 0.00030327767470624613, + "n": 5, + "std": 0.0004992791583911452 + }, + "scope": "full_domain", + "scope_label": "global", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.38916402243638726 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.1971458843464264 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.3681553641584426 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.3415199178431181 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.4724120147971174 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.0006305504019789734 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.00030327767470624613 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.0005820655534941249 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.00038899196042053183 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.0015512677798392085 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.01841620626151013 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.018581463856786766 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.024598770061496925 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.02585193889541716 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.015582982831138383 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.3233671389763082 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.19137568834619276 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.3224566312114399 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.29397699288019324 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.3412019506417814 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.4075802286978974 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.21572734820321318 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.3927541342199395 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.36737185673853523 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.4879949976282558 + } + ] + }, + "strict_f1": { + "max": 0.02585193889541716, + "mean": 0.02060627238126987, + "min": 0.015582982831138383, + "n": 5, + "std": 0.004404166549372983 + }, + "thresholds": [ + 0.005, + 0.01, + 0.02 + ], + "tolerance_f1": { + "max": 0.3412019506417814, + "mean": 0.2944756804111831, + "min": 0.19137568834619276, + "n": 5, + "std": 0.06006423929515853 + }, + "union_f1": { + "max": 0.4879949976282558, + "mean": 0.3742857130975682, + "min": 0.21572734820321318, + "n": 5, + "std": 0.09945849036660906 + } + }, + { + "difference": { + "max": 0.572037609102215, + "mean": 0.44831513328161643, + "min": 0.32645966955454164, + "n": 5, + "std": 0.09076276520267298 + }, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.005397249431087365, + "mean": 0.002825764321757198, + "min": 0.001496487582863362, + "n": 5, + "std": 0.0014965985811042732 + }, + "scope": "train_fire_top05pct", + "scope_label": "top 5%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.4907581884518499 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.32645966955454164 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.42865099938480467 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.42366919991467084 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.572037609102215 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.0024215889977243494 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0025873157217769865 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.002226179875333927 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.001496487582863362 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.005397249431087365 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.05801376597836775 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.06853879105188007 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.07774936061381074 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.07710843373493977 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.06425702811244981 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.45453465315678554 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.348489928787613 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.46103410735073025 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.4329934518241347 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.44689699140777084 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.5487719544302176 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.39499846060642174 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.5064003599986154 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.5007776336496106 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.6362946372146648 + } + ] + }, + "strict_f1": { + "max": 0.07774936061381074, + "mean": 0.06913347589828964, + "min": 0.05801376597836775, + "n": 5, + "std": 0.008450070468719153 + }, + "thresholds": [ + 0.005, + 0.01 + ], + "tolerance_f1": { + "max": 0.46103410735073025, + "mean": 0.42878982650540687, + "min": 0.348489928787613, + "n": 5, + "std": 0.04608715441056064 + }, + "union_f1": { + "max": 0.6362946372146648, + "mean": 0.5174486091799061, + "min": 0.39499846060642174, + "n": 5, + "std": 0.08732060818913379 + } + }, + { + "difference": { + "max": 0.5674882177936724, + "mean": 0.43934631826412546, + "min": 0.29747106554895125, + "n": 5, + "std": 0.0991562015356911 + }, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.002728307113881468, + "mean": 0.001524438507964777, + "min": 0.0007507173246264965, + "n": 5, + "std": 0.0007698172196033509 + }, + "scope": "train_fire_top10pct", + "scope_label": "top 10%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.48639654658611775 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.29747106554895125 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.42494003459095464 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.42043572680093133 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.5674882177936724 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.0012293459978232908 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.001793311566241219 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.00112051053725141 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.0007507173246264965 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.002728307113881468 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.05730937348227295 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.05725646123260437 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.07711821410451548 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.07655502392344497 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.06358911647814125 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.4503813713224908 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.31417824074074074 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.4570756272994282 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.4297532170909699 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.4435187312985661 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.5437059200683907 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.3547275267815556 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.5020582486954701 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.4969907507243763 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.6310773342718137 + } + ] + }, + "strict_f1": { + "max": 0.07711821410451548, + "mean": 0.0663656378441958, + "min": 0.05725646123260437, + "n": 5, + "std": 0.009901307723431575 + }, + "thresholds": [ + 0.005, + 0.01 + ], + "tolerance_f1": { + "max": 0.4570756272994282, + "mean": 0.41898143755043915, + "min": 0.31417824074074074, + "n": 5, + "std": 0.05945364415661427 + }, + "union_f1": { + "max": 0.6310773342718137, + "mean": 0.5057119561083213, + "min": 0.3547275267815556, + "n": 5, + "std": 0.10005714147402384 + } + }, + { + "difference": { + "max": 0.5375844952591261, + "mean": 0.4019246057868907, + "min": 0.22027935343108077, + "n": 5, + "std": 0.11678824405387903 + }, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.0013673469387755102, + "mean": 0.0006557823129251701, + "min": 0.00035250463821892396, + "n": 5, + "std": 0.00041445475311508546 + }, + "scope": "train_fire_top20pct", + "scope_label": "top 20%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.4576561443425643 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.22027935343108077 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.39876530350195205 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.3953377323997304 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.5375844952591261 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.0006233766233766234 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.00035250463821892396 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.0005602968460111318 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.0003753865182436611 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.0013673469387755102 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.0544280442804428 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.047398843930635835 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.07357212003872217 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.0724391624221845 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.06170275882527441 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.42554776574916503 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.25536195152859176 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.42986611450839224 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.40493051697817745 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.4259185655133553 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.5120841886230071 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.2676781973617166 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.4723374235406742 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.4677768948219149 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.5992872540844005 + } + ] + }, + "strict_f1": { + "max": 0.07357212003872217, + "mean": 0.06190818589945194, + "min": 0.047398843930635835, + "n": 5, + "std": 0.011329878082076436 + }, + "thresholds": [ + 0.005, + 0.01, + 0.02 + ], + "tolerance_f1": { + "max": 0.42986611450839224, + "mean": 0.38832498285553635, + "min": 0.25536195152859176, + "n": 5, + "std": 0.07496564230623358 + }, + "union_f1": { + "max": 0.5992872540844005, + "mean": 0.4638327916863426, + "min": 0.2676781973617166, + "n": 5, + "std": 0.12169697847221474 + } + }, + { + "difference": { + "max": 0.2611249045411472, + "mean": 0.23038172350103436, + "min": 0.14280219707769823, + "n": 5, + "std": 0.04932517127284255 + }, + "label": "Aurora", + "model_tag": "aurora", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.7199758812615955, + "mean": 0.5558489053803339, + "min": 0.21716512059369203, + "n": 5, + "std": 0.1975239087458258 + }, + "scope": "full_domain", + "scope_label": "global", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.14280219707769823 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.2611249045411472 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.2520341870433926 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.25176783806212244 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.24417949078081141 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.21716512059369203 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.6671256648113791 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.5915943104514533 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.5833835497835498 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.7199758812615955 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.0004938156417811271 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0007124228455695601 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.0007076278562951189 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.0007150400212604667 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.0006488102455003172 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.05024333401294363 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.09742777619989017 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.09343380358557288 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.09353511619384626 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.09040502340162183 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.14329601271947937 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.2618373273867167 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.2527418148996877 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.25248287808338293 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.24482830102631173 + } + ] + }, + "strict_f1": { + "max": 0.0007150400212604667, + "mean": 0.000655543322081318, + "min": 0.0004938156417811271, + "n": 5, + "std": 9.445781201845953e-05 + }, + "thresholds": [ + 0.0002701166733984173, + 0.00028167077107355, + 0.00028207036666572094, + 0.0002832056488841772, + 0.0002848515287041664 + ], + "tolerance_f1": { + "max": 0.09742777619989017, + "mean": 0.08500901067877495, + "min": 0.05024333401294363, + "n": 5, + "std": 0.01959379095008962 + }, + "union_f1": { + "max": 0.2618373273867167, + "mean": 0.23103726682311568, + "min": 0.14329601271947937, + "n": 5, + "std": 0.04941752137736377 + } + }, + { + "difference": { + "max": 0.4699697200479976, + "mean": 0.3449747587619155, + "min": 0.21988543507354893, + "n": 5, + "std": 0.10372797772336852 + }, + "label": "Aurora", + "model_tag": "aurora", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.921086870485802, + "mean": 0.572238052834669, + "min": 0.08275452656574651, + "n": 5, + "std": 0.35339959438797136 + }, + "scope": "train_fire_top05pct", + "scope_label": "top 5%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.4699697200479976 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.38950157948139946 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.25643683344302254 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.21988543507354893 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.3890802257636093 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.08275452656574651 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.9150341347580885 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.48403581676066093 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.4582789156030474 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.921086870485802 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.026484309350641822 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.00565521517042468 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.00618071474679909 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.0053561461508737284 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.005618158058227469 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.24351172066402033 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.16052688500868942 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.10264712599745741 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.08994268846280176 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.16005632828519903 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.4964540293986394 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.39515679465182413 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.26261754818982164 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.22524158122442267 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.39469838382183675 + } + ] + }, + "strict_f1": { + "max": 0.026484309350641822, + "mean": 0.009858908695393357, + "min": 0.0053561461508737284, + "n": 5, + "std": 0.009298695250165839 + }, + "thresholds": [ + 0.00026978866662830114, + 0.0002819437358994037, + 0.00028234132332727313, + 0.00028413085965439677, + 0.0002847809228114784 + ], + "tolerance_f1": { + "max": 0.24351172066402033, + "mean": 0.1513369496836336, + "min": 0.08994268846280176, + "n": 5, + "std": 0.06082068957436595 + }, + "union_f1": { + "max": 0.4964540293986394, + "mean": 0.35483366745730893, + "min": 0.22524158122442267, + "n": 5, + "std": 0.11019233024683481 + } + }, + { + "difference": { + "max": 0.46744943745251477, + "mean": 0.2975149719293163, + "min": 0.2284351905540453, + "n": 5, + "std": 0.09865622266744696 + }, + "label": "Aurora", + "model_tag": "aurora", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.9565152864351439, + "mean": 0.687941773028594, + "min": 0.04137726328287326, + "n": 5, + "std": 0.37695842871011653 + }, + "scope": "train_fire_top10pct", + "scope_label": "top 10%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.46744943745251477 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.2284351905540453 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.28330440732569195 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.2799506337602843 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.2284351905540453 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.04137726328287326 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.9565152864351439 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.735743791431681 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.749557237558128 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.9565152864351439 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.026476637409113293 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.00275028568108307 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.0035136704264394467 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.0034589138681033788 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.00275028568108307 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.24290193233931348 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.08722255267001126 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.11052705132032803 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.10903157063807417 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.08722255267001126 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.49392607486162804 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.23118547623512836 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.2868180777521314 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.28340954762838766 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.23118547623512836 + } + ] + }, + "strict_f1": { + "max": 0.026476637409113293, + "mean": 0.007789958613164452, + "min": 0.00275028568108307, + "n": 5, + "std": 0.01045266903938076 + }, + "thresholds": [ + 0.00026914896443486214, + 0.00028129236306995153, + 0.00028190662851557136, + 0.000282647815765813, + 0.0002847809228114784 + ], + "tolerance_f1": { + "max": 0.24290193233931348, + "mean": 0.12738113192754766, + "min": 0.08722255267001126, + "n": 5, + "std": 0.06555769355379562 + }, + "union_f1": { + "max": 0.49392607486162804, + "mean": 0.30530493054248076, + "min": 0.23118547623512836, + "n": 5, + "std": 0.10884219611087281 + } + }, + { + "difference": { + "max": 0.44473130599773897, + "mean": 0.24278971528664223, + "min": 0.16866850620027352, + "n": 5, + "std": 0.11494327446365726 + }, + "label": "Aurora", + "model_tag": "aurora", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.9746196660482375, + "mean": 0.6728910327767471, + "min": 0.02069016697588126, + "n": 5, + "std": 0.38399138452827886 + }, + "scope": "train_fire_top20pct", + "scope_label": "top 20%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.44473130599773897 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.17276769792528876 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.21405981286832887 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.2137212534415811 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.16866850620027352 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.02069016697588126 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.9366672850958565 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.7109764996907854 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.7215015460729747 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.9746196660482375 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.026403975040443728 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0014870516331508276 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.00196209796760366 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.0019489171350575599 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.001471035812113219 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.23725772191057362 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.06425881062262738 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.08141778117479781 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.08112237765113199 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.06246138809231819 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.4711352810381827 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.1742547495584396 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.21602191083593253 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.21567017057663868 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.17013954201238674 + } + ] + }, + "strict_f1": { + "max": 0.026403975040443728, + "mean": 0.0066546155176738, + "min": 0.001471035812113219, + "n": 5, + "std": 0.011042800084784003 + }, + "thresholds": [ + 0.0002, + 0.0002701335761230439, + 0.00028130068676546216, + 0.0002826936251949519, + 0.0002847809228114784 + ], + "tolerance_f1": { + "max": 0.23725772191057362, + "mean": 0.1053036158902898, + "min": 0.06246138809231819, + "n": 5, + "std": 0.07430895639255175 + }, + "union_f1": { + "max": 0.4711352810381827, + "mean": 0.24944433080431602, + "min": 0.17013954201238674, + "n": 5, + "std": 0.12584449940748835 + } + }, + { + "difference": { + "max": 0.664031416861268, + "mean": 0.5980261490686182, + "min": 0.46922312530335947, + "n": 5, + "std": 0.07545417523567537 + }, + "label": "ClimaX", + "model_tag": "climax", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.12421632653061225, + "mean": 0.056763042671614104, + "min": 0.025392455163883734, + "n": 5, + "std": 0.03926077114304812 + }, + "scope": "full_domain", + "scope_label": "global", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.6362769893231447 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.46922312530335947 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.615867982755411 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.664031416861268 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.604731231099908 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.041071861471861475 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.025392455163883734 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.037147804576376 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.055986765615337046 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.12421632653061225 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.003394935139973715 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0027501821392557145 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.004637761701086003 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.003717439410136951 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.002897888198757764 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.3112148878371163 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.24162386260287075 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.31289270592627744 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.33641526956922785 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.2855276491036767 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.6396719244631184 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.4719733074426152 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.620505744456497 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.6677488562714049 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.6076291192986658 + } + ] + }, + "strict_f1": { + "max": 0.004637761701086003, + "mean": 0.00347964131784203, + "min": 0.0027501821392557145, + "n": 5, + "std": 0.0007542402263439388 + }, + "thresholds": [ + 0.0005393968409013734, + 0.001021229882830197, + 0.0013572092986540852, + 0.001442534703828549, + 0.0014632086704014654 + ], + "tolerance_f1": { + "max": 0.33641526956922785, + "mean": 0.2975348750078338, + "min": 0.24162386260287075, + "n": 5, + "std": 0.03607251580095264 + }, + "union_f1": { + "max": 0.6677488562714049, + "mean": 0.6015057903864602, + "min": 0.4719733074426152, + "n": 5, + "std": 0.07586493774699653 + } + }, + { + "difference": { + "max": 0.7272887839717904, + "mean": 0.6792494176719048, + "min": 0.6028322641867834, + "n": 5, + "std": 0.05726305372578923 + }, + "label": "ClimaX", + "model_tag": "climax", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.3090951815573365, + "mean": 0.24009102602156923, + "min": 0.13341248639556744, + "n": 5, + "std": 0.0928155581273325 + }, + "scope": "train_fire_top05pct", + "scope_label": "top 5%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.6341272298959009 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.7267635176891648 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.6028322641867834 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.7272887839717904 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.7052352926158846 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.13341248639556744 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.3068887899475611 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.14358118135945386 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.3090951815573365 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.3074774908479272 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.011492789729228419 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.013106997858124742 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.014246556800108287 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.013585576434738445 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.012252516711603196 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.3186326966868806 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.3655334236435691 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.32161191611096424 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.3665570668664193 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.3566174282966785 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.6456200196251293 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.7398705155472896 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.6170788209868917 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.7408743604065289 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.7174878093274878 + } + ] + }, + "strict_f1": { + "max": 0.014246556800108287, + "mean": 0.012936887506760619, + "min": 0.011492789729228419, + "n": 5, + "std": 0.0010861342333245315 + }, + "thresholds": [ + 0.0005436268660297483, + 0.0006166165592580956, + 0.0007465386633598005, + 0.0014632082671483923, + 0.0014882655838293205 + ], + "tolerance_f1": { + "max": 0.3665570668664193, + "mean": 0.34579050632090236, + "min": 0.3186326966868806, + "n": 5, + "std": 0.02377186314441771 + }, + "union_f1": { + "max": 0.7408743604065289, + "mean": 0.6921863051786654, + "min": 0.6170788209868917, + "n": 5, + "std": 0.057214573939719605 + } + }, + { + "difference": { + "max": 0.7230514919507499, + "mean": 0.6731908103073884, + "min": 0.5999371340636104, + "n": 5, + "std": 0.05553767153675633 + }, + "label": "ClimaX", + "model_tag": "climax", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.15408504007123774, + "mean": 0.1184503314534481, + "min": 0.06670624319778372, + "n": 5, + "std": 0.04506675919516373 + }, + "scope": "train_fire_top10pct", + "scope_label": "top 10%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.6312315933404747 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.7230353684802844 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.5999371340636104 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.7230514919507499 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.6886984637018222 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.06670624319778372 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.15368556446027506 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.07179059067972693 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.15408504007123774 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.14598421885821708 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.011490700169087834 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.01308555880921415 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.014244146704560834 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.013497596536465794 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.010293697838827224 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.3179193199949169 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.36399388919405107 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.3208182317552862 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.36524448333846904 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.34872837593596695 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.6427222935095626 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.7361209272894985 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.6141812807681712 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.7365490884872157 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.6989921615406495 + } + ] + }, + "strict_f1": { + "max": 0.014244146704560834, + "mean": 0.012522340011631166, + "min": 0.010293697838827224, + "n": 5, + "std": 0.0016018805893483327 + }, + "thresholds": [ + 0.0006294738670532621, + 0.0006776005131691771, + 0.0007519771209341887, + 0.0014632082671483923, + 0.0014882655838293205 + ], + "tolerance_f1": { + "max": 0.36524448333846904, + "mean": 0.34334086004373804, + "min": 0.3179193199949169, + "n": 5, + "std": 0.022852052082273543 + }, + "union_f1": { + "max": 0.7365490884872157, + "mean": 0.6857131503190195, + "min": 0.6141812807681712, + "n": 5, + "std": 0.05537732130173235 + } + }, + { + "difference": { + "max": 0.6825790409197952, + "mean": 0.5903626491479447, + "min": 0.47288261881572224, + "n": 5, + "std": 0.07589123629900595 + }, + "label": "ClimaX", + "model_tag": "climax", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.12288311688311689, + "mean": 0.057441929499072356, + "min": 0.020547928262213976, + "n": 5, + "std": 0.04134148942976998 + }, + "scope": "train_fire_top20pct", + "scope_label": "top 20%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.5984324485861181 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.47288261881572224 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.6150773048223223 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.6825790409197952 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.5828418325957657 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.030718614718614718 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.020547928262213976 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.041087198515769946 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.07197278911564625 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.12288311688311689 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.011488825936418005 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.010760193101843772 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.013757803485310216 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.008694912370256934 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.006734646906365392 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.30575189275627285 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.24066482674710968 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.32901058451573606 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.35193063179186285 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.2833418180729231 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.6099212745225361 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.48364281191756603 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.6288351083076325 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.691273953290052 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.5895764795021311 + } + ] + }, + "strict_f1": { + "max": 0.013757803485310216, + "mean": 0.010287276360038863, + "min": 0.006734646906365392, + "n": 5, + "std": 0.002686481400804917 + }, + "thresholds": [ + 0.0006820998141378855, + 0.0009633156281329466, + 0.0013572138352511622, + 0.0013755693906898965, + 0.0015083511573617753 + ], + "tolerance_f1": { + "max": 0.35193063179186285, + "mean": 0.3021399507767809, + "min": 0.24066482674710968, + "n": 5, + "std": 0.042856591284064 + }, + "union_f1": { + "max": 0.691273953290052, + "mean": 0.6006499255079836, + "min": 0.48364281191756603, + "n": 5, + "std": 0.07567424048289119 + } + }, + { + "difference": { + "max": 0.36515720594022893, + "mean": 0.2802079369511431, + "min": 0.17556822794728488, + "n": 5, + "std": 0.06925745403651407 + }, + "label": "DLWP", + "model_tag": "dlwp", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.0589282622139765, + "mean": 0.04322077922077922, + "min": 0.02625751391465677, + "n": 5, + "std": 0.015619298009606992 + }, + "scope": "full_domain", + "scope_label": "global", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.26339192421292845 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.3044358568583716 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.2924864697969017 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.17556822794728488 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.36515720594022893 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0589282622139765 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.057648732220160794 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.04519196042053185 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.02625751391465677 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.028077427334570193 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0016304484151205278 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.001786130143937306 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.0019600476944939 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.001008153911497155 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.002078902199880332 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.14442504878529014 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.16383974771641738 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.17299721297318826 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.09460750931776095 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.16987085515333694 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.265022372628049 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.3062219870023089 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.2944465174913956 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.17657638185878205 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.36723610814010926 + } + ] + }, + "strict_f1": { + "max": 0.002078902199880332, + "mean": 0.001692736472985844, + "min": 0.001008153911497155, + "n": 5, + "std": 0.0004188778823216006 + }, + "thresholds": [ + 0.08, + 0.1 + ], + "tolerance_f1": { + "max": 0.17299721297318826, + "mean": 0.14914807478919873, + "min": 0.09460750931776095, + "n": 5, + "std": 0.032446265301716536 + }, + "union_f1": { + "max": 0.36723610814010926, + "mean": 0.281900673424129, + "min": 0.17657638185878205, + "n": 5, + "std": 0.06965835259895045 + } + }, + { + "difference": { + "max": 0.5906566149800531, + "mean": 0.5365418706210278, + "min": 0.45626799482364494, + "n": 5, + "std": 0.0547523973572839 + }, + "label": "DLWP", + "model_tag": "dlwp", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.16420797467102008, + "mean": 0.09400069258929455, + "min": 0.04792965271593945, + "n": 5, + "std": 0.05080713313175061 + }, + "scope": "train_fire_top05pct", + "scope_label": "top 5%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.45626799482364494 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.5668021940668209 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.5053802602935323 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.5636022889410879 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.5906566149800531 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.06984763035519936 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.05733155238943306 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.04792965271593945 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.1306866528148808 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.16420797467102008 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.016864097224592905 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.01972516815912186 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.024960845732184808 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.017035016422646553 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.011684287006420426 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.28520211660517514 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.3500819147736832 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.35323298220195054 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.3101459291213085 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.2874938355126855 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.47313209204823786 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.5865273622259427 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.5303411060257172 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.5806373053637345 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.6023409019864735 + } + ] + }, + "strict_f1": { + "max": 0.024960845732184808, + "mean": 0.01805388290899331, + "min": 0.011684287006420426, + "n": 5, + "std": 0.004835410031272547 + }, + "thresholds": [ + 0.05, + 0.08, + 0.1 + ], + "tolerance_f1": { + "max": 0.35323298220195054, + "mean": 0.31723135564296057, + "min": 0.28520211660517514, + "n": 5, + "std": 0.032922909485364145 + }, + "union_f1": { + "max": 0.6023409019864735, + "mean": 0.5545957535300212, + "min": 0.47313209204823786, + "n": 5, + "std": 0.05291976314318387 + } + }, + { + "difference": { + "max": 0.5537110636994692, + "mean": 0.45515834191308435, + "min": 0.3591438068453628, + "n": 5, + "std": 0.07792651255459898 + }, + "label": "DLWP", + "model_tag": "dlwp", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.11266572672405263, + "mean": 0.049700455130107846, + "min": 0.025880577817354308, + "n": 5, + "std": 0.03684905017300331 + }, + "scope": "train_fire_top10pct", + "scope_label": "top 10%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.3591438068453628 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.5537110636994692 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.486215877804802 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.3946007462885974 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.48212021492719026 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.052691204115959234 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.02979989116453943 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.025880577817354308 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.02746487582863362 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.11266572672405263 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.011312372621310312 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.018998410174880763 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.023191305534082125 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.018477139910622206 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.008571862589787548 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.2160234001333052 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.3409571337826908 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.33689092573275314 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.2606560686639405 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.22837595071441363 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.3704561794666731 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.57270947387435 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.5094071833388841 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.41307788619921965 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.4906920775169778 + } + ] + }, + "strict_f1": { + "max": 0.023191305534082125, + "mean": 0.016110218166136593, + "min": 0.008571862589787548, + "n": 5, + "std": 0.005998581011452884 + }, + "thresholds": [ + 0.05, + 0.08, + 0.1 + ], + "tolerance_f1": { + "max": 0.3409571337826908, + "mean": 0.27658069580542066, + "min": 0.2160234001333052, + "n": 5, + "std": 0.05921597516408298 + }, + "union_f1": { + "max": 0.57270947387435, + "mean": 0.4712685600792209, + "min": 0.3704561794666731, + "n": 5, + "std": 0.08011144927032286 + } + }, + { + "difference": { + "max": 0.44734760227295295, + "mean": 0.33405352297889834, + "min": 0.26320327785336994, + "n": 5, + "std": 0.0787597726808095 + }, + "label": "DLWP", + "model_tag": "dlwp", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.04168893011750155, + "mean": 0.019198021026592453, + "min": 0.0027860235003092145, + "n": 5, + "std": 0.014678247188046134 + }, + "scope": "train_fire_top20pct", + "scope_label": "top 20%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.2695667418377301 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.44734760227295295 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.26320327785336994 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.30932859219900066 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.38082140073143794 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.04168893011750155 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.010166975881261595 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.0027860235003092145 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.021262213976499692 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.02008596165739023 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.007379212786746584 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.01931818181818182 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.028949691085613415 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.01226752201682564 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.008323671928416423 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.1600767886426284 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.28953047944835925 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.20397124965652888 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.19661692001462533 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.19681923783547378 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.2769459546244767 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.4666657840911348 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.29215296893898335 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.3215961142158263 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.3891450726598544 + } + ] + }, + "strict_f1": { + "max": 0.028949691085613415, + "mean": 0.015247655927156777, + "min": 0.007379212786746584, + "n": 5, + "std": 0.008986797239635393 + }, + "thresholds": [ + 0.08, + 0.1, + 0.2 + ], + "tolerance_f1": { + "max": 0.28953047944835925, + "mean": 0.2094029351195231, + "min": 0.1600767886426284, + "n": 5, + "std": 0.047970894595050655 + }, + "union_f1": { + "max": 0.4666657840911348, + "mean": 0.3493011789060551, + "min": 0.2769459546244767, + "n": 5, + "std": 0.0784709395452888 + } + }, + { + "difference": { + "max": 0.5017259192233794, + "mean": 0.39777501610341254, + "min": 0.2830040701913329, + "n": 5, + "std": 0.09342259085449434 + }, + "label": "FCN", + "model_tag": "fcn", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.03192430426716141, + "mean": 0.01520235003092146, + "min": 0.0016064316635745208, + "n": 5, + "std": 0.013446242178130542 + }, + "scope": "full_domain", + "scope_label": "global", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.4822930267328773 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.3840989936961055 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.33775307067336774 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.2830040701913329 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.5017259192233794 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.027048113790970933 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.03192430426716141 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.009495980210265924 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.0016064316635745208 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.005936920222634508 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0026467855605157606 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.002068681772914111 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.00208290997764682 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.0033447053314602988 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.00400312243549969 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.23211789308126832 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.19094441931639336 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.16485748333864078 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.16083569292964747 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.22655113161542761 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.48493981229339306 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.38616767546901964 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.33983598065101456 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.2863487755227932 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.5057290416588791 + } + ] + }, + "strict_f1": { + "max": 0.00400312243549969, + "mean": 0.002829241015607336, + "min": 0.002068681772914111, + "n": 5, + "std": 0.0008385182499278472 + }, + "thresholds": [ + 0.15, + 0.2, + 0.3, + 0.5 + ], + "tolerance_f1": { + "max": 0.23211789308126832, + "mean": 0.19506132405627552, + "min": 0.16083569292964747, + "n": 5, + "std": 0.03341186919631261 + }, + "union_f1": { + "max": 0.5057290416588791, + "mean": 0.4006042571190199, + "min": 0.2863487755227932, + "n": 5, + "std": 0.09370061309902981 + } + }, + { + "difference": { + "max": 0.6396713821034217, + "mean": 0.5268011783824539, + "min": 0.46051807014713964, + "n": 5, + "std": 0.0743888677938 + }, + "label": "FCN", + "model_tag": "fcn", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.07633570792520035, + "mean": 0.047855941426733946, + "min": 0.01251113089937667, + "n": 5, + "std": 0.029409395848307603 + }, + "scope": "train_fire_top05pct", + "scope_label": "top 5%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.5645980289087711 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.46051807014713964 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.48713449736377723 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.4820839133891601 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.6396713821034217 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.06537300880577818 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.06528643514395963 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.01251113089937667 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.0197734243593549 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.07633570792520035 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.01222529471692621 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.012095158293562608 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.024537870112874206 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.01569234169521494 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.016606090988845718 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.3028035236186743 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.25572856051368525 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.27999632336313063 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.300679830289768 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.3296389749733903 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.5768233236256973 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.47261322844070225 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.5116723674766515 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.49777625508437506 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.6562774730922674 + } + ] + }, + "strict_f1": { + "max": 0.024537870112874206, + "mean": 0.016231351161484737, + "min": 0.012095158293562608, + "n": 5, + "std": 0.00506423236804874 + }, + "thresholds": [ + 0.15, + 0.2, + 0.3 + ], + "tolerance_f1": { + "max": 0.3296389749733903, + "mean": 0.2937694425517297, + "min": 0.25572856051368525, + "n": 5, + "std": 0.0276260699067859 + }, + "union_f1": { + "max": 0.6562774730922674, + "mean": 0.5430325295439387, + "min": 0.47261322844070225, + "n": 5, + "std": 0.0740889720448565 + } + }, + { + "difference": { + "max": 0.5525178187961205, + "mean": 0.4227336789163064, + "min": 0.30442339402679647, + "n": 5, + "std": 0.0902513347968921 + }, + "label": "FCN", + "model_tag": "fcn", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.049730384881765115, + "mean": 0.0272573463935886, + "min": 0.00802908875037103, + "n": 5, + "std": 0.01635259738911405 + }, + "scope": "train_fire_top10pct", + "scope_label": "top 10%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.4452328295737776 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.30442339402679647 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.4238672329193606 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.387627119265477 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.5525178187961205 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.030024982685267636 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.03344711586029484 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.00802908875037103 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.015055159790244386 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.049730384881765115 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.008918001736248123 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.006403642961329112 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.019849146486701073 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.010726695875509895 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.012986069049061175 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.2270292295163525 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.16176542623012913 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.23420382391023928 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.2256264421482596 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.2724598040570218 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.4541508313100257 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.3108270369881256 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.4437163794060617 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.3983538151409869 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.5655038878451817 + } + ] + }, + "strict_f1": { + "max": 0.019849146486701073, + "mean": 0.011776711221769876, + "min": 0.006403642961329112, + "n": 5, + "std": 0.005117894394726336 + }, + "thresholds": [ + 0.15, + 0.2, + 0.3 + ], + "tolerance_f1": { + "max": 0.2724598040570218, + "mean": 0.22421694517240046, + "min": 0.16176542623012913, + "n": 5, + "std": 0.03980326313080014 + }, + "union_f1": { + "max": 0.5655038878451817, + "mean": 0.4345103901380763, + "min": 0.3108270369881256, + "n": 5, + "std": 0.09251305768823673 + } + }, + { + "difference": { + "max": 0.4298567373898511, + "mean": 0.3308974628182499, + "min": 0.2121721179401987, + "n": 5, + "std": 0.07927520828147509 + }, + "label": "FCN", + "model_tag": "fcn", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.01035930735930736, + "mean": 0.008367841682127397, + "min": 0.005085961657390229, + "n": 5, + "std": 0.0023583622377585994 + }, + "scope": "train_fire_top20pct", + "scope_label": "top 20%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.3226698082083355 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.2121721179401987 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.3647698181559645 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.3250188323968997 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.4298567373898511 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.009654916512059368 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.01035930735930736 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.005085961657390229 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.010080395794681509 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.006658627087198516 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.008704984497972811 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.004578192172408018 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.016197783461210574 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.00859106529209622 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.011738073279114613 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.1596269732904185 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.10554731176319206 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.19512358967789506 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.18647162974553239 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.20219147661108805 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.3313747927063083 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.2167503101126067 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.3809676016171751 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.33360989768899596 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.4415948106689657 + } + ] + }, + "strict_f1": { + "max": 0.016197783461210574, + "mean": 0.009962019740560447, + "min": 0.004578192172408018, + "n": 5, + "std": 0.004315216342666275 + }, + "thresholds": [ + 0.2, + 0.3 + ], + "tolerance_f1": { + "max": 0.20219147661108805, + "mean": 0.1697921962176252, + "min": 0.10554731176319206, + "n": 5, + "std": 0.03937109759451842 + }, + "union_f1": { + "max": 0.4415948106689657, + "mean": 0.3408594825588104, + "min": 0.2167503101126067, + "n": 5, + "std": 0.08261638767751367 + } + }, + { + "difference": { + "max": 0.43726313068468925, + "mean": 0.23840960437970468, + "min": 0.06735555938074993, + "n": 5, + "std": 0.13573634707074209 + }, + "label": "FengWu", + "model_tag": "fengwu", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.012518243661100804, + "mean": 0.004276685219542362, + "min": 0.00042881880024737165, + "n": 5, + "std": 0.004829522662220775 + }, + "scope": "full_domain", + "scope_label": "global", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.18397565646547948 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.2798111779709097 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.2236424973966951 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.06735555938074993 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.43726313068468925 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0016931354359925788 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.004433395176252319 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.0023098330241187385 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.00042881880024737165 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.012518243661100804 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.00268370607028754 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.003438698584843275 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.0018414421399496025 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.001842299189388357 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.0032567291196169464 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.09710793321083327 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.1569315132056949 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.10327893388535589 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.042906126190853267 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.20002472134774485 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.186659362535767 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.283249876555753 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.2254839395366447 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.06919785857013828 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.4405198598043062 + } + ] + }, + "strict_f1": { + "max": 0.003438698584843275, + "mean": 0.0026125750208171442, + "min": 0.0018414421399496025, + "n": 5, + "std": 0.0007567108180601367 + }, + "thresholds": [ + 0.2, + 0.3, + 0.5 + ], + "tolerance_f1": { + "max": 0.20002472134774485, + "mean": 0.12004984556809643, + "min": 0.042906126190853267, + "n": 5, + "std": 0.06023907872870804 + }, + "union_f1": { + "max": 0.4405198598043062, + "mean": 0.24102217940052184, + "min": 0.06919785857013828, + "n": 5, + "std": 0.13629274822774434 + } + }, + { + "difference": { + "max": 0.31898912374476807, + "mean": 0.28535994798223646, + "min": 0.20932175095916816, + "n": 5, + "std": 0.04769577870036795 + }, + "label": "FengWu", + "model_tag": "fengwu", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.010032650638171565, + "mean": 0.00600375977045612, + "min": 0.0030746017611556348, + "n": 5, + "std": 0.0030413161402819498 + }, + "scope": "train_fire_top05pct", + "scope_label": "top 5%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.20932175095916816 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.31898912374476807 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.26699212025153246 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.3168732218355133 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.3146235231202003 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0030746017611556348 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.008098347679825863 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.0032452755515979025 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.005567923221529633 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.010032650638171565 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.013054830287206266 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.01848001848001848 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.01267427122940431 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.0205686630369026 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.013695949911954608 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.12305221580797777 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.21207047330588913 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.13288915578187116 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.1866312847963942 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.15917085028122405 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.22237658124637444 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.3374691422247865 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.27966639148093675 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.3374418848724159 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.3283194730321549 + } + ] + }, + "strict_f1": { + "max": 0.0205686630369026, + "mean": 0.015694746589097253, + "min": 0.01267427122940431, + "n": 5, + "std": 0.003591671570543508 + }, + "thresholds": [ + 0.2, + 0.3 + ], + "tolerance_f1": { + "max": 0.21207047330588913, + "mean": 0.16276279599467128, + "min": 0.12305221580797777, + "n": 5, + "std": 0.03702437856278332 + }, + "union_f1": { + "max": 0.3374691422247865, + "mean": 0.3010546945713337, + "min": 0.22237658124637444, + "n": 5, + "std": 0.05010267654001306 + } + }, + { + "difference": { + "max": 0.3141659158267793, + "mean": 0.2294269435105075, + "min": 0.10438312130672354, + "n": 5, + "std": 0.0818631441187217 + }, + "label": "FengWu", + "model_tag": "fengwu", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.005720045513010785, + "mean": 0.002608835460571881, + "min": 0.0004947066389630949, + "n": 5, + "std": 0.001935467532779746 + }, + "scope": "train_fire_top10pct", + "scope_label": "top 10%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.19895488431057093 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.28149494747245746 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.24813584863600618 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.3141659158267793 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.10438312130672354 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0019998515880083113 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.005720045513010785 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.0019738794894627485 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.0028556940734144652 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.0004947066389630949 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.011185682326621925 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.014059753954305799 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.011273957158962795 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.02015411973918198 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.005460750853242321 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.11778287307219655 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.17969557482138315 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.1191832198916516 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.1840127014362662 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.04684227229450968 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.21014056663719285 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.29555470142676327 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.259409805794969 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.33432003556596124 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.10984387215996587 + } + ] + }, + "strict_f1": { + "max": 0.02015411973918198, + "mean": 0.012426852806462965, + "min": 0.005460750853242321, + "n": 5, + "std": 0.005333227666358275 + }, + "thresholds": [ + 0.2, + 0.3, + 0.5 + ], + "tolerance_f1": { + "max": 0.1840127014362662, + "mean": 0.12950332830320144, + "min": 0.04684227229450968, + "n": 5, + "std": 0.05605188948381181 + }, + "union_f1": { + "max": 0.33432003556596124, + "mean": 0.24185379631697046, + "min": 0.10984387215996587, + "n": 5, + "std": 0.0868538356837431 + } + }, + { + "difference": { + "max": 0.2991917676543021, + "mean": 0.21666770769632157, + "min": 0.1019380452757028, + "n": 5, + "std": 0.07443767874640678 + }, + "label": "FengWu", + "model_tag": "fengwu", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.0035126777983920843, + "mean": 0.0015006802721088436, + "min": 0.0003116883116883117, + "n": 5, + "std": 0.0012055357942519695 + }, + "scope": "train_fire_top20pct", + "scope_label": "top 20%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.19412591498868878 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.2536326799641239 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.2344501305987902 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.2991917676543021 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.1019380452757028 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0010822510822510823 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.0035126777983920843 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.0010810142238713667 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.0015157699443413728 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.0003116883116883117 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.010309278350515464 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.011695906432748539 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.010316368638239341 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.018831348656881748 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.004807692307692307 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.11130870584431331 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.1579992352547558 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.11041562343515275 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.17388386719173746 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.04393081279657125 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.20443519333920424 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.26532858639687246 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.24476649923702953 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.31802311631118385 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.1067457375833951 + } + ] + }, + "strict_f1": { + "max": 0.018831348656881748, + "mean": 0.01119211887721548, + "min": 0.004807692307692307, + "n": 5, + "std": 0.005022919520207697 + }, + "thresholds": [ + 0.2, + 0.3, + 0.5 + ], + "tolerance_f1": { + "max": 0.17388386719173746, + "mean": 0.11950764890450612, + "min": 0.04393081279657125, + "n": 5, + "std": 0.05074475609731675 + }, + "union_f1": { + "max": 0.31802311631118385, + "mean": 0.22785982657353704, + "min": 0.1067457375833951, + "n": 5, + "std": 0.07911462942092182 + } + }, + { + "difference": { + "max": 0.45054053460571325, + "mean": 0.36911429006032814, + "min": 0.217453250561222, + "n": 5, + "std": 0.09432711805206372 + }, + "label": "FuXi", + "model_tag": "fuxi", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.008687322201607915, + "mean": 0.004504539270253556, + "min": 0.001522943722943723, + "n": 5, + "std": 0.00277275768973151 + }, + "scope": "full_domain", + "scope_label": "global", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.44532955766011967 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.3742624367300788 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.217453250561222 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.35798567074450705 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.45054053460571325 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.005430797773654916 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.004230179344465059 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.001522943722943723 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.0026514533085961658 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.008687322201607915 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0033573873422136956 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.003097173828881146 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.0029424127784783522 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.005897939994871356 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.003573506191307238 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.2682493391647262 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.20441214417386547 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.13539446199130012 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.22566599998043468 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.21789178988640112 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.44868694500233336 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.37735961055895995 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.22039566333970034 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.3638836107393784 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.4541140407970205 + } + ] + }, + "strict_f1": { + "max": 0.005897939994871356, + "mean": 0.0037736840271503577, + "min": 0.0029424127784783522, + "n": 5, + "std": 0.0012118699857535593 + }, + "thresholds": [ + 0.2, + 0.3 + ], + "tolerance_f1": { + "max": 0.2682493391647262, + "mean": 0.21032274703934553, + "min": 0.13539446199130012, + "n": 5, + "std": 0.04821145875766877 + }, + "union_f1": { + "max": 0.4541140407970205, + "mean": 0.37288797408747854, + "min": 0.22039566333970034, + "n": 5, + "std": 0.09446975287636859 + } + }, + { + "difference": { + "max": 0.6138219771674631, + "mean": 0.5190011895614883, + "min": 0.3862603850081952, + "n": 5, + "std": 0.08687800582865718 + }, + "label": "FuXi", + "model_tag": "fuxi", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.0665751459384585, + "mean": 0.029315326011675073, + "min": 0.004068962105471455, + "n": 5, + "std": 0.026391672443609994 + }, + "scope": "train_fire_top05pct", + "scope_label": "top 5%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.6138219771674631 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.530575061359281 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.3862603850081952 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.49281035455029043 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.5715381697222122 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0665751459384585 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.024220837043633127 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.007007519540912239 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.004068962105471455 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.04470416542990007 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.016660707901322847 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.02138840232322301 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.02109053497942387 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.030370370370370364 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.012024257632789628 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.38200179460101663 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.30180702036802187 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.26086649919553234 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.3499638832422191 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.30008183753413636 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.6304826850687859 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.551963463682504 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.4073509199876191 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.5231807249206608 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.5835624273550017 + } + ] + }, + "strict_f1": { + "max": 0.030370370370370364, + "mean": 0.020306854641425946, + "min": 0.012024257632789628, + "n": 5, + "std": 0.006799890593691795 + }, + "thresholds": [ + 0.1, + 0.15, + 0.2, + 0.3 + ], + "tolerance_f1": { + "max": 0.38200179460101663, + "mean": 0.31894420698818526, + "min": 0.26086649919553234, + "n": 5, + "std": 0.047331205244476655 + }, + "union_f1": { + "max": 0.6304826850687859, + "mean": 0.5393080442029143, + "min": 0.4073509199876191, + "n": 5, + "std": 0.08382153402586862 + } + }, + { + "difference": { + "max": 0.45700386933894555, + "mean": 0.38559721262174607, + "min": 0.23030507895417707, + "n": 5, + "std": 0.09774430533636969 + }, + "label": "FuXi", + "model_tag": "fuxi", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.01593821114079351, + "mean": 0.00519664588898783, + "min": 0.0016448995745522905, + "n": 5, + "std": 0.006074020737200343 + }, + "scope": "train_fire_top10pct", + "scope_label": "top 10%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.34740284065122157 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.43990346446961154 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.23030507895417707 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.45700386933894555 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.45337080969477445 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0019404867913327397 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.0040170179083803305 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.0016448995745522905 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.002442614029880281 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.01593821114079351 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.011389521640091117 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.019476002782286114 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.016701461377870565 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.026973684210526316 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.008170871559633027 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.2251054852320675 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.26166119072060445 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.1656142112861716 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.3231296638237823 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.22512986809908905 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.35879236229131267 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.45937946725189766 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.24700654033204764 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.48397755354947186 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.4615416812544075 + } + ] + }, + "strict_f1": { + "max": 0.026973684210526316, + "mean": 0.016542308314081426, + "min": 0.008170871559633027, + "n": 5, + "std": 0.007315696478545316 + }, + "thresholds": [ + 0.2, + 0.3 + ], + "tolerance_f1": { + "max": 0.3231296638237823, + "mean": 0.24012808383234296, + "min": 0.1656142112861716, + "n": 5, + "std": 0.05778381935942338 + }, + "union_f1": { + "max": 0.48397755354947186, + "mean": 0.4021395209358275, + "min": 0.24700654033204764, + "n": 5, + "std": 0.09930723729972839 + } + }, + { + "difference": { + "max": 0.43708558281057147, + "mean": 0.3536682043586449, + "min": 0.21606998197866278, + "n": 5, + "std": 0.09922328477614482 + }, + "label": "FuXi", + "model_tag": "fuxi", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.011240568954854669, + "mean": 0.003620779220779221, + "min": 0.0009029066171923315, + "n": 5, + "std": 0.004346389540674709 + }, + "scope": "train_fire_top20pct", + "scope_label": "top 20%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.2820322009484037 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.40169955480281794 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.21606998197866278 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.4314537012527686 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.43708558281057147 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0013902288188002473 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.0031719233147804574 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.0009029066171923315 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.0013982683982683983 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.011240568954854669 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.008802816901408451 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.013674670058832883 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.015267175572519083 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.0239695995323005 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.006516342573438146 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.18265891116039673 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.25519274478475146 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.15340459969134643 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.30046102485294895 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.20602494796886814 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.29083501784981214 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.4153742248616508 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.23133715755118187 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.4554233007850691 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.4436019253840096 + } + ] + }, + "strict_f1": { + "max": 0.0239695995323005, + "mean": 0.013646120927699812, + "min": 0.006516342573438146, + "n": 5, + "std": 0.006773001125535398 + }, + "thresholds": [ + 0.2, + 0.3 + ], + "tolerance_f1": { + "max": 0.30046102485294895, + "mean": 0.21954844569166232, + "min": 0.15340459969134643, + "n": 5, + "std": 0.058600538898278126 + }, + "union_f1": { + "max": 0.4554233007850691, + "mean": 0.3673143252863447, + "min": 0.23133715755118187, + "n": 5, + "std": 0.10028910465565644 + } + }, + { + "difference": { + "max": 0.24742507585853188, + "mean": 0.2199448505015782, + "min": 0.1624675064035813, + "n": 5, + "std": 0.03728443961138173 + }, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.8827594310451453, + "mean": 0.6338042300556587, + "min": 0.21222844774273344, + "n": 5, + "std": 0.2507607336766575 + }, + "scope": "full_domain", + "scope_label": "global", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.1624675064035813 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.24742507585853188 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.2016630331978699 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.24307011894354136 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.24509851810436645 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.21222844774273344 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.7071789734075449 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.8827594310451453 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.655947804576376 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.7109064935064935 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.0005518596739603732 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0006577494583008349 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.0005395660023552084 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.000687991383709005 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.0006508233558970649 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.06046181002685303 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.09170559714407109 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.07610963330033767 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.08961425169301492 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.09112517773927671 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.16301936607754167 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.2480828253168327 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.20220259920022512 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.24375811032725037 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.2457493414602635 + } + ] + }, + "strict_f1": { + "max": 0.000687991383709005, + "mean": 0.0006175979748444972, + "min": 0.0005395660023552084, + "n": 5, + "std": 6.723431511345116e-05 + }, + "thresholds": [ + 0.00024381681578233838, + 0.0002703598001971841, + 0.00027441306156106293, + 0.0002784867538139224, + 0.00027882601716555655 + ], + "tolerance_f1": { + "max": 0.09170559714407109, + "mean": 0.08180329398071068, + "min": 0.06046181002685303, + "n": 5, + "std": 0.013544748931105265 + }, + "union_f1": { + "max": 0.2480828253168327, + "mean": 0.22056244847642267, + "min": 0.16301936607754167, + "n": 5, + "std": 0.03734327197007739 + } + }, + { + "difference": { + "max": 0.4598793069893796, + "mean": 0.3807330813931281, + "min": 0.27868986084470576, + "n": 5, + "std": 0.06491382662211008 + }, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.9213416444048679, + "mean": 0.6241896705253784, + "min": 0.08085485307212822, + "n": 5, + "std": 0.41065405269695054 + }, + "scope": "train_fire_top05pct", + "scope_label": "top 5%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.4598793069893796 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.38914191998978065 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.3868339716949791 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.27868986084470576 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.38912034744679525 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.08085485307212822 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.9213416444048679 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.9162387454239636 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.28134213911150685 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.9211709706144257 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.026612927125626056 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.005648734389013078 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.00568010509540423 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.009651904247534756 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.005639067545640032 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.24179585238367046 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.1601110414083686 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.16063421440737302 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.1110438397786047 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.16012924166858108 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.4864922341150057 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.39479065437879374 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.3925140767903833 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.2883417650922405 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.3947594149924353 + } + ] + }, + "strict_f1": { + "max": 0.026612927125626056, + "mean": 0.010646547680643631, + "min": 0.005639067545640032, + "n": 5, + "std": 0.009091660121962298 + }, + "thresholds": [ + 0.0002431461471132934, + 0.0002697413437999785, + 0.00027438640245236456, + 0.0002780304348561913, + 0.00028155450127087533 + ], + "tolerance_f1": { + "max": 0.24179585238367046, + "mean": 0.16674283792931957, + "min": 0.1110438397786047, + "n": 5, + "std": 0.04706476664025482 + }, + "union_f1": { + "max": 0.4864922341150057, + "mean": 0.3913796290737717, + "min": 0.2883417650922405, + "n": 5, + "std": 0.07015617234020738 + } + }, + { + "difference": { + "max": 0.4574594224967701, + "mean": 0.2340286562655959, + "min": 0.02818447450809833, + "n": 5, + "std": 0.15198406003212697 + }, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.9565152864351439, + "mean": 0.5820859305431878, + "min": 0.0007742158899772435, + "n": 5, + "std": 0.5127551339472974 + }, + "scope": "train_fire_top10pct", + "scope_label": "top 10%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.4574594224967701 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.2284351905540453 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.2276584353760274 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.02818447450809833 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.2284057583930383 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.04042742653606411 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.9565152864351439 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.9562827743148313 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.0007742158899772435 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.9564299495399228 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.02660504251473943 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.00275028568108307 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.002750953470140531 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.0011827321111768183 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.0027453654202759313 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.2411933776075557 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.08722255267001126 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.08724283500251707 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.019439944457301548 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.08722526453259725 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.4840644650115095 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.23118547623512836 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.23040938884616793 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.029367206619275145 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.23115112381331424 + } + ] + }, + "strict_f1": { + "max": 0.02660504251473943, + "mean": 0.007206875839483156, + "min": 0.0011827321111768183, + "n": 5, + "std": 0.010865089659576754 + }, + "thresholds": [ + 0.00024276715703308582, + 0.0002697413437999785, + 0.00027438640245236456, + 0.0002780304348561913, + 0.000285720539977774 + ], + "tolerance_f1": { + "max": 0.2411933776075557, + "mean": 0.10446479485399655, + "min": 0.019439944457301548, + "n": 5, + "std": 0.08187646601609166 + }, + "union_f1": { + "max": 0.4840644650115095, + "mean": 0.24123553210507903, + "min": 0.029367206619275145, + "n": 5, + "std": 0.1613799726515661 + } + }, + { + "difference": { + "max": 0.43562410120810985, + "mean": 0.20557509446670016, + "min": 0.08847200309630879, + "n": 5, + "std": 0.13312438976032662 + }, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.9827272727272728, + "mean": 0.6610079158936302, + "min": 0.0202152133580705, + "n": 5, + "std": 0.4509300379554766 + }, + "scope": "train_fire_top20pct", + "scope_label": "top 20%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.43562410120810985 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.16739547174950506 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.16835043410148648 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.08847200309630879 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.16803346217809073 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.0202152133580705 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.9827272727272728 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.9766338899196042 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.34681447124304265 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.9786487322201608 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.026530371070668874 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0014589084597825474 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.0014680041534393378 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.0021531819937041672 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.0014649840398829276 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.23561614605975847 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.06196203978961812 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.062336581960243484 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.031681254572138745 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.062212235501567396 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.4621544722787787 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.1688543802092876 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.16981843825492582 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.09062518509001295 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.16949844621797366 + } + ] + }, + "strict_f1": { + "max": 0.026530371070668874, + "mean": 0.006615089943495571, + "min": 0.0014589084597825474, + "n": 5, + "std": 0.011136980463397231 + }, + "thresholds": [ + 0.0002, + 0.0002300136402482167, + 0.00023887513088993728, + 0.00027438640245236456, + 0.00028135377215221524 + ], + "tolerance_f1": { + "max": 0.23561614605975847, + "mean": 0.09076165157666524, + "min": 0.031681254572138745, + "n": 5, + "std": 0.08204539789375309 + }, + "union_f1": { + "max": 0.4621544722787787, + "mean": 0.21219018441019574, + "min": 0.09062518509001295, + "n": 5, + "std": 0.1438368307990196 + } + }, + { + "difference": { + "max": 0.44278198559528875, + "mean": 0.26295780114154454, + "min": 0.13447416377324056, + "n": 5, + "std": 0.12366414839800409 + }, + "label": "Pangu-Weather", + "model_tag": "pangu_weather_h24", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.015876932591218307, + "mean": 0.004966580086580087, + "min": 0.0013461966604823748, + "n": 5, + "std": 0.006165207367586338 + }, + "scope": "full_domain", + "scope_label": "global", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.1610876335708797 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.2698875125828892 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.44278198559528875 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.3065577101854246 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.13447416377324056 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0013768707482993197 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.0034029684601113172 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.015876932591218307 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.0028299319727891157 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.0013461966604823748 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.00412434125105018 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.0018999796430752525 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.002025689424980434 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.00458918723078781 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.0015570260801868433 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.08503531719948608 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.11199020227536667 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.17658327178527805 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.14940756501428915 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.04422040571422221 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.1652119748219299 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.27178749222596443 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.4448076750202692 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.31114689741621243 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.1360311898534274 + } + ] + }, + "strict_f1": { + "max": 0.00458918723078781, + "mean": 0.002839244726016104, + "min": 0.0015570260801868433, + "n": 5, + "std": 0.0014055201553382893 + }, + "thresholds": [ + 0.1, + 0.3, + 0.5 + ], + "tolerance_f1": { + "max": 0.17658327178527805, + "mean": 0.11344735239772843, + "min": 0.04422040571422221, + "n": 5, + "std": 0.05215606001713916 + }, + "union_f1": { + "max": 0.4448076750202692, + "mean": 0.26579704586756064, + "min": 0.1360311898534274, + "n": 5, + "std": 0.12358583427828769 + } + }, + { + "difference": { + "max": 0.5419792852745725, + "mean": 0.34929776788574307, + "min": 0.2627312378871571, + "n": 5, + "std": 0.1137924578244252 + }, + "label": "Pangu-Weather", + "model_tag": "pangu_weather_h24", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.02707034728406055, + "mean": 0.014698228950232511, + "min": 0.00599584446423271, + "n": 5, + "std": 0.008898365132157035 + }, + "scope": "train_fire_top05pct", + "scope_label": "top 5%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.27422409940357173 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.2627312378871571 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.5419792852745725 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.3107866198278387 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.35676759703557515 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.019518650440288908 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.00599584446423271 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.02707034728406055 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.006668645493222519 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.014237657069357871 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.00961323496534764 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.006323656223052602 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.014334527877323108 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.012796587576646227 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.010571134928791661 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.13752896018113672 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.1236361475439318 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.23786880092608761 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.15839502058327895 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.16460271440438795 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.2838373343689194 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.26905489411020966 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.5563138131518957 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.3235832074044849 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.3673387319643668 + } + ] + }, + "strict_f1": { + "max": 0.014334527877323108, + "mean": 0.010727828314232248, + "min": 0.006323656223052602, + "n": 5, + "std": 0.0030802455090206595 + }, + "thresholds": [ + 0.1, + 0.3 + ], + "tolerance_f1": { + "max": 0.23786880092608761, + "mean": 0.16440632872776462, + "min": 0.1236361475439318, + "n": 5, + "std": 0.04420834148717055 + }, + "union_f1": { + "max": 0.5563138131518957, + "mean": 0.36002559619997526, + "min": 0.26905489411020966, + "n": 5, + "std": 0.11617972461830063 + } + }, + { + "difference": { + "max": 0.467280448823381, + "mean": 0.24433320781190368, + "min": 0.11744148944724705, + "n": 5, + "std": 0.14096512937898192 + }, + "label": "Pangu-Weather", + "model_tag": "pangu_weather_h24", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.018105026219451866, + "mean": 0.006657019887206886, + "min": 0.002389433066191748, + "n": 5, + "std": 0.006467521103800908 + }, + "scope": "train_fire_top10pct", + "scope_label": "top 10%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.14003318233886966 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.2112720683625158 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.467280448823381 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.285638850087505 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.11744148944724705 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0035594142673394677 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.004856782428020184 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.018105026219451866 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.004374443455031167 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.002389433066191748 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.005579507988840983 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.004407051282051282 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.010952623535404991 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.010430247718383311 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.002669336002669336 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.07891517659613233 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.09804613935969869 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.19299696406724362 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.14381976498868418 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.03347640061525648 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.14561269032771065 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.2156791196445671 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.478233072358786 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.2960690978058883 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.12011082544991639 + } + ] + }, + "strict_f1": { + "max": 0.010952623535404991, + "mean": 0.006807753305469981, + "min": 0.002669336002669336, + "n": 5, + "std": 0.0036980018094160414 + }, + "thresholds": [ + 0.1, + 0.3, + 0.5 + ], + "tolerance_f1": { + "max": 0.19299696406724362, + "mean": 0.10945088912540306, + "min": 0.03347640061525648, + "n": 5, + "std": 0.06122864367998089 + }, + "union_f1": { + "max": 0.478233072358786, + "mean": 0.2511409611173737, + "min": 0.12011082544991639, + "n": 5, + "std": 0.14418045901223492 + } + }, + { + "difference": { + "max": 0.31356600176953114, + "mean": 0.2043983344887205, + "min": 0.11926427213402073, + "n": 5, + "std": 0.08451922309197234 + }, + "label": "Pangu-Weather", + "model_tag": "pangu_weather_h24", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.004520098948670378, + "mean": 0.0027991341991341996, + "min": 0.0014211502782931354, + "n": 5, + "std": 0.0012806432499773903 + }, + "scope": "train_fire_top20pct", + "scope_label": "top 20%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.13031562548755435 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.19289957921827447 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.31356600176953114 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.2659461938342217 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.11926427213402073 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0019332096474953618 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.0036982065553494126 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.004520098948670378 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.002423005565862709 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.0014211502782931354 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0051329911339244045 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.003081232492997199 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.003306175463454953 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.0094525403702245 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.002313475997686524 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.07335569213391166 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.0918614020950846 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.10046960132658891 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.1330139941503748 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.036969834429575875 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.13544861662147875 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.19598081171127166 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.3168721772329861 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.2753987342044462 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.12157774813170726 + } + ] + }, + "strict_f1": { + "max": 0.0094525403702245, + "mean": 0.004657283091657516, + "min": 0.002313475997686524, + "n": 5, + "std": 0.002873310351989516 + }, + "thresholds": [ + 0.15, + 0.3, + 0.5 + ], + "tolerance_f1": { + "max": 0.1330139941503748, + "mean": 0.08713410482710718, + "min": 0.036969834429575875, + "n": 5, + "std": 0.03539578269908524 + }, + "union_f1": { + "max": 0.3168721772329861, + "mean": 0.209055617580378, + "min": 0.12157774813170726, + "n": 5, + "std": 0.08553820555850469 + } + }, + { + "difference": { + "max": 0.4653493031095141, + "mean": 0.3536303833961917, + "min": 0.21893843540252053, + "n": 5, + "std": 0.0907743328516622 + }, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.029129251700680273, + "mean": 0.010800964749536178, + "min": 0.0012771799628942485, + "n": 5, + "std": 0.011307793167633017 + }, + "scope": "full_domain", + "scope_label": "global", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.21893843540252053 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.3981469865854522 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.3350135537819443 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.35070363810152716 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.4653493031095141 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0012771799628942485 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.01413716759431045 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.00479208410636982 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.004669140383426098 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.029129251700680273 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0037437942540896885 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.0019783244452090142 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.0014740203906154037 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.003978746443051044 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.0026024230327071516 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.12218430629860097 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.1809729686535962 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.14188194761227685 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.18301623358471802 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.2264875300093167 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.22268222965661022 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.40012531103066123 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.3364875741725597 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.3546823845445782 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.46795172614222125 + } + ] + }, + "strict_f1": { + "max": 0.003978746443051044, + "mean": 0.0027554617131344606, + "min": 0.0014740203906154037, + "n": 5, + "std": 0.0010888849544285033 + }, + "thresholds": [ + 0.15, + 0.2, + 0.3 + ], + "tolerance_f1": { + "max": 0.2264875300093167, + "mean": 0.17090859723170176, + "min": 0.12218430629860097, + "n": 5, + "std": 0.04047686720194535 + }, + "union_f1": { + "max": 0.46795172614222125, + "mean": 0.35638584510932614, + "min": 0.22268222965661022, + "n": 5, + "std": 0.09032661075096597 + } + }, + { + "difference": { + "max": 0.5794332814358789, + "mean": 0.42057790363243475, + "min": 0.21672193002919604, + "n": 5, + "std": 0.13059925591392604 + }, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.056067576926882357, + "mean": 0.02054862966261007, + "min": 0.004447412684278223, + "n": 5, + "std": 0.02189321852493811 + }, + "scope": "train_fire_top05pct", + "scope_label": "top 5%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.21672193002919604 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.4579044245182327 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.42661251656646776 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.4222173656123986 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.5794332814358789 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.004447412684278223 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.027451271396062134 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.006297615514000198 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.008479271791827446 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.056067576926882357 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.010515247108307044 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.011848926191063935 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.012218828103304637 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.017845192951148784 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.015850265576258328 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.12664654352782623 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.23514322359982898 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.1893747086942956 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.2501388218123639 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.3098064830268362 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.22723717713750308 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.46975335070929664 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.4388313446697724 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.44006255856354737 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.5952835470121373 + } + ] + }, + "strict_f1": { + "max": 0.017845192951148784, + "mean": 0.013655691986016547, + "min": 0.010515247108307044, + "n": 5, + "std": 0.003064288092894302 + }, + "thresholds": [ + 0.15, + 0.2, + 0.3 + ], + "tolerance_f1": { + "max": 0.3098064830268362, + "mean": 0.2222219561322302, + "min": 0.12664654352782623, + "n": 5, + "std": 0.068612974451766 + }, + "union_f1": { + "max": 0.5952835470121373, + "mean": 0.4342335956184513, + "min": 0.22723717713750308, + "n": 5, + "std": 0.13238331958267657 + } + }, + { + "difference": { + "max": 0.5050483949122122, + "mean": 0.3743938492792621, + "min": 0.18892752891362113, + "n": 5, + "std": 0.11526082075130117 + }, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.03677772830711388, + "mean": 0.014029385574354408, + "min": 0.0033887404768972, + "n": 5, + "std": 0.014738630997131075 + }, + "scope": "train_fire_top10pct", + "scope_label": "top 10%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.18892752891362113 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.37162889777394226 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.4013551577246312 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.40500926707190377 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.5050483949122122 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0033887404768972 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.021221678044919364 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.004044226773523301 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.004714554269318294 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.03677772830711388 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.00788436268068331 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.008011413520632135 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.010149942329873126 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.016403526758253024 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.012206999545484059 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.1079162350962175 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.18336249012390426 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.1623425297604306 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.23804674197160172 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.25501453744070673 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.19681189159430446 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.3796403112945744 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.41150510005450436 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.4214127938301568 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.5172553944576963 + } + ] + }, + "strict_f1": { + "max": 0.016403526758253024, + "mean": 0.010931248966985132, + "min": 0.00788436268068331, + "n": 5, + "std": 0.003535277885116421 + }, + "thresholds": [ + 0.15, + 0.2, + 0.3 + ], + "tolerance_f1": { + "max": 0.25501453744070673, + "mean": 0.18933650687857217, + "min": 0.1079162350962175, + "n": 5, + "std": 0.0593294088865109 + }, + "union_f1": { + "max": 0.5172553944576963, + "mean": 0.3853250982462472, + "min": 0.19681189159430446, + "n": 5, + "std": 0.1172207993601839 + } + }, + { + "difference": { + "max": 0.44522188799641144, + "mean": 0.3368440115053533, + "min": 0.17487748330642924, + "n": 5, + "std": 0.10133432810494146 + }, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.026268398268398267, + "mean": 0.010103277674706247, + "min": 0.001932591218305504, + "n": 5, + "std": 0.011083627907574344 + }, + "scope": "train_fire_top20pct", + "scope_label": "top 20%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.17487748330642924 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.3180949408491859 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.36502027762951006 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.3810054677452299 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.44522188799641144 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.001932591218305504 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.017168212739641312 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.0025015460729746444 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.0026456400742115027 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.026268398268398267 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0070011668611435224 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.005117388748660143 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.008453410182516812 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.01471129091577786 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.008937574479787332 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.09937369519832985 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.14938342628742876 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.15039592221466835 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.2249788440916092 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.226729727407505 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.18187865016757276 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 17, + "value": 0.323212329597846 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 37, + "value": 0.3734736878120269 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 57, + "value": 0.3957167586610078 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 97, + "value": 0.45415946247619876 + } + ] + }, + "strict_f1": { + "max": 0.01471129091577786, + "mean": 0.008844166237577134, + "min": 0.005117388748660143, + "n": 5, + "std": 0.0036010444504776733 + }, + "thresholds": [ + 0.15, + 0.2, + 0.3 + ], + "tolerance_f1": { + "max": 0.226729727407505, + "mean": 0.17017232303990823, + "min": 0.09937369519832985, + "n": 5, + "std": 0.05485942193604147 + }, + "union_f1": { + "max": 0.45415946247619876, + "mean": 0.3456881777429305, + "min": 0.18187865016757276, + "n": 5, + "std": 0.10293157698741554 + } + }, + { + "difference": { + "max": 0.2215775413347868, + "mean": 0.20130053743009468, + "min": 0.17211673370575886, + "n": 5, + "std": 0.018296989279232255 + }, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.9588571428571429, + "mean": 0.6197105751391466, + "min": 0.22581076066790354, + "n": 5, + "std": 0.30910050227102126 + }, + "scope": "full_domain", + "scope_label": "global", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.2215775413347868 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.20983886056574044 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.17211673370575886 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.20276874034790468 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.20020081119628272 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.5006159554730983 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.9099814471243043 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.22581076066790354 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.5032875695732839 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.9588571428571429 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.0006040339888888368 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.000532941187181528 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.0005777939978365554 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.0005418761442097979 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.0005057825179019543 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.07892160427759505 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.07586161173658511 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.06182533987250242 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.06950848282026648 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.07212988048730175 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.22218157532367563 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.21037180175292197 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.17269452770359542 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.2033106164921145 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.20070659371418467 + } + ] + }, + "strict_f1": { + "max": 0.0006040339888888368, + "mean": 0.0005524855672037344, + "min": 0.0005057825179019543, + "n": 5, + "std": 3.864412600940357e-05 + }, + "thresholds": [ + 0.0002606518683023751, + 0.00028123805532231927, + 0.00028278783429414034, + 0.00028816089616157115, + 0.00028831177041865885 + ], + "tolerance_f1": { + "max": 0.07892160427759505, + "mean": 0.07164938383885015, + "min": 0.06182533987250242, + "n": 5, + "std": 0.006556565946161047 + }, + "union_f1": { + "max": 0.22218157532367563, + "mean": 0.20185302299729843, + "min": 0.17269452770359542, + "n": 5, + "std": 0.018299428665013905 + } + }, + { + "difference": { + "max": 0.46663155596188977, + "mean": 0.4116737119637831, + "min": 0.38844131515497954, + "n": 5, + "std": 0.034845536781985785 + }, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.9234194122885129, + "mean": 0.5741169486494508, + "min": 0.013950727218759276, + "n": 5, + "std": 0.47898702418670663 + }, + "scope": "train_fire_top05pct", + "scope_label": "top 5%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.38844131515497954 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.38844131515497954 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.46663155596188977 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.4264130583920869 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.38844131515497954 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.9234194122885129 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.9234194122885129 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.08637577916295637 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.013950727218759276 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.9234194122885129 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.005636060100166944 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.005636060100166944 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.02590687977762335 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.027781926811053025 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.005636060100166944 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.15977956000729543 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.15977956000729543 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.23967146506700618 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.24416957191197788 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.15977956000729543 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.39407737525514647 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.39407737525514647 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.4925384357395131 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.4541949852031399 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.39407737525514647 + } + ] + }, + "strict_f1": { + "max": 0.027781926811053025, + "mean": 0.01411939737783544, + "min": 0.005636060100166944, + "n": 5, + "std": 0.011635188932059573 + }, + "thresholds": [ + 0.00018521470337873325, + 0.0002438416049699299, + 0.0002606456328066997, + 0.0002606526104500517, + 0.0002883114793803543 + ], + "tolerance_f1": { + "max": 0.24416957191197788, + "mean": 0.19263594340017406, + "min": 0.15977956000729543, + "n": 5, + "std": 0.04501855438328491 + }, + "union_f1": { + "max": 0.4925384357395131, + "mean": 0.4257931093416185, + "min": 0.39407737525514647, + "n": 5, + "std": 0.04549524292260173 + } + }, + { + "difference": { + "max": 0.46415551438013897, + "mean": 0.31453640656868304, + "min": 0.2282163144290469, + "n": 5, + "std": 0.11905339505419958 + }, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.9575541703769664, + "mean": 0.5845651528643514, + "min": 0.006975363609379638, + "n": 5, + "std": 0.5108966939133424 + }, + "scope": "train_fire_top10pct", + "scope_label": "top 10%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.2282163144290469 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.2282163144290469 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.46415551438013897 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.4238775751761356 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.2282163144290469 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.9575541703769664 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.9575541703769664 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.043187889581478185 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.006975363609379638 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.9575541703769664 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.002747305898968793 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.002747305898968793 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.02589968042239822 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.02774049217002237 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.002747305898968793 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.08713204456539597 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.08713204456539597 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.23908194736539978 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.24342288270264262 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.08713204456539597 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.2309636203280157 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.2309636203280157 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.4900551948025372 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.451618067346158 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.2309636203280157 + } + ] + }, + "strict_f1": { + "max": 0.02774049217002237, + "mean": 0.012376418057865393, + "min": 0.002747305898968793, + "n": 5, + "std": 0.013201257527312249 + }, + "thresholds": [ + 0.00018521470337873325, + 0.0002438416049699299, + 0.0002606456328066997, + 0.0002606526104500517, + 0.0002883114793803543 + ], + "tolerance_f1": { + "max": 0.24342288270264262, + "mean": 0.14878019275284607, + "min": 0.08713204456539597, + "n": 5, + "std": 0.08442915398703313 + }, + "union_f1": { + "max": 0.4900551948025372, + "mean": 0.32691282462654847, + "min": 0.2309636203280157, + "n": 5, + "std": 0.13208480223417152 + } + }, + { + "difference": { + "max": 0.44182468955354215, + "mean": 0.2697996825176341, + "min": 0.16866850620027352, + "n": 5, + "std": 0.13922359360588243 + }, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.9746196660482375, + "mean": 0.5897884972170686, + "min": 0.0034879406307977735, + "n": 5, + "std": 0.5269906679651803 + }, + "scope": "train_fire_top20pct", + "scope_label": "top 20%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.16866850620027352 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.16866850620027352 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.44182468955354215 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.4011682044338078 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.16866850620027352 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.9746196660482375 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.9746196660482375 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.021595547309833023 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.0034879406307977735 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.9746196660482375 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.001471035812113219 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.001471035812113219 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.025831485587583147 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.027352941176470587 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.001471035812113219 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.06246138809231819 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.06246138809231819 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.23362286716686329 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.2365506772536198 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.06246138809231819 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.17013954201238674 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.17013954201238674 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.4676561751411253 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.4285211456102784 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.17013954201238674 + } + ] + }, + "strict_f1": { + "max": 0.027352941176470587, + "mean": 0.011519506840078677, + "min": 0.001471035812113219, + "n": 5, + "std": 0.013769946337526496 + }, + "thresholds": [ + 0.0002, + 0.0002438417504890822, + 0.000260645778325852, + 0.0002606526104500517, + 0.0002883114793803543 + ], + "tolerance_f1": { + "max": 0.2365506772536198, + "mean": 0.13151154173948754, + "min": 0.06246138809231819, + "n": 5, + "std": 0.09455648302229831 + }, + "union_f1": { + "max": 0.4676561751411253, + "mean": 0.2813191893577128, + "min": 0.17013954201238674, + "n": 5, + "std": 0.15286646962866243 + } + }, + { + "difference": { + "max": 0.6194077454910121, + "mean": 0.5861093281854558, + "min": 0.5480197595988799, + "n": 5, + "std": 0.026945326610355286 + }, + "label": "Reference", + "model_tag": "reference", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.021380705009276437, + "mean": 0.016807866419294992, + "min": 0.012341125541125542, + "n": 5, + "std": 0.0036836114267075385 + }, + "scope": "full_domain", + "scope_label": "global", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.5480197595988799 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.601348973225457 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.6194077454910121 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.5755105867935725 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.5862595758183573 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.016084972170686457 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.014596289424860853 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.019636239950525665 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.012341125541125542 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.021380705009276437 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.004136112962850736 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.006568199246490847 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.0045296167247386755 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.004855561780634761 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.0026426577586601383 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.2985122125101588 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.29937890424270136 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.3056545098251043 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.30820726968158313 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.2756667293290642 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.5521558725617307 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.6079171724719479 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.6239373622157508 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.5803661485742072 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.5889022335770174 + } + ] + }, + "strict_f1": { + "max": 0.006568199246490847, + "mean": 0.004546429694675032, + "min": 0.0026426577586601383, + "n": 5, + "std": 0.0014120999298872481 + }, + "thresholds": [ + 0.95 + ], + "tolerance_f1": { + "max": 0.30820726968158313, + "mean": 0.29748392511772237, + "min": 0.2756667293290642, + "n": 5, + "std": 0.012868439357804603 + }, + "union_f1": { + "max": 0.6239373622157508, + "mean": 0.5906557578801308, + "min": 0.5521558725617307, + "n": 5, + "std": 0.02737158059714251 + } + }, + { + "difference": { + "max": 0.7136230759527924, + "mean": 0.6926756278808944, + "min": 0.6696707231715443, + "n": 5, + "std": 0.0199602553757671 + }, + "label": "Reference", + "model_tag": "reference", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.04860987434451371, + "mean": 0.030619372711981795, + "min": 0.022801028989809045, + "n": 5, + "std": 0.010925458415897562 + }, + "scope": "train_fire_top05pct", + "scope_label": "top 5%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.6696707231715443 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.7136230759527924 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.7095179230119556 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.6744974001959292 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.6960690170722506 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.023750865736618186 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.02449292569506283 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.033442168793905214 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.022801028989809045 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.04860987434451371 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.03321760345312939 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.04873596787441818 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.038147512864494 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.0332911515623479 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.02462935239291061 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.39555976109165375 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.40145293653054454 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.4063154181985604 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.3895272012200133 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.3702273155255325 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.7028883266246737 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.7623590438272105 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.7476654358764496 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.707788551758277 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.7206983694651612 + } + ] + }, + "strict_f1": { + "max": 0.04873596787441818, + "mean": 0.03560431762946002, + "min": 0.02462935239291061, + "n": 5, + "std": 0.008809158358157916 + }, + "thresholds": [ + 0.9, + 0.95 + ], + "tolerance_f1": { + "max": 0.4063154181985604, + "mean": 0.3926165265132609, + "min": 0.3702273155255325, + "n": 5, + "std": 0.014010934164100928 + }, + "union_f1": { + "max": 0.7623590438272105, + "mean": 0.7282799455103544, + "min": 0.7028883266246737, + "n": 5, + "std": 0.025784171509335416 + } + }, + { + "difference": { + "max": 0.7104053704635315, + "mean": 0.6896286190689807, + "min": 0.6666121634162013, + "n": 5, + "std": 0.019887948249560174 + }, + "label": "Reference", + "model_tag": "reference", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.024304937172256853, + "mean": 0.015309686355990898, + "min": 0.011400514494904522, + "n": 5, + "std": 0.005462729207948781 + }, + "scope": "train_fire_top10pct", + "scope_label": "top 10%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.6666121634162013 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.7104053704635315 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.7063341380484894 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.6715102385713686 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.6932811848453125 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.011875432868309093 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.012246462847531414 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.016721084396952607 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.011400514494904522 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.024304937172256853 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.033186462923033656 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0486915291328531 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.0381213575591361 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.033258776621608484 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.02461746391852102 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.3945515652074689 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.4004617740449798 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.4052951105632299 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.38853542457185924 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.369479820868587 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.699798626339235 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.7590968995963846 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.7444554956076256 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.7047690151929771 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.7178986487638336 + } + ] + }, + "strict_f1": { + "max": 0.0486915291328531, + "mean": 0.03557511803103047, + "min": 0.02461746391852102, + "n": 5, + "std": 0.008798648622457047 + }, + "thresholds": [ + 0.9, + 0.95 + ], + "tolerance_f1": { + "max": 0.4052951105632299, + "mean": 0.391664739051225, + "min": 0.369479820868587, + "n": 5, + "std": 0.013905680686091083 + }, + "union_f1": { + "max": 0.7590968995963846, + "mean": 0.7252037371000112, + "min": 0.699798626339235, + "n": 5, + "std": 0.025669721937652358 + } + }, + { + "difference": { + "max": 0.6811711568125168, + "mean": 0.6619276528174679, + "min": 0.6388534040739275, + "n": 5, + "std": 0.019273149109644262 + }, + "label": "Reference", + "model_tag": "reference", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.012153370439084725, + "mean": 0.0076554112554112555, + "min": 0.005700680272108844, + "n": 5, + "std": 0.0027315673026463 + }, + "scope": "train_fire_top20pct", + "scope_label": "top 20%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.6388534040739275 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.6811711568125168 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.677407206441351 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.6443640032843302 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.6678424934752141 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.0059381570810142234 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.006123685837971553 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.008361162646876932 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.005700680272108844 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.012153370439084725 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.032893514216688347 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.04827336828783222 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.03787465940054495 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.03295432645981885 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.024505093215452623 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.3852239693382854 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.39128423640663973 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.39585180444145995 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.37935948303093225 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.36252633147981417 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.6717469182906158 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.7294445251003491 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.7152818658418959 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.6773183297441491 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.6923475866906666 + } + ] + }, + "strict_f1": { + "max": 0.04827336828783222, + "mean": 0.035300192316067396, + "min": 0.024505093215452623, + "n": 5, + "std": 0.008700007560989128 + }, + "thresholds": [ + 0.9, + 0.95 + ], + "tolerance_f1": { + "max": 0.39585180444145995, + "mean": 0.3828491649394263, + "min": 0.36252633147981417, + "n": 5, + "std": 0.012952356930876381 + }, + "union_f1": { + "max": 0.7294445251003491, + "mean": 0.6972278451335353, + "min": 0.6717469182906158, + "n": 5, + "std": 0.02466447592239507 + } + }, + { + "difference": { + "max": 0.26027328981051695, + "mean": 0.2231910699925796, + "min": 0.12941486750349504, + "n": 5, + "std": 0.05417763955331957 + }, + "label": "StormCast", + "model_tag": "stormcast", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.7873692022263451, + "mean": 0.6065068645640074, + "min": 0.3248534322820037, + "n": 5, + "std": 0.17489509665246694 + }, + "scope": "full_domain", + "scope_label": "global", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.12941486750349504 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.24941796232451194 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.26027328981051695 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.2529130183198567 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.22393621200451744 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.3248534322820037 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.7037476808905381 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.62891280148423 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.5876512059369202 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.7873692022263451 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.00043296285460247506 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0006714969233599232 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.0007179477702826634 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.0007136363665052921 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.0005917233172122231 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.04396525865017935 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.09264506128556889 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.09668169596405224 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.09390595738710937 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.08255863031622387 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.12984783035809752 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.25008945924787185 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.2609912375807996 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.253626654686362 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.22452793532172965 + } + ] + }, + "strict_f1": { + "max": 0.0007179477702826634, + "mean": 0.0006255534463925153, + "min": 0.00043296285460247506, + "n": 5, + "std": 0.00011899761411994709 + }, + "thresholds": [ + 0.0002731947461143136, + 0.0002782046503853053, + 0.0002819171058945358, + 0.0002892915508709848, + 0.000301085616229102 + ], + "tolerance_f1": { + "max": 0.09668169596405224, + "mean": 0.08195132072062675, + "min": 0.04396525865017935, + "n": 5, + "std": 0.021894994384989055 + }, + "union_f1": { + "max": 0.2609912375807996, + "mean": 0.2238166234389721, + "min": 0.12984783035809752, + "n": 5, + "std": 0.05429431793170512 + } + }, + { + "difference": { + "max": 0.46737645889869145, + "mean": 0.35228429923041954, + "min": 0.2505963038813504, + "n": 5, + "std": 0.09181595284950021 + }, + "label": "StormCast", + "model_tag": "stormcast", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.9234194122885129, + "mean": 0.5760166221430691, + "min": 0.09290590679726922, + "n": 5, + "std": 0.3529211324352006 + }, + "scope": "train_fire_top05pct", + "scope_label": "top 5%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.46737645889869145 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.38914191998978065 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.2505963038813504 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.2658654982272957 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.38844131515497954 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.09290590679726922 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.9213416444048679 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.43306619174829325 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.5093499554764025 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.9234194122885129 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.023876731839958566 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.005648734389013078 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.006835665824509611 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.005865442686314773 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.005636060100166944 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.23852766585861737 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.1601110414083686 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.10250537509480662 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.10517133941552075 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.15977956000729543 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.49125319073865004 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.39479065437879374 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.25743196970586 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.2717309409136105 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.39407737525514647 + } + ] + }, + "strict_f1": { + "max": 0.023876731839958566, + "mean": 0.009572526967992595, + "min": 0.005636060100166944, + "n": 5, + "std": 0.00801147738269429 + }, + "thresholds": [ + 0.0002716928720474243, + 0.00027877578395418823, + 0.000282936030998826, + 0.0002883674460463226, + 0.000301085616229102 + ], + "tolerance_f1": { + "max": 0.23852766585861737, + "mean": 0.15321899635692177, + "min": 0.10250537509480662, + "n": 5, + "std": 0.05533660427697729 + }, + "union_f1": { + "max": 0.49125319073865004, + "mean": 0.36185682619841214, + "min": 0.25743196970586, + "n": 5, + "std": 0.09733119467598991 + } + }, + { + "difference": { + "max": 0.4648919131947094, + "mean": 0.2974639722933117, + "min": 0.2282163144290469, + "n": 5, + "std": 0.09749379807889277 + }, + "label": "StormCast", + "model_tag": "stormcast", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.9575541703769664, + "mean": 0.6807658058771149, + "min": 0.04645295339863461, + "n": 5, + "std": 0.37389945609232794 + }, + "scope": "train_fire_top10pct", + "scope_label": "top 10%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.4648919131947094 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.2284351905540453 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.2840988865146931 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.2816775567740636 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.2282163144290469 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.04645295339863461 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.9565152864351439 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.7009498367468091 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.7423567824280202 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.9575541703769664 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.023870550161812297 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.00275028568108307 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.00356800070444239 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.003485751823118051 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.002747305898968793 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.23793892088542526 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.08722255267001126 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.1112312688602769 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.10982263054722373 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.08713204456539597 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.4887624633565217 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.23118547623512836 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.2876668872191355 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.28516330859718164 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.2309636203280157 + } + ] + }, + "strict_f1": { + "max": 0.023870550161812297, + "mean": 0.0072843788538849204, + "min": 0.002747305898968793, + "n": 5, + "std": 0.009280155500075412 + }, + "thresholds": [ + 0.0002716928720474243, + 0.00027818421949632466, + 0.00028128057601861656, + 0.0002883674460463226, + 0.000301085616229102 + ], + "tolerance_f1": { + "max": 0.23793892088542526, + "mean": 0.12666948350566662, + "min": 0.08713204456539597, + "n": 5, + "std": 0.06328963590343438 + }, + "union_f1": { + "max": 0.4887624633565217, + "mean": 0.30474835114719656, + "min": 0.2309636203280157, + "n": 5, + "std": 0.1065272580858829 + } + }, + { + "difference": { + "max": 0.4425322203556718, + "mean": 0.24080298432225694, + "min": 0.16790877204956894, + "n": 5, + "std": 0.11498838284648957 + }, + "label": "StormCast", + "model_tag": "stormcast", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "predicted_positive_rate": { + "max": 0.979443413729128, + "mean": 0.6783970315398886, + "min": 0.020630797773654917, + "n": 5, + "std": 0.39241013872037767 + }, + "scope": "train_fire_top20pct", + "scope_label": "top 20%", + "seed_values": { + "difference": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.4425322203556718 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.16790877204956894 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.21164756227103257 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.21401759488544256 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.16790877204956894 + } + ], + "predicted_positive_rate": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.020630797773654917 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.979443413729128 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.6907606679035251 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.7217068645640075 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.979443413729128 + } + ], + "strict_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.022074159907300116 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.0014637962799384197 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.0020248273888312522 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.001948363237912615 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.0014637962799384197 + } + ], + "tolerance_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.23443811051591282 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.062163327857870446 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.08079366172039935 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.08122801407099456 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.062163327857870446 + } + ], + "union_f1": [ + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 1, + "value": 0.4646063802629719 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 7, + "value": 0.16937256832950737 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 42, + "value": 0.21367238965986382 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 99, + "value": 0.21596595812335517 + }, + { + "path": "REDACTED_LOCAL_RESULT_PATH", + "seed": 123, + "value": 0.16937256832950737 + } + ] + }, + "strict_f1": { + "max": 0.022074159907300116, + "mean": 0.005794988618784165, + "min": 0.0014637962799384197, + "n": 5, + "std": 0.009104127010779225 + }, + "thresholds": [ + 0.00021927931811660528, + 0.0002497032401151955, + 0.0002781348885037005, + 0.0002812395105138421, + 0.00030111230444163084 + ], + "tolerance_f1": { + "max": 0.23443811051591282, + "mean": 0.10415728840460951, + "min": 0.062163327857870446, + "n": 5, + "std": 0.07343651804127242 + }, + "union_f1": { + "max": 0.4646063802629719, + "mean": 0.2465979729410411, + "min": 0.16937256832950737, + "n": 5, + "std": 0.12397339055627195 + } + } + ], + "threshold_max_ppr": 0.999999, + "threshold_min_ppr": 1e-06, + "threshold_selector": "strict_f1", + "tolerance_combo": "t0_s8", + "union_combo": "t3_s8" +} diff --git a/artifacts/results/fireprone_contract_progression_table.generated.tex b/artifacts/results/fireprone_contract_progression_table.generated.tex new file mode 100644 index 0000000000000000000000000000000000000000..5616b6c2609fad6179e9e544b196c4e853f7c569 --- /dev/null +++ b/artifacts/results/fireprone_contract_progression_table.generated.tex @@ -0,0 +1,69 @@ +\begin{table*}[t] + \centering + \scriptsize + \setlength{\tabcolsep}{4pt} + \caption{Occupancy scores across global and fire-prone scopes. Global uses the full validation/test domain; top-\(k\) rows use train-defined fire-prone masks from historical fire frequency. Values are \(F_1\) percentages from the same validation-selected strict threshold. Tolerance is spatial-only; union adds temporal and spatial matching. Difference is union minus strict. Rows report five-seed mean with small std. Values use four decimals.} + \label{tab:fireprone_contract_progression} + \begin{adjustbox}{max width=\textwidth} + \begin{tabular}{@{}llcccc@{}} + \toprule + Backbone & Scope & Strict \(F_1\uparrow\) & Tolerance \(F_1\uparrow\) & Union \(F_1\uparrow\) & Difference \(\uparrow\) \\ + \midrule + \textcolor{blue}{FireWx-FM ref.} & global & \ms{0.4550}{0.1410} & \ms{29.7480}{1.2870} & \ms{59.0660}{2.7370} & \ms{58.6110}{2.6950} \\ + & top 5\% & \ms{3.5600}{0.8810} & \ms{39.2620}{1.4010} & \ms{72.8280}{2.5780} & \ms{69.2680}{1.9960} \\ + & top 10\% & \ms{3.5580}{0.8800} & \ms{39.1660}{1.3910} & \ms{72.5200}{2.5670} & \ms{68.9630}{1.9890} \\ + & top 20\% & \ms{3.5300}{0.8700} & \ms{38.2850}{1.2950} & \ms{69.7230}{2.4660} & \ms{66.1930}{1.9270} \\ + \addlinespace[1pt] + Prithvi-WxC & global & \ms{0.0550}{0.0040} & \ms{7.1600}{0.6600} & \ms{20.1900}{1.8300} & \ms{20.1300}{1.8300} \\ + & top 5\% & \ms{1.4100}{1.1600} & \ms{19.2600}{4.5000} & \ms{42.5800}{4.5500} & \ms{41.1700}{3.4800} \\ + & top 10\% & \ms{1.2400}{1.3200} & \ms{14.8800}{8.4400} & \ms{32.6900}{13.2100} & \ms{31.4500}{11.9100} \\ + & top 20\% & \ms{1.1500}{1.3800} & \ms{13.1500}{9.4600} & \ms{28.1300}{15.2900} & \ms{26.9800}{13.9200} \\ + \addlinespace[1pt] + Aurora & global & \ms{0.0700}{0.0100} & \ms{8.5000}{1.9600} & \ms{23.1000}{4.9400} & \ms{23.0400}{4.9300} \\ + & top 5\% & \ms{0.9900}{0.9300} & \ms{15.1300}{6.0800} & \ms{35.4800}{11.0200} & \ms{34.5000}{10.3700} \\ + & top 10\% & \ms{0.7800}{1.0500} & \ms{12.7400}{6.5600} & \ms{30.5300}{10.8800} & \ms{29.7500}{9.8700} \\ + & top 20\% & \ms{0.6700}{1.1000} & \ms{10.5300}{7.4300} & \ms{24.9400}{12.5800} & \ms{24.2800}{11.4900} \\ + \addlinespace[1pt] + ClimaX & global & \ms{0.3500}{0.0800} & \ms{29.7500}{3.6100} & \ms{60.1500}{7.5900} & \ms{59.8000}{7.5500} \\ + & top 5\% & \ms{1.2900}{0.1100} & \ms{34.5800}{2.3800} & \ms{69.2200}{5.7200} & \ms{67.9200}{5.7300} \\ + & top 10\% & \ms{1.2500}{0.1600} & \ms{34.3300}{2.2900} & \ms{68.5700}{5.5400} & \ms{67.3200}{5.5500} \\ + & top 20\% & \ms{1.0300}{0.2700} & \ms{30.2100}{4.2900} & \ms{60.0600}{7.5700} & \ms{59.0400}{7.5900} \\ + \addlinespace[1pt] + StormCast & global & \ms{0.0560}{0.0110} & \ms{8.2000}{2.1900} & \ms{22.3800}{5.4300} & \ms{22.3200}{5.4200} \\ + & top 5\% & \ms{0.9600}{0.8000} & \ms{15.3200}{5.5300} & \ms{36.1900}{9.7300} & \ms{35.2300}{9.1800} \\ + & top 10\% & \ms{0.7300}{0.9300} & \ms{12.6700}{6.3300} & \ms{30.4700}{10.6500} & \ms{29.7500}{9.7500} \\ + & top 20\% & \ms{0.5800}{0.9100} & \ms{10.4200}{7.3400} & \ms{24.6600}{12.4000} & \ms{24.0800}{11.5000} \\ + \addlinespace[1pt] + AlphaEarth & global & \ms{2.0600}{0.4400} & \ms{29.4500}{6.0100} & \ms{37.4300}{9.9500} & \ms{35.3700}{10.0300} \\ + & top 5\% & \ms{6.9100}{0.8500} & \ms{42.8800}{4.6100} & \ms{51.7400}{8.7300} & \ms{44.8300}{9.0800} \\ + & top 10\% & \ms{6.6400}{0.9900} & \ms{41.9000}{5.9500} & \ms{50.5700}{10.0100} & \ms{43.9300}{9.9200} \\ + & top 20\% & \ms{6.1900}{1.1300} & \ms{38.8300}{7.5000} & \ms{46.3800}{12.1700} & \ms{40.1900}{11.6800} \\ + \addlinespace[1pt] + DLWP & global & \ms{0.1700}{0.0400} & \ms{14.9100}{3.2400} & \ms{28.1900}{6.9700} & \ms{28.0200}{6.9300} \\ + & top 5\% & \ms{1.8100}{0.4800} & \ms{31.7200}{3.2900} & \ms{55.4600}{5.2900} & \ms{53.6500}{5.4800} \\ + & top 10\% & \ms{1.6100}{0.6000} & \ms{27.6600}{5.9200} & \ms{47.1300}{8.0100} & \ms{45.5200}{7.7900} \\ + & top 20\% & \ms{1.5200}{0.9000} & \ms{20.9400}{4.8000} & \ms{34.9300}{7.8500} & \ms{33.4100}{7.8800} \\ + \addlinespace[1pt] + FCN & global & \ms{0.2800}{0.0800} & \ms{19.5100}{3.3400} & \ms{40.0600}{9.3700} & \ms{39.7800}{9.3400} \\ + & top 5\% & \ms{1.6200}{0.5100} & \ms{29.3800}{2.7600} & \ms{54.3000}{7.4100} & \ms{52.6800}{7.4400} \\ + & top 10\% & \ms{1.1800}{0.5100} & \ms{22.4200}{3.9800} & \ms{43.4500}{9.2500} & \ms{42.2700}{9.0300} \\ + & top 20\% & \ms{1.0000}{0.4300} & \ms{16.9800}{3.9400} & \ms{34.0900}{8.2600} & \ms{33.0900}{7.9300} \\ + \addlinespace[1pt] + FengWu & global & \ms{0.2600}{0.0800} & \ms{12.0000}{6.0200} & \ms{24.1000}{13.6300} & \ms{23.8400}{13.5700} \\ + & top 5\% & \ms{1.5700}{0.3600} & \ms{16.2800}{3.7000} & \ms{30.1100}{5.0100} & \ms{28.5400}{4.7700} \\ + & top 10\% & \ms{1.2400}{0.5300} & \ms{12.9500}{5.6100} & \ms{24.1900}{8.6900} & \ms{22.9400}{8.1900} \\ + & top 20\% & \ms{1.1200}{0.5000} & \ms{11.9500}{5.0700} & \ms{22.7900}{7.9100} & \ms{21.6700}{7.4400} \\ + \addlinespace[1pt] + FuXi & global & \ms{0.3800}{0.1200} & \ms{21.0300}{4.8200} & \ms{37.2900}{9.4500} & \ms{36.9100}{9.4300} \\ + & top 5\% & \ms{2.0300}{0.6800} & \ms{31.8900}{4.7300} & \ms{53.9300}{8.3800} & \ms{51.9000}{8.6900} \\ + & top 10\% & \ms{1.6500}{0.7300} & \ms{24.0100}{5.7800} & \ms{40.2100}{9.9300} & \ms{38.5600}{9.7700} \\ + & top 20\% & \ms{1.3600}{0.6800} & \ms{21.9500}{5.8600} & \ms{36.7300}{10.0300} & \ms{35.3700}{9.9200} \\ + \addlinespace[1pt] + Pangu-Weather & global & \ms{0.2800}{0.1100} & \ms{17.0900}{4.0500} & \ms{35.6400}{9.0300} & \ms{35.3600}{9.0800} \\ + & top 5\% & \ms{1.3700}{0.3100} & \ms{22.2200}{6.8600} & \ms{43.4200}{13.2400} & \ms{42.0600}{13.0600} \\ + & top 10\% & \ms{1.0900}{0.3500} & \ms{18.9300}{5.9300} & \ms{38.5300}{11.7200} & \ms{37.4400}{11.5300} \\ + & top 20\% & \ms{0.8800}{0.3600} & \ms{17.0200}{5.4900} & \ms{34.5700}{10.2900} & \ms{33.6800}{10.1300} \\ + \bottomrule + \end{tabular} + \end{adjustbox} +\end{table*} diff --git a/artifacts/results/selection_regret_all_backbones_20260504.csv b/artifacts/results/selection_regret_all_backbones_20260504.csv new file mode 100644 index 0000000000000000000000000000000000000000..ea8a02982a8c66171b1344716be24df46cd272c8 --- /dev/null +++ b/artifacts/results/selection_regret_all_backbones_20260504.csv @@ -0,0 +1,25 @@ +model_tag,label,scope,n,seeds,exact_regret_mean,exact_regret_std,tolerated_regret_mean,tolerated_regret_std,union_regret_mean,union_regret_std +reference,Reference,global,5,1 7 42 99 123,0.0,0.0,0.08783024981138902,0.09670495645481135,0.08783024981138902,0.09670495645481135 +reference,Reference,fire_prone,5,1 7 42 99 123,0.0,0.0,0.03402707057672223,0.032044658643147844,0.03402707057672223,0.032044658643147844 +prithvi_wxc,Prithvi-WxC,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 +prithvi_wxc,Prithvi-WxC,fire_prone,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 +aurora,Aurora,global,5,1 7 42 99 123,0.00020004882767231798,0.00026703384456332115,0.09851983041506818,0.1298781980037557,0.09851983041506818,0.1298781980037557 +aurora,Aurora,fire_prone,5,1 7 42 99 123,0.008202508825959588,0.01834136732088763,0.14391889430974364,0.32121904665016227,0.14391889430974364,0.32121904665016227 +climax,ClimaX,global,5,1 7 42 99 123,3.0287686240700486e-06,4.147312242167625e-06,0.0012959969982639485,0.0017746169760203706,0.0012959969982639485,0.0017746169760203706 +climax,ClimaX,fire_prone,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 +stormcast,StormCast,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 +stormcast,StormCast,fire_prone,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 +pangu_weather,Pangu-Weather,global,5,1 7 42 99 123,0.00013033979247265275,0.0002685372203690466,0.048806713097574374,0.10733308684741971,0.048806713097574374,0.10733308684741971 +pangu_weather,Pangu-Weather,fire_prone,5,1 7 42 99 123,0.027875386332505546,0.02348779386900393,0.43111948243387105,0.39355644251497235,0.43111948243387105,0.39355644251497235 +dlwp,DLWP,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 +dlwp,DLWP,fire_prone,5,1 7 42 99 123,0.0007702319787454587,0.0010995336594539604,0.043265915053601556,0.04332331365579739,0.043265915053601556,0.04332331365579739 +fcn,FCN,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 +fcn,FCN,fire_prone,5,1 7 42 99 123,5.960229415004348e-06,1.3327478133443526e-05,0.011679805987441694,0.019872372458657642,0.011679805987441694,0.019872372458657642 +fengwu,FengWu,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 +fengwu,FengWu,fire_prone,5,1 7 42 99 123,0.0006908222234409067,0.0011910586589384115,0.005222389249812243,0.0062394095558402415,0.005222389249812243,0.0062394095558402415 +fuxi,FuXi,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 +fuxi,FuXi,fire_prone,5,1 7 42 99 123,0.0,0.0,0.0010839188523199318,0.0017288780545672386,0.0010839188523199318,0.0017288780545672386 +pangu6,Pangu-Weather,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 +pangu6,Pangu-Weather,fire_prone,5,1 7 42 99 123,0.0007280423771922354,0.001178746460551365,0.0018491271881979853,0.0032630386057089294,0.0018491271881979853,0.0032630386057089294 +alphaearth,AlphaEarth,global,5,1 7 42 99 123,0.0,0.0,0.1722171037486726,0.08849214830495522,0.1722171037486726,0.08849214830495522 +alphaearth,AlphaEarth,fire_prone,5,1 7 42 99 123,0.0,0.0,0.038803552655092256,0.0594825313313219,0.038803552655092256,0.0594825313313219 diff --git a/artifacts/results/selection_regret_all_backbones_20260504.json b/artifacts/results/selection_regret_all_backbones_20260504.json new file mode 100644 index 0000000000000000000000000000000000000000..3fc2ab353adfc8eb74aaa156550b344da6ecd27c --- /dev/null +++ b/artifacts/results/selection_regret_all_backbones_20260504.json @@ -0,0 +1,5196 @@ +{ + "issues": [], + "notes": [ + "Reference global exact_regret has true zero displayed std; table omitted", + "Reference fire_prone exact_regret has true zero displayed std; table omitted", + "Prithvi-WxC global exact_regret has true zero displayed std; table omitted", + "Prithvi-WxC global tolerated_regret has true zero displayed std; table omitted", + "Prithvi-WxC global union_regret has true zero displayed std; table omitted", + "Prithvi-WxC fire_prone exact_regret has true zero displayed std; table omitted", + "Prithvi-WxC fire_prone tolerated_regret has true zero displayed std; table omitted", + "Prithvi-WxC fire_prone union_regret has true zero displayed std; table omitted", + "ClimaX fire_prone exact_regret has true zero displayed std; table omitted", + "ClimaX fire_prone tolerated_regret has true zero displayed std; table omitted", + "ClimaX fire_prone union_regret has true zero displayed std; table omitted", + "StormCast global exact_regret has true zero displayed std; table omitted", + "StormCast global tolerated_regret has true zero displayed std; table omitted", + "StormCast global union_regret has true zero displayed std; table omitted", + "StormCast fire_prone exact_regret has true zero displayed std; table omitted", + "StormCast fire_prone tolerated_regret has true zero displayed std; table omitted", + "StormCast fire_prone union_regret has true zero displayed std; table omitted", + "DLWP global exact_regret has true zero displayed std; table omitted", + "DLWP global tolerated_regret has true zero displayed std; table omitted", + "DLWP global union_regret has true zero displayed std; table omitted", + "FCN global exact_regret has true zero displayed std; table omitted", + "FCN global tolerated_regret has true zero displayed std; table omitted", + "FCN global union_regret has true zero displayed std; table omitted", + "FengWu global exact_regret has true zero displayed std; table omitted", + "FengWu global tolerated_regret has true zero displayed std; table omitted", + "FengWu global union_regret has true zero displayed std; table omitted", + "FuXi global exact_regret has true zero displayed std; table omitted", + "FuXi global tolerated_regret has true zero displayed std; table omitted", + "FuXi global union_regret has true zero displayed std; table omitted", + "FuXi fire_prone exact_regret has true zero displayed std; table omitted", + "Pangu-Weather global exact_regret has true zero displayed std; table omitted", + "Pangu-Weather global tolerated_regret has true zero displayed std; table omitted", + "Pangu-Weather global union_regret has true zero displayed std; table omitted", + "AlphaEarth global exact_regret has true zero displayed std; table omitted", + "AlphaEarth fire_prone exact_regret has true zero displayed std; table omitted" + ], + "rows": [ + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.004193290734824281", + "exact_ranking_score": "0.004193290734824281", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0056438127090301", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0056438127090301", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0006549130347299629", + "ranking_selected_val_pr_auc": "0.0007883068263073185", + "scope": "global", + "seed": 1, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.6337627266658229", + "tolerated_ranking_score": "0.40561891947698747", + "tolerated_regret": 0.22814380718883542, + "tolerated_test_gap": "0.22814380718883542", + "tolerated_val_decision_score": "0.6238973446702573", + "tolerated_val_gap": "0.1715500493543587", + "tolerated_val_ranking_score": "0.45234729531589857", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.6337627266658229", + "union_ranking_score": "0.40561891947698747", + "union_regret": 0.22814380718883542, + "union_test_gap": "0.22814380718883542", + "union_val_decision_score": "0.6238973446702573", + "union_val_gap": "0.1715500493543587", + "union_val_ranking_score": "0.45234729531589857" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.09715762273901808", + "exact_ranking_score": "0.09715762273901808", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.13462469733656174", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.13462469733656174", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.04390614036305261", + "ranking_selected_val_pr_auc": "0.05734457583552234", + "scope": "fire_prone", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7863702028280513", + "tolerated_ranking_score": "0.7863702028280513", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.758785818734687", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.758785818734687", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7863702028280513", + "union_ranking_score": "0.7863702028280513", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.758785818734687", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.758785818734687" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.006959088991986505", + "exact_ranking_score": "0.006959088991986505", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.006399069226294357", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.006399069226294357", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0007047712457371991", + "ranking_selected_val_pr_auc": "0.0007434343102653092", + "scope": "global", + "seed": 123, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.5604635792586619", + "tolerated_ranking_score": "0.4427625907752311", + "tolerated_regret": 0.11770098848343075, + "tolerated_test_gap": "0.11770098848343075", + "tolerated_val_decision_score": "0.5441785824390972", + "tolerated_val_gap": "0.056405166058839995", + "tolerated_val_ranking_score": "0.48777341638025723", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.5604635792586619", + "union_ranking_score": "0.4427625907752311", + "union_regret": 0.11770098848343075, + "union_test_gap": "0.11770098848343075", + "union_val_decision_score": "0.5441785824390972", + "union_val_gap": "0.056405166058839995", + "union_val_ranking_score": "0.48777341638025723" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.10531544957774468", + "exact_ranking_score": "0.10531544957774468", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.12267343485617596", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.12267343485617596", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.045809049876129763", + "ranking_selected_val_pr_auc": "0.05470980065861018", + "scope": "fire_prone", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7789089693560928", + "tolerated_ranking_score": "0.7789089693560928", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.749825256630312", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.749825256630312", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7789089693560928", + "union_ranking_score": "0.7789089693560928", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.749825256630312", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.749825256630312" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.004809747755451047", + "exact_ranking_score": "0.004809747755451047", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.004359099620457705", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.004359099620457705", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0005634701573991865", + "ranking_selected_val_pr_auc": "0.0005920938943373892", + "scope": "global", + "seed": 42, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.6812131506751973", + "tolerated_ranking_score": "0.4087444681515033", + "tolerated_regret": 0.272468682523694, + "tolerated_test_gap": "0.272468682523694", + "tolerated_val_decision_score": "0.7447325774946227", + "tolerated_val_gap": "0.3021550064919567", + "tolerated_val_ranking_score": "0.44257757100266604", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.6812131506751973", + "union_ranking_score": "0.4087444681515033", + "union_regret": 0.272468682523694, + "union_test_gap": "0.272468682523694", + "union_val_decision_score": "0.7447325774946227", + "union_val_gap": "0.3021550064919567", + "union_val_ranking_score": "0.44257757100266604" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.08702469619756958", + "exact_ranking_score": "0.08702469619756958", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.10515392597717052", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.10515392597717052", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.038083070948941686", + "ranking_selected_val_pr_auc": "0.04505214599515043", + "scope": "fire_prone", + "seed": 42, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.8461131676361712", + "tolerated_ranking_score": "0.7112901458230849", + "tolerated_regret": 0.13482302181308625, + "tolerated_test_gap": "0.13482302181308625", + "tolerated_val_decision_score": "0.806760850261024", + "tolerated_val_gap": "0.07815518654517495", + "tolerated_val_ranking_score": "0.728605663715849", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.8461131676361712", + "union_ranking_score": "0.7112901458230849", + "union_regret": 0.13482302181308625, + "union_test_gap": "0.13482302181308625", + "union_val_decision_score": "0.806760850261024", + "union_val_gap": "0.07815518654517495", + "union_val_ranking_score": "0.728605663715849" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.010460251046025104", + "exact_ranking_score": "0.010460251046025104", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.009547591069330199", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.009547591069330199", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.001005722733868245", + "ranking_selected_val_pr_auc": "0.0009777524224237678", + "scope": "global", + "seed": 7, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.6691395427484861", + "tolerated_ranking_score": "0.6184842128568402", + "tolerated_regret": 0.050655329891645895, + "tolerated_test_gap": "0.050655329891645895", + "tolerated_val_decision_score": "0.677404587086621", + "tolerated_val_gap": "0.011527156971746999", + "tolerated_val_ranking_score": "0.665877430114874", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.6691395427484861", + "union_ranking_score": "0.6184842128568402", + "union_regret": 0.050655329891645895, + "union_test_gap": "0.050655329891645895", + "union_val_decision_score": "0.677404587086621", + "union_val_gap": "0.011527156971746999", + "union_val_ranking_score": "0.665877430114874" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.12514898688915374", + "exact_ranking_score": "0.12514898688915374", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.14325685506435365", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.14325685506435365", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.05955397140893137", + "ranking_selected_val_pr_auc": "0.06058697421048699", + "scope": "fire_prone", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.8294499693616161", + "tolerated_ranking_score": "0.8294499693616161", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.8107581951083846", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.8107581951083846", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.8294499693616161", + "union_ranking_score": "0.8294499693616161", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.8107581951083846", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.8107581951083846" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.006780481898534931", + "exact_ranking_score": "0.006780481898534931", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.005964994898359627", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.005964994898359627", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0006577120081349608", + "ranking_selected_val_pr_auc": "0.0007340310681749519", + "scope": "global", + "seed": 99, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.5842714676652996", + "tolerated_ranking_score": "0.3921547570095426", + "tolerated_regret": 0.19211671065575697, + "tolerated_test_gap": "0.19211671065575697", + "tolerated_val_decision_score": "0.5663548190132099", + "tolerated_val_gap": "0.10549666519652506", + "tolerated_val_ranking_score": "0.4608581538166849", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.5842714676652996", + "union_ranking_score": "0.3921547570095426", + "union_regret": 0.19211671065575697, + "union_test_gap": "0.19211671065575697", + "union_val_decision_score": "0.5663548190132099", + "union_val_gap": "0.10549666519652506", + "union_val_ranking_score": "0.4608581538166849" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.10365251727541955", + "exact_ranking_score": "0.10365251727541955", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.12251941328731666", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.12251941328731666", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0458102699699856", + "ranking_selected_val_pr_auc": "0.0554648890254577", + "scope": "fire_prone", + "seed": 99, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.8350245452333586", + "tolerated_ranking_score": "0.7758298037709835", + "tolerated_regret": 0.05919474146237502, + "tolerated_test_gap": "0.05919474146237502", + "tolerated_val_decision_score": "0.7736506356603177", + "tolerated_val_gap": "0.027586681737492236", + "tolerated_val_ranking_score": "0.7460639539228254", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.8350245452333586", + "union_ranking_score": "0.7758298037709835", + "union_regret": 0.05919474146237502, + "union_test_gap": "0.05919474146237502", + "union_val_decision_score": "0.7736506356603177", + "union_val_gap": "0.027586681737492236", + "union_val_ranking_score": "0.7460639539228254" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.00048497822606044473", + "exact_regret": 7.571921560175121e-06, + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_pr_auc": "0.00024254792826221397", + "scope": "global", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.23755358049655212", + "tolerated_regret": 0.0032399924956598714, + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.23755358049655212", + "union_regret": 0.0032399924956598714 + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_pr_auc": "0.02093558282208589", + "scope": "fire_prone", + "seed": 1, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0 + }, + { + "exact_decision_head": "pixel MLP head", + "exact_decision_head_arch": "pixel_mlp", + "exact_decision_score": "0.0005311562430574265", + "exact_ranking_score": "0.0005311562430574265", + "exact_regret": 0.0, + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_pr_auc": "0.00031683315961488916", + "scope": "global", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.23647112940979162", + "tolerated_regret": 0.004322443582420371, + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.23647112940979162", + "union_regret": 0.004322443582420371 + }, + { + "exact_decision_head": "pixel MLP head", + "exact_decision_head_arch": "pixel_mlp", + "exact_decision_score": "0.04541038665445361", + "exact_ranking_score": "0.04541038665445361", + "exact_regret": 0.0, + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_pr_auc": "0.03014151567817997", + "scope": "fire_prone", + "seed": 123, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7175172112337449", + "tolerated_regret": 0.0010152594893735323, + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7175172112337449", + "union_regret": 0.0010152594893735323 + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.00048497822606044473", + "exact_regret": 7.571921560175121e-06, + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_pr_auc": "0.0002876372030063385", + "scope": "global", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.0", + "tolerated_regret": 0.240793572992212, + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.0", + "union_regret": 0.240793572992212 + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_pr_auc": "0.02613792907867929", + "scope": "fire_prone", + "seed": 42, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0 + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.0", + "exact_regret": 0.0004925501476206198, + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_pr_auc": "0.00027660331739269843", + "scope": "global", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.23734400297937533", + "tolerated_regret": 0.0034495700128366613, + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.23734400297937533", + "union_regret": 0.0034495700128366613 + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.0413500618483831", + "exact_ranking_score": "0.0413500618483831", + "exact_regret": 0.0, + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_pr_auc": "0.024802820904513342", + "scope": "fire_prone", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7184857293868923", + "tolerated_regret": 4.674133622617482e-05, + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7184857293868923", + "union_regret": 4.674133622617482e-05 + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.0", + "exact_regret": 0.0004925501476206198, + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_pr_auc": "0.00024254792826221397", + "scope": "global", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.0", + "tolerated_regret": 0.240793572992212, + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.0", + "union_regret": 0.240793572992212 + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.0", + "exact_regret": 0.04101254412979794, + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_pr_auc": "0.02093558282208589", + "scope": "fire_prone", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.0", + "tolerated_regret": 0.7185324707231184, + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.0", + "union_regret": 0.7185324707231184 + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.00048497822606044473", + "exact_regret": 7.571921560175121e-06, + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_pr_auc": "0.0002543464414550104", + "scope": "global", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.23755358049655212", + "tolerated_regret": 0.0032399924956598714, + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.23755358049655212", + "union_regret": 0.0032399924956598714 + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_pr_auc": "0.02281272244151735", + "scope": "fire_prone", + "seed": 1, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0 + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.0004925501476206198", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0004727121748000204", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0004727121748000204", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00025952340213823634", + "ranking_selected_val_pr_auc": "0.00028511175967657057", + "scope": "global", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.240793572992212", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.2179368807313983", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.240793572992212", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.2179368807313983" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.024930358757410707", + "ranking_selected_val_pr_auc": "0.03159877521821671", + "scope": "fire_prone", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.0004925501476206198", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0004727121748000204", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0004727121748000204", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00023723426605001756", + "ranking_selected_val_pr_auc": "0.00023206733426223207", + "scope": "global", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.240793572992212", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.2179368807313983", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.240793572992212", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.2179368807313983" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.021516770872035896", + "ranking_selected_val_pr_auc": "0.023234418719582723", + "scope": "fire_prone", + "seed": 42, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.00048497822606044473", + "exact_regret": 7.571921560175121e-06, + "exact_test_gap": "7.571921560175121e-06", + "exact_val_decision_score": "0.0004727121748000204", + "exact_val_gap": "7.267025482891767e-06", + "exact_val_ranking_score": "0.00046544514931712863", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.00025423546642937565", + "ranking_selected_val_pr_auc": "0.00023120715224267713", + "scope": "global", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.23755358049655212", + "tolerated_regret": 0.0032399924956598714, + "tolerated_test_gap": "0.0032399924956598714", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.00299097714352789", + "tolerated_val_ranking_score": "0.2149459035878704", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.23755358049655212", + "union_regret": 0.0032399924956598714, + "union_test_gap": "0.0032399924956598714", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.00299097714352789", + "union_val_ranking_score": "0.2149459035878704" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.021317405351800135", + "ranking_selected_val_pr_auc": "0.02320587721933335", + "scope": "fire_prone", + "seed": 7, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.00048696535779102983", + "exact_ranking_score": "0.00048696535779102983", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.00046735226565276225", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.00046735226565276225", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0002340075376021003", + "ranking_selected_val_pr_auc": "0.00023166839498120457", + "scope": "global", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.2384111045733381", + "tolerated_ranking_score": "0.2384111045733381", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.21573180756095836", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.21573180756095836", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.2384111045733381", + "union_ranking_score": "0.2384111045733381", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.21573180756095836", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.21573180756095836" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.02099123219536693", + "ranking_selected_val_pr_auc": "0.023142108180759017", + "scope": "fire_prone", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.00487022180273714", + "exact_ranking_score": "0.00487022180273714", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0013194321729398599", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0013194321729398599", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0006257446338466172", + "ranking_selected_val_pr_auc": "0.00041081035639410466", + "scope": "global", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.38023285660836226", + "tolerated_ranking_score": "0.38023285660836226", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.40408253980815234", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.40408253980815234", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.38023285660836226", + "union_ranking_score": "0.38023285660836226", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.40408253980815234", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.40408253980815234" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.043506471331489265", + "exact_ranking_score": "0.043506471331489265", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04461624715984538", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04461624715984538", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.019747545663020845", + "ranking_selected_val_pr_auc": "0.019740435565559492", + "scope": "fire_prone", + "seed": 1, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7280364139105968", + "tolerated_ranking_score": "0.7280364139105968", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7120828721033882", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7120828721033882", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7280364139105968", + "union_ranking_score": "0.7280364139105968", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7120828721033882", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7120828721033882" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0025026814444047193", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0025026814444047193", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0007129763973023342", + "ranking_selected_val_pr_auc": "0.0004370316771032176", + "scope": "global", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.3797689460796109", + "tolerated_ranking_score": "0.3797689460796109", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.4257234764346952", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.4257234764346952", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.3797689460796109", + "union_ranking_score": "0.3797689460796109", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.4257234764346952", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.4257234764346952" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.04249854057209574", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.001485996442297799, + "exact_test_gap": "0.001485996442297799", + "exact_val_decision_score": "0.04371409485637943", + "exact_val_gap": "0.0011821959321862904", + "exact_val_ranking_score": "0.04253189892419314", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.021118517500793188", + "ranking_selected_val_pr_auc": "0.02303236785318257", + "scope": "fire_prone", + "seed": 123, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7321459738801157", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.013613503156997275, + "tolerated_test_gap": "0.013613503156997275", + "tolerated_val_decision_score": "0.7128995538153613", + "tolerated_val_gap": "0.022224140467747677", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7321459738801157", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.013613503156997275, + "union_test_gap": "0.013613503156997275", + "union_val_decision_score": "0.7128995538153613", + "union_val_gap": "0.022224140467747677", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0020120724346076456", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0020120724346076456", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0006203713852571992", + "ranking_selected_val_pr_auc": "0.0004707444097300293", + "scope": "global", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.3405125814370199", + "tolerated_ranking_score": "0.3405125814370199", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.474350305376108", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.474350305376108", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.3405125814370199", + "union_ranking_score": "0.3405125814370199", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.474350305376108", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.474350305376108" + }, + { + "exact_decision_head": "pixel MLP head", + "exact_decision_head_arch": "pixel_mlp", + "exact_decision_score": "0.04637177602565815", + "exact_ranking_score": "0.04637177602565815", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.044021164021164016", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.044021164021164016", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.020762153794205103", + "ranking_selected_val_pr_auc": "0.020894537350314525", + "scope": "fire_prone", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7728400679088107", + "tolerated_ranking_score": "0.6606900017471591", + "tolerated_regret": 0.11215006616165157, + "tolerated_test_gap": "0.11215006616165157", + "tolerated_val_decision_score": "0.7122584178150692", + "tolerated_val_gap": "0.009110393702543962", + "tolerated_val_ranking_score": "0.7031480241125252", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7728400679088107", + "union_ranking_score": "0.6606900017471591", + "union_regret": 0.11215006616165157, + "union_test_gap": "0.11215006616165157", + "union_val_decision_score": "0.7122584178150692", + "union_val_gap": "0.009110393702543962", + "union_val_ranking_score": "0.7031480241125252" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0010056989607777405", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0010056989607777405", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0005264872646085452", + "ranking_selected_val_pr_auc": "0.00035284338324857376", + "scope": "global", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.3432315705541329", + "tolerated_ranking_score": "0.3432315705541329", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.3469162743670964", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.3469162743670964", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.3432315705541329", + "union_ranking_score": "0.3432315705541329", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.3469162743670964", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.3469162743670964" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.043377707581227436", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0023651634514294945, + "exact_test_gap": "0.0023651634514294945", + "exact_val_decision_score": "0.04303681559882578", + "exact_val_gap": "0.0005049166746326425", + "exact_val_ranking_score": "0.04253189892419314", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.018519739310339497", + "ranking_selected_val_pr_auc": "0.01953230111607746", + "scope": "fire_prone", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.762536895087842", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.044004424364723516, + "tolerated_test_gap": "0.044004424364723516", + "tolerated_val_decision_score": "0.7274867592832499", + "tolerated_val_gap": "0.036811345935636286", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.762536895087842", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.044004424364723516, + "union_test_gap": "0.044004424364723516", + "union_val_decision_score": "0.7274867592832499", + "union_val_gap": "0.036811345935636286", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.002074688796680498", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.002074688796680498", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0007477128447471452", + "ranking_selected_val_pr_auc": "0.00047085132311599663", + "scope": "global", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.3979559626836394", + "tolerated_ranking_score": "0.3979559626836394", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.4277982050028173", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.4277982050028173", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.3979559626836394", + "union_ranking_score": "0.3979559626836394", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.4277982050028173", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.4277982050028173" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.04819843096725701", + "exact_ranking_score": "0.04819843096725701", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.05605272285813389", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.05605272285813389", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.02136633936888583", + "ranking_selected_val_pr_auc": "0.024917289620763165", + "scope": "fire_prone", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7653346239087763", + "tolerated_ranking_score": "0.7187730423241409", + "tolerated_regret": 0.046561581584635414, + "tolerated_test_gap": "0.046561581584635414", + "tolerated_val_decision_score": "0.7238142274110074", + "tolerated_val_gap": "0.032313773339499696", + "tolerated_val_ranking_score": "0.6915004540715077", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7653346239087763", + "union_ranking_score": "0.7187730423241409", + "union_regret": 0.046561581584635414, + "union_test_gap": "0.046561581584635414", + "union_val_decision_score": "0.7238142274110074", + "union_val_gap": "0.032313773339499696", + "union_val_ranking_score": "0.6915004540715077" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0009319664492078285", + "exact_ranking_score": "0.0009319664492078285", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.004399472063352399", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.004399472063352399", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00037256097806901117", + "ranking_selected_val_pr_auc": "0.00045069184813668276", + "scope": "global", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.31167484413093016", + "tolerated_ranking_score": "0.31167484413093016", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.3052875742065006", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.3052875742065006", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.31167484413093016", + "union_ranking_score": "0.31167484413093016", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.3052875742065006", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.3052875742065006" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.042068766252528166", + "exact_ranking_score": "0.042068766252528166", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.043150583715087926", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.043150583715087926", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.01844011667219625", + "ranking_selected_val_pr_auc": "0.019762912264716534", + "scope": "fire_prone", + "seed": 1, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7182175622542595", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "-0.0003149084688589543", + "tolerated_val_decision_score": "0.6906967717983766", + "tolerated_val_gap": "2.135845076300935e-05", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7182175622542595", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "-0.0003149084688589543", + "union_val_decision_score": "0.6906967717983766", + "union_val_gap": "2.135845076300935e-05", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0006535947712418301", + "exact_ranking_score": "0.0006535947712418301", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0037359900373599006", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0037359900373599006", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.000387412312932838", + "ranking_selected_val_pr_auc": "0.0004364526611974742", + "scope": "global", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.3096850885545486", + "tolerated_ranking_score": "0.3096850885545486", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.2992790399255972", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.2992790399255972", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.3096850885545486", + "union_ranking_score": "0.3096850885545486", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.2992790399255972", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.2992790399255972" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04104234527687296", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 2.980114707502174e-05, + "exact_test_gap": "2.980114707502174e-05", + "exact_val_decision_score": "0.04257237303415807", + "exact_val_gap": "4.047410996493167e-05", + "exact_val_ranking_score": "0.04253189892419314", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.019943278646881796", + "ranking_selected_val_pr_auc": "0.0240285163314118", + "scope": "fire_prone", + "seed": 123, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7310509974227326", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.012518526699614174, + "tolerated_test_gap": "0.012518526699614174", + "tolerated_val_decision_score": "0.7058203125", + "tolerated_val_gap": "0.01514489915238637", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7310509974227326", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.012518526699614174, + "union_test_gap": "0.012518526699614174", + "union_val_decision_score": "0.7058203125", + "union_val_gap": "0.01514489915238637", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0007027406886858749", + "exact_ranking_score": "0.0007027406886858749", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0031868882312770315", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0031868882312770315", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00041063897933390575", + "ranking_selected_val_pr_auc": "0.00042489429428535383", + "scope": "global", + "seed": 42, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.2870596305028149", + "tolerated_ranking_score": "0.31987973649439366", + "tolerated_regret": 0.0, + "tolerated_test_gap": "-0.03282010599157875", + "tolerated_val_decision_score": "0.31159367647460456", + "tolerated_val_gap": "0.0054887421624861266", + "tolerated_val_ranking_score": "0.30610493431211844", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.2870596305028149", + "union_ranking_score": "0.31987973649439366", + "union_regret": 0.0, + "union_test_gap": "-0.03282010599157875", + "union_val_decision_score": "0.31159367647460456", + "union_val_gap": "0.0054887421624861266", + "union_val_ranking_score": "0.30610493431211844" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0414596444738876", + "exact_ranking_score": "0.0414596444738876", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04356664102845841", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04356664102845841", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.018030815062946615", + "ranking_selected_val_pr_auc": "0.018978807537809048", + "scope": "fire_prone", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7197180735022655", + "tolerated_ranking_score": "0.7197180735022655", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6914092197111065", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6914092197111065", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7197180735022655", + "union_ranking_score": "0.7197180735022655", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6914092197111065", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6914092197111065" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.001086071137659517", + "exact_ranking_score": "0.001086071137659517", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.003349460364719018", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.003349460364719018", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0003268363416054406", + "ranking_selected_val_pr_auc": "0.00039156287297214436", + "scope": "global", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.3051941376005135", + "tolerated_ranking_score": "0.3051941376005135", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.2948179787588662", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.2948179787588662", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.3051941376005135", + "union_ranking_score": "0.3051941376005135", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.2948179787588662", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.2948179787588662" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.040288313241635454", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "-0.0007242308881624873", + "exact_val_decision_score": "0.04306006752022422", + "exact_val_gap": "0.0005281685960310811", + "exact_val_ranking_score": "0.04253189892419314", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.02050876208409485", + "ranking_selected_val_pr_auc": "0.02590116599560572", + "scope": "fire_prone", + "seed": 7, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7644129739607127", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.045880503237594294, + "tolerated_test_gap": "0.045880503237594294", + "tolerated_val_decision_score": "0.6962583126352054", + "tolerated_val_gap": "0.005582899287591814", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7644129739607127", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.045880503237594294, + "union_test_gap": "0.045880503237594294", + "union_val_decision_score": "0.6962583126352054", + "union_val_gap": "0.005582899287591814", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0018159806295399514", + "exact_ranking_score": "0.0018159806295399514", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.003241491085899514", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.003241491085899514", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00038120453362995967", + "ranking_selected_val_pr_auc": "0.0004356191579614426", + "scope": "global", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.3054145960271247", + "tolerated_ranking_score": "0.3054145960271247", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.2930313844339525", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.2930313844339525", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.3054145960271247", + "union_ranking_score": "0.3054145960271247", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.2930313844339525", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.2930313844339525" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.042822140550172624", + "exact_ranking_score": "0.042822140550172624", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.043420983295568316", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.043420983295568316", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.029098665712304895", + "ranking_selected_val_pr_auc": "0.032221638150467385", + "scope": "fire_prone", + "seed": 99, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.726408418760773", + "tolerated_ranking_score": "0.726408418760773", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6933831038489982", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6933831038489982", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.726408418760773", + "union_ranking_score": "0.726408418760773", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6933831038489982", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6933831038489982" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0031948881789137383", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0031948881789137383", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.000398077435365184", + "ranking_selected_val_pr_auc": "0.00047223676755013246", + "scope": "global", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.31071390711162444", + "tolerated_ranking_score": "0.31071390711162444", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.3130246199713141", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.3130246199713141", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.31071390711162444", + "union_ranking_score": "0.31071390711162444", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.3130246199713141", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.3130246199713141" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04452825597664091", + "exact_ranking_score": "0.04452825597664091", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0455414882958375", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0455414882958375", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.022736128443885992", + "ranking_selected_val_pr_auc": "0.021286665509494304", + "scope": "fire_prone", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7269980510116651", + "tolerated_ranking_score": "0.7269980510116651", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7088464801269441", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7088464801269441", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7269980510116651", + "union_ranking_score": "0.7269980510116651", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7088464801269441", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7088464801269441" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.003656307129798903", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.003656307129798903", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0004116035724473925", + "ranking_selected_val_pr_auc": "0.0004377390722357372", + "scope": "global", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.3145618361221859", + "tolerated_ranking_score": "0.3145618361221859", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.3129112248106385", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.3129112248106385", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.3145618361221859", + "union_ranking_score": "0.3145618361221859", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.3129112248106385", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.3129112248106385" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04171607200603644", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0007035278762385008, + "exact_test_gap": "0.0007035278762385008", + "exact_val_decision_score": "0.043619564341543335", + "exact_val_gap": "0.0010876654173501976", + "exact_val_ranking_score": "0.04253189892419314", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.020495144103834257", + "ranking_selected_val_pr_auc": "0.023246350918909517", + "scope": "fire_prone", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7251252524331628", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.006592781710044404, + "tolerated_test_gap": "0.006592781710044404", + "tolerated_val_decision_score": "0.7015002586652871", + "tolerated_val_gap": "0.01082484531767347", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7251252524331628", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.006592781710044404, + "union_test_gap": "0.006592781710044404", + "union_val_decision_score": "0.7015002586652871", + "union_val_gap": "0.01082484531767347", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.001528350909368791", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.001528350909368791", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00036987379624400454", + "ranking_selected_val_pr_auc": "0.00039852867365630293", + "scope": "global", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.31219058559732665", + "tolerated_ranking_score": "0.31219058559732665", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.31476872634310843", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.31476872634310843", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.31219058559732665", + "union_ranking_score": "0.31219058559732665", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.31476872634310843", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.31476872634310843" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04574838388861263", + "exact_ranking_score": "0.04574838388861263", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04613821138211382", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04613821138211382", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.021165185967854567", + "ranking_selected_val_pr_auc": "0.020671268187642432", + "scope": "fire_prone", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7265705731122933", + "tolerated_ranking_score": "0.7265705731122933", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7074814597750377", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7074814597750377", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7265705731122933", + "union_ranking_score": "0.7265705731122933", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7074814597750377", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7074814597750377" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0027012425715829285", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0027012425715829285", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00036963872610239895", + "ranking_selected_val_pr_auc": "0.00042054736055892957", + "scope": "global", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.30641904273669174", + "tolerated_ranking_score": "0.30641904273669174", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.30868675618673724", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.30868675618673724", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.30641904273669174", + "union_ranking_score": "0.30641904273669174", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.30868675618673724", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.30868675618673724" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.043763127370763974", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0027505832409660327, + "exact_test_gap": "0.0027505832409660327", + "exact_val_decision_score": "0.04404735872043984", + "exact_val_gap": "0.0015154597962467042", + "exact_val_ranking_score": "0.04253189892419314", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.016801706970978273", + "ranking_selected_val_pr_auc": "0.022509307119807316", + "scope": "fire_prone", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.722901134194411", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.004368663471292611, + "tolerated_test_gap": "0.004368663471292611", + "tolerated_val_decision_score": "0.7033268834654516", + "tolerated_val_gap": "0.01265147011783796", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.722901134194411", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.004368663471292611, + "union_test_gap": "0.004368663471292611", + "union_val_decision_score": "0.7033268834654516", + "union_val_gap": "0.01265147011783796", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0041356492969396195", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0041356492969396195", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00042782651526874734", + "ranking_selected_val_pr_auc": "0.0004271990954031648", + "scope": "global", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.3111214819309595", + "tolerated_ranking_score": "0.3111214819309595", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.30674980181224504", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.30674980181224504", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.3111214819309595", + "union_ranking_score": "0.3111214819309595", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.30674980181224504", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.30674980181224504" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04882017900732303", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04882017900732303", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.02199779031689959", + "ranking_selected_val_pr_auc": "0.030768274290330223", + "scope": "fire_prone", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7336829717908426", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.015150501067724198, + "tolerated_test_gap": "0.015150501067724198", + "tolerated_val_decision_score": "0.6987796297897932", + "tolerated_val_gap": "0.008104216442179624", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7336829717908426", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.015150501067724198, + "union_test_gap": "0.015150501067724198", + "union_val_decision_score": "0.6987796297897932", + "union_val_gap": "0.008104216442179624", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0006866311182961118", + "exact_ranking_score": "0.0006866311182961118", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0008811590083408097", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0008811590083408097", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0003596048414560045", + "ranking_selected_val_pr_auc": "0.0004121169333774252", + "scope": "global", + "seed": 1, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.21516044416153002", + "tolerated_ranking_score": "0.3091927111996169", + "tolerated_regret": 0.0, + "tolerated_test_gap": "-0.09403226703808687", + "tolerated_val_decision_score": "0.3118144595732139", + "tolerated_val_gap": "0.001671993688703799", + "tolerated_val_ranking_score": "0.3101424658845101", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.21516044416153002", + "union_ranking_score": "0.3091927111996169", + "union_regret": 0.0, + "union_test_gap": "-0.09403226703808687", + "union_val_decision_score": "0.3118144595732139", + "union_val_gap": "0.001671993688703799", + "union_val_ranking_score": "0.3101424658845101" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.043196160341303", + "exact_ranking_score": "0.043196160341303", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04576053854784421", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04576053854784421", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.01904942261850253", + "ranking_selected_val_pr_auc": "0.020787588115504", + "scope": "fire_prone", + "seed": 1, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7261195534617713", + "tolerated_ranking_score": "0.7246636456247585", + "tolerated_regret": 0.001455907837012771, + "tolerated_test_gap": "0.001455907837012771", + "tolerated_val_decision_score": "0.7033365957825429", + "tolerated_val_gap": "0.003706992036648815", + "tolerated_val_ranking_score": "0.6996296037458941", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7261195534617713", + "union_ranking_score": "0.7246636456247585", + "union_regret": 0.001455907837012771, + "union_test_gap": "0.001455907837012771", + "union_val_decision_score": "0.7033365957825429", + "union_val_gap": "0.003706992036648815", + "union_val_ranking_score": "0.6996296037458941" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0009888839378897987", + "exact_ranking_score": "0.0009888839378897987", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0010117798317730948", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0010117798317730948", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00039999784430104664", + "ranking_selected_val_pr_auc": "0.00043073606570656576", + "scope": "global", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.3096924811079745", + "tolerated_ranking_score": "0.3096924811079745", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.31039074285584656", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.31039074285584656", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.3096924811079745", + "union_ranking_score": "0.3096924811079745", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.31039074285584656", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.31039074285584656" + }, + { + "exact_decision_head": "pixel MLP head", + "exact_decision_head_arch": "pixel_mlp", + "exact_decision_score": "0.018543768748295608", + "exact_ranking_score": "0.018543768748295608", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04600811907983762", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04600811907983762", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.021045021724179047", + "ranking_selected_val_pr_auc": "0.025380726482200633", + "scope": "fire_prone", + "seed": 123, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7224961571477053", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0039636864245868875, + "tolerated_test_gap": "0.0039636864245868875", + "tolerated_val_decision_score": "0.6975587357465372", + "tolerated_val_gap": "0.00688332239892353", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7224961571477053", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0039636864245868875, + "union_test_gap": "0.0039636864245868875", + "union_val_decision_score": "0.6975587357465372", + "union_val_gap": "0.00688332239892353", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0003188165529554294", + "exact_ranking_score": "0.0003188165529554294", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0011176098051633573", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0011176098051633573", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0003830459807690152", + "ranking_selected_val_pr_auc": "0.00042901589997107405", + "scope": "global", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.316668570748256", + "tolerated_ranking_score": "0.316668570748256", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.3180456774666437", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.3180456774666437", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.316668570748256", + "union_ranking_score": "0.316668570748256", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.3180456774666437", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.3180456774666437" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04603830266616599", + "exact_ranking_score": "0.04603830266616599", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04428715218026559", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04428715218026559", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.020363596550511454", + "ranking_selected_val_pr_auc": "0.020674507603092498", + "scope": "fire_prone", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7054843984273774", + "tolerated_ranking_score": "0.7054843984273774", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.72648949860566", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.72648949860566", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7054843984273774", + "union_ranking_score": "0.7054843984273774", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.72648949860566", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.72648949860566" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0008450613619815959", + "exact_ranking_score": "0.0008450613619815959", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0009152735719220856", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0009152735719220856", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00037118462783325537", + "ranking_selected_val_pr_auc": "0.0004157684867215522", + "scope": "global", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.3062654921252336", + "tolerated_ranking_score": "0.3062654921252336", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.30677382687191324", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.30677382687191324", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.3062654921252336", + "union_ranking_score": "0.3062654921252336", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.30677382687191324", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.30677382687191324" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.044572342126298965", + "exact_ranking_score": "0.044572342126298965", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04500911929271383", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04500911929271383", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.018717347905238046", + "ranking_selected_val_pr_auc": "0.020626878606552698", + "scope": "fire_prone", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7235687421646468", + "tolerated_ranking_score": "0.7235687421646468", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6980281690140845", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6980281690140845", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7235687421646468", + "union_ranking_score": "0.7235687421646468", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6980281690140845", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6980281690140845" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0007771032641515431", + "exact_ranking_score": "0.0007771032641515431", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0008510302291201489", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0008510302291201489", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0003758107890486513", + "ranking_selected_val_pr_auc": "0.0004065107971511574", + "scope": "global", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.3108110703043797", + "tolerated_ranking_score": "0.3108110703043797", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.3116979187141017", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.3116979187141017", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.3108110703043797", + "union_ranking_score": "0.3108110703043797", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.3116979187141017", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.3116979187141017" + }, + { + "exact_decision_head": "pixel MLP head", + "exact_decision_head_arch": "pixel_mlp", + "exact_decision_score": "0.030284377692970574", + "exact_ranking_score": "0.030284377692970574", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.051651931130758486", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.051651931130758486", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.02225455497934009", + "ranking_selected_val_pr_auc": "0.02778239852030042", + "scope": "fire_prone", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7183426482806338", + "tolerated_ranking_score": "0.7203102915557309", + "tolerated_regret": 0.0, + "tolerated_test_gap": "-0.0019676432750971307", + "tolerated_val_decision_score": "0.7124343844122053", + "tolerated_val_gap": "0.02087441927176159", + "tolerated_val_ranking_score": "0.6915599651404437", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7183426482806338", + "union_ranking_score": "0.7203102915557309", + "union_regret": 0.0, + "union_test_gap": "-0.0019676432750971307", + "union_val_decision_score": "0.7124343844122053", + "union_val_gap": "0.02087441927176159", + "union_val_ranking_score": "0.6915599651404437" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.00048497822606044473", + "exact_regret": 7.571921560175121e-06, + "exact_test_gap": "7.571921560175121e-06", + "exact_val_decision_score": "0.0004727121748000204", + "exact_val_gap": "7.267025482891767e-06", + "exact_val_ranking_score": "0.00046544514931712863", + "family": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.00024254792826221397", + "ranking_selected_val_pr_auc": "0.00023277674706246134", + "scope": "global", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.23755358049655212", + "tolerated_regret": 0.0032399924956598714, + "tolerated_test_gap": "0.0032399924956598714", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.00299097714352789", + "tolerated_val_ranking_score": "0.2149459035878704", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.23755358049655212", + "union_regret": 0.0032399924956598714, + "union_test_gap": "0.0032399924956598714", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.00299097714352789", + "union_val_ranking_score": "0.2149459035878704" + }, + { + "exact_decision_head": "pixel MLP head", + "exact_decision_head_arch": "pixel_mlp", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.014813375233921266", + "ranking_selected_val_pr_auc": "0.02263545197493654", + "scope": "fire_prone", + "seed": 1, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "pixel MLP head", + "exact_decision_head_arch": "pixel_mlp", + "exact_decision_score": "0.0005266170481980583", + "exact_ranking_score": "0.0004925501476206198", + "exact_regret": 3.406690057743849e-05, + "exact_test_gap": "3.406690057743849e-05", + "exact_val_decision_score": "0.00047637431559568154", + "exact_val_gap": "3.6621407956611337e-06", + "exact_val_ranking_score": "0.0004727121748000204", + "family": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0003300754798792237", + "ranking_selected_val_pr_auc": "0.00024271959698351982", + "scope": "global", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.240793572992212", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.2179368807313983", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.240793572992212", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.2179368807313983" + }, + { + "exact_decision_head": "pixel MLP head", + "exact_decision_head_arch": "pixel_mlp", + "exact_decision_score": "0.045738515901060074", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.004725971771262133, + "exact_test_gap": "0.004725971771262133", + "exact_val_decision_score": "0.04344902498318937", + "exact_val_gap": "0.0009171260589962357", + "exact_val_ranking_score": "0.04253189892419314", + "family": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.02093558282208589", + "ranking_selected_val_pr_auc": "0.0217280163599182", + "scope": "fire_prone", + "seed": 123, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.0006100601402256502", + "exact_ranking_score": "0.0", + "exact_regret": 0.0006100601402256502, + "exact_test_gap": "0.0006100601402256502", + "exact_val_decision_score": "0.00048444832931275107", + "exact_val_gap": "0.00048444832931275107", + "exact_val_ranking_score": "0.0", + "family": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.00021817709863716954", + "ranking_selected_val_pr_auc": "0.00025116618856577995", + "scope": "global", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.0", + "tolerated_regret": 0.240793572992212, + "tolerated_test_gap": "0.240793572992212", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.2179368807313983", + "tolerated_val_ranking_score": "0.0", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.0", + "union_regret": 0.240793572992212, + "union_test_gap": "0.240793572992212", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.2179368807313983", + "union_val_ranking_score": "0.0" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.04813290893564165", + "exact_ranking_score": "0.0", + "exact_regret": 0.04813290893564165, + "exact_test_gap": "0.04813290893564165", + "exact_val_decision_score": "0.04370736036385822", + "exact_val_gap": "0.04370736036385822", + "exact_val_ranking_score": "0.0", + "family": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.02188826028256454", + "ranking_selected_val_pr_auc": "0.032209470774519654", + "scope": "fire_prone", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.0", + "tolerated_regret": 0.7185324707231184, + "tolerated_test_gap": "0.7185324707231184", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.6906754133476136", + "tolerated_val_ranking_score": "0.0", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.0", + "union_regret": 0.7185324707231184, + "union_test_gap": "0.7185324707231184", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.6906754133476136", + "union_val_ranking_score": "0.0" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.00035527516330545857", + "exact_ranking_score": "0.0004925501476206198", + "exact_regret": 0.0, + "exact_test_gap": "-0.00013727498431516128", + "exact_val_decision_score": "0.0005024040170237669", + "exact_val_gap": "2.9691842223746458e-05", + "exact_val_ranking_score": "0.0004727121748000204", + "family": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00028045576992618193", + "ranking_selected_val_pr_auc": "0.00024387937757633073", + "scope": "global", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.240793572992212", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.2179368807313983", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.240793572992212", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.2179368807313983" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.04550550682582603", + "exact_ranking_score": "0.0", + "exact_regret": 0.04550550682582603, + "exact_test_gap": "0.04550550682582603", + "exact_val_decision_score": "0.04319677350880917", + "exact_val_gap": "0.04319677350880917", + "exact_val_ranking_score": "0.0", + "family": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.029883397445312192", + "ranking_selected_val_pr_auc": "0.02789152422119005", + "scope": "fire_prone", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.0", + "tolerated_regret": 0.7185324707231184, + "tolerated_test_gap": "0.7185324707231184", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.6906754133476136", + "tolerated_val_ranking_score": "0.0", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.0", + "union_regret": 0.7185324707231184, + "union_test_gap": "0.7185324707231184", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.6906754133476136", + "union_val_ranking_score": "0.0" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004887421693148924", + "exact_ranking_score": "0.0004887421693148924", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0004690575308283202", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0004690575308283202", + "family": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0003408299850116487", + "ranking_selected_val_pr_auc": "0.00024250312441493891", + "scope": "global", + "seed": 99, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.229804399271568", + "tolerated_ranking_score": "0.23917716245227552", + "tolerated_regret": 0.0, + "tolerated_test_gap": "-0.00937276318070751", + "tolerated_val_decision_score": "0.2256312133013075", + "tolerated_val_gap": "0.009197267184311658", + "tolerated_val_ranking_score": "0.21643394611699585", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.229804399271568", + "union_ranking_score": "0.23917716245227552", + "union_regret": 0.0, + "union_test_gap": "-0.00937276318070751", + "union_val_decision_score": "0.2256312133013075", + "union_val_gap": "0.009197267184311658", + "union_val_ranking_score": "0.21643394611699585" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.0", + "exact_regret": 0.04101254412979794, + "exact_test_gap": "0.04101254412979794", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.04253189892419314", + "exact_val_ranking_score": "0.0", + "family": "Pangu-Weather", + "model_tag": "pangu_weather", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.02093558282208589", + "ranking_selected_val_pr_auc": "0.0217280163599182", + "scope": "fire_prone", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.0", + "tolerated_regret": 0.7185324707231184, + "tolerated_test_gap": "0.7185324707231184", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.6906754133476136", + "tolerated_val_ranking_score": "0.0", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.0", + "union_regret": 0.7185324707231184, + "union_test_gap": "0.7185324707231184", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.6906754133476136", + "union_val_ranking_score": "0.0" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.004253509145044662", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.004253509145044662", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00042881385578730365", + "ranking_selected_val_pr_auc": "0.0004901027948814223", + "scope": "global", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.32066974948800614", + "tolerated_ranking_score": "0.32066974948800614", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.3120952399779768", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.3120952399779768", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.32066974948800614", + "union_ranking_score": "0.32066974948800614", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.3120952399779768", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.3120952399779768" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04418397717161179", + "exact_ranking_score": "0.04418397717161179", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04535556523362161", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04535556523362161", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.02328829578760066", + "ranking_selected_val_pr_auc": "0.02010310006243177", + "scope": "fire_prone", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7231388400090598", + "tolerated_ranking_score": "0.7231388400090598", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.698733466652633", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.698733466652633", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7231388400090598", + "union_ranking_score": "0.7231388400090598", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.698733466652633", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.698733466652633" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0042863266180882984", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0042863266180882984", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00043738577276574255", + "ranking_selected_val_pr_auc": "0.0004554894460132563", + "scope": "global", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.31812983009681495", + "tolerated_ranking_score": "0.31812983009681495", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.3113263265858816", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.3113263265858816", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.31812983009681495", + "union_ranking_score": "0.31812983009681495", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.3113263265858816", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.3113263265858816" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04194314737445728", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0009306032446593365, + "exact_test_gap": "0.0009306032446593365", + "exact_val_decision_score": "0.04314380355334118", + "exact_val_gap": "0.0006119046291480426", + "exact_val_ranking_score": "0.04253189892419314", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.02254610440161657", + "ranking_selected_val_pr_auc": "0.026053751384712866", + "scope": "fire_prone", + "seed": 123, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7261721555350364", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.007639684811918013, + "tolerated_test_gap": "0.007639684811918013", + "tolerated_val_decision_score": "0.7019436364687074", + "tolerated_val_gap": "0.011268223121093768", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7261721555350364", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.007639684811918013, + "union_test_gap": "0.007639684811918013", + "union_val_decision_score": "0.7019436364687074", + "union_val_gap": "0.011268223121093768", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0024802356223841263", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0024802356223841263", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00040651309469793043", + "ranking_selected_val_pr_auc": "0.0004045310999550776", + "scope": "global", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.32154249424354786", + "tolerated_ranking_score": "0.32154249424354786", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.31489341288382094", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.31489341288382094", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.32154249424354786", + "union_ranking_score": "0.32154249424354786", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.31489341288382094", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.31489341288382094" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0430065972893146", + "exact_ranking_score": "0.0430065972893146", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.044111302847282145", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.044111302847282145", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.021420904714594617", + "ranking_selected_val_pr_auc": "0.019667925605377917", + "scope": "fire_prone", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7241279907754397", + "tolerated_ranking_score": "0.7241279907754397", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6965685921876124", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6965685921876124", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7241279907754397", + "union_ranking_score": "0.7241279907754397", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6965685921876124", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6965685921876124" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.003894080996884735", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.003894080996884735", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00038395193539280824", + "ranking_selected_val_pr_auc": "0.0005470853500668261", + "scope": "global", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.31120670593952293", + "tolerated_ranking_score": "0.31120670593952293", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.30691963220059004", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.30691963220059004", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.31120670593952293", + "union_ranking_score": "0.31120670593952293", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.30691963220059004", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.30691963220059004" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04372215277109978", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0027096086413018403, + "exact_test_gap": "0.0027096086413018403", + "exact_val_decision_score": "0.043845404352062356", + "exact_val_gap": "0.0013135054278692188", + "exact_val_ranking_score": "0.04253189892419314", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.018664183428008234", + "ranking_selected_val_pr_auc": "0.024661788918260808", + "scope": "fire_prone", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.719461376864001", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0009289061408825905, + "tolerated_test_gap": "0.0009289061408825905", + "tolerated_val_decision_score": "0.6947253251451462", + "tolerated_val_gap": "0.004049911797532557", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.719461376864001", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0009289061408825905, + "union_test_gap": "0.0009289061408825905", + "union_val_decision_score": "0.6947253251451462", + "union_val_gap": "0.004049911797532557", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.002408671216378964", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.002408671216378964", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0004450373086921994", + "ranking_selected_val_pr_auc": "0.000447244495874373", + "scope": "global", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.3214547875801752", + "tolerated_ranking_score": "0.3214547875801752", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.3045294255465288", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.3045294255465288", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.3214547875801752", + "union_ranking_score": "0.3214547875801752", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.3045294255465288", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.3045294255465288" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.0044004400440044", + "exact_ranking_score": "0.0044004400440044", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.06142857142857142", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.06142857142857142", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.024740444457605402", + "ranking_selected_val_pr_auc": "0.036676155849720546", + "scope": "fire_prone", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.719015307962107", + "tolerated_ranking_score": "0.7183382629739177", + "tolerated_regret": 0.0006770449881893237, + "tolerated_test_gap": "0.0006770449881893237", + "tolerated_val_decision_score": "0.6913348632305398", + "tolerated_val_gap": "0.0006380914321632014", + "tolerated_val_ranking_score": "0.6906967717983766", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.719015307962107", + "union_ranking_score": "0.7183382629739177", + "union_regret": 0.0006770449881893237, + "union_test_gap": "0.0006770449881893237", + "union_val_decision_score": "0.6913348632305398", + "union_val_gap": "0.0006380914321632014", + "union_val_ranking_score": "0.6906967717983766" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.0004925501476206198", + "exact_regret": 0.0, + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_pr_auc": "0.0002463357401629007", + "scope": "global", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.240793572992212", + "tolerated_regret": 0.0, + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.240793572992212", + "union_regret": 0.0 + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_pr_auc": "0.02093558282208589", + "scope": "fire_prone", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0 + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.0004925501476206198", + "exact_regret": 0.0, + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_pr_auc": "0.0002463357401629007", + "scope": "global", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.240793572992212", + "tolerated_regret": 0.0, + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.240793572992212", + "union_regret": 0.0 + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_pr_auc": "0.02093558282208589", + "scope": "fire_prone", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0 + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.0004925501476206198", + "exact_regret": 0.0, + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_pr_auc": "0.0002463357401629007", + "scope": "global", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.240793572992212", + "tolerated_regret": 0.0, + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.240793572992212", + "union_regret": 0.0 + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_pr_auc": "0.02093558282208589", + "scope": "fire_prone", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0 + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.0004925501476206198", + "exact_regret": 0.0, + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_pr_auc": "0.0002463357401629007", + "scope": "global", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.240793572992212", + "tolerated_regret": 0.0, + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.240793572992212", + "union_regret": 0.0 + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_pr_auc": "0.02093558282208589", + "scope": "fire_prone", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0 + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.00048497822606044473", + "exact_ranking_score": "0.00048497822606044473", + "exact_regret": 0.0, + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_pr_auc": "0.0002454330184381399", + "scope": "global", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.23755358049655212", + "tolerated_ranking_score": "0.23755358049655212", + "tolerated_regret": 0.0, + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.23755358049655212", + "union_ranking_score": "0.23755358049655212", + "union_regret": 0.0 + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_pr_auc": "0.02093558282208589", + "scope": "fire_prone", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0 + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.017134572294714646", + "exact_ranking_score": "0.017134572294714646", + "exact_regret": 0.0, + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_pr_auc": "0.002624341503354088", + "scope": "global", + "seed": 1, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7286109603326707", + "tolerated_ranking_score": "0.6186566066408326", + "tolerated_regret": 0.10995435369183815, + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7286109603326707", + "union_ranking_score": "0.6186566066408326", + "union_regret": 0.10995435369183815 + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.1611624834874505", + "exact_ranking_score": "0.1611624834874505", + "exact_regret": 0.0, + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_pr_auc": "0.10204224118176683", + "scope": "fire_prone", + "seed": 1, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.8685988450931266", + "tolerated_ranking_score": "0.799032457577039", + "tolerated_regret": 0.06956638751608757, + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.8685988450931266", + "union_ranking_score": "0.799032457577039", + "union_regret": 0.06956638751608757 + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.019320660641944532", + "exact_ranking_score": "0.019320660641944532", + "exact_regret": 0.0, + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_pr_auc": "0.0025603887793133394", + "scope": "global", + "seed": 123, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.735221546471909", + "tolerated_ranking_score": "0.5009707461135111", + "tolerated_regret": 0.23425080035839785, + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.735221546471909", + "union_ranking_score": "0.5009707461135111", + "union_regret": 0.23425080035839785 + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.19165378670788252", + "exact_ranking_score": "0.19165378670788252", + "exact_regret": 0.0, + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_pr_auc": "0.11551882470432043", + "scope": "fire_prone", + "seed": 123, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.8567215417854325", + "tolerated_ranking_score": "0.8066689866810086", + "tolerated_regret": 0.05005255510442386, + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.8567215417854325", + "union_ranking_score": "0.8066689866810086", + "union_regret": 0.05005255510442386 + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.025515210991167808", + "exact_ranking_score": "0.025515210991167808", + "exact_regret": 0.0, + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_pr_auc": "0.0032729435045747413", + "scope": "global", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7239943741463363", + "tolerated_ranking_score": "0.7239943741463363", + "tolerated_regret": 0.0, + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7239943741463363", + "union_ranking_score": "0.7239943741463363", + "union_regret": 0.0 + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.19225806451612906", + "exact_ranking_score": "0.19225806451612906", + "exact_regret": 0.0, + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_pr_auc": "0.12048427320762313", + "scope": "fire_prone", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.8358151221553631", + "tolerated_ranking_score": "0.8358151221553631", + "tolerated_regret": 0.0, + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.8358151221553631", + "union_ranking_score": "0.8358151221553631", + "union_regret": 0.0 + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.02287675150128682", + "exact_ranking_score": "0.02287675150128682", + "exact_regret": 0.0, + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_pr_auc": "0.0030317345997110698", + "scope": "global", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.5002193417313746", + "tolerated_ranking_score": "0.5002193417313746", + "tolerated_regret": 0.0, + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.5002193417313746", + "union_ranking_score": "0.5002193417313746", + "union_regret": 0.0 + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.22885572139303484", + "exact_ranking_score": "0.22885572139303484", + "exact_regret": 0.0, + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_pr_auc": "0.1323902067230726", + "scope": "fire_prone", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.8027807546359551", + "tolerated_ranking_score": "0.8027807546359551", + "tolerated_regret": 0.0, + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.8027807546359551", + "union_ranking_score": "0.8027807546359551", + "union_regret": 0.0 + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.021876258220373104", + "exact_ranking_score": "0.021876258220373104", + "exact_regret": 0.0, + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_pr_auc": "0.0031953098868323193", + "scope": "global", + "seed": 99, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7647466876509988", + "tolerated_ranking_score": "0.6698005926442897", + "tolerated_regret": 0.09494609500670914, + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7647466876509988", + "union_ranking_score": "0.6698005926442897", + "union_regret": 0.09494609500670914 + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.19477124183006533", + "exact_ranking_score": "0.19477124183006533", + "exact_regret": 0.0, + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_pr_auc": "0.11947246238005743", + "scope": "fire_prone", + "seed": 99, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.9039923296574045", + "tolerated_ranking_score": "0.8534759193943048", + "tolerated_regret": 0.05051641026309972, + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.9039923296574045", + "union_ranking_score": "0.8534759193943048", + "union_regret": 0.05051641026309972 + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.0004925501476206198", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0004727121748000204", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0004727121748000204", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0002463357401629007", + "ranking_selected_val_pr_auc": "0.00023641196480702655", + "scope": "global", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.240793572992212", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.2179368807313983", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.240793572992212", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.2179368807313983" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.02093558282208589", + "ranking_selected_val_pr_auc": "0.0217280163599182", + "scope": "fire_prone", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.0004925501476206198", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0004727121748000204", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0004727121748000204", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0002463357401629007", + "ranking_selected_val_pr_auc": "0.00023641196480702655", + "scope": "global", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.240793572992212", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.2179368807313983", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.240793572992212", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.2179368807313983" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.02093558282208589", + "ranking_selected_val_pr_auc": "0.0217280163599182", + "scope": "fire_prone", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.0004925501476206198", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0004727121748000204", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0004727121748000204", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0002463357401629007", + "ranking_selected_val_pr_auc": "0.00023641196480702655", + "scope": "global", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.240793572992212", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.2179368807313983", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.240793572992212", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.2179368807313983" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.02093558282208589", + "ranking_selected_val_pr_auc": "0.0217280163599182", + "scope": "fire_prone", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.0004925501476206198", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0004727121748000204", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0004727121748000204", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0002463357401629007", + "ranking_selected_val_pr_auc": "0.00023641196480702655", + "scope": "global", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.240793572992212", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.2179368807313983", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.240793572992212", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.2179368807313983" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.02093558282208589", + "ranking_selected_val_pr_auc": "0.0217280163599182", + "scope": "fire_prone", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004905319945803844", + "exact_ranking_score": "0.0004905319945803844", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0004707752858483998", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0004707752858483998", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0002463357401629007", + "ranking_selected_val_pr_auc": "0.00023641196480702655", + "scope": "global", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.2399251001974223", + "tolerated_ranking_score": "0.2399251001974223", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.21714067007331003", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.21714067007331003", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.2399251001974223", + "union_ranking_score": "0.2399251001974223", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.21714067007331003", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.21714067007331003" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.02093558282208589", + "ranking_selected_val_pr_auc": "0.0217280163599182", + "scope": "fire_prone", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + } + ], + "summary": [ + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "Reference", + "model_tag": "reference", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "global", + "scope_label": "\\(\\Omega=\\)global", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.23425080035839785, + "mean": 0.08783024981138902, + "min": 0.0, + "n": 5, + "std": 0.09670495645481135 + }, + "union_regret": { + "max": 0.23425080035839785, + "mean": 0.08783024981138902, + "min": 0.0, + "n": 5, + "std": 0.09670495645481135 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "Reference", + "model_tag": "reference", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "fire_prone", + "scope_label": "\\(\\Omega=\\)top 20\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.06956638751608757, + "mean": 0.03402707057672223, + "min": 0.0, + "n": 5, + "std": 0.032044658643147844 + }, + "union_regret": { + "max": 0.06956638751608757, + "mean": 0.03402707057672223, + "min": 0.0, + "n": 5, + "std": 0.032044658643147844 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "global", + "scope_label": "\\(\\Omega=\\)global", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "union_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "fire_prone", + "scope_label": "\\(\\Omega=\\)top 20\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "union_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + } + }, + { + "exact_regret": { + "max": 0.0004925501476206198, + "mean": 0.00020004882767231798, + "min": 0.0, + "n": 5, + "std": 0.00026703384456332115 + }, + "label": "Aurora", + "model_tag": "aurora", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "global", + "scope_label": "\\(\\Omega=\\)global", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.240793572992212, + "mean": 0.09851983041506818, + "min": 0.0032399924956598714, + "n": 5, + "std": 0.1298781980037557 + }, + "union_regret": { + "max": 0.240793572992212, + "mean": 0.09851983041506818, + "min": 0.0032399924956598714, + "n": 5, + "std": 0.1298781980037557 + } + }, + { + "exact_regret": { + "max": 0.04101254412979794, + "mean": 0.008202508825959588, + "min": 0.0, + "n": 5, + "std": 0.01834136732088763 + }, + "label": "Aurora", + "model_tag": "aurora", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "fire_prone", + "scope_label": "\\(\\Omega=\\)top 20\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.7185324707231184, + "mean": 0.14391889430974364, + "min": 0.0, + "n": 5, + "std": 0.32121904665016227 + }, + "union_regret": { + "max": 0.7185324707231184, + "mean": 0.14391889430974364, + "min": 0.0, + "n": 5, + "std": 0.32121904665016227 + } + }, + { + "exact_regret": { + "max": 7.571921560175121e-06, + "mean": 3.0287686240700486e-06, + "min": 0.0, + "n": 5, + "std": 4.147312242167625e-06 + }, + "label": "ClimaX", + "model_tag": "climax", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "global", + "scope_label": "\\(\\Omega=\\)global", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0032399924956598714, + "mean": 0.0012959969982639485, + "min": 0.0, + "n": 5, + "std": 0.0017746169760203706 + }, + "union_regret": { + "max": 0.0032399924956598714, + "mean": 0.0012959969982639485, + "min": 0.0, + "n": 5, + "std": 0.0017746169760203706 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "ClimaX", + "model_tag": "climax", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "fire_prone", + "scope_label": "\\(\\Omega=\\)top 20\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "union_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "StormCast", + "model_tag": "stormcast", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "global", + "scope_label": "\\(\\Omega=\\)global", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "union_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "StormCast", + "model_tag": "stormcast", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "fire_prone", + "scope_label": "\\(\\Omega=\\)top 20\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "union_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + } + }, + { + "exact_regret": { + "max": 0.0006100601402256502, + "mean": 0.00013033979247265275, + "min": 0.0, + "n": 5, + "std": 0.0002685372203690466 + }, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "global", + "scope_label": "\\(\\Omega=\\)global", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.240793572992212, + "mean": 0.048806713097574374, + "min": 0.0, + "n": 5, + "std": 0.10733308684741971 + }, + "union_regret": { + "max": 0.240793572992212, + "mean": 0.048806713097574374, + "min": 0.0, + "n": 5, + "std": 0.10733308684741971 + } + }, + { + "exact_regret": { + "max": 0.04813290893564165, + "mean": 0.027875386332505546, + "min": 0.0, + "n": 5, + "std": 0.02348779386900393 + }, + "label": "Pangu-Weather", + "model_tag": "pangu_weather", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "fire_prone", + "scope_label": "\\(\\Omega=\\)top 20\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.7185324707231184, + "mean": 0.43111948243387105, + "min": 0.0, + "n": 5, + "std": 0.39355644251497235 + }, + "union_regret": { + "max": 0.7185324707231184, + "mean": 0.43111948243387105, + "min": 0.0, + "n": 5, + "std": 0.39355644251497235 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "DLWP", + "model_tag": "dlwp", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "global", + "scope_label": "\\(\\Omega=\\)global", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "union_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + } + }, + { + "exact_regret": { + "max": 0.0023651634514294945, + "mean": 0.0007702319787454587, + "min": 0.0, + "n": 5, + "std": 0.0010995336594539604 + }, + "label": "DLWP", + "model_tag": "dlwp", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "fire_prone", + "scope_label": "\\(\\Omega=\\)top 20\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.11215006616165157, + "mean": 0.043265915053601556, + "min": 0.0, + "n": 5, + "std": 0.04332331365579739 + }, + "union_regret": { + "max": 0.11215006616165157, + "mean": 0.043265915053601556, + "min": 0.0, + "n": 5, + "std": 0.04332331365579739 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "FCN", + "model_tag": "fcn", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "global", + "scope_label": "\\(\\Omega=\\)global", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "union_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + } + }, + { + "exact_regret": { + "max": 2.980114707502174e-05, + "mean": 5.960229415004348e-06, + "min": 0.0, + "n": 5, + "std": 1.3327478133443526e-05 + }, + "label": "FCN", + "model_tag": "fcn", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "fire_prone", + "scope_label": "\\(\\Omega=\\)top 20\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.045880503237594294, + "mean": 0.011679805987441694, + "min": 0.0, + "n": 5, + "std": 0.019872372458657642 + }, + "union_regret": { + "max": 0.045880503237594294, + "mean": 0.011679805987441694, + "min": 0.0, + "n": 5, + "std": 0.019872372458657642 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "FengWu", + "model_tag": "fengwu", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "global", + "scope_label": "\\(\\Omega=\\)global", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "union_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + } + }, + { + "exact_regret": { + "max": 0.0027505832409660327, + "mean": 0.0006908222234409067, + "min": 0.0, + "n": 5, + "std": 0.0011910586589384115 + }, + "label": "FengWu", + "model_tag": "fengwu", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "fire_prone", + "scope_label": "\\(\\Omega=\\)top 20\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.015150501067724198, + "mean": 0.005222389249812243, + "min": 0.0, + "n": 5, + "std": 0.0062394095558402415 + }, + "union_regret": { + "max": 0.015150501067724198, + "mean": 0.005222389249812243, + "min": 0.0, + "n": 5, + "std": 0.0062394095558402415 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "FuXi", + "model_tag": "fuxi", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "global", + "scope_label": "\\(\\Omega=\\)global", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "union_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "FuXi", + "model_tag": "fuxi", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "fire_prone", + "scope_label": "\\(\\Omega=\\)top 20\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0039636864245868875, + "mean": 0.0010839188523199318, + "min": 0.0, + "n": 5, + "std": 0.0017288780545672386 + }, + "union_regret": { + "max": 0.0039636864245868875, + "mean": 0.0010839188523199318, + "min": 0.0, + "n": 5, + "std": 0.0017288780545672386 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "global", + "scope_label": "\\(\\Omega=\\)global", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "union_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + } + }, + { + "exact_regret": { + "max": 0.0027096086413018403, + "mean": 0.0007280423771922354, + "min": 0.0, + "n": 5, + "std": 0.001178746460551365 + }, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "fire_prone", + "scope_label": "\\(\\Omega=\\)top 20\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.007639684811918013, + "mean": 0.0018491271881979853, + "min": 0.0, + "n": 5, + "std": 0.0032630386057089294 + }, + "union_regret": { + "max": 0.007639684811918013, + "mean": 0.0018491271881979853, + "min": 0.0, + "n": 5, + "std": 0.0032630386057089294 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "global", + "scope_label": "\\(\\Omega=\\)global", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.272468682523694, + "mean": 0.1722171037486726, + "min": 0.050655329891645895, + "n": 5, + "std": 0.08849214830495522 + }, + "union_regret": { + "max": 0.272468682523694, + "mean": 0.1722171037486726, + "min": 0.050655329891645895, + "n": 5, + "std": 0.08849214830495522 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "fire_prone", + "scope_label": "\\(\\Omega=\\)top 20\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.13482302181308625, + "mean": 0.038803552655092256, + "min": 0.0, + "n": 5, + "std": 0.0594825313313219 + }, + "union_regret": { + "max": 0.13482302181308625, + "mean": 0.038803552655092256, + "min": 0.0, + "n": 5, + "std": 0.0594825313313219 + } + } + ] +} diff --git a/artifacts/results/selection_regret_full_head_table.generated.tex b/artifacts/results/selection_regret_full_head_table.generated.tex new file mode 100644 index 0000000000000000000000000000000000000000..31f1a16d8792acc16ee2d4f27c119f27955008f4 --- /dev/null +++ b/artifacts/results/selection_regret_full_head_table.generated.tex @@ -0,0 +1,2 @@ +% Full per-head rows are kept in the supplementary CSV files. +% The manuscript uses the all-backbone selection-regret summaries instead. diff --git a/artifacts/results/selection_regret_head_metrics.csv b/artifacts/results/selection_regret_head_metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..94dbb805d21454ea846dedb8eb5ed3c693660a93 --- /dev/null +++ b/artifacts/results/selection_regret_head_metrics.csv @@ -0,0 +1,241 @@ +family,model_tag,scope,seed,selected_by,head_label,test_pr_auc,exact_f1,tolerated_f1,union_f1 +AlphaEarth,alphaearth,fire_prone,1,PR-AUC,shallow spatial adapter,0.04390614036305261,0.09715762273901808,0.7863702028280513,0.7863702028280513 +AlphaEarth,alphaearth,fire_prone,1,decision,shallow spatial adapter,,,,0.7863702028280513 +AlphaEarth,alphaearth,fire_prone,7,PR-AUC,shallow spatial adapter,0.05955397140893137,0.12514898688915374,0.8294499693616161,0.8294499693616161 +AlphaEarth,alphaearth,fire_prone,7,decision,shallow spatial adapter,,,,0.8294499693616161 +AlphaEarth,alphaearth,fire_prone,42,PR-AUC,shallow spatial adapter,0.038083070948941686,0.08702469619756958,0.7112901458230849,0.7112901458230849 +AlphaEarth,alphaearth,fire_prone,42,decision,pixel MLP head,,,,0.8461131676361712 +AlphaEarth,alphaearth,fire_prone,99,PR-AUC,shallow spatial adapter,0.0458102699699856,0.10365251727541955,0.7758298037709835,0.7758298037709835 +AlphaEarth,alphaearth,fire_prone,99,decision,pixel MLP head,,,,0.8350245452333586 +AlphaEarth,alphaearth,fire_prone,123,PR-AUC,shallow spatial adapter,0.045809049876129763,0.10531544957774468,0.7789089693560928,0.7789089693560928 +AlphaEarth,alphaearth,fire_prone,123,decision,shallow spatial adapter,,,,0.7789089693560928 +AlphaEarth,alphaearth,global,1,PR-AUC,shallow spatial adapter,0.0006549130347299629,0.004193290734824281,0.40561891947698747,0.40561891947698747 +AlphaEarth,alphaearth,global,1,decision,pixel MLP head,,,,0.6337627266658229 +AlphaEarth,alphaearth,global,7,PR-AUC,shallow spatial adapter,0.001005722733868245,0.010460251046025104,0.6184842128568402,0.6184842128568402 +AlphaEarth,alphaearth,global,7,decision,pixel MLP head,,,,0.6691395427484861 +AlphaEarth,alphaearth,global,42,PR-AUC,shallow spatial adapter,0.0005634701573991865,0.004809747755451047,0.4087444681515033,0.4087444681515033 +AlphaEarth,alphaearth,global,42,decision,pixel MLP head,,,,0.6812131506751973 +AlphaEarth,alphaearth,global,99,PR-AUC,shallow spatial adapter,0.0006577120081349608,0.006780481898534931,0.3921547570095426,0.3921547570095426 +AlphaEarth,alphaearth,global,99,decision,pixel MLP head,,,,0.5842714676652996 +AlphaEarth,alphaearth,global,123,PR-AUC,shallow spatial adapter,0.0007047712457371991,0.006959088991986505,0.4427625907752311,0.4427625907752311 +AlphaEarth,alphaearth,global,123,decision,pixel MLP head,,,,0.5604635792586619 +Aurora,aurora,fire_prone,1,PR-AUC,linear probe,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184 +Aurora,aurora,fire_prone,1,decision,linear probe,,,,0.7185324707231184 +Aurora,aurora,fire_prone,7,PR-AUC,linear probe,0.024802820904513342,0.0413500618483831,0.7184857293868923,0.7184857293868923 +Aurora,aurora,fire_prone,7,decision,shallow spatial adapter,,,,0.7185324707231184 +Aurora,aurora,fire_prone,42,PR-AUC,linear probe,0.02613792907867929,0.04101254412979794,0.7185324707231184,0.7185324707231184 +Aurora,aurora,fire_prone,42,decision,linear probe,,,,0.7185324707231184 +Aurora,aurora,fire_prone,99,PR-AUC,linear probe,0.02093558282208589,0.0,0.0,0.0 +Aurora,aurora,fire_prone,99,decision,shallow spatial adapter,,,,0.7185324707231184 +Aurora,aurora,fire_prone,123,PR-AUC,pixel MLP head,0.03014151567817997,0.04541038665445361,0.7175172112337449,0.7175172112337449 +Aurora,aurora,fire_prone,123,decision,linear probe,,,,0.7185324707231184 +Aurora,aurora,global,1,PR-AUC,linear probe,0.00024254792826221397,0.00048497822606044473,0.23755358049655212,0.23755358049655212 +Aurora,aurora,global,1,decision,shallow spatial adapter,,,,0.240793572992212 +Aurora,aurora,global,7,PR-AUC,linear probe,0.00027660331739269843,0.0,0.23734400297937533,0.23734400297937533 +Aurora,aurora,global,7,decision,shallow spatial adapter,,,,0.240793572992212 +Aurora,aurora,global,42,PR-AUC,linear probe,0.0002876372030063385,0.00048497822606044473,0.0,0.0 +Aurora,aurora,global,42,decision,shallow spatial adapter,,,,0.240793572992212 +Aurora,aurora,global,99,PR-AUC,linear probe,0.00024254792826221397,0.0,0.0,0.0 +Aurora,aurora,global,99,decision,shallow spatial adapter,,,,0.240793572992212 +Aurora,aurora,global,123,PR-AUC,pixel MLP head,0.00031683315961488916,0.0005311562430574265,0.23647112940979162,0.23647112940979162 +Aurora,aurora,global,123,decision,shallow spatial adapter,,,,0.240793572992212 +ClimaX,climax,fire_prone,1,PR-AUC,linear probe,0.02281272244151735,0.04101254412979794,0.7185324707231184,0.7185324707231184 +ClimaX,climax,fire_prone,1,decision,linear probe,,,,0.7185324707231184 +ClimaX,climax,fire_prone,7,PR-AUC,linear probe,0.021317405351800135,0.04101254412979794,0.7185324707231184,0.7185324707231184 +ClimaX,climax,fire_prone,7,decision,linear probe,,,,0.7185324707231184 +ClimaX,climax,fire_prone,42,PR-AUC,linear probe,0.021516770872035896,0.04101254412979794,0.7185324707231184,0.7185324707231184 +ClimaX,climax,fire_prone,42,decision,linear probe,,,,0.7185324707231184 +ClimaX,climax,fire_prone,99,PR-AUC,shallow spatial adapter,0.02099123219536693,0.04101254412979794,0.7185324707231184,0.7185324707231184 +ClimaX,climax,fire_prone,99,decision,shallow spatial adapter,,,,0.7185324707231184 +ClimaX,climax,fire_prone,123,PR-AUC,shallow spatial adapter,0.024930358757410707,0.04101254412979794,0.7185324707231184,0.7185324707231184 +ClimaX,climax,fire_prone,123,decision,shallow spatial adapter,,,,0.7185324707231184 +ClimaX,climax,global,1,PR-AUC,linear probe,0.0002543464414550104,0.00048497822606044473,0.23755358049655212,0.23755358049655212 +ClimaX,climax,global,1,decision,shallow spatial adapter,,,,0.240793572992212 +ClimaX,climax,global,7,PR-AUC,pixel MLP head,0.00025423546642937565,0.00048497822606044473,0.23755358049655212,0.23755358049655212 +ClimaX,climax,global,7,decision,shallow spatial adapter,,,,0.240793572992212 +ClimaX,climax,global,42,PR-AUC,shallow spatial adapter,0.00023723426605001756,0.0004925501476206198,0.240793572992212,0.240793572992212 +ClimaX,climax,global,42,decision,shallow spatial adapter,,,,0.240793572992212 +ClimaX,climax,global,99,PR-AUC,shallow spatial adapter,0.0002340075376021003,0.00048696535779102983,0.2384111045733381,0.2384111045733381 +ClimaX,climax,global,99,decision,shallow spatial adapter,,,,0.2384111045733381 +ClimaX,climax,global,123,PR-AUC,shallow spatial adapter,0.00025952340213823634,0.0004925501476206198,0.240793572992212,0.240793572992212 +ClimaX,climax,global,123,decision,shallow spatial adapter,,,,0.240793572992212 +DLWP,dlwp,fire_prone,1,PR-AUC,linear probe,0.019747545663020845,0.043506471331489265,0.7280364139105968,0.7280364139105968 +DLWP,dlwp,fire_prone,1,decision,linear probe,,,,0.7280364139105968 +DLWP,dlwp,fire_prone,7,PR-AUC,pixel MLP head,0.018519739310339497,0.04101254412979794,0.7185324707231184,0.7185324707231184 +DLWP,dlwp,fire_prone,7,decision,shallow spatial adapter,,,,0.762536895087842 +DLWP,dlwp,fire_prone,42,PR-AUC,pixel MLP head,0.020762153794205103,0.04637177602565815,0.6606900017471591,0.6606900017471591 +DLWP,dlwp,fire_prone,42,decision,shallow spatial adapter,,,,0.7728400679088107 +DLWP,dlwp,fire_prone,99,PR-AUC,linear probe,0.02136633936888583,0.04819843096725701,0.7187730423241409,0.7187730423241409 +DLWP,dlwp,fire_prone,99,decision,shallow spatial adapter,,,,0.7653346239087763 +DLWP,dlwp,fire_prone,123,PR-AUC,pixel MLP head,0.021118517500793188,0.04101254412979794,0.7185324707231184,0.7185324707231184 +DLWP,dlwp,fire_prone,123,decision,linear probe,,,,0.7321459738801157 +DLWP,dlwp,global,1,PR-AUC,shallow spatial adapter,0.0006257446338466172,0.00487022180273714,0.38023285660836226,0.38023285660836226 +DLWP,dlwp,global,1,decision,shallow spatial adapter,,,,0.38023285660836226 +DLWP,dlwp,global,7,PR-AUC,shallow spatial adapter,0.0005264872646085452,0.0,0.3432315705541329,0.3432315705541329 +DLWP,dlwp,global,7,decision,shallow spatial adapter,,,,0.3432315705541329 +DLWP,dlwp,global,42,PR-AUC,shallow spatial adapter,0.0006203713852571992,0.0,0.3405125814370199,0.3405125814370199 +DLWP,dlwp,global,42,decision,shallow spatial adapter,,,,0.3405125814370199 +DLWP,dlwp,global,99,PR-AUC,shallow spatial adapter,0.0007477128447471452,0.0,0.3979559626836394,0.3979559626836394 +DLWP,dlwp,global,99,decision,shallow spatial adapter,,,,0.3979559626836394 +DLWP,dlwp,global,123,PR-AUC,shallow spatial adapter,0.0007129763973023342,0.0,0.3797689460796109,0.3797689460796109 +DLWP,dlwp,global,123,decision,shallow spatial adapter,,,,0.3797689460796109 +FCN,fcn,fire_prone,1,PR-AUC,shallow spatial adapter,0.01844011667219625,0.042068766252528166,0.7185324707231184,0.7185324707231184 +FCN,fcn,fire_prone,1,decision,linear probe,,,,0.7182175622542595 +FCN,fcn,fire_prone,7,PR-AUC,pixel MLP head,0.02050876208409485,0.04101254412979794,0.7185324707231184,0.7185324707231184 +FCN,fcn,fire_prone,7,decision,linear probe,,,,0.7644129739607127 +FCN,fcn,fire_prone,42,PR-AUC,shallow spatial adapter,0.018030815062946615,0.0414596444738876,0.7197180735022655,0.7197180735022655 +FCN,fcn,fire_prone,42,decision,shallow spatial adapter,,,,0.7197180735022655 +FCN,fcn,fire_prone,99,PR-AUC,linear probe,0.029098665712304895,0.042822140550172624,0.726408418760773,0.726408418760773 +FCN,fcn,fire_prone,99,decision,linear probe,,,,0.726408418760773 +FCN,fcn,fire_prone,123,PR-AUC,pixel MLP head,0.019943278646881796,0.04101254412979794,0.7185324707231184,0.7185324707231184 +FCN,fcn,fire_prone,123,decision,linear probe,,,,0.7310509974227326 +FCN,fcn,global,1,PR-AUC,shallow spatial adapter,0.00037256097806901117,0.0009319664492078285,0.31167484413093016,0.31167484413093016 +FCN,fcn,global,1,decision,shallow spatial adapter,,,,0.31167484413093016 +FCN,fcn,global,7,PR-AUC,shallow spatial adapter,0.0003268363416054406,0.001086071137659517,0.3051941376005135,0.3051941376005135 +FCN,fcn,global,7,decision,shallow spatial adapter,,,,0.3051941376005135 +FCN,fcn,global,42,PR-AUC,shallow spatial adapter,0.00041063897933390575,0.0007027406886858749,0.31987973649439366,0.31987973649439366 +FCN,fcn,global,42,decision,pixel MLP head,,,,0.2870596305028149 +FCN,fcn,global,99,PR-AUC,shallow spatial adapter,0.00038120453362995967,0.0018159806295399514,0.3054145960271247,0.3054145960271247 +FCN,fcn,global,99,decision,shallow spatial adapter,,,,0.3054145960271247 +FCN,fcn,global,123,PR-AUC,shallow spatial adapter,0.000387412312932838,0.0006535947712418301,0.3096850885545486,0.3096850885545486 +FCN,fcn,global,123,decision,shallow spatial adapter,,,,0.3096850885545486 +FengWu,fengwu,fire_prone,1,PR-AUC,shallow spatial adapter,0.022736128443885992,0.04452825597664091,0.7269980510116651,0.7269980510116651 +FengWu,fengwu,fire_prone,1,decision,shallow spatial adapter,,,,0.7269980510116651 +FengWu,fengwu,fire_prone,7,PR-AUC,pixel MLP head,0.016801706970978273,0.04101254412979794,0.7185324707231184,0.7185324707231184 +FengWu,fengwu,fire_prone,7,decision,shallow spatial adapter,,,,0.722901134194411 +FengWu,fengwu,fire_prone,42,PR-AUC,shallow spatial adapter,0.021165185967854567,0.04574838388861263,0.7265705731122933,0.7265705731122933 +FengWu,fengwu,fire_prone,42,decision,shallow spatial adapter,,,,0.7265705731122933 +FengWu,fengwu,fire_prone,99,PR-AUC,linear probe,0.02199779031689959,0.0,0.7185324707231184,0.7185324707231184 +FengWu,fengwu,fire_prone,99,decision,shallow spatial adapter,,,,0.7336829717908426 +FengWu,fengwu,fire_prone,123,PR-AUC,pixel MLP head,0.020495144103834257,0.04101254412979794,0.7185324707231184,0.7185324707231184 +FengWu,fengwu,fire_prone,123,decision,shallow spatial adapter,,,,0.7251252524331628 +FengWu,fengwu,global,1,PR-AUC,shallow spatial adapter,0.000398077435365184,0.0,0.31071390711162444,0.31071390711162444 +FengWu,fengwu,global,1,decision,shallow spatial adapter,,,,0.31071390711162444 +FengWu,fengwu,global,7,PR-AUC,shallow spatial adapter,0.00036963872610239895,0.0,0.30641904273669174,0.30641904273669174 +FengWu,fengwu,global,7,decision,shallow spatial adapter,,,,0.30641904273669174 +FengWu,fengwu,global,42,PR-AUC,shallow spatial adapter,0.00036987379624400454,0.0,0.31219058559732665,0.31219058559732665 +FengWu,fengwu,global,42,decision,shallow spatial adapter,,,,0.31219058559732665 +FengWu,fengwu,global,99,PR-AUC,shallow spatial adapter,0.00042782651526874734,0.0,0.3111214819309595,0.3111214819309595 +FengWu,fengwu,global,99,decision,shallow spatial adapter,,,,0.3111214819309595 +FengWu,fengwu,global,123,PR-AUC,shallow spatial adapter,0.0004116035724473925,0.0,0.3145618361221859,0.3145618361221859 +FengWu,fengwu,global,123,decision,shallow spatial adapter,,,,0.3145618361221859 +FuXi,fuxi,fire_prone,1,PR-AUC,shallow spatial adapter,0.01904942261850253,0.043196160341303,0.7246636456247585,0.7246636456247585 +FuXi,fuxi,fire_prone,1,decision,linear probe,,,,0.7261195534617713 +FuXi,fuxi,fire_prone,7,PR-AUC,shallow spatial adapter,0.018717347905238046,0.044572342126298965,0.7235687421646468,0.7235687421646468 +FuXi,fuxi,fire_prone,7,decision,shallow spatial adapter,,,,0.7235687421646468 +FuXi,fuxi,fire_prone,42,PR-AUC,shallow spatial adapter,0.020363596550511454,0.04603830266616599,0.7054843984273774,0.7054843984273774 +FuXi,fuxi,fire_prone,42,decision,shallow spatial adapter,,,,0.7054843984273774 +FuXi,fuxi,fire_prone,99,PR-AUC,pixel MLP head,0.02225455497934009,0.030284377692970574,0.7203102915557309,0.7203102915557309 +FuXi,fuxi,fire_prone,99,decision,shallow spatial adapter,,,,0.7183426482806338 +FuXi,fuxi,fire_prone,123,PR-AUC,pixel MLP head,0.021045021724179047,0.018543768748295608,0.7185324707231184,0.7185324707231184 +FuXi,fuxi,fire_prone,123,decision,linear probe,,,,0.7224961571477053 +FuXi,fuxi,global,1,PR-AUC,shallow spatial adapter,0.0003596048414560045,0.0006866311182961118,0.3091927111996169,0.3091927111996169 +FuXi,fuxi,global,1,decision,linear probe,,,,0.21516044416153002 +FuXi,fuxi,global,7,PR-AUC,shallow spatial adapter,0.00037118462783325537,0.0008450613619815959,0.3062654921252336,0.3062654921252336 +FuXi,fuxi,global,7,decision,shallow spatial adapter,,,,0.3062654921252336 +FuXi,fuxi,global,42,PR-AUC,shallow spatial adapter,0.0003830459807690152,0.0003188165529554294,0.316668570748256,0.316668570748256 +FuXi,fuxi,global,42,decision,shallow spatial adapter,,,,0.316668570748256 +FuXi,fuxi,global,99,PR-AUC,shallow spatial adapter,0.0003758107890486513,0.0007771032641515431,0.3108110703043797,0.3108110703043797 +FuXi,fuxi,global,99,decision,shallow spatial adapter,,,,0.3108110703043797 +FuXi,fuxi,global,123,PR-AUC,shallow spatial adapter,0.00039999784430104664,0.0009888839378897987,0.3096924811079745,0.3096924811079745 +FuXi,fuxi,global,123,decision,shallow spatial adapter,,,,0.3096924811079745 +Pangu-Weather,pangu_weather,fire_prone,1,PR-AUC,pixel MLP head,0.014813375233921266,0.04101254412979794,0.7185324707231184,0.7185324707231184 +Pangu-Weather,pangu_weather,fire_prone,1,decision,pixel MLP head,,,,0.7185324707231184 +Pangu-Weather,pangu_weather,fire_prone,7,PR-AUC,pixel MLP head,0.029883397445312192,0.0,0.0,0.0 +Pangu-Weather,pangu_weather,fire_prone,7,decision,shallow spatial adapter,,,,0.7185324707231184 +Pangu-Weather,pangu_weather,fire_prone,42,PR-AUC,pixel MLP head,0.02188826028256454,0.0,0.0,0.0 +Pangu-Weather,pangu_weather,fire_prone,42,decision,shallow spatial adapter,,,,0.7185324707231184 +Pangu-Weather,pangu_weather,fire_prone,99,PR-AUC,linear probe,0.02093558282208589,0.0,0.0,0.0 +Pangu-Weather,pangu_weather,fire_prone,99,decision,shallow spatial adapter,,,,0.7185324707231184 +Pangu-Weather,pangu_weather,fire_prone,123,PR-AUC,linear probe,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184 +Pangu-Weather,pangu_weather,fire_prone,123,decision,linear probe,,,,0.7185324707231184 +Pangu-Weather,pangu_weather,global,1,PR-AUC,linear probe,0.00024254792826221397,0.00048497822606044473,0.23755358049655212,0.23755358049655212 +Pangu-Weather,pangu_weather,global,1,decision,shallow spatial adapter,,,,0.240793572992212 +Pangu-Weather,pangu_weather,global,7,PR-AUC,shallow spatial adapter,0.00028045576992618193,0.0004925501476206198,0.240793572992212,0.240793572992212 +Pangu-Weather,pangu_weather,global,7,decision,shallow spatial adapter,,,,0.240793572992212 +Pangu-Weather,pangu_weather,global,42,PR-AUC,pixel MLP head,0.00021817709863716954,0.0,0.0,0.0 +Pangu-Weather,pangu_weather,global,42,decision,shallow spatial adapter,,,,0.240793572992212 +Pangu-Weather,pangu_weather,global,99,PR-AUC,shallow spatial adapter,0.0003408299850116487,0.0004887421693148924,0.23917716245227552,0.23917716245227552 +Pangu-Weather,pangu_weather,global,99,decision,pixel MLP head,,,,0.229804399271568 +Pangu-Weather,pangu_weather,global,123,PR-AUC,shallow spatial adapter,0.0003300754798792237,0.0004925501476206198,0.240793572992212,0.240793572992212 +Pangu-Weather,pangu_weather,global,123,decision,shallow spatial adapter,,,,0.240793572992212 +Pangu-Weather,pangu6,fire_prone,1,PR-AUC,shallow spatial adapter,0.02328829578760066,0.04418397717161179,0.7231388400090598,0.7231388400090598 +Pangu-Weather,pangu6,fire_prone,1,decision,shallow spatial adapter,,,,0.7231388400090598 +Pangu-Weather,pangu6,fire_prone,7,PR-AUC,pixel MLP head,0.018664183428008234,0.04101254412979794,0.7185324707231184,0.7185324707231184 +Pangu-Weather,pangu6,fire_prone,7,decision,shallow spatial adapter,,,,0.719461376864001 +Pangu-Weather,pangu6,fire_prone,42,PR-AUC,shallow spatial adapter,0.021420904714594617,0.0430065972893146,0.7241279907754397,0.7241279907754397 +Pangu-Weather,pangu6,fire_prone,42,decision,shallow spatial adapter,,,,0.7241279907754397 +Pangu-Weather,pangu6,fire_prone,99,PR-AUC,linear probe,0.024740444457605402,0.0044004400440044,0.7183382629739177,0.7183382629739177 +Pangu-Weather,pangu6,fire_prone,99,decision,shallow spatial adapter,,,,0.719015307962107 +Pangu-Weather,pangu6,fire_prone,123,PR-AUC,pixel MLP head,0.02254610440161657,0.04101254412979794,0.7185324707231184,0.7185324707231184 +Pangu-Weather,pangu6,fire_prone,123,decision,linear probe,,,,0.7261721555350364 +Pangu-Weather,pangu6,global,1,PR-AUC,shallow spatial adapter,0.00042881385578730365,0.0,0.32066974948800614,0.32066974948800614 +Pangu-Weather,pangu6,global,1,decision,shallow spatial adapter,,,,0.32066974948800614 +Pangu-Weather,pangu6,global,7,PR-AUC,shallow spatial adapter,0.00038395193539280824,0.0,0.31120670593952293,0.31120670593952293 +Pangu-Weather,pangu6,global,7,decision,shallow spatial adapter,,,,0.31120670593952293 +Pangu-Weather,pangu6,global,42,PR-AUC,shallow spatial adapter,0.00040651309469793043,0.0,0.32154249424354786,0.32154249424354786 +Pangu-Weather,pangu6,global,42,decision,shallow spatial adapter,,,,0.32154249424354786 +Pangu-Weather,pangu6,global,99,PR-AUC,shallow spatial adapter,0.0004450373086921994,0.0,0.3214547875801752,0.3214547875801752 +Pangu-Weather,pangu6,global,99,decision,shallow spatial adapter,,,,0.3214547875801752 +Pangu-Weather,pangu6,global,123,PR-AUC,shallow spatial adapter,0.00043738577276574255,0.0,0.31812983009681495,0.31812983009681495 +Pangu-Weather,pangu6,global,123,decision,shallow spatial adapter,,,,0.31812983009681495 +Prithvi-WxC,prithvi_wxc,fire_prone,1,PR-AUC,shallow spatial adapter,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184 +Prithvi-WxC,prithvi_wxc,fire_prone,1,decision,shallow spatial adapter,,,,0.7185324707231184 +Prithvi-WxC,prithvi_wxc,fire_prone,7,PR-AUC,shallow spatial adapter,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184 +Prithvi-WxC,prithvi_wxc,fire_prone,7,decision,shallow spatial adapter,,,,0.7185324707231184 +Prithvi-WxC,prithvi_wxc,fire_prone,42,PR-AUC,pixel MLP head,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184 +Prithvi-WxC,prithvi_wxc,fire_prone,42,decision,shallow spatial adapter,,,,0.7185324707231184 +Prithvi-WxC,prithvi_wxc,fire_prone,99,PR-AUC,linear probe,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184 +Prithvi-WxC,prithvi_wxc,fire_prone,99,decision,shallow spatial adapter,,,,0.7185324707231184 +Prithvi-WxC,prithvi_wxc,fire_prone,123,PR-AUC,shallow spatial adapter,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184 +Prithvi-WxC,prithvi_wxc,fire_prone,123,decision,shallow spatial adapter,,,,0.7185324707231184 +Prithvi-WxC,prithvi_wxc,global,1,PR-AUC,shallow spatial adapter,0.0002463357401629007,0.0004925501476206198,0.240793572992212,0.240793572992212 +Prithvi-WxC,prithvi_wxc,global,1,decision,shallow spatial adapter,,,,0.240793572992212 +Prithvi-WxC,prithvi_wxc,global,7,PR-AUC,shallow spatial adapter,0.0002463357401629007,0.0004925501476206198,0.240793572992212,0.240793572992212 +Prithvi-WxC,prithvi_wxc,global,7,decision,shallow spatial adapter,,,,0.240793572992212 +Prithvi-WxC,prithvi_wxc,global,42,PR-AUC,shallow spatial adapter,0.0002463357401629007,0.0004925501476206198,0.240793572992212,0.240793572992212 +Prithvi-WxC,prithvi_wxc,global,42,decision,shallow spatial adapter,,,,0.240793572992212 +Prithvi-WxC,prithvi_wxc,global,99,PR-AUC,shallow spatial adapter,0.0002454330184381399,0.00048497822606044473,0.23755358049655212,0.23755358049655212 +Prithvi-WxC,prithvi_wxc,global,99,decision,shallow spatial adapter,,,,0.23755358049655212 +Prithvi-WxC,prithvi_wxc,global,123,PR-AUC,shallow spatial adapter,0.0002463357401629007,0.0004925501476206198,0.240793572992212,0.240793572992212 +Prithvi-WxC,prithvi_wxc,global,123,decision,shallow spatial adapter,,,,0.240793572992212 +Reference,reference,fire_prone,1,PR-AUC,shallow spatial adapter,0.10204224118176683,0.1611624834874505,0.799032457577039,0.799032457577039 +Reference,reference,fire_prone,1,decision,linear probe,,,,0.8685988450931266 +Reference,reference,fire_prone,7,PR-AUC,shallow spatial adapter,0.1323902067230726,0.22885572139303484,0.8027807546359551,0.8027807546359551 +Reference,reference,fire_prone,7,decision,shallow spatial adapter,,,,0.8027807546359551 +Reference,reference,fire_prone,42,PR-AUC,shallow spatial adapter,0.12048427320762313,0.19225806451612906,0.8358151221553631,0.8358151221553631 +Reference,reference,fire_prone,42,decision,shallow spatial adapter,,,,0.8358151221553631 +Reference,reference,fire_prone,99,PR-AUC,shallow spatial adapter,0.11947246238005743,0.19477124183006533,0.8534759193943048,0.8534759193943048 +Reference,reference,fire_prone,99,decision,linear probe,,,,0.9039923296574045 +Reference,reference,fire_prone,123,PR-AUC,shallow spatial adapter,0.11551882470432043,0.19165378670788252,0.8066689866810086,0.8066689866810086 +Reference,reference,fire_prone,123,decision,pixel MLP head,,,,0.8567215417854325 +Reference,reference,global,1,PR-AUC,shallow spatial adapter,0.002624341503354088,0.017134572294714646,0.6186566066408326,0.6186566066408326 +Reference,reference,global,1,decision,linear probe,,,,0.7286109603326707 +Reference,reference,global,7,PR-AUC,shallow spatial adapter,0.0030317345997110698,0.02287675150128682,0.5002193417313746,0.5002193417313746 +Reference,reference,global,7,decision,shallow spatial adapter,,,,0.5002193417313746 +Reference,reference,global,42,PR-AUC,shallow spatial adapter,0.0032729435045747413,0.025515210991167808,0.7239943741463363,0.7239943741463363 +Reference,reference,global,42,decision,shallow spatial adapter,,,,0.7239943741463363 +Reference,reference,global,99,PR-AUC,shallow spatial adapter,0.0031953098868323193,0.021876258220373104,0.6698005926442897,0.6698005926442897 +Reference,reference,global,99,decision,linear probe,,,,0.7647466876509988 +Reference,reference,global,123,PR-AUC,shallow spatial adapter,0.0025603887793133394,0.019320660641944532,0.5009707461135111,0.5009707461135111 +Reference,reference,global,123,decision,pixel MLP head,,,,0.735221546471909 +StormCast,stormcast,fire_prone,1,PR-AUC,linear probe,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184 +StormCast,stormcast,fire_prone,1,decision,shallow spatial adapter,,,,0.7185324707231184 +StormCast,stormcast,fire_prone,7,PR-AUC,linear probe,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184 +StormCast,stormcast,fire_prone,7,decision,shallow spatial adapter,,,,0.7185324707231184 +StormCast,stormcast,fire_prone,42,PR-AUC,linear probe,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184 +StormCast,stormcast,fire_prone,42,decision,shallow spatial adapter,,,,0.7185324707231184 +StormCast,stormcast,fire_prone,99,PR-AUC,pixel MLP head,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184 +StormCast,stormcast,fire_prone,99,decision,shallow spatial adapter,,,,0.7185324707231184 +StormCast,stormcast,fire_prone,123,PR-AUC,linear probe,0.02093558282208589,0.04101254412979794,0.7185324707231184,0.7185324707231184 +StormCast,stormcast,fire_prone,123,decision,shallow spatial adapter,,,,0.7185324707231184 +StormCast,stormcast,global,1,PR-AUC,shallow spatial adapter,0.0002463357401629007,0.0004925501476206198,0.240793572992212,0.240793572992212 +StormCast,stormcast,global,1,decision,shallow spatial adapter,,,,0.240793572992212 +StormCast,stormcast,global,7,PR-AUC,shallow spatial adapter,0.0002463357401629007,0.0004925501476206198,0.240793572992212,0.240793572992212 +StormCast,stormcast,global,7,decision,shallow spatial adapter,,,,0.240793572992212 +StormCast,stormcast,global,42,PR-AUC,shallow spatial adapter,0.0002463357401629007,0.0004925501476206198,0.240793572992212,0.240793572992212 +StormCast,stormcast,global,42,decision,shallow spatial adapter,,,,0.240793572992212 +StormCast,stormcast,global,99,PR-AUC,shallow spatial adapter,0.0002463357401629007,0.0004905319945803844,0.2399251001974223,0.2399251001974223 +StormCast,stormcast,global,99,decision,shallow spatial adapter,,,,0.2399251001974223 +StormCast,stormcast,global,123,PR-AUC,shallow spatial adapter,0.0002463357401629007,0.0004925501476206198,0.240793572992212,0.240793572992212 +StormCast,stormcast,global,123,decision,shallow spatial adapter,,,,0.240793572992212 diff --git a/artifacts/results/selection_regret_main_table.generated.tex b/artifacts/results/selection_regret_main_table.generated.tex new file mode 100644 index 0000000000000000000000000000000000000000..62e3549c0d2599148e0a5383f92988f050f2976c --- /dev/null +++ b/artifacts/results/selection_regret_main_table.generated.tex @@ -0,0 +1,24 @@ +\begin{table*}[!t] + \centering + \small + \setlength{\tabcolsep}{4pt} + \caption{Fixed-feature selection-regret check across evaluation scopes. Values are percentage-point regret \(\delta = D(h_D)-D(h_R)\) under union-\(F_1\), where \(h_R\) is selected by PR-AUC and \(h_D\) by the decision metric. Top-\(k\) columns use train-defined fire-prone scopes. Rows report mean with small std over five seeds; \(0.0000\) means the two selectors give the same decision score for all seeds.} + \label{tab:selection_regret_diagnostic} + \begin{tabular}{lcccc} + \toprule + \textbf{Feature source} & \textbf{\(\Omega=\)global} & \textbf{\(\Omega=\)top 5\%} & \textbf{\(\Omega=\)top 10\%} & \textbf{\(\Omega=\)top 20\%} \\ + \midrule + \textcolor{blue}{FireWx-FM ref.} & \ms{7.3831}{7.4536} & \ms{0.3664}{0.6812} & \ms{1.2275}{1.2665} & \ms{2.9385}{2.7513} \\ + Prithvi-WxC & 0.0000 & 0.0000 & 0.0000 & 0.0000 \\ + Aurora & \ms{4.9455}{10.6974} & \ms{15.4283}{34.4987} & \ms{13.9934}{31.2903} & \ms{14.3706}{32.1337} \\ + ClimaX & \ms{0.1296}{0.1775} & 0.0000 & 0.0000 & 0.0000 \\ + StormCast & 0.0000 & 0.0000 & 0.0000 & 0.0000 \\ + DLWP & 0.0000 & \ms{1.6716}{1.6079} & \ms{2.8465}{2.6938} & \ms{4.4634}{4.3561} \\ + FCN & 0.0000 & \ms{0.4510}{1.0071} & \ms{0.4200}{0.9390} & \ms{1.1680}{1.9872} \\ + FengWu & 0.0000 & \ms{0.8796}{0.5532} & \ms{0.4023}{0.5511} & \ms{0.5222}{0.6239} \\ + FuXi & 0.0000 & \ms{1.3545}{2.0970} & \ms{0.1656}{0.3703} & \ms{0.2833}{0.3681} \\ + Pangu-Weather & 0.0000 & \ms{0.7593}{0.8974} & \ms{0.3048}{0.5054} & \ms{0.1868}{0.3255} \\ + AlphaEarth & \ms{17.2217}{8.8492} & \ms{6.3846}{4.9653} & \ms{6.5738}{6.8970} & \ms{3.8804}{5.9483} \\ + \bottomrule + \end{tabular} +\end{table*} diff --git a/artifacts/results/selection_regret_per_seed.csv b/artifacts/results/selection_regret_per_seed.csv new file mode 100644 index 0000000000000000000000000000000000000000..c6166c5bb3e2d4d3511716055b7615ee11e4e624 --- /dev/null +++ b/artifacts/results/selection_regret_per_seed.csv @@ -0,0 +1,121 @@ +family,model_tag,scope,seed,ranking_head,decision_head,ranking_pr_auc,ranking_union_f1,decision_union_f1,regret,top1_agreement +AlphaEarth,alphaearth,fire_prone,1,shallow spatial adapter,shallow spatial adapter,0.04390614036305261,0.7863702028280513,0.7863702028280513,0.0,True +AlphaEarth,alphaearth,fire_prone,7,shallow spatial adapter,shallow spatial adapter,0.05955397140893137,0.8294499693616161,0.8294499693616161,0.0,True +AlphaEarth,alphaearth,fire_prone,42,shallow spatial adapter,pixel MLP head,0.038083070948941686,0.7112901458230849,0.8461131676361712,0.13482302181308625,False +AlphaEarth,alphaearth,fire_prone,99,shallow spatial adapter,pixel MLP head,0.0458102699699856,0.7758298037709835,0.8350245452333586,0.05919474146237502,False +AlphaEarth,alphaearth,fire_prone,123,shallow spatial adapter,shallow spatial adapter,0.045809049876129763,0.7789089693560928,0.7789089693560928,0.0,True +AlphaEarth,alphaearth,global,1,shallow spatial adapter,pixel MLP head,0.0006549130347299629,0.40561891947698747,0.6337627266658229,0.22814380718883542,False +AlphaEarth,alphaearth,global,7,shallow spatial adapter,pixel MLP head,0.001005722733868245,0.6184842128568402,0.6691395427484861,0.050655329891645895,False +AlphaEarth,alphaearth,global,42,shallow spatial adapter,pixel MLP head,0.0005634701573991865,0.4087444681515033,0.6812131506751973,0.272468682523694,False +AlphaEarth,alphaearth,global,99,shallow spatial adapter,pixel MLP head,0.0006577120081349608,0.3921547570095426,0.5842714676652996,0.19211671065575697,False +AlphaEarth,alphaearth,global,123,shallow spatial adapter,pixel MLP head,0.0007047712457371991,0.4427625907752311,0.5604635792586619,0.11770098848343075,False +Aurora,aurora,fire_prone,1,linear probe,linear probe,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,True +Aurora,aurora,fire_prone,7,linear probe,shallow spatial adapter,0.024802820904513342,0.7184857293868923,0.7185324707231184,4.674133622617482e-05,False +Aurora,aurora,fire_prone,42,linear probe,linear probe,0.02613792907867929,0.7185324707231184,0.7185324707231184,0.0,True +Aurora,aurora,fire_prone,99,linear probe,shallow spatial adapter,0.02093558282208589,0.0,0.7185324707231184,0.7185324707231184,False +Aurora,aurora,fire_prone,123,pixel MLP head,linear probe,0.03014151567817997,0.7175172112337449,0.7185324707231184,0.0010152594893735323,False +Aurora,aurora,global,1,linear probe,shallow spatial adapter,0.00024254792826221397,0.23755358049655212,0.240793572992212,0.0032399924956598714,False +Aurora,aurora,global,7,linear probe,shallow spatial adapter,0.00027660331739269843,0.23734400297937533,0.240793572992212,0.0034495700128366613,False +Aurora,aurora,global,42,linear probe,shallow spatial adapter,0.0002876372030063385,0.0,0.240793572992212,0.240793572992212,False +Aurora,aurora,global,99,linear probe,shallow spatial adapter,0.00024254792826221397,0.0,0.240793572992212,0.240793572992212,False +Aurora,aurora,global,123,pixel MLP head,shallow spatial adapter,0.00031683315961488916,0.23647112940979162,0.240793572992212,0.004322443582420371,False +ClimaX,climax,fire_prone,1,linear probe,linear probe,0.02281272244151735,0.7185324707231184,0.7185324707231184,0.0,True +ClimaX,climax,fire_prone,7,linear probe,linear probe,0.021317405351800135,0.7185324707231184,0.7185324707231184,0.0,True +ClimaX,climax,fire_prone,42,linear probe,linear probe,0.021516770872035896,0.7185324707231184,0.7185324707231184,0.0,True +ClimaX,climax,fire_prone,99,shallow spatial adapter,shallow spatial adapter,0.02099123219536693,0.7185324707231184,0.7185324707231184,0.0,True +ClimaX,climax,fire_prone,123,shallow spatial adapter,shallow spatial adapter,0.024930358757410707,0.7185324707231184,0.7185324707231184,0.0,True +ClimaX,climax,global,1,linear probe,shallow spatial adapter,0.0002543464414550104,0.23755358049655212,0.240793572992212,0.0032399924956598714,False +ClimaX,climax,global,7,pixel MLP head,shallow spatial adapter,0.00025423546642937565,0.23755358049655212,0.240793572992212,0.0032399924956598714,False +ClimaX,climax,global,42,shallow spatial adapter,shallow spatial adapter,0.00023723426605001756,0.240793572992212,0.240793572992212,0.0,True +ClimaX,climax,global,99,shallow spatial adapter,shallow spatial adapter,0.0002340075376021003,0.2384111045733381,0.2384111045733381,0.0,True +ClimaX,climax,global,123,shallow spatial adapter,shallow spatial adapter,0.00025952340213823634,0.240793572992212,0.240793572992212,0.0,True +DLWP,dlwp,fire_prone,1,linear probe,linear probe,0.019747545663020845,0.7280364139105968,0.7280364139105968,0.0,True +DLWP,dlwp,fire_prone,7,pixel MLP head,shallow spatial adapter,0.018519739310339497,0.7185324707231184,0.762536895087842,0.044004424364723516,False +DLWP,dlwp,fire_prone,42,pixel MLP head,shallow spatial adapter,0.020762153794205103,0.6606900017471591,0.7728400679088107,0.11215006616165157,False +DLWP,dlwp,fire_prone,99,linear probe,shallow spatial adapter,0.02136633936888583,0.7187730423241409,0.7653346239087763,0.046561581584635414,False +DLWP,dlwp,fire_prone,123,pixel MLP head,linear probe,0.021118517500793188,0.7185324707231184,0.7321459738801157,0.013613503156997275,False +DLWP,dlwp,global,1,shallow spatial adapter,shallow spatial adapter,0.0006257446338466172,0.38023285660836226,0.38023285660836226,0.0,True +DLWP,dlwp,global,7,shallow spatial adapter,shallow spatial adapter,0.0005264872646085452,0.3432315705541329,0.3432315705541329,0.0,True +DLWP,dlwp,global,42,shallow spatial adapter,shallow spatial adapter,0.0006203713852571992,0.3405125814370199,0.3405125814370199,0.0,True +DLWP,dlwp,global,99,shallow spatial adapter,shallow spatial adapter,0.0007477128447471452,0.3979559626836394,0.3979559626836394,0.0,True +DLWP,dlwp,global,123,shallow spatial adapter,shallow spatial adapter,0.0007129763973023342,0.3797689460796109,0.3797689460796109,0.0,True +FCN,fcn,fire_prone,1,shallow spatial adapter,linear probe,0.01844011667219625,0.7185324707231184,0.7182175622542595,0.0,False +FCN,fcn,fire_prone,7,pixel MLP head,linear probe,0.02050876208409485,0.7185324707231184,0.7644129739607127,0.045880503237594294,False +FCN,fcn,fire_prone,42,shallow spatial adapter,shallow spatial adapter,0.018030815062946615,0.7197180735022655,0.7197180735022655,0.0,True +FCN,fcn,fire_prone,99,linear probe,linear probe,0.029098665712304895,0.726408418760773,0.726408418760773,0.0,True +FCN,fcn,fire_prone,123,pixel MLP head,linear probe,0.019943278646881796,0.7185324707231184,0.7310509974227326,0.012518526699614174,False +FCN,fcn,global,1,shallow spatial adapter,shallow spatial adapter,0.00037256097806901117,0.31167484413093016,0.31167484413093016,0.0,True +FCN,fcn,global,7,shallow spatial adapter,shallow spatial adapter,0.0003268363416054406,0.3051941376005135,0.3051941376005135,0.0,True +FCN,fcn,global,42,shallow spatial adapter,pixel MLP head,0.00041063897933390575,0.31987973649439366,0.2870596305028149,0.0,False +FCN,fcn,global,99,shallow spatial adapter,shallow spatial adapter,0.00038120453362995967,0.3054145960271247,0.3054145960271247,0.0,True +FCN,fcn,global,123,shallow spatial adapter,shallow spatial adapter,0.000387412312932838,0.3096850885545486,0.3096850885545486,0.0,True +FengWu,fengwu,fire_prone,1,shallow spatial adapter,shallow spatial adapter,0.022736128443885992,0.7269980510116651,0.7269980510116651,0.0,True +FengWu,fengwu,fire_prone,7,pixel MLP head,shallow spatial adapter,0.016801706970978273,0.7185324707231184,0.722901134194411,0.004368663471292611,False +FengWu,fengwu,fire_prone,42,shallow spatial adapter,shallow spatial adapter,0.021165185967854567,0.7265705731122933,0.7265705731122933,0.0,True +FengWu,fengwu,fire_prone,99,linear probe,shallow spatial adapter,0.02199779031689959,0.7185324707231184,0.7336829717908426,0.015150501067724198,False +FengWu,fengwu,fire_prone,123,pixel MLP head,shallow spatial adapter,0.020495144103834257,0.7185324707231184,0.7251252524331628,0.006592781710044404,False +FengWu,fengwu,global,1,shallow spatial adapter,shallow spatial adapter,0.000398077435365184,0.31071390711162444,0.31071390711162444,0.0,True +FengWu,fengwu,global,7,shallow spatial adapter,shallow spatial adapter,0.00036963872610239895,0.30641904273669174,0.30641904273669174,0.0,True +FengWu,fengwu,global,42,shallow spatial adapter,shallow spatial adapter,0.00036987379624400454,0.31219058559732665,0.31219058559732665,0.0,True +FengWu,fengwu,global,99,shallow spatial adapter,shallow spatial adapter,0.00042782651526874734,0.3111214819309595,0.3111214819309595,0.0,True +FengWu,fengwu,global,123,shallow spatial adapter,shallow spatial adapter,0.0004116035724473925,0.3145618361221859,0.3145618361221859,0.0,True +FuXi,fuxi,fire_prone,1,shallow spatial adapter,linear probe,0.01904942261850253,0.7246636456247585,0.7261195534617713,0.001455907837012771,False +FuXi,fuxi,fire_prone,7,shallow spatial adapter,shallow spatial adapter,0.018717347905238046,0.7235687421646468,0.7235687421646468,0.0,True +FuXi,fuxi,fire_prone,42,shallow spatial adapter,shallow spatial adapter,0.020363596550511454,0.7054843984273774,0.7054843984273774,0.0,True +FuXi,fuxi,fire_prone,99,pixel MLP head,shallow spatial adapter,0.02225455497934009,0.7203102915557309,0.7183426482806338,0.0,False +FuXi,fuxi,fire_prone,123,pixel MLP head,linear probe,0.021045021724179047,0.7185324707231184,0.7224961571477053,0.0039636864245868875,False +FuXi,fuxi,global,1,shallow spatial adapter,linear probe,0.0003596048414560045,0.3091927111996169,0.21516044416153002,0.0,False +FuXi,fuxi,global,7,shallow spatial adapter,shallow spatial adapter,0.00037118462783325537,0.3062654921252336,0.3062654921252336,0.0,True +FuXi,fuxi,global,42,shallow spatial adapter,shallow spatial adapter,0.0003830459807690152,0.316668570748256,0.316668570748256,0.0,True +FuXi,fuxi,global,99,shallow spatial adapter,shallow spatial adapter,0.0003758107890486513,0.3108110703043797,0.3108110703043797,0.0,True +FuXi,fuxi,global,123,shallow spatial adapter,shallow spatial adapter,0.00039999784430104664,0.3096924811079745,0.3096924811079745,0.0,True +Pangu-Weather,pangu_weather,fire_prone,1,pixel MLP head,pixel MLP head,0.014813375233921266,0.7185324707231184,0.7185324707231184,0.0,True +Pangu-Weather,pangu_weather,fire_prone,7,pixel MLP head,shallow spatial adapter,0.029883397445312192,0.0,0.7185324707231184,0.7185324707231184,False +Pangu-Weather,pangu_weather,fire_prone,42,pixel MLP head,shallow spatial adapter,0.02188826028256454,0.0,0.7185324707231184,0.7185324707231184,False +Pangu-Weather,pangu_weather,fire_prone,99,linear probe,shallow spatial adapter,0.02093558282208589,0.0,0.7185324707231184,0.7185324707231184,False +Pangu-Weather,pangu_weather,fire_prone,123,linear probe,linear probe,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,True +Pangu-Weather,pangu_weather,global,1,linear probe,shallow spatial adapter,0.00024254792826221397,0.23755358049655212,0.240793572992212,0.0032399924956598714,False +Pangu-Weather,pangu_weather,global,7,shallow spatial adapter,shallow spatial adapter,0.00028045576992618193,0.240793572992212,0.240793572992212,0.0,True +Pangu-Weather,pangu_weather,global,42,pixel MLP head,shallow spatial adapter,0.00021817709863716954,0.0,0.240793572992212,0.240793572992212,False +Pangu-Weather,pangu_weather,global,99,shallow spatial adapter,pixel MLP head,0.0003408299850116487,0.23917716245227552,0.229804399271568,0.0,False +Pangu-Weather,pangu_weather,global,123,shallow spatial adapter,shallow spatial adapter,0.0003300754798792237,0.240793572992212,0.240793572992212,0.0,True +Pangu-Weather,pangu6,fire_prone,1,shallow spatial adapter,shallow spatial adapter,0.02328829578760066,0.7231388400090598,0.7231388400090598,0.0,True +Pangu-Weather,pangu6,fire_prone,7,pixel MLP head,shallow spatial adapter,0.018664183428008234,0.7185324707231184,0.719461376864001,0.0009289061408825905,False +Pangu-Weather,pangu6,fire_prone,42,shallow spatial adapter,shallow spatial adapter,0.021420904714594617,0.7241279907754397,0.7241279907754397,0.0,True +Pangu-Weather,pangu6,fire_prone,99,linear probe,shallow spatial adapter,0.024740444457605402,0.7183382629739177,0.719015307962107,0.0006770449881893237,False +Pangu-Weather,pangu6,fire_prone,123,pixel MLP head,linear probe,0.02254610440161657,0.7185324707231184,0.7261721555350364,0.007639684811918013,False +Pangu-Weather,pangu6,global,1,shallow spatial adapter,shallow spatial adapter,0.00042881385578730365,0.32066974948800614,0.32066974948800614,0.0,True +Pangu-Weather,pangu6,global,7,shallow spatial adapter,shallow spatial adapter,0.00038395193539280824,0.31120670593952293,0.31120670593952293,0.0,True +Pangu-Weather,pangu6,global,42,shallow spatial adapter,shallow spatial adapter,0.00040651309469793043,0.32154249424354786,0.32154249424354786,0.0,True +Pangu-Weather,pangu6,global,99,shallow spatial adapter,shallow spatial adapter,0.0004450373086921994,0.3214547875801752,0.3214547875801752,0.0,True +Pangu-Weather,pangu6,global,123,shallow spatial adapter,shallow spatial adapter,0.00043738577276574255,0.31812983009681495,0.31812983009681495,0.0,True +Prithvi-WxC,prithvi_wxc,fire_prone,1,shallow spatial adapter,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,True +Prithvi-WxC,prithvi_wxc,fire_prone,7,shallow spatial adapter,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,True +Prithvi-WxC,prithvi_wxc,fire_prone,42,pixel MLP head,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,False +Prithvi-WxC,prithvi_wxc,fire_prone,99,linear probe,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,False +Prithvi-WxC,prithvi_wxc,fire_prone,123,shallow spatial adapter,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,True +Prithvi-WxC,prithvi_wxc,global,1,shallow spatial adapter,shallow spatial adapter,0.0002463357401629007,0.240793572992212,0.240793572992212,0.0,True +Prithvi-WxC,prithvi_wxc,global,7,shallow spatial adapter,shallow spatial adapter,0.0002463357401629007,0.240793572992212,0.240793572992212,0.0,True +Prithvi-WxC,prithvi_wxc,global,42,shallow spatial adapter,shallow spatial adapter,0.0002463357401629007,0.240793572992212,0.240793572992212,0.0,True +Prithvi-WxC,prithvi_wxc,global,99,shallow spatial adapter,shallow spatial adapter,0.0002454330184381399,0.23755358049655212,0.23755358049655212,0.0,True +Prithvi-WxC,prithvi_wxc,global,123,shallow spatial adapter,shallow spatial adapter,0.0002463357401629007,0.240793572992212,0.240793572992212,0.0,True +Reference,reference,fire_prone,1,shallow spatial adapter,linear probe,0.10204224118176683,0.799032457577039,0.8685988450931266,0.06956638751608757,False +Reference,reference,fire_prone,7,shallow spatial adapter,shallow spatial adapter,0.1323902067230726,0.8027807546359551,0.8027807546359551,0.0,True +Reference,reference,fire_prone,42,shallow spatial adapter,shallow spatial adapter,0.12048427320762313,0.8358151221553631,0.8358151221553631,0.0,True +Reference,reference,fire_prone,99,shallow spatial adapter,linear probe,0.11947246238005743,0.8534759193943048,0.9039923296574045,0.05051641026309972,False +Reference,reference,fire_prone,123,shallow spatial adapter,pixel MLP head,0.11551882470432043,0.8066689866810086,0.8567215417854325,0.05005255510442386,False +Reference,reference,global,1,shallow spatial adapter,linear probe,0.002624341503354088,0.6186566066408326,0.7286109603326707,0.10995435369183815,False +Reference,reference,global,7,shallow spatial adapter,shallow spatial adapter,0.0030317345997110698,0.5002193417313746,0.5002193417313746,0.0,True +Reference,reference,global,42,shallow spatial adapter,shallow spatial adapter,0.0032729435045747413,0.7239943741463363,0.7239943741463363,0.0,True +Reference,reference,global,99,shallow spatial adapter,linear probe,0.0031953098868323193,0.6698005926442897,0.7647466876509988,0.09494609500670914,False +Reference,reference,global,123,shallow spatial adapter,pixel MLP head,0.0025603887793133394,0.5009707461135111,0.735221546471909,0.23425080035839785,False +StormCast,stormcast,fire_prone,1,linear probe,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,False +StormCast,stormcast,fire_prone,7,linear probe,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,False +StormCast,stormcast,fire_prone,42,linear probe,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,False +StormCast,stormcast,fire_prone,99,pixel MLP head,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,False +StormCast,stormcast,fire_prone,123,linear probe,shallow spatial adapter,0.02093558282208589,0.7185324707231184,0.7185324707231184,0.0,False +StormCast,stormcast,global,1,shallow spatial adapter,shallow spatial adapter,0.0002463357401629007,0.240793572992212,0.240793572992212,0.0,True +StormCast,stormcast,global,7,shallow spatial adapter,shallow spatial adapter,0.0002463357401629007,0.240793572992212,0.240793572992212,0.0,True +StormCast,stormcast,global,42,shallow spatial adapter,shallow spatial adapter,0.0002463357401629007,0.240793572992212,0.240793572992212,0.0,True +StormCast,stormcast,global,99,shallow spatial adapter,shallow spatial adapter,0.0002463357401629007,0.2399251001974223,0.2399251001974223,0.0,True +StormCast,stormcast,global,123,shallow spatial adapter,shallow spatial adapter,0.0002463357401629007,0.240793572992212,0.240793572992212,0.0,True diff --git a/artifacts/results/selection_regret_rq2_figure_values.csv b/artifacts/results/selection_regret_rq2_figure_values.csv new file mode 100644 index 0000000000000000000000000000000000000000..66605affbdfa7fdc9e132b4749da594f89863727 --- /dev/null +++ b/artifacts/results/selection_regret_rq2_figure_values.csv @@ -0,0 +1,12 @@ +feature_source,global_mean_pp,global_std_pp,top20_mean_pp,top20_std_pp +FireWx-FM ref.,7.3831,7.4536,2.9385,2.7513 +Prithvi-WxC,0.0000,0.0000,0.0000,0.0000 +Aurora,4.9455,10.6974,14.3706,32.1337 +ClimaX,0.1296,0.1775,0.0000,0.0000 +StormCast,0.0000,0.0000,0.0000,0.0000 +DLWP,0.0000,0.0000,4.4634,4.3561 +FCN,0.0000,0.0000,1.1680,1.9872 +FengWu,0.0000,0.0000,0.5222,0.6239 +FuXi,0.0000,0.0000,0.2833,0.3681 +Pangu-Weather,0.0000,0.0000,0.1868,0.3255 +AlphaEarth,17.2217,8.8492,3.8804,5.9483 diff --git a/artifacts/results/selection_regret_scope_sweep_20260505.csv b/artifacts/results/selection_regret_scope_sweep_20260505.csv new file mode 100644 index 0000000000000000000000000000000000000000..b2d3d4305cf14014566b7a7ab80542c3846b3fba --- /dev/null +++ b/artifacts/results/selection_regret_scope_sweep_20260505.csv @@ -0,0 +1,45 @@ +model_tag,label,scope,scope_label,n,seeds,exact_regret_mean,exact_regret_std,exact_regret_min,exact_regret_max,tolerated_regret_mean,tolerated_regret_std,tolerated_regret_min,tolerated_regret_max,union_regret_mean,union_regret_std,union_regret_min,union_regret_max +reference,FireWx-FM ref.,global,\(\Omega=\)global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.07383089600948442,0.07453636071372995,0.0,0.17497107865629713,0.07383089600948442,0.07453636071372995,0.0,0.17497107865629713 +reference,FireWx-FM ref.,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.003663718115532055,0.006812231244812292,0.0,0.015676762201120686,0.003663718115532055,0.006812231244812292,0.0,0.015676762201120686 +reference,FireWx-FM ref.,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.012275489592085752,0.012665162001740834,0.0,0.02670880922526031,0.012275489592085752,0.012665162001740834,0.0,0.02670880922526031 +reference,FireWx-FM ref.,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.029384646387840017,0.02751315335001922,0.0,0.05675140203555318,0.029384646387840017,0.02751315335001922,0.0,0.05675140203555318 +prithvi_wxc,Prithvi-WxC,global,\(\Omega=\)global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +prithvi_wxc,Prithvi-WxC,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +prithvi_wxc,Prithvi-WxC,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +prithvi_wxc,Prithvi-WxC,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +aurora,Aurora,global,\(\Omega=\)global,5,1 7 42 99 123,0.00010153879814819402,0.00021861477435572763,0.0,0.0004925501476206198,0.04945471159670635,0.10697394238964528,0.0,0.240793572992212,0.04945471159670635,0.10697394238964528,0.0,0.240793572992212 +aurora,Aurora,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.023667154505540456,0.05292136630837888,0.0,0.11833577252770228,0.1542829840966487,0.34498724021162547,0.0,0.7714149204832434,0.1542829840966487,0.34498724021162547,0.0,0.7714149204832434 +aurora,Aurora,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.014651279478173606,0.032761256870543834,0.0,0.07325639739086803,0.1399343063221699,0.31290262132065055,0.0,0.6996715316108496,0.1399343063221699,0.31290262132065055,0.0,0.6996715316108496 +aurora,Aurora,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.008202508825959588,0.01834136732088763,0.0,0.04101254412979794,0.1437064941446237,0.32133748971555404,0.0,0.7185324707231184,0.1437064941446237,0.32133748971555404,0.0,0.7185324707231184 +climax,ClimaX,global,\(\Omega=\)global,5,1 7 42 99 123,3.0287686240700486e-06,4.147312242167625e-06,0.0,7.571921560175121e-06,0.0012959969982639485,0.0017746169760203706,0.0,0.0032399924956598714,0.0012959969982639485,0.0017746169760203706,0.0,0.0032399924956598714 +climax,ClimaX,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +climax,ClimaX,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +climax,ClimaX,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +stormcast,StormCast,global,\(\Omega=\)global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +stormcast,StormCast,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +stormcast,StormCast,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +stormcast,StormCast,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +dlwp,DLWP,global,\(\Omega=\)global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +dlwp,DLWP,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.0048037709215293075,0.006217185202152866,0.0,0.015203005078871956,0.016716228534155796,0.016079313546074458,0.0,0.03305057342744666,0.016716228534155796,0.016079313546074458,0.0,0.03305057342744666 +dlwp,DLWP,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.0017281632798742507,0.002514722758075371,0.0,0.005523780499856246,0.02846514801700826,0.026938012702643194,0.0,0.053927677500854476,0.02846514801700826,0.026938012702643194,0.0,0.053927677500854476 +dlwp,DLWP,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.0007702319787454587,0.0010995336594539604,0.0,0.0023651634514294945,0.04463354681768479,0.04356064433532197,0.0,0.11215006616165157,0.04463354681768479,0.04356064433532197,0.0,0.11215006616165157 +fcn,FCN,global,\(\Omega=\)global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fcn,FCN,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.0006342898232943345,0.0009899554165032742,0.0,0.002257520679520411,0.004509624980300697,0.010070611656609236,0.0,0.022524473456150496,0.004509624980300697,0.010070611656609236,0.0,0.022524473456150496 +fcn,FCN,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.00021156854817603877,0.0004730816556225618,0.0,0.0010578427408801938,0.004199537050817615,0.009390450319657174,0.0,0.020997685254088072,0.004199537050817615,0.009390450319657174,0.0,0.020997685254088072 +fcn,FCN,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,5.754560074337778e-06,1.2867587506825515e-05,0.0,2.877280037168889e-05,0.011679805987441694,0.019872372458657642,0.0,0.045880503237594294,0.011679805987441694,0.019872372458657642,0.0,0.045880503237594294 +fengwu,FengWu,global,\(\Omega=\)global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fengwu,FengWu,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.0005029843170376968,0.0008109166521114917,0.0,0.0018628094907809783,0.008795951947678215,0.005532321338017505,0.0,0.01484136735033148,0.008795951947678215,0.005532321338017505,0.0,0.01484136735033148 +fengwu,FengWu,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.000495228089292582,0.0007349190216431337,0.0,0.0016387212062008855,0.00402300475984525,0.005510851442075993,0.0,0.010273937098576491,0.00402300475984525,0.005510851442075993,0.0,0.010273937098576491 +fengwu,FengWu,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.0006908222234409067,0.0011910586589384115,0.0,0.0027505832409660327,0.005222389249812243,0.0062394095558402415,0.0,0.015150501067724198,0.005222389249812243,0.0062394095558402415,0.0,0.015150501067724198 +fuxi,FuXi,global,\(\Omega=\)global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +fuxi,FuXi,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.002973545331200933,0.0023946274991058026,0.0010927807990139538,0.007024214143542151,0.013545122545609134,0.02097023683418404,0.0,0.050156261654859424,0.013545122545609134,0.02097023683418404,0.0,0.050156261654859424 +fuxi,FuXi,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.001383793743586542,0.0019248128430711165,0.0,0.003938013087198336,0.0016559834970027332,0.0037028916689159307,0.0,0.008279917485013666,0.0016559834970027332,0.0037028916689159307,0.0,0.008279917485013666 +fuxi,FuXi,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.00283318355751887,0.0036808289681375247,0.0,0.008746323525994693,0.00283318355751887,0.0036808289681375247,0.0,0.008746323525994693 +pangu6,Pangu-Weather,global,\(\Omega=\)global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +pangu6,Pangu-Weather,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.003154674487241463,0.002546125713211599,0.0,0.005711594157251587,0.007592888149777122,0.00897418737588444,0.0,0.019790633919317124,0.007592888149777122,0.00897418737588444,0.0,0.019790633919317124 +pangu6,Pangu-Weather,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.0017345627303725214,0.0019305189318827886,0.0,0.004535321555179647,0.003047840737004992,0.005053805614558161,0.0,0.011660780793438352,0.003047840737004992,0.005053805614558161,0.0,0.011660780793438352 +pangu6,Pangu-Weather,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.0007280423771922354,0.001178746460551365,0.0,0.0027096086413018403,0.0018679847512695024,0.0032548337047755126,0.0,0.007639684811918013,0.0018679847512695024,0.0032548337047755126,0.0,0.007639684811918013 +alphaearth,AlphaEarth,global,\(\Omega=\)global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.1722171037486726,0.08849214830495522,0.050655329891645895,0.272468682523694,0.1722171037486726,0.08849214830495522,0.050655329891645895,0.272468682523694 +alphaearth,AlphaEarth,top5,\(\Omega=\)top 5\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.06384618090125256,0.04965276403138872,0.0,0.1365277562230962,0.06384618090125256,0.04965276403138872,0.0,0.1365277562230962 +alphaearth,AlphaEarth,top10,\(\Omega=\)top 10\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.06573776411084173,0.06897015340160571,0.0,0.1615566566666954,0.06573776411084173,0.06897015340160571,0.0,0.1615566566666954 +alphaearth,AlphaEarth,top20,\(\Omega=\)top 20\%,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.038803552655092256,0.0594825313313219,0.0,0.13482302181308625,0.038803552655092256,0.0594825313313219,0.0,0.13482302181308625 diff --git a/artifacts/results/selection_regret_scope_sweep_20260505.generated.tex b/artifacts/results/selection_regret_scope_sweep_20260505.generated.tex new file mode 100644 index 0000000000000000000000000000000000000000..397761ed00823310be4af0041178534708d7573d --- /dev/null +++ b/artifacts/results/selection_regret_scope_sweep_20260505.generated.tex @@ -0,0 +1,24 @@ +\begin{table*}[!t] + \centering + \small + \setlength{\tabcolsep}{4pt} + \caption{Fixed-feature selection-regret sweep across evaluation scopes. Values are percentage-point regret \(\delta = D(h_D)-D(h_R)\) under union-\(F_1\). Top-\(k\) scopes are train-defined fire-prone masks. Rows report mean with small std over five seeds.} + \label{tab:selection_regret_scope_sweep} + \begin{tabular}{lcccc} + \toprule + \textbf{Feature source} & \textbf{\(\Omega=\)global} & \textbf{\(\Omega=\)top 5\%} & \textbf{\(\Omega=\)top 10\%} & \textbf{\(\Omega=\)top 20\%} \\ + \midrule + \textcolor{blue}{FireWx-FM ref.} & \ms{7.3831}{7.4536} & \ms{0.3664}{0.6812} & \ms{1.2275}{1.2665} & \ms{2.9385}{2.7513} \\ + Prithvi-WxC & 0.0000 & 0.0000 & 0.0000 & 0.0000 \\ + Aurora & \ms{4.9455}{10.6974} & \ms{15.4283}{34.4987} & \ms{13.9934}{31.2903} & \ms{14.3706}{32.1337} \\ + ClimaX & \ms{0.1296}{0.1775} & 0.0000 & 0.0000 & 0.0000 \\ + StormCast & 0.0000 & 0.0000 & 0.0000 & 0.0000 \\ + DLWP & 0.0000 & \ms{1.6716}{1.6079} & \ms{2.8465}{2.6938} & \ms{4.4634}{4.3561} \\ + FCN & 0.0000 & \ms{0.4510}{1.0071} & \ms{0.4200}{0.9390} & \ms{1.1680}{1.9872} \\ + FengWu & 0.0000 & \ms{0.8796}{0.5532} & \ms{0.4023}{0.5511} & \ms{0.5222}{0.6239} \\ + FuXi & 0.0000 & \ms{1.3545}{2.0970} & \ms{0.1656}{0.3703} & \ms{0.2833}{0.3681} \\ + Pangu-Weather & 0.0000 & \ms{0.7593}{0.8974} & \ms{0.3048}{0.5054} & \ms{0.1868}{0.3255} \\ + AlphaEarth & \ms{17.2217}{8.8492} & \ms{6.3846}{4.9653} & \ms{6.5738}{6.8970} & \ms{3.8804}{5.9483} \\ + \bottomrule + \end{tabular} +\end{table*} diff --git a/artifacts/results/selection_regret_scope_sweep_20260505.json b/artifacts/results/selection_regret_scope_sweep_20260505.json new file mode 100644 index 0000000000000000000000000000000000000000..3d57532d393728a04d354cb2a0b5b451e0221690 --- /dev/null +++ b/artifacts/results/selection_regret_scope_sweep_20260505.json @@ -0,0 +1,10274 @@ +{ + "issues": [], + "notes": [ + "FireWx-FM ref. global exact_regret has true zero displayed std", + "FireWx-FM ref. top5 exact_regret has true zero displayed std", + "FireWx-FM ref. top10 exact_regret has true zero displayed std", + "FireWx-FM ref. top20 exact_regret has true zero displayed std", + "Prithvi-WxC global exact_regret has true zero displayed std", + "Prithvi-WxC global tolerated_regret has true zero displayed std", + "Prithvi-WxC global union_regret has true zero displayed std", + "Prithvi-WxC top5 exact_regret has true zero displayed std", + "Prithvi-WxC top5 tolerated_regret has true zero displayed std", + "Prithvi-WxC top5 union_regret has true zero displayed std", + "Prithvi-WxC top10 exact_regret has true zero displayed std", + "Prithvi-WxC top10 tolerated_regret has true zero displayed std", + "Prithvi-WxC top10 union_regret has true zero displayed std", + "Prithvi-WxC top20 exact_regret has true zero displayed std", + "Prithvi-WxC top20 tolerated_regret has true zero displayed std", + "Prithvi-WxC top20 union_regret has true zero displayed std", + "ClimaX top5 exact_regret has true zero displayed std", + "ClimaX top5 tolerated_regret has true zero displayed std", + "ClimaX top5 union_regret has true zero displayed std", + "ClimaX top10 exact_regret has true zero displayed std", + "ClimaX top10 tolerated_regret has true zero displayed std", + "ClimaX top10 union_regret has true zero displayed std", + "ClimaX top20 exact_regret has true zero displayed std", + "ClimaX top20 tolerated_regret has true zero displayed std", + "ClimaX top20 union_regret has true zero displayed std", + "StormCast global exact_regret has true zero displayed std", + "StormCast global tolerated_regret has true zero displayed std", + "StormCast global union_regret has true zero displayed std", + "StormCast top5 exact_regret has true zero displayed std", + "StormCast top5 tolerated_regret has true zero displayed std", + "StormCast top5 union_regret has true zero displayed std", + "StormCast top10 exact_regret has true zero displayed std", + "StormCast top10 tolerated_regret has true zero displayed std", + "StormCast top10 union_regret has true zero displayed std", + "StormCast top20 exact_regret has true zero displayed std", + "StormCast top20 tolerated_regret has true zero displayed std", + "StormCast top20 union_regret has true zero displayed std", + "DLWP global exact_regret has true zero displayed std", + "DLWP global tolerated_regret has true zero displayed std", + "DLWP global union_regret has true zero displayed std", + "FCN global exact_regret has true zero displayed std", + "FCN global tolerated_regret has true zero displayed std", + "FCN global union_regret has true zero displayed std", + "FengWu global exact_regret has true zero displayed std", + "FengWu global tolerated_regret has true zero displayed std", + "FengWu global union_regret has true zero displayed std", + "FuXi global exact_regret has true zero displayed std", + "FuXi global tolerated_regret has true zero displayed std", + "FuXi global union_regret has true zero displayed std", + "FuXi top20 exact_regret has true zero displayed std", + "Pangu-Weather global exact_regret has true zero displayed std", + "Pangu-Weather global tolerated_regret has true zero displayed std", + "Pangu-Weather global union_regret has true zero displayed std", + "AlphaEarth global exact_regret has true zero displayed std", + "AlphaEarth top5 exact_regret has true zero displayed std", + "AlphaEarth top10 exact_regret has true zero displayed std", + "AlphaEarth top20 exact_regret has true zero displayed std" + ], + "rows": [ + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.004193290734824281", + "exact_ranking_score": "0.004193290734824281", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0056438127090301", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0056438127090301", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0006549130347299629", + "ranking_selected_val_pr_auc": "0.0007883068263073185", + "scope": "global", + "seed": 1, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.6337627266658229", + "tolerated_ranking_score": "0.40561891947698747", + "tolerated_regret": 0.22814380718883542, + "tolerated_test_gap": "0.22814380718883542", + "tolerated_val_decision_score": "0.6238973446702573", + "tolerated_val_gap": "0.1715500493543587", + "tolerated_val_ranking_score": "0.45234729531589857", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.6337627266658229", + "union_ranking_score": "0.40561891947698747", + "union_regret": 0.22814380718883542, + "union_test_gap": "0.22814380718883542", + "union_val_decision_score": "0.6238973446702573", + "union_val_gap": "0.1715500493543587", + "union_val_ranking_score": "0.45234729531589857" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.2053654024051804", + "exact_ranking_score": "0.2053654024051804", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.24264049955396969", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.24264049955396969", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.13521999470870394", + "ranking_selected_val_pr_auc": "0.1530074531653213", + "scope": "top5", + "seed": 1, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.9218545130329889", + "tolerated_ranking_score": "0.8564974369370675", + "tolerated_regret": 0.06535707609592145, + "tolerated_test_gap": "0.06535707609592145", + "tolerated_val_decision_score": "0.8921006948278831", + "tolerated_val_gap": "0.06576085909428331", + "tolerated_val_ranking_score": "0.8263398357335998", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.9218545130329889", + "union_ranking_score": "0.8564974369370675", + "union_regret": 0.06535707609592145, + "union_test_gap": "0.06535707609592145", + "union_val_decision_score": "0.8921006948278831", + "union_val_gap": "0.06576085909428331", + "union_val_ranking_score": "0.8263398357335998" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.16019417475728157", + "exact_ranking_score": "0.16019417475728157", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.18440523619806493", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.18440523619806493", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.08541592232377082", + "ranking_selected_val_pr_auc": "0.09854032446284237", + "scope": "top10", + "seed": 1, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.8506993653336619", + "tolerated_ranking_score": "0.7839805825242719", + "tolerated_regret": 0.06671878280938992, + "tolerated_test_gap": "0.06671878280938992", + "tolerated_val_decision_score": "0.8034698049616", + "tolerated_val_gap": "0.04257704982233934", + "tolerated_val_ranking_score": "0.7608927551392607", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.8506993653336619", + "union_ranking_score": "0.7839805825242719", + "union_regret": 0.06671878280938992, + "union_test_gap": "0.06671878280938992", + "union_val_decision_score": "0.8034698049616", + "union_val_gap": "0.04257704982233934", + "union_val_ranking_score": "0.7608927551392607" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.09715762273901808", + "exact_ranking_score": "0.09715762273901808", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.13462469733656174", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.13462469733656174", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.04390614036305261", + "ranking_selected_val_pr_auc": "0.05734457583552234", + "scope": "top20", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7863702028280513", + "tolerated_ranking_score": "0.7863702028280513", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.758785818734687", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.758785818734687", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7863702028280513", + "union_ranking_score": "0.7863702028280513", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.758785818734687", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.758785818734687" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.006959088991986505", + "exact_ranking_score": "0.006959088991986505", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.006399069226294357", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.006399069226294357", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0007047712457371991", + "ranking_selected_val_pr_auc": "0.0007434343102653092", + "scope": "global", + "seed": 123, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.5604635792586619", + "tolerated_ranking_score": "0.4427625907752311", + "tolerated_regret": 0.11770098848343075, + "tolerated_test_gap": "0.11770098848343075", + "tolerated_val_decision_score": "0.5441785824390972", + "tolerated_val_gap": "0.056405166058839995", + "tolerated_val_ranking_score": "0.48777341638025723", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.5604635792586619", + "union_ranking_score": "0.4427625907752311", + "union_regret": 0.11770098848343075, + "union_test_gap": "0.11770098848343075", + "union_val_decision_score": "0.5441785824390972", + "union_val_gap": "0.056405166058839995", + "union_val_ranking_score": "0.48777341638025723" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.1978021978021978", + "exact_ranking_score": "0.1978021978021978", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.2180376610505451", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.2180376610505451", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.13463750657454468", + "ranking_selected_val_pr_auc": "0.14286946914705156", + "scope": "top5", + "seed": 123, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.8743168479997309", + "tolerated_ranking_score": "0.8305909861908366", + "tolerated_regret": 0.04372586180889426, + "tolerated_test_gap": "0.04372586180889426", + "tolerated_val_decision_score": "0.8625937780701644", + "tolerated_val_gap": "0.05250190173735958", + "tolerated_val_ranking_score": "0.8100918763328048", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.8743168479997309", + "union_ranking_score": "0.8305909861908366", + "union_regret": 0.04372586180889426, + "union_test_gap": "0.04372586180889426", + "union_val_decision_score": "0.8625937780701644", + "union_val_gap": "0.05250190173735958", + "union_val_ranking_score": "0.8100918763328048" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.15615615615615616", + "exact_ranking_score": "0.15615615615615616", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.1708413615928067", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.1708413615928067", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0877757525096481", + "ranking_selected_val_pr_auc": "0.09333059447315038", + "scope": "top10", + "seed": 123, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.6977392921485441", + "tolerated_ranking_score": "0.7665340693262388", + "tolerated_regret": 0.0, + "tolerated_test_gap": "-0.06879477717769467", + "tolerated_val_decision_score": "0.7559010025052879", + "tolerated_val_gap": "0.0006607651392849245", + "tolerated_val_ranking_score": "0.755240237366003", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.6977392921485441", + "union_ranking_score": "0.7665340693262388", + "union_regret": 0.0, + "union_test_gap": "-0.06879477717769467", + "union_val_decision_score": "0.7559010025052879", + "union_val_gap": "0.0006607651392849245", + "union_val_ranking_score": "0.755240237366003" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.10531544957774468", + "exact_ranking_score": "0.10531544957774468", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.12267343485617596", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.12267343485617596", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.045809049876129763", + "ranking_selected_val_pr_auc": "0.05470980065861018", + "scope": "top20", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7789089693560928", + "tolerated_ranking_score": "0.7789089693560928", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.749825256630312", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.749825256630312", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7789089693560928", + "union_ranking_score": "0.7789089693560928", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.749825256630312", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.749825256630312" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.004809747755451047", + "exact_ranking_score": "0.004809747755451047", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.004359099620457705", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.004359099620457705", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0005634701573991865", + "ranking_selected_val_pr_auc": "0.0005920938943373892", + "scope": "global", + "seed": 42, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.6812131506751973", + "tolerated_ranking_score": "0.4087444681515033", + "tolerated_regret": 0.272468682523694, + "tolerated_test_gap": "0.272468682523694", + "tolerated_val_decision_score": "0.7447325774946227", + "tolerated_val_gap": "0.3021550064919567", + "tolerated_val_ranking_score": "0.44257757100266604", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.6812131506751973", + "union_ranking_score": "0.4087444681515033", + "union_regret": 0.272468682523694, + "union_test_gap": "0.272468682523694", + "union_val_decision_score": "0.7447325774946227", + "union_val_gap": "0.3021550064919567", + "union_val_ranking_score": "0.44257757100266604" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.1818181818181818", + "exact_ranking_score": "0.1818181818181818", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.20273972602739726", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.20273972602739726", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.11776218604745192", + "ranking_selected_val_pr_auc": "0.12790468093676954", + "scope": "top5", + "seed": 42, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.9079426767063397", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 0.1365277562230962, + "tolerated_test_gap": "0.1365277562230962", + "tolerated_val_decision_score": "0.8560076563904758", + "tolerated_val_gap": "0.058814205003907216", + "tolerated_val_ranking_score": "0.7971934513865686", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.9079426767063397", + "union_ranking_score": "0.7714149204832434", + "union_regret": 0.1365277562230962, + "union_test_gap": "0.1365277562230962", + "union_val_decision_score": "0.8560076563904758", + "union_val_gap": "0.058814205003907216", + "union_val_ranking_score": "0.7971934513865686" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.14064516129032256", + "exact_ranking_score": "0.14064516129032256", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.1536748329621381", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.1536748329621381", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.07431134992726657", + "ranking_selected_val_pr_auc": "0.07868604702405124", + "scope": "top10", + "seed": 42, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.860840238051614", + "tolerated_ranking_score": "0.6992835813849186", + "tolerated_regret": 0.1615566566666954, + "tolerated_test_gap": "0.1615566566666954", + "tolerated_val_decision_score": "0.8130323993938946", + "tolerated_val_gap": "0.0827924722936264", + "tolerated_val_ranking_score": "0.7302399271002682", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.860840238051614", + "union_ranking_score": "0.6992835813849186", + "union_regret": 0.1615566566666954, + "union_test_gap": "0.1615566566666954", + "union_val_decision_score": "0.8130323993938946", + "union_val_gap": "0.0827924722936264", + "union_val_ranking_score": "0.7302399271002682" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.08702469619756958", + "exact_ranking_score": "0.08702469619756958", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.10515392597717052", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.10515392597717052", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.038083070948941686", + "ranking_selected_val_pr_auc": "0.04505214599515043", + "scope": "top20", + "seed": 42, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.8461131676361712", + "tolerated_ranking_score": "0.7112901458230849", + "tolerated_regret": 0.13482302181308625, + "tolerated_test_gap": "0.13482302181308625", + "tolerated_val_decision_score": "0.806760850261024", + "tolerated_val_gap": "0.07815518654517495", + "tolerated_val_ranking_score": "0.728605663715849", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.8461131676361712", + "union_ranking_score": "0.7112901458230849", + "union_regret": 0.13482302181308625, + "union_test_gap": "0.13482302181308625", + "union_val_decision_score": "0.806760850261024", + "union_val_gap": "0.07815518654517495", + "union_val_ranking_score": "0.728605663715849" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.010460251046025104", + "exact_ranking_score": "0.010460251046025104", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.009547591069330199", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.009547591069330199", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.001005722733868245", + "ranking_selected_val_pr_auc": "0.0009777524224237678", + "scope": "global", + "seed": 7, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.6691395427484861", + "tolerated_ranking_score": "0.6184842128568402", + "tolerated_regret": 0.050655329891645895, + "tolerated_test_gap": "0.050655329891645895", + "tolerated_val_decision_score": "0.677404587086621", + "tolerated_val_gap": "0.011527156971746999", + "tolerated_val_ranking_score": "0.665877430114874", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.6691395427484861", + "union_ranking_score": "0.6184842128568402", + "union_regret": 0.050655329891645895, + "union_test_gap": "0.050655329891645895", + "union_val_decision_score": "0.677404587086621", + "union_val_gap": "0.011527156971746999", + "union_val_ranking_score": "0.665877430114874" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.18705957719410635", + "exact_ranking_score": "0.18705957719410635", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.22265122265122267", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.22265122265122267", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.1415020173172751", + "ranking_selected_val_pr_auc": "0.13899023094495766", + "scope": "top5", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.8671946832706918", + "tolerated_ranking_score": "0.8671946832706918", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.8765714285714286", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.8765714285714286", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.8671946832706918", + "union_ranking_score": "0.8671946832706918", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.8765714285714286", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.8765714285714286" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.15882806476484196", + "exact_ranking_score": "0.15882806476484196", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.17872968980797638", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.17872968980797638", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0993207020173454", + "ranking_selected_val_pr_auc": "0.09562560899249391", + "scope": "top10", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.8345528858733409", + "tolerated_ranking_score": "0.8345528858733409", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.8241729549074388", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.8241729549074388", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.8345528858733409", + "union_ranking_score": "0.8345528858733409", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.8241729549074388", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.8241729549074388" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.12514898688915374", + "exact_ranking_score": "0.12514898688915374", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.14325685506435365", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.14325685506435365", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.05955397140893137", + "ranking_selected_val_pr_auc": "0.06058697421048699", + "scope": "top20", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.8294499693616161", + "tolerated_ranking_score": "0.8294499693616161", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.8107581951083846", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.8107581951083846", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.8294499693616161", + "union_ranking_score": "0.8294499693616161", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.8107581951083846", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.8107581951083846" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.006780481898534931", + "exact_ranking_score": "0.006780481898534931", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.005964994898359627", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.005964994898359627", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0006577120081349608", + "ranking_selected_val_pr_auc": "0.0007340310681749519", + "scope": "global", + "seed": 99, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.5842714676652996", + "tolerated_ranking_score": "0.3921547570095426", + "tolerated_regret": 0.19211671065575697, + "tolerated_test_gap": "0.19211671065575697", + "tolerated_val_decision_score": "0.5663548190132099", + "tolerated_val_gap": "0.10549666519652506", + "tolerated_val_ranking_score": "0.4608581538166849", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.5842714676652996", + "union_ranking_score": "0.3921547570095426", + "union_regret": 0.19211671065575697, + "union_test_gap": "0.19211671065575697", + "union_val_decision_score": "0.5663548190132099", + "union_val_gap": "0.10549666519652506", + "union_val_ranking_score": "0.4608581538166849" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.1867517956903432", + "exact_ranking_score": "0.1867517956903432", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.2253944402704733", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.2253944402704733", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.13302539422913043", + "ranking_selected_val_pr_auc": "0.14641451099665195", + "scope": "top5", + "seed": 99, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.898839683642517", + "tolerated_ranking_score": "0.8252194732641661", + "tolerated_regret": 0.07362021037835087, + "tolerated_test_gap": "0.07362021037835087", + "tolerated_val_decision_score": "0.8737527215993107", + "tolerated_val_gap": "0.06839730093782548", + "tolerated_val_ranking_score": "0.8053554206614852", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.898839683642517", + "union_ranking_score": "0.8252194732641661", + "union_regret": 0.07362021037835087, + "union_test_gap": "0.07362021037835087", + "union_val_decision_score": "0.8737527215993107", + "union_val_gap": "0.06839730093782548", + "union_val_ranking_score": "0.8053554206614852" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.15477084898572502", + "exact_ranking_score": "0.15477084898572502", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.16870573084352863", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.16870573084352863", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.08656943374833374", + "ranking_selected_val_pr_auc": "0.09321280420976051", + "scope": "top10", + "seed": 99, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.8655328563033635", + "tolerated_ranking_score": "0.7651194752252402", + "tolerated_regret": 0.10041338107812336, + "tolerated_test_gap": "0.10041338107812336", + "tolerated_val_decision_score": "0.8092827685342129", + "tolerated_val_gap": "0.06382807430932869", + "tolerated_val_ranking_score": "0.7454546942248842", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.8655328563033635", + "union_ranking_score": "0.7651194752252402", + "union_regret": 0.10041338107812336, + "union_test_gap": "0.10041338107812336", + "union_val_decision_score": "0.8092827685342129", + "union_val_gap": "0.06382807430932869", + "union_val_ranking_score": "0.7454546942248842" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.10365251727541955", + "exact_ranking_score": "0.10365251727541955", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.12251941328731666", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.12251941328731666", + "family": "AlphaEarth", + "model_tag": "alphaearth", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0458102699699856", + "ranking_selected_val_pr_auc": "0.0554648890254577", + "scope": "top20", + "seed": 99, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.8350245452333586", + "tolerated_ranking_score": "0.7758298037709835", + "tolerated_regret": 0.05919474146237502, + "tolerated_test_gap": "0.05919474146237502", + "tolerated_val_decision_score": "0.7736506356603177", + "tolerated_val_gap": "0.027586681737492236", + "tolerated_val_ranking_score": "0.7460639539228254", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.8350245452333586", + "union_ranking_score": "0.7758298037709835", + "union_regret": 0.05919474146237502, + "union_test_gap": "0.05919474146237502", + "union_val_decision_score": "0.7736506356603177", + "union_val_gap": "0.027586681737492236", + "union_val_ranking_score": "0.7460639539228254" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.00048497822606044473", + "exact_regret": 7.571921560175121e-06, + "exact_test_gap": "7.571921560175121e-06", + "exact_val_decision_score": "0.0004727121748000204", + "exact_val_gap": "7.267025482891767e-06", + "exact_val_ranking_score": "0.00046544514931712863", + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.00017433024575177084", + "ranking_selected_val_pr_auc": "0.00039761796201498743", + "scope": "global", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.23755358049655212", + "tolerated_regret": 0.0032399924956598714, + "tolerated_test_gap": "0.0032399924956598714", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.00299097714352789", + "tolerated_val_ranking_score": "0.2149459035878704", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.23755358049655212", + "union_regret": 0.0032399924956598714, + "union_test_gap": "0.0032399924956598714", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.00299097714352789", + "union_val_ranking_score": "0.2149459035878704" + }, + { + "exact_decision_head": "pixel MLP head", + "exact_decision_head_arch": "pixel_mlp", + "exact_decision_score": "0.11833577252770228", + "exact_ranking_score": "0.11833577252770228", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.11931877546755824", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.11931877546755824", + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.043017434088706286", + "ranking_selected_val_pr_auc": "0.18453515297332748", + "scope": "top5", + "seed": 1, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.7714149204832434", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7971934513865686", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7971934513865686", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.7714149204832434", + "union_ranking_score": "0.7714149204832434", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7971934513865686", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7971934513865686" + }, + { + "exact_decision_head": "pixel MLP head", + "exact_decision_head_arch": "pixel_mlp", + "exact_decision_score": "0.07325639739086803", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.07460890493381468", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.07460890493381468", + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.025652132600196966", + "ranking_selected_val_pr_auc": "0.11087183021305622", + "scope": "top10", + "seed": 1, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.6996715316108496", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7015182768065465", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.6996715316108496", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7015182768065465", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "pixel MLP head", + "exact_decision_head_arch": "pixel_mlp", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.014185996780547955", + "ranking_selected_val_pr_auc": "0.059710932216298356", + "scope": "top20", + "seed": 1, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.0004925501476206198", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0004727121748000204", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0004727121748000204", + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0001870694786133523", + "ranking_selected_val_pr_auc": "0.00044346166657711726", + "scope": "global", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.240793572992212", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.2179368807313983", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.240793572992212", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.2179368807313983" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.11833577252770228", + "exact_ranking_score": "0.11833577252770228", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.11931877546755824", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.11931877546755824", + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0476637592736729", + "ranking_selected_val_pr_auc": "0.19978759776073984", + "scope": "top5", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7714149204832434", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7971934513865686", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7971934513865686", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7714149204832434", + "union_ranking_score": "0.7714149204832434", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7971934513865686", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7971934513865686" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07325639739086803", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.07460890493381468", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.07460890493381468", + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.028202726028592855", + "ranking_selected_val_pr_auc": "0.12040472446461332", + "scope": "top10", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.6996715316108496", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7015182768065465", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.6996715316108496", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7015182768065465", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.015530707998309533", + "ranking_selected_val_pr_auc": "0.06432554845990393", + "scope": "top20", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.0", + "exact_regret": 0.0004925501476206198, + "exact_test_gap": "0.0004925501476206198", + "exact_val_decision_score": "0.0004727121748000204", + "exact_val_gap": "0.0004727121748000204", + "exact_val_ranking_score": "0.0", + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.000198182392518492", + "ranking_selected_val_pr_auc": "0.0002767362365663907", + "scope": "global", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.0", + "tolerated_regret": 0.240793572992212, + "tolerated_test_gap": "0.240793572992212", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.2179368807313983", + "tolerated_val_ranking_score": "0.0", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.0", + "union_regret": 0.240793572992212, + "union_test_gap": "0.240793572992212", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.2179368807313983", + "union_val_ranking_score": "0.0" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.11833577252770228", + "exact_ranking_score": "0.0", + "exact_regret": 0.11833577252770228, + "exact_test_gap": "0.11833577252770228", + "exact_val_decision_score": "0.11931877546755824", + "exact_val_gap": "0.11931877546755824", + "exact_val_ranking_score": "0.0", + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.047074707804185205", + "ranking_selected_val_pr_auc": "0.11187203617000477", + "scope": "top5", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7714149204832434", + "tolerated_ranking_score": "0.0", + "tolerated_regret": 0.7714149204832434, + "tolerated_test_gap": "0.7714149204832434", + "tolerated_val_decision_score": "0.7971934513865686", + "tolerated_val_gap": "0.7971934513865686", + "tolerated_val_ranking_score": "0.0", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7714149204832434", + "union_ranking_score": "0.0", + "union_regret": 0.7714149204832434, + "union_test_gap": "0.7714149204832434", + "union_val_decision_score": "0.7971934513865686", + "union_val_gap": "0.7971934513865686", + "union_val_ranking_score": "0.0" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07325639739086803", + "exact_ranking_score": "0.0", + "exact_regret": 0.07325639739086803, + "exact_test_gap": "0.07325639739086803", + "exact_val_decision_score": "0.07460890493381468", + "exact_val_gap": "0.07460890493381468", + "exact_val_ranking_score": "0.0", + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.027753810085825968", + "ranking_selected_val_pr_auc": "0.06589784404428278", + "scope": "top10", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.6996715316108496", + "tolerated_ranking_score": "0.0", + "tolerated_regret": 0.6996715316108496, + "tolerated_test_gap": "0.6996715316108496", + "tolerated_val_decision_score": "0.7015182768065465", + "tolerated_val_gap": "0.7015182768065465", + "tolerated_val_ranking_score": "0.0", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.6996715316108496", + "union_ranking_score": "0.0", + "union_regret": 0.6996715316108496, + "union_test_gap": "0.6996715316108496", + "union_val_decision_score": "0.7015182768065465", + "union_val_gap": "0.7015182768065465", + "union_val_ranking_score": "0.0" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.0", + "exact_regret": 0.04101254412979794, + "exact_test_gap": "0.04101254412979794", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.04253189892419314", + "exact_val_ranking_score": "0.0", + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.015401540261000917", + "ranking_selected_val_pr_auc": "0.03525031596119595", + "scope": "top20", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.0", + "tolerated_regret": 0.7185324707231184, + "tolerated_test_gap": "0.7185324707231184", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.6906754133476136", + "tolerated_val_ranking_score": "0.0", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.0", + "union_regret": 0.7185324707231184, + "union_test_gap": "0.7185324707231184", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.6906754133476136", + "union_val_ranking_score": "0.0" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0004925501476206198", + "exact_regret": 0.0, + "exact_test_gap": "-0.0004925501476206198", + "exact_val_decision_score": "0.0005036333549176199", + "exact_val_gap": "3.092118011759949e-05", + "exact_val_ranking_score": "0.0004727121748000204", + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0002120945228028488", + "ranking_selected_val_pr_auc": "0.000380637430664089", + "scope": "global", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.240793572992212", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.2179368807313983", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.240793572992212", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.2179368807313983" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.11833577252770228", + "exact_ranking_score": "0.11833577252770228", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.11931877546755824", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.11931877546755824", + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.05983564956422643", + "ranking_selected_val_pr_auc": "0.15910946905394902", + "scope": "top5", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7714149204832434", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7971934513865686", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7971934513865686", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7714149204832434", + "union_ranking_score": "0.7714149204832434", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7971934513865686", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7971934513865686" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07325639739086803", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.07460890493381468", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.07460890493381468", + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.03479712897675945", + "ranking_selected_val_pr_auc": "0.0949024678484547", + "scope": "top10", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.6996715316108496", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7015182768065465", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.6996715316108496", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7015182768065465", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.01908165402331067", + "ranking_selected_val_pr_auc": "0.05014988402480292", + "scope": "top20", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.00048497822606044473", + "exact_regret": 7.571921560175121e-06, + "exact_test_gap": "7.571921560175121e-06", + "exact_val_decision_score": "0.0004727121748000204", + "exact_val_gap": "7.267025482891767e-06", + "exact_val_ranking_score": "0.00046544514931712863", + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.00016818557842652088", + "ranking_selected_val_pr_auc": "0.0004365856110391047", + "scope": "global", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.23755358049655212", + "tolerated_regret": 0.0032399924956598714, + "tolerated_test_gap": "0.0032399924956598714", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.0025778293487095094", + "tolerated_val_ranking_score": "0.2153590513826888", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.23755358049655212", + "union_regret": 0.0032399924956598714, + "union_test_gap": "0.0032399924956598714", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.0025778293487095094", + "union_val_ranking_score": "0.2153590513826888" + }, + { + "exact_decision_head": "pixel MLP head", + "exact_decision_head_arch": "pixel_mlp", + "exact_decision_score": "0.11833577252770228", + "exact_ranking_score": "0.11833577252770228", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.1250696689332293", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.1250696689332293", + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.04280276014558355", + "ranking_selected_val_pr_auc": "0.20552291444972312", + "scope": "top5", + "seed": 99, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.7714149204832434", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7985410856039477", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7985410856039477", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.7714149204832434", + "union_ranking_score": "0.7714149204832434", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7985410856039477", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7985410856039477" + }, + { + "exact_decision_head": "pixel MLP head", + "exact_decision_head_arch": "pixel_mlp", + "exact_decision_score": "0.07325639739086803", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.07801114444920705", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.07801114444920705", + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.025614133200453124", + "ranking_selected_val_pr_auc": "0.12487452965433544", + "scope": "top10", + "seed": 99, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.6996715316108496", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7022486149835246", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7022486149835246", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.6996715316108496", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7022486149835246", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7022486149835246" + }, + { + "exact_decision_head": "pixel MLP head", + "exact_decision_head_arch": "pixel_mlp", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04437663936620095", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04437663936620095", + "family": "Aurora", + "model_tag": "aurora", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.01412932353396452", + "ranking_selected_val_pr_auc": "0.06804297819080397", + "scope": "top20", + "seed": 99, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.692216769941617", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.692216769941617", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.692216769941617", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.692216769941617" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.0004925501476206198", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0004727121748000204", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0004727121748000204", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00023234964096770515", + "ranking_selected_val_pr_auc": "0.0002601351527039687", + "scope": "global", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.240793572992212", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.2179368807313983", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.240793572992212", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.2179368807313983" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.11833577252770228", + "exact_ranking_score": "0.11833577252770228", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.11931877546755824", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.11931877546755824", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.055890306674778144", + "ranking_selected_val_pr_auc": "0.0694777315217738", + "scope": "top5", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7714149204832434", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7971934513865686", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7971934513865686", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7714149204832434", + "union_ranking_score": "0.7714149204832434", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7971934513865686", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7971934513865686" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07325639739086803", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.07460890493381468", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.07460890493381468", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.03345772284683149", + "ranking_selected_val_pr_auc": "0.042554440111266106", + "scope": "top10", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.6996715316108496", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7015182768065465", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.6996715316108496", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7015182768065465", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.01871363925580095", + "ranking_selected_val_pr_auc": "0.024212450415123626", + "scope": "top20", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.0004925501476206198", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0004727121748000204", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0004727121748000204", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00023488826232608934", + "ranking_selected_val_pr_auc": "0.00022945923295652787", + "scope": "global", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.240793572992212", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.2179368807313983", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.240793572992212", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.2179368807313983" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.11833577252770228", + "exact_ranking_score": "0.11833577252770228", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.11931877546755824", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.11931877546755824", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.06647102886669706", + "ranking_selected_val_pr_auc": "0.07019151291101249", + "scope": "top5", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7714149204832434", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7971934513865686", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7971934513865686", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7714149204832434", + "union_ranking_score": "0.7714149204832434", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7971934513865686", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7971934513865686" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07325639739086803", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.07460890493381468", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.07460890493381468", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.03955291143531272", + "ranking_selected_val_pr_auc": "0.04218583124525834", + "scope": "top10", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.6996715316108496", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7015182768065465", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.6996715316108496", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7015182768065465", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.021395640000026753", + "ranking_selected_val_pr_auc": "0.02312815465123813", + "scope": "top20", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.00048497822606044473", + "exact_regret": 7.571921560175121e-06, + "exact_test_gap": "7.571921560175121e-06", + "exact_val_decision_score": "0.0004727121748000204", + "exact_val_gap": "7.267025482891767e-06", + "exact_val_ranking_score": "0.00046544514931712863", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.00023439386477689915", + "ranking_selected_val_pr_auc": "0.0002283896136939234", + "scope": "global", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.23755358049655212", + "tolerated_regret": 0.0032399924956598714, + "tolerated_test_gap": "0.0032399924956598714", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.00299097714352789", + "tolerated_val_ranking_score": "0.2149459035878704", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.23755358049655212", + "union_regret": 0.0032399924956598714, + "union_test_gap": "0.0032399924956598714", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.00299097714352789", + "union_val_ranking_score": "0.2149459035878704" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.11833577252770228", + "exact_ranking_score": "0.11833577252770228", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.11931877546755824", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.11931877546755824", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.06646486382553912", + "ranking_selected_val_pr_auc": "0.07019337939369524", + "scope": "top5", + "seed": 42, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7714149204832434", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7971934513865686", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7971934513865686", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7714149204832434", + "union_ranking_score": "0.7714149204832434", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7971934513865686", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7971934513865686" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.07325639739086803", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.07460890493381468", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.07460890493381468", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.039777194719281345", + "ranking_selected_val_pr_auc": "0.042300836914518636", + "scope": "top10", + "seed": 42, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.6996715316108496", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7015182768065465", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.6996715316108496", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7015182768065465", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.021516770872035896", + "ranking_selected_val_pr_auc": "0.023234418719582723", + "scope": "top20", + "seed": 42, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.00048497822606044473", + "exact_regret": 7.571921560175121e-06, + "exact_test_gap": "7.571921560175121e-06", + "exact_val_decision_score": "0.0004727121748000204", + "exact_val_gap": "7.267025482891767e-06", + "exact_val_ranking_score": "0.00046544514931712863", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.00025423546642937565", + "ranking_selected_val_pr_auc": "0.00023120715224267713", + "scope": "global", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.23755358049655212", + "tolerated_regret": 0.0032399924956598714, + "tolerated_test_gap": "0.0032399924956598714", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.00299097714352789", + "tolerated_val_ranking_score": "0.2149459035878704", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.23755358049655212", + "union_regret": 0.0032399924956598714, + "union_test_gap": "0.0032399924956598714", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.00299097714352789", + "union_val_ranking_score": "0.2149459035878704" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.11833577252770228", + "exact_ranking_score": "0.11833577252770228", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.11931877546755824", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.11931877546755824", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.06582645251185222", + "ranking_selected_val_pr_auc": "0.07011813439504147", + "scope": "top5", + "seed": 7, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7714149204832434", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7971934513865686", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7971934513865686", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7714149204832434", + "union_ranking_score": "0.7714149204832434", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7971934513865686", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7971934513865686" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.07325639739086803", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.07460890493381468", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.07460890493381468", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.03938368107690723", + "ranking_selected_val_pr_auc": "0.042249808498221526", + "scope": "top10", + "seed": 7, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.6996715316108496", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7015182768065465", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.6996715316108496", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7015182768065465", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.021317405351800135", + "ranking_selected_val_pr_auc": "0.02320587721933335", + "scope": "top20", + "seed": 7, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.00048696535779102983", + "exact_ranking_score": "0.00048696535779102983", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.00046735226565276225", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.00046735226565276225", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0002540711776626832", + "ranking_selected_val_pr_auc": "0.00022678877479338636", + "scope": "global", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.2384111045733381", + "tolerated_ranking_score": "0.2384111045733381", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.21573180756095836", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.21573180756095836", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.2384111045733381", + "union_ranking_score": "0.2384111045733381", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.21573180756095836", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.21573180756095836" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.11833577252770228", + "exact_ranking_score": "0.11833577252770228", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.11931877546755824", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.11931877546755824", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.06558789102558008", + "ranking_selected_val_pr_auc": "0.0697713050234347", + "scope": "top5", + "seed": 99, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7714149204832434", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7971934513865686", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7971934513865686", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7714149204832434", + "union_ranking_score": "0.7714149204832434", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7971934513865686", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7971934513865686" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.07325639739086803", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.07460890493381468", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.07460890493381468", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.039026227487447884", + "ranking_selected_val_pr_auc": "0.04197164598362404", + "scope": "top10", + "seed": 99, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.6996715316108496", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7015182768065465", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.6996715316108496", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7015182768065465", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "ClimaX", + "model_tag": "climax", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.021159149554047907", + "ranking_selected_val_pr_auc": "0.023032008375736605", + "scope": "top20", + "seed": 99, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0053407639614536165", + "exact_ranking_score": "0.0053407639614536165", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0016902678135446794", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0016902678135446794", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0006824870760177812", + "ranking_selected_val_pr_auc": "0.00042302773922457535", + "scope": "global", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.3560602558354403", + "tolerated_ranking_score": "0.3560602558354403", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.3983991584038598", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.3983991584038598", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.3560602558354403", + "union_ranking_score": "0.3560602558354403", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.3983991584038598", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.3983991584038598" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.12001093493712411", + "exact_ranking_score": "0.12001093493712411", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.1225081551286698", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.1225081551286698", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.05689342648649434", + "ranking_selected_val_pr_auc": "0.05377805405038012", + "scope": "top5", + "seed": 1, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7748819895562846", + "tolerated_ranking_score": "0.7748819895562846", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.8195894738438443", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.8195894738438443", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7748819895562846", + "union_ranking_score": "0.7748819895562846", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.8195894738438443", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.8195894738438443" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.078002052685597", + "exact_ranking_score": "0.078002052685597", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.07851948205522379", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.07851948205522379", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.03678683052582872", + "ranking_selected_val_pr_auc": "0.03453391215275296", + "scope": "top10", + "seed": 1, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.723747005823518", + "tolerated_ranking_score": "0.723747005823518", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7278456297612906", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7278456297612906", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.723747005823518", + "union_ranking_score": "0.723747005823518", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7278456297612906", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7278456297612906" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.043506471331489265", + "exact_ranking_score": "0.043506471331489265", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04461624715984538", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04461624715984538", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.019747545663020845", + "ranking_selected_val_pr_auc": "0.019740435565559492", + "scope": "top20", + "seed": 1, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7280364139105968", + "tolerated_ranking_score": "0.7280364139105968", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7120828721033882", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7120828721033882", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7280364139105968", + "union_ranking_score": "0.7280364139105968", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7120828721033882", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7120828721033882" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0020376974019358125", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0020376974019358125", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0007448529317390826", + "ranking_selected_val_pr_auc": "0.00044761016558069356", + "scope": "global", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.3859134431294053", + "tolerated_ranking_score": "0.3859134431294053", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.4280727908627396", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.4280727908627396", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.3859134431294053", + "union_ranking_score": "0.3859134431294053", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.4280727908627396", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.4280727908627396" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.13353877760657423", + "exact_ranking_score": "0.11833577252770228", + "exact_regret": 0.015203005078871956, + "exact_test_gap": "0.015203005078871956", + "exact_val_decision_score": "0.12187299550994227", + "exact_val_gap": "0.0025542200423840228", + "exact_val_ranking_score": "0.11931877546755824", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.06589923575225648", + "ranking_selected_val_pr_auc": "0.06980014500652997", + "scope": "top5", + "seed": 123, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7910976805607687", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 0.01968276007752523, + "tolerated_test_gap": "0.01968276007752523", + "tolerated_val_decision_score": "0.8156849318846232", + "tolerated_val_gap": "0.018491480498054602", + "tolerated_val_ranking_score": "0.7971934513865686", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7910976805607687", + "union_ranking_score": "0.7714149204832434", + "union_regret": 0.01968276007752523, + "union_test_gap": "0.01968276007752523", + "union_val_decision_score": "0.8156849318846232", + "union_val_gap": "0.018491480498054602", + "union_val_ranking_score": "0.7971934513865686" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.07637343329038304", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.0031170358995150083, + "exact_test_gap": "0.0031170358995150083", + "exact_val_decision_score": "0.07697248688577239", + "exact_val_gap": "0.0023635819519577106", + "exact_val_ranking_score": "0.07460890493381468", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.03734242132263064", + "ranking_selected_val_pr_auc": "0.04123208357155757", + "scope": "top10", + "seed": 123, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7355445123185741", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.03587298070772449, + "tolerated_test_gap": "0.03587298070772449", + "tolerated_val_decision_score": "0.7311417346046021", + "tolerated_val_gap": "0.02962345779805553", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7355445123185741", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.03587298070772449, + "union_test_gap": "0.03587298070772449", + "union_val_decision_score": "0.7311417346046021", + "union_val_gap": "0.02962345779805553", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.04249854057209574", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.001485996442297799, + "exact_test_gap": "0.001485996442297799", + "exact_val_decision_score": "0.04371409485637943", + "exact_val_gap": "0.0011821959321862904", + "exact_val_ranking_score": "0.04253189892419314", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.021118517500793188", + "ranking_selected_val_pr_auc": "0.02303236785318257", + "scope": "top20", + "seed": 123, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7321459738801157", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.013613503156997275, + "tolerated_test_gap": "0.013613503156997275", + "tolerated_val_decision_score": "0.7128995538153613", + "tolerated_val_gap": "0.022224140467747677", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7321459738801157", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.013613503156997275, + "union_test_gap": "0.013613503156997275", + "union_val_decision_score": "0.7128995538153613", + "union_val_gap": "0.022224140467747677", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0020120724346076456", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0020120724346076456", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0006203713852571992", + "ranking_selected_val_pr_auc": "0.0004707444097300293", + "scope": "global", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.3405125814370199", + "tolerated_ranking_score": "0.3405125814370199", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.474350305376108", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.474350305376108", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.3405125814370199", + "union_ranking_score": "0.3405125814370199", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.474350305376108", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.474350305376108" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.1227865796831314", + "exact_ranking_score": "0.11833577252770228", + "exact_regret": 0.004450807155429129, + "exact_test_gap": "0.004450807155429129", + "exact_val_decision_score": "0.1210664835793569", + "exact_val_gap": "0.0017477081117986543", + "exact_val_ranking_score": "0.11931877546755824", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.058186924071015014", + "ranking_selected_val_pr_auc": "0.055593579382965747", + "scope": "top5", + "seed": 42, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.7049597657509027", + "tolerated_ranking_score": "0.7049597657509027", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.8131649530621368", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.8131649530621368", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.7049597657509027", + "union_ranking_score": "0.7049597657509027", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.8131649530621368", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.8131649530621368" + }, + { + "exact_decision_head": "pixel MLP head", + "exact_decision_head_arch": "pixel_mlp", + "exact_decision_score": "0.0811232449297972", + "exact_ranking_score": "0.0811232449297972", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.07607950651130911", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.07607950651130911", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.03845833667153789", + "ranking_selected_val_pr_auc": "0.036365711264814124", + "scope": "top10", + "seed": 42, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.6763081901295341", + "tolerated_ranking_score": "0.6763081901295341", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7265137957010752", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7265137957010752", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.6763081901295341", + "union_ranking_score": "0.6763081901295341", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7265137957010752", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7265137957010752" + }, + { + "exact_decision_head": "pixel MLP head", + "exact_decision_head_arch": "pixel_mlp", + "exact_decision_score": "0.04637177602565815", + "exact_ranking_score": "0.04637177602565815", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.044021164021164016", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.044021164021164016", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.020762153794205103", + "ranking_selected_val_pr_auc": "0.020894537350314525", + "scope": "top20", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7728400679088107", + "tolerated_ranking_score": "0.6606900017471591", + "tolerated_regret": 0.11215006616165157, + "tolerated_test_gap": "0.11215006616165157", + "tolerated_val_decision_score": "0.7122584178150692", + "tolerated_val_gap": "0.009110393702543962", + "tolerated_val_ranking_score": "0.7031480241125252", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7728400679088107", + "union_ranking_score": "0.6606900017471591", + "union_regret": 0.11215006616165157, + "union_test_gap": "0.11215006616165157", + "union_val_decision_score": "0.7122584178150692", + "union_val_gap": "0.009110393702543962", + "union_val_ranking_score": "0.7031480241125252" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0010056989607777405", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0010056989607777405", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0005264872646085452", + "ranking_selected_val_pr_auc": "0.00035284338324857376", + "scope": "global", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.3432315705541329", + "tolerated_ranking_score": "0.3432315705541329", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.3469162743670964", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.3469162743670964", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.3432315705541329", + "union_ranking_score": "0.3432315705541329", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.3469162743670964", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.3469162743670964" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.12270081490104773", + "exact_ranking_score": "0.11833577252770228", + "exact_regret": 0.004365042373345451, + "exact_test_gap": "0.004365042373345451", + "exact_val_decision_score": "0.12128657583203037", + "exact_val_gap": "0.001967800364472125", + "exact_val_ranking_score": "0.11931877546755824", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.05890031091946385", + "ranking_selected_val_pr_auc": "0.05786838028004154", + "scope": "top5", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.8044654939106901", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 0.03305057342744666, + "tolerated_test_gap": "0.03305057342744666", + "tolerated_val_decision_score": "0.8071267256214067", + "tolerated_val_gap": "0.009933274234838141", + "tolerated_val_ranking_score": "0.7971934513865686", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.8044654939106901", + "union_ranking_score": "0.7714149204832434", + "union_regret": 0.03305057342744666, + "union_test_gap": "0.03305057342744666", + "union_val_decision_score": "0.8071267256214067", + "union_val_gap": "0.009933274234838141", + "union_val_ranking_score": "0.7971934513865686" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07878017789072428", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.005523780499856246, + "exact_test_gap": "0.005523780499856246", + "exact_val_decision_score": "0.07624569272001312", + "exact_val_gap": "0.0016367877861984403", + "exact_val_ranking_score": "0.07460890493381468", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.03308736705588518", + "ranking_selected_val_pr_auc": "0.03446421323907367", + "scope": "top10", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7521966134873119", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.05252508187646232, + "tolerated_test_gap": "0.05252508187646232", + "tolerated_val_decision_score": "0.7227897891452092", + "tolerated_val_gap": "0.021271512338662646", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7521966134873119", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.05252508187646232, + "union_test_gap": "0.05252508187646232", + "union_val_decision_score": "0.7227897891452092", + "union_val_gap": "0.021271512338662646", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.043377707581227436", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0023651634514294945, + "exact_test_gap": "0.0023651634514294945", + "exact_val_decision_score": "0.04303681559882578", + "exact_val_gap": "0.0005049166746326425", + "exact_val_ranking_score": "0.04253189892419314", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.018519739310339497", + "ranking_selected_val_pr_auc": "0.01953230111607746", + "scope": "top20", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.762536895087842", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.044004424364723516, + "tolerated_test_gap": "0.044004424364723516", + "tolerated_val_decision_score": "0.7274867592832499", + "tolerated_val_gap": "0.036811345935636286", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.762536895087842", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.044004424364723516, + "union_test_gap": "0.044004424364723516", + "union_val_decision_score": "0.7274867592832499", + "union_val_gap": "0.036811345935636286", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0016250253910217347", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0016250253910217347", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0007269780629045868", + "ranking_selected_val_pr_auc": "0.0004442608756800871", + "scope": "global", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.3494091553610714", + "tolerated_ranking_score": "0.3494091553610714", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.41347824096521096", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.41347824096521096", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.3494091553610714", + "union_ranking_score": "0.3494091553610714", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.41347824096521096", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.41347824096521096" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.14089481946624804", + "exact_ranking_score": "0.14089481946624804", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.15619694397283532", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.15619694397283532", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.0672622238447139", + "ranking_selected_val_pr_auc": "0.07740571441869959", + "scope": "top5", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.8022627296490505", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 0.030847809165807094, + "tolerated_test_gap": "0.030847809165807094", + "tolerated_val_decision_score": "0.8078282610569877", + "tolerated_val_gap": "0.010634809670419076", + "tolerated_val_ranking_score": "0.7971934513865686", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.8022627296490505", + "union_ranking_score": "0.7714149204832434", + "union_regret": 0.030847809165807094, + "union_test_gap": "0.030847809165807094", + "union_val_decision_score": "0.8078282610569877", + "union_val_gap": "0.010634809670419076", + "union_val_ranking_score": "0.7971934513865686" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.08289870985451551", + "exact_ranking_score": "0.08289870985451551", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.09763863093658795", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.09763863093658795", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.037689588394968596", + "ranking_selected_val_pr_auc": "0.0445507563162313", + "scope": "top10", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7536939930764294", + "tolerated_ranking_score": "0.6997663155755749", + "tolerated_regret": 0.053927677500854476, + "tolerated_test_gap": "0.053927677500854476", + "tolerated_val_decision_score": "0.7243805445090241", + "tolerated_val_gap": "0.02259882240347566", + "tolerated_val_ranking_score": "0.7017817221055485", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7536939930764294", + "union_ranking_score": "0.6997663155755749", + "union_regret": 0.053927677500854476, + "union_test_gap": "0.053927677500854476", + "union_val_decision_score": "0.7243805445090241", + "union_val_gap": "0.02259882240347566", + "union_val_ranking_score": "0.7017817221055485" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.04819843096725701", + "exact_ranking_score": "0.04819843096725701", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.05605272285813389", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.05605272285813389", + "family": "DLWP", + "model_tag": "dlwp", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.02136633936888583", + "ranking_selected_val_pr_auc": "0.024917289620763165", + "scope": "top20", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7721727827291924", + "tolerated_ranking_score": "0.7187730423241409", + "tolerated_regret": 0.05339974040505158, + "tolerated_test_gap": "0.05339974040505158", + "tolerated_val_decision_score": "0.7238375070567806", + "tolerated_val_gap": "0.032337052985272896", + "tolerated_val_ranking_score": "0.6915004540715077", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7721727827291924", + "union_ranking_score": "0.7187730423241409", + "union_regret": 0.05339974040505158, + "union_test_gap": "0.05339974040505158", + "union_val_decision_score": "0.7238375070567806", + "union_val_gap": "0.032337052985272896", + "union_val_ranking_score": "0.6915004540715077" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.000946969696969697", + "exact_ranking_score": "0.000946969696969697", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.004184100418410042", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.004184100418410042", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0003658682156259175", + "ranking_selected_val_pr_auc": "0.0004302667397755488", + "scope": "global", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.31130402819686975", + "tolerated_ranking_score": "0.31130402819686975", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.3016823150717557", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.3016823150717557", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.31130402819686975", + "union_ranking_score": "0.31130402819686975", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.3016823150717557", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.3016823150717557" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.12059329320722269", + "exact_ranking_score": "0.11833577252770228", + "exact_regret": 0.002257520679520411, + "exact_test_gap": "0.002257520679520411", + "exact_val_decision_score": "0.12220272049144364", + "exact_val_gap": "0.0028839450238853925", + "exact_val_ranking_score": "0.11931877546755824", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.0689177522952707", + "ranking_selected_val_pr_auc": "0.06437104063472687", + "scope": "top5", + "seed": 1, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7714385719285964", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 2.365144535299013e-05, + "tolerated_test_gap": "2.365144535299013e-05", + "tolerated_val_decision_score": "0.7976559342583861", + "tolerated_val_gap": "0.00046248287181749337", + "tolerated_val_ranking_score": "0.7971934513865686", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7714385719285964", + "union_ranking_score": "0.7714149204832434", + "union_regret": 2.365144535299013e-05, + "union_test_gap": "2.365144535299013e-05", + "union_val_decision_score": "0.7976559342583861", + "union_val_gap": "0.00046248287181749337", + "union_val_ranking_score": "0.7971934513865686" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07431424013174823", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.0010578427408801938, + "exact_test_gap": "0.0010578427408801938", + "exact_val_decision_score": "0.07578837633567892", + "exact_val_gap": "0.0011794714018642377", + "exact_val_ranking_score": "0.07460890493381468", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.03927901160149114", + "ranking_selected_val_pr_auc": "0.03716454445172032", + "scope": "top10", + "seed": 1, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.6756347251105248", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.0, + "tolerated_test_gap": "-0.024036806500324803", + "tolerated_val_decision_score": "0.7162675437027928", + "tolerated_val_gap": "0.014749266896246227", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.6756347251105248", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.0, + "union_test_gap": "-0.024036806500324803", + "union_val_decision_score": "0.7162675437027928", + "union_val_gap": "0.014749266896246227", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.042069180430040515", + "exact_ranking_score": "0.042069180430040515", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04316055094101566", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04316055094101566", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.017947733209238474", + "ranking_selected_val_pr_auc": "0.019698135675792587", + "scope": "top20", + "seed": 1, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7182175622542595", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "-0.0003149084688589543", + "tolerated_val_decision_score": "0.6906967717983766", + "tolerated_val_gap": "2.135845076300935e-05", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7182175622542595", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "-0.0003149084688589543", + "union_val_decision_score": "0.6906967717983766", + "union_val_gap": "2.135845076300935e-05", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0006047777441790143", + "exact_ranking_score": "0.0006047777441790143", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0034982508745627187", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0034982508745627187", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0004059591718395402", + "ranking_selected_val_pr_auc": "0.00044924149635682735", + "scope": "global", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.3100602050525378", + "tolerated_ranking_score": "0.3100602050525378", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.30078743758851223", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.30078743758851223", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.3100602050525378", + "union_ranking_score": "0.3100602050525378", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.30078743758851223", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.30078743758851223" + }, + { + "exact_decision_head": "pixel MLP head", + "exact_decision_head_arch": "pixel_mlp", + "exact_decision_score": "0.11833577252770228", + "exact_ranking_score": "0.11833577252770228", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.11931877546755824", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.11931877546755824", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.058898489734440555", + "ranking_selected_val_pr_auc": "0.07226448398929738", + "scope": "top5", + "seed": 123, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.793939393939394", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 0.022524473456150496, + "tolerated_test_gap": "0.022524473456150496", + "tolerated_val_decision_score": "0.8166754936823787", + "tolerated_val_gap": "0.019482042295810076", + "tolerated_val_ranking_score": "0.7971934513865686", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.793939393939394", + "union_ranking_score": "0.7714149204832434", + "union_regret": 0.022524473456150496, + "union_test_gap": "0.022524473456150496", + "union_val_decision_score": "0.8166754936823787", + "union_val_gap": "0.019482042295810076", + "union_val_ranking_score": "0.7971934513865686" + }, + { + "exact_decision_head": "pixel MLP head", + "exact_decision_head_arch": "pixel_mlp", + "exact_decision_score": "0.07325639739086803", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.07460890493381468", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.07460890493381468", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.03552513494863446", + "ranking_selected_val_pr_auc": "0.042774542140641386", + "scope": "top10", + "seed": 123, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7206692168649377", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.020997685254088072, + "tolerated_test_gap": "0.020997685254088072", + "tolerated_val_decision_score": "0.7230799450009822", + "tolerated_val_gap": "0.021561668194435613", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7206692168649377", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.020997685254088072, + "union_test_gap": "0.020997685254088072", + "union_val_decision_score": "0.7230799450009822", + "union_val_gap": "0.021561668194435613", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04104131693016963", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 2.877280037168889e-05, + "exact_test_gap": "2.877280037168889e-05", + "exact_val_decision_score": "0.04257343918258997", + "exact_val_gap": "4.154025839683134e-05", + "exact_val_ranking_score": "0.04253189892419314", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.01989120112622579", + "ranking_selected_val_pr_auc": "0.024039539026323706", + "scope": "top20", + "seed": 123, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7310509974227326", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.012518526699614174, + "tolerated_test_gap": "0.012518526699614174", + "tolerated_val_decision_score": "0.7058203125", + "tolerated_val_gap": "0.01514489915238637", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7310509974227326", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.012518526699614174, + "union_test_gap": "0.012518526699614174", + "union_val_decision_score": "0.7058203125", + "union_val_gap": "0.01514489915238637", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0030852294639413806", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0030852294639413806", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00036191656354256076", + "ranking_selected_val_pr_auc": "0.0004136690018862186", + "scope": "global", + "seed": 42, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.2870596305028149", + "tolerated_ranking_score": "0.31490583114252463", + "tolerated_regret": 0.0, + "tolerated_test_gap": "-0.027846200639709717", + "tolerated_val_decision_score": "0.31159367647460456", + "tolerated_val_gap": "0.009129115930882226", + "tolerated_val_ranking_score": "0.30246456054372234", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.2870596305028149", + "union_ranking_score": "0.31490583114252463", + "union_regret": 0.0, + "union_test_gap": "-0.027846200639709717", + "union_val_decision_score": "0.31159367647460456", + "union_val_gap": "0.009129115930882226", + "union_val_ranking_score": "0.30246456054372234" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.11846022968949382", + "exact_ranking_score": "0.11754630125254256", + "exact_regret": 0.0009139284369512612, + "exact_test_gap": "0.0009139284369512612", + "exact_val_decision_score": "0.12063219008708741", + "exact_val_gap": "0.0033196374333384765", + "exact_val_ranking_score": "0.11731255265374893", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.058729135567964254", + "ranking_selected_val_pr_auc": "0.05944644145445249", + "scope": "top5", + "seed": 42, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7714385719285964", + "tolerated_ranking_score": "0.7714385719285964", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7976559342583861", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7976559342583861", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7714385719285964", + "union_ranking_score": "0.7714385719285964", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7976559342583861", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7976559342583861" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07337579617834394", + "exact_ranking_score": "0.07337579617834394", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.07542516569901865", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.07542516569901865", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.031759062375776324", + "ranking_selected_val_pr_auc": "0.034979775498206175", + "scope": "top10", + "seed": 42, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.49088319481106213", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.0, + "tolerated_test_gap": "-0.20878833679978748", + "tolerated_val_decision_score": "0.7133624143149648", + "tolerated_val_gap": "0.011793150245700712", + "tolerated_val_ranking_score": "0.7015692640692641", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.49088319481106213", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.0, + "union_test_gap": "-0.20878833679978748", + "union_val_decision_score": "0.7133624143149648", + "union_val_gap": "0.011793150245700712", + "union_val_ranking_score": "0.7015692640692641" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04137542565993292", + "exact_ranking_score": "0.04137542565993292", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.043099210968826795", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.043099210968826795", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.017928450700376908", + "ranking_selected_val_pr_auc": "0.01946275083543708", + "scope": "top20", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7189733167245789", + "tolerated_ranking_score": "0.7189733167245789", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6912897822445562", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6912897822445562", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7189733167245789", + "union_ranking_score": "0.7189733167245789", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6912897822445562", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6912897822445562" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.001086071137659517", + "exact_ranking_score": "0.001086071137659517", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.003349460364719018", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.003349460364719018", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0003268363416054406", + "ranking_selected_val_pr_auc": "0.00039156287297214436", + "scope": "global", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.3051941376005135", + "tolerated_ranking_score": "0.3051941376005135", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.2948179787588662", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.2948179787588662", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.3051941376005135", + "union_ranking_score": "0.3051941376005135", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.2948179787588662", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.2948179787588662" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.13187490371283314", + "exact_ranking_score": "0.13187490371283314", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.12086776859504132", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.12086776859504132", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.07575936549127547", + "ranking_selected_val_pr_auc": "0.0704268488739946", + "scope": "top5", + "seed": 7, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7714065672477771", + "tolerated_ranking_score": "0.7714065672477771", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7976559342583861", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7976559342583861", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7714065672477771", + "union_ranking_score": "0.7714065672477771", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7976559342583861", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7976559342583861" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0705085565701155", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.0, + "exact_test_gap": "-0.0027478408207525368", + "exact_val_decision_score": "0.07484534819623935", + "exact_val_gap": "0.00023644326242466818", + "exact_val_ranking_score": "0.07460890493381468", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.03431519870355593", + "ranking_selected_val_pr_auc": "0.04346900418958381", + "scope": "top10", + "seed": 7, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.6993745656706046", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.0, + "tolerated_test_gap": "-0.00029696594024497536", + "tolerated_val_decision_score": "0.7015653241482795", + "tolerated_val_gap": "4.7047341732997516e-05", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.6993745656706046", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.0, + "union_test_gap": "-0.00029696594024497536", + "union_val_decision_score": "0.7015653241482795", + "union_val_gap": "4.7047341732997516e-05", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.040288313241635454", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "-0.0007242308881624873", + "exact_val_decision_score": "0.04306006752022422", + "exact_val_gap": "0.0005281685960310811", + "exact_val_ranking_score": "0.04253189892419314", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.02050876208409485", + "ranking_selected_val_pr_auc": "0.02590116599560572", + "scope": "top20", + "seed": 7, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7644129739607127", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.045880503237594294, + "tolerated_test_gap": "0.045880503237594294", + "tolerated_val_decision_score": "0.6962583126352054", + "tolerated_val_gap": "0.005582899287591814", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7644129739607127", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.045880503237594294, + "union_test_gap": "0.045880503237594294", + "union_val_decision_score": "0.6962583126352054", + "union_val_gap": "0.005582899287591814", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.001445086705202312", + "exact_ranking_score": "0.001445086705202312", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0036289408029031526", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0036289408029031526", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00037608715321531055", + "ranking_selected_val_pr_auc": "0.0004397940562535402", + "scope": "global", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.3060314081779048", + "tolerated_ranking_score": "0.3060314081779048", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.29429605373755735", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.29429605373755735", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.3060314081779048", + "union_ranking_score": "0.3060314081779048", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.29429605373755735", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.29429605373755735" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.12161541991739329", + "exact_ranking_score": "0.12161541991739329", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.12163050624589085", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.12163050624589085", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.09974311720982598", + "ranking_selected_val_pr_auc": "0.10326265767079287", + "scope": "top5", + "seed": 99, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7714065672477771", + "tolerated_ranking_score": "0.7714065672477771", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7976559342583861", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7976559342583861", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7714065672477771", + "union_ranking_score": "0.7714065672477771", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7976559342583861", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7976559342583861" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.07668301382077575", + "exact_ranking_score": "0.07668301382077575", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.07647961072618607", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.07647961072618607", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.058967493704484314", + "ranking_selected_val_pr_auc": "0.06262653319620513", + "scope": "top10", + "seed": 99, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7063465223762824", + "tolerated_ranking_score": "0.7063465223762824", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7021793497677742", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7021793497677742", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7063465223762824", + "union_ranking_score": "0.7063465223762824", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7021793497677742", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7021793497677742" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.042822140550172624", + "exact_ranking_score": "0.042822140550172624", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.043420983295568316", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.043420983295568316", + "family": "FCN", + "model_tag": "fcn", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.029098665712304895", + "ranking_selected_val_pr_auc": "0.032221638150467385", + "scope": "top20", + "seed": 99, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.726408418760773", + "tolerated_ranking_score": "0.726408418760773", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6933831038489982", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6933831038489982", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.726408418760773", + "union_ranking_score": "0.726408418760773", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6933831038489982", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6933831038489982" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0031948881789137383", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0031948881789137383", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.000398077435365184", + "ranking_selected_val_pr_auc": "0.00047223676755013246", + "scope": "global", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.31071390711162444", + "tolerated_ranking_score": "0.31071390711162444", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.3130246199713141", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.3130246199713141", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.31071390711162444", + "union_ranking_score": "0.31071390711162444", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.3130246199713141", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.3130246199713141" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.12371632233901173", + "exact_ranking_score": "0.12371632233901173", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.12572845629262244", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.12572845629262244", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.06541062880458605", + "ranking_selected_val_pr_auc": "0.06522920883515884", + "scope": "top5", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7834980065869301", + "tolerated_ranking_score": "0.7834980065869301", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.8136773785024538", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.8136773785024538", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7834980065869301", + "union_ranking_score": "0.7834980065869301", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.8136773785024538", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.8136773785024538" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07892387357895363", + "exact_ranking_score": "0.07892387357895363", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.08108271403540572", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.08108271403540572", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.04061009784825756", + "ranking_selected_val_pr_auc": "0.040143560685329066", + "scope": "top10", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7136104648359283", + "tolerated_ranking_score": "0.7136104648359283", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7207024927160893", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7207024927160893", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7136104648359283", + "union_ranking_score": "0.7136104648359283", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7207024927160893", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7207024927160893" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04452825597664091", + "exact_ranking_score": "0.04452825597664091", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0455414882958375", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0455414882958375", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.022736128443885992", + "ranking_selected_val_pr_auc": "0.021286665509494304", + "scope": "top20", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7269980510116651", + "tolerated_ranking_score": "0.7269980510116651", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7088464801269441", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7088464801269441", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7269980510116651", + "union_ranking_score": "0.7269980510116651", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7088464801269441", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7088464801269441" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.003656307129798903", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.003656307129798903", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0004116035724473925", + "ranking_selected_val_pr_auc": "0.0004377390722357372", + "scope": "global", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.3145618361221859", + "tolerated_ranking_score": "0.3145618361221859", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.3129112248106385", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.3129112248106385", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.3145618361221859", + "union_ranking_score": "0.3145618361221859", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.3129112248106385", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.3129112248106385" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.11817781690140844", + "exact_ranking_score": "0.11833577252770228", + "exact_regret": 0.0, + "exact_test_gap": "-0.00015795562629383753", + "exact_val_decision_score": "0.12163664665332445", + "exact_val_gap": "0.002317871185766207", + "exact_val_ranking_score": "0.11931877546755824", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.06604572317905974", + "ranking_selected_val_pr_auc": "0.0732912945840262", + "scope": "top5", + "seed": 123, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7862562878335749", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 0.01484136735033148, + "tolerated_test_gap": "0.01484136735033148", + "tolerated_val_decision_score": "0.8131524008350731", + "tolerated_val_gap": "0.01595894944850451", + "tolerated_val_ranking_score": "0.7971934513865686", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7862562878335749", + "union_ranking_score": "0.7714149204832434", + "union_regret": 0.01484136735033148, + "union_test_gap": "0.01484136735033148", + "union_val_decision_score": "0.8131524008350731", + "union_val_gap": "0.01595894944850451", + "union_val_ranking_score": "0.7971934513865686" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07409381663113006", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.0008374192402620245, + "exact_test_gap": "0.0008374192402620245", + "exact_val_decision_score": "0.07709239276319321", + "exact_val_gap": "0.0024834878293785306", + "exact_val_ranking_score": "0.07460890493381468", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.038869102384950775", + "ranking_selected_val_pr_auc": "0.04346587500401188", + "scope": "top10", + "seed": 123, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7099454687094261", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.010273937098576491, + "tolerated_test_gap": "0.010273937098576491", + "tolerated_val_decision_score": "0.7148757170172084", + "tolerated_val_gap": "0.013357440210661875", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7099454687094261", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.010273937098576491, + "union_test_gap": "0.010273937098576491", + "union_val_decision_score": "0.7148757170172084", + "union_val_gap": "0.013357440210661875", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04171607200603644", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0007035278762385008, + "exact_test_gap": "0.0007035278762385008", + "exact_val_decision_score": "0.043619564341543335", + "exact_val_gap": "0.0010876654173501976", + "exact_val_ranking_score": "0.04253189892419314", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.020495144103834257", + "ranking_selected_val_pr_auc": "0.023246350918909517", + "scope": "top20", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7251252524331628", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.006592781710044404, + "tolerated_test_gap": "0.006592781710044404", + "tolerated_val_decision_score": "0.7015002586652871", + "tolerated_val_gap": "0.01082484531767347", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7251252524331628", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.006592781710044404, + "union_test_gap": "0.006592781710044404", + "union_val_decision_score": "0.7015002586652871", + "union_val_gap": "0.01082484531767347", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.001528350909368791", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.001528350909368791", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00036987379624400454", + "ranking_selected_val_pr_auc": "0.00039852867365630293", + "scope": "global", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.31219058559732665", + "tolerated_ranking_score": "0.31219058559732665", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.31476872634310843", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.31476872634310843", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.31219058559732665", + "union_ranking_score": "0.31219058559732665", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.31476872634310843", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.31476872634310843" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.12233412322274881", + "exact_ranking_score": "0.12047131373196783", + "exact_regret": 0.0018628094907809783, + "exact_test_gap": "0.0018628094907809783", + "exact_val_decision_score": "0.12295192958700069", + "exact_val_gap": "0.004349514799095983", + "exact_val_ranking_score": "0.1186024147879047", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.08182944446297657", + "ranking_selected_val_pr_auc": "0.06677724388642242", + "scope": "top5", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7830574200952944", + "tolerated_ranking_score": "0.77137519758586", + "tolerated_regret": 0.011682222509434403, + "tolerated_test_gap": "0.011682222509434403", + "tolerated_val_decision_score": "0.8181423987511924", + "tolerated_val_gap": "0.020669744358910225", + "tolerated_val_ranking_score": "0.7974726543922822", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7830574200952944", + "union_ranking_score": "0.77137519758586", + "union_regret": 0.011682222509434403, + "union_test_gap": "0.011682222509434403", + "union_val_decision_score": "0.8181423987511924", + "union_val_gap": "0.020669744358910225", + "union_val_ranking_score": "0.7974726543922822" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0804971885173128", + "exact_ranking_score": "0.0804971885173128", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.08148543948704247", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.08148543948704247", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.03819635082226218", + "ranking_selected_val_pr_auc": "0.038941681913664185", + "scope": "top10", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7115204223666349", + "tolerated_ranking_score": "0.7115204223666349", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7256034753011779", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7256034753011779", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7115204223666349", + "union_ranking_score": "0.7115204223666349", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7256034753011779", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7256034753011779" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04574838388861263", + "exact_ranking_score": "0.04574838388861263", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04613821138211382", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04613821138211382", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.021165185967854567", + "ranking_selected_val_pr_auc": "0.020671268187642432", + "scope": "top20", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7265705731122933", + "tolerated_ranking_score": "0.7265705731122933", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7074814597750377", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7074814597750377", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7265705731122933", + "union_ranking_score": "0.7265705731122933", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7074814597750377", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7074814597750377" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0027012425715829285", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0027012425715829285", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00036963872610239895", + "ranking_selected_val_pr_auc": "0.00042054736055892957", + "scope": "global", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.30641904273669174", + "tolerated_ranking_score": "0.30641904273669174", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.30868675618673724", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.30868675618673724", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.30641904273669174", + "union_ranking_score": "0.30641904273669174", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.30868675618673724", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.30868675618673724" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.11923365086160762", + "exact_ranking_score": "0.120447564721369", + "exact_regret": 0.0, + "exact_test_gap": "-0.001213913859761384", + "exact_val_decision_score": "0.12151351351351353", + "exact_val_gap": "0.003267042862839506", + "exact_val_ranking_score": "0.11824647065067402", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.055750221614838426", + "ranking_selected_val_pr_auc": "0.06442862811569598", + "scope": "top5", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.779811109738517", + "tolerated_ranking_score": "0.77137519758586", + "tolerated_regret": 0.008435912152656933, + "tolerated_test_gap": "0.008435912152656933", + "tolerated_val_decision_score": "0.8111111111111111", + "tolerated_val_gap": "0.013638456718828906", + "tolerated_val_ranking_score": "0.7974726543922822", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.779811109738517", + "union_ranking_score": "0.77137519758586", + "union_regret": 0.008435912152656933, + "union_test_gap": "0.008435912152656933", + "union_val_decision_score": "0.8111111111111111", + "union_val_gap": "0.013638456718828906", + "union_val_ranking_score": "0.7974726543922822" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07741464305276612", + "exact_ranking_score": "0.07741464305276612", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.07768666378938284", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.07768666378938284", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0356468915576778", + "ranking_selected_val_pr_auc": "0.03730439444340619", + "scope": "top10", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7094619133450868", + "tolerated_ranking_score": "0.7094619133450868", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7188508982531667", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7188508982531667", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7094619133450868", + "union_ranking_score": "0.7094619133450868", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7188508982531667", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7188508982531667" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.043763127370763974", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0027505832409660327, + "exact_test_gap": "0.0027505832409660327", + "exact_val_decision_score": "0.04404735872043984", + "exact_val_gap": "0.0015154597962467042", + "exact_val_ranking_score": "0.04253189892419314", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.016801706970978273", + "ranking_selected_val_pr_auc": "0.022509307119807316", + "scope": "top20", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.722901134194411", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.004368663471292611, + "tolerated_test_gap": "0.004368663471292611", + "tolerated_val_decision_score": "0.7033268834654516", + "tolerated_val_gap": "0.01265147011783796", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.722901134194411", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.004368663471292611, + "union_test_gap": "0.004368663471292611", + "union_val_decision_score": "0.7033268834654516", + "union_val_gap": "0.01265147011783796", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0041356492969396195", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0041356492969396195", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00042782651526874734", + "ranking_selected_val_pr_auc": "0.0004271990954031648", + "scope": "global", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.3111214819309595", + "tolerated_ranking_score": "0.3111214819309595", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.30674980181224504", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.30674980181224504", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.3111214819309595", + "union_ranking_score": "0.3111214819309595", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.30674980181224504", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.30674980181224504" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.12190521103513322", + "exact_ranking_score": "0.12125309894072571", + "exact_regret": 0.000652112094407506, + "exact_test_gap": "0.000652112094407506", + "exact_val_decision_score": "0.12268433321064899", + "exact_val_gap": "0.0024291153021381295", + "exact_val_ranking_score": "0.12025521790851086", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.06535531985010146", + "ranking_selected_val_pr_auc": "0.07922769803833127", + "scope": "top5", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7803954553118283", + "tolerated_ranking_score": "0.77137519758586", + "tolerated_regret": 0.009020257725968261, + "tolerated_test_gap": "0.009020257725968261", + "tolerated_val_decision_score": "0.804416403785489", + "tolerated_val_gap": "0.006943749393206744", + "tolerated_val_ranking_score": "0.7974726543922822", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7803954553118283", + "union_ranking_score": "0.77137519758586", + "union_regret": 0.009020257725968261, + "union_test_gap": "0.009020257725968261", + "union_val_decision_score": "0.804416403785489", + "union_val_gap": "0.006943749393206744", + "union_val_ranking_score": "0.7974726543922822" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07714516687839242", + "exact_ranking_score": "0.07550644567219153", + "exact_regret": 0.0016387212062008855, + "exact_test_gap": "0.0016387212062008855", + "exact_val_decision_score": "0.0780302129178066", + "exact_val_gap": "0.0027111964223195256", + "exact_val_ranking_score": "0.07531901649548707", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.04049928566381453", + "ranking_selected_val_pr_auc": "0.051914070260881295", + "scope": "top10", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7095126183114994", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.00984108670064976, + "tolerated_test_gap": "0.00984108670064976", + "tolerated_val_decision_score": "0.7096515175847117", + "tolerated_val_gap": "0.00813324077816513", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7095126183114994", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.00984108670064976, + "union_test_gap": "0.00984108670064976", + "union_val_decision_score": "0.7096515175847117", + "union_val_gap": "0.00813324077816513", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04882017900732303", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04882017900732303", + "family": "FengWu", + "model_tag": "fengwu", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.02199779031689959", + "ranking_selected_val_pr_auc": "0.030768274290330223", + "scope": "top20", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7336829717908426", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.015150501067724198, + "tolerated_test_gap": "0.015150501067724198", + "tolerated_val_decision_score": "0.6987796297897932", + "tolerated_val_gap": "0.008104216442179624", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7336829717908426", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.015150501067724198, + "union_test_gap": "0.015150501067724198", + "union_val_decision_score": "0.6987796297897932", + "union_val_gap": "0.008104216442179624", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0006866311182961118", + "exact_ranking_score": "0.0006866311182961118", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0008811590083408097", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0008811590083408097", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0003596048414560045", + "ranking_selected_val_pr_auc": "0.0004121169333774252", + "scope": "global", + "seed": 1, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.21516044416153002", + "tolerated_ranking_score": "0.3091927111996169", + "tolerated_regret": 0.0, + "tolerated_test_gap": "-0.09403226703808687", + "tolerated_val_decision_score": "0.3118144595732139", + "tolerated_val_gap": "0.001671993688703799", + "tolerated_val_ranking_score": "0.3101424658845101", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.21516044416153002", + "union_ranking_score": "0.3091927111996169", + "union_regret": 0.0, + "union_test_gap": "-0.09403226703808687", + "union_val_decision_score": "0.3118144595732139", + "union_val_gap": "0.001671993688703799", + "union_val_ranking_score": "0.3101424658845101" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.12601927353595255", + "exact_ranking_score": "0.1189950593924104", + "exact_regret": 0.007024214143542151, + "exact_test_gap": "0.007024214143542151", + "exact_val_decision_score": "0.12394433210419888", + "exact_val_gap": "0.004525739612303029", + "exact_val_ranking_score": "0.11941859249189585", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.06032490713940969", + "ranking_selected_val_pr_auc": "0.0614132176206002", + "scope": "top5", + "seed": 1, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.8241199478487613", + "tolerated_ranking_score": "0.7739636861939019", + "tolerated_regret": 0.050156261654859424, + "tolerated_test_gap": "0.050156261654859424", + "tolerated_val_decision_score": "0.8658519277773397", + "tolerated_val_gap": "0.06831247872679147", + "tolerated_val_ranking_score": "0.7975394490505482", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.8241199478487613", + "union_ranking_score": "0.7739636861939019", + "union_regret": 0.050156261654859424, + "union_test_gap": "0.050156261654859424", + "union_val_decision_score": "0.8658519277773397", + "union_val_gap": "0.06831247872679147", + "union_val_ranking_score": "0.7975394490505482" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07447712973984016", + "exact_ranking_score": "0.07447712973984016", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.08072936959401543", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.08072936959401543", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.03373982859419053", + "ranking_selected_val_pr_auc": "0.037115310159824", + "scope": "top10", + "seed": 1, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.6935334872979214", + "tolerated_ranking_score": "0.7046101309049516", + "tolerated_regret": 0.0, + "tolerated_test_gap": "-0.011076643607030201", + "tolerated_val_decision_score": "0.754546876833418", + "tolerated_val_gap": "0.04256271595760763", + "tolerated_val_ranking_score": "0.7119841608758104", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.6935334872979214", + "union_ranking_score": "0.7046101309049516", + "union_regret": 0.0, + "union_test_gap": "-0.011076643607030201", + "union_val_decision_score": "0.754546876833418", + "union_val_gap": "0.04256271595760763", + "union_val_ranking_score": "0.7119841608758104" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.043196160341303", + "exact_ranking_score": "0.043196160341303", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04576053854784421", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04576053854784421", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.01904942261850253", + "ranking_selected_val_pr_auc": "0.020787588115504", + "scope": "top20", + "seed": 1, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7261195534617713", + "tolerated_ranking_score": "0.7246636456247585", + "tolerated_regret": 0.001455907837012771, + "tolerated_test_gap": "0.001455907837012771", + "tolerated_val_decision_score": "0.7033365957825429", + "tolerated_val_gap": "0.003706992036648815", + "tolerated_val_ranking_score": "0.6996296037458941", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7261195534617713", + "union_ranking_score": "0.7246636456247585", + "union_regret": 0.001455907837012771, + "union_test_gap": "0.001455907837012771", + "union_val_decision_score": "0.7033365957825429", + "union_val_gap": "0.003706992036648815", + "union_val_ranking_score": "0.6996296037458941" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0009888839378897987", + "exact_ranking_score": "0.0009888839378897987", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0010117798317730948", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0010117798317730948", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00039999784430104664", + "ranking_selected_val_pr_auc": "0.00043073606570656576", + "scope": "global", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.3096924811079745", + "tolerated_ranking_score": "0.3096924811079745", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.31039074285584656", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.31039074285584656", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.3096924811079745", + "union_ranking_score": "0.3096924811079745", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.31039074285584656", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.31039074285584656" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.12121896162528217", + "exact_ranking_score": "0.11833577252770228", + "exact_regret": 0.002883189097579897, + "exact_test_gap": "0.002883189097579897", + "exact_val_decision_score": "0.1213910761154856", + "exact_val_gap": "0.00207230064792735", + "exact_val_ranking_score": "0.11931877546755824", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.06578907632994217", + "ranking_selected_val_pr_auc": "0.0774014225555978", + "scope": "top5", + "seed": 123, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7834545900433254", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 0.012039669560081956, + "tolerated_test_gap": "0.012039669560081956", + "tolerated_val_decision_score": "0.8107372400756144", + "tolerated_val_gap": "0.013543788689045821", + "tolerated_val_ranking_score": "0.7971934513865686", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7834545900433254", + "union_ranking_score": "0.7714149204832434", + "union_regret": 0.012039669560081956, + "union_test_gap": "0.012039669560081956", + "union_val_decision_score": "0.8107372400756144", + "union_val_gap": "0.013543788689045821", + "union_val_ranking_score": "0.7971934513865686" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07623735302160241", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.002980955630734375, + "exact_test_gap": "0.002980955630734375", + "exact_val_decision_score": "0.07630118890356671", + "exact_val_gap": "0.0016922839697520314", + "exact_val_ranking_score": "0.07460890493381468", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.03878602989774424", + "ranking_selected_val_pr_auc": "0.04567884589440755", + "scope": "top10", + "seed": 123, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7079514490958633", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.008279917485013666, + "tolerated_test_gap": "0.008279917485013666", + "tolerated_val_decision_score": "0.7130764176104518", + "tolerated_val_gap": "0.01155814080390527", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7079514490958633", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.008279917485013666, + "union_test_gap": "0.008279917485013666", + "union_val_decision_score": "0.7130764176104518", + "union_val_gap": "0.01155814080390527", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "pixel MLP head", + "exact_decision_head_arch": "pixel_mlp", + "exact_decision_score": "0.018543768748295608", + "exact_ranking_score": "0.018543768748295608", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04600811907983762", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04600811907983762", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.021045021724179047", + "ranking_selected_val_pr_auc": "0.025380726482200633", + "scope": "top20", + "seed": 123, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7224961571477053", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0039636864245868875, + "tolerated_test_gap": "0.0039636864245868875", + "tolerated_val_decision_score": "0.6975587357465372", + "tolerated_val_gap": "0.00688332239892353", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7224961571477053", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0039636864245868875, + "union_test_gap": "0.0039636864245868875", + "union_val_decision_score": "0.6975587357465372", + "union_val_gap": "0.00688332239892353", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0003188165529554294", + "exact_ranking_score": "0.0003188165529554294", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0011176098051633573", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0011176098051633573", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0003830459807690152", + "ranking_selected_val_pr_auc": "0.00042901589997107405", + "scope": "global", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.316668570748256", + "tolerated_ranking_score": "0.316668570748256", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.3180456774666437", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.3180456774666437", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.316668570748256", + "union_ranking_score": "0.316668570748256", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.3180456774666437", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.3180456774666437" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.12472723096359249", + "exact_ranking_score": "0.12343429742655432", + "exact_regret": 0.001292933537038174, + "exact_test_gap": "0.001292933537038174", + "exact_val_decision_score": "0.12229812968977491", + "exact_val_gap": "0.0007780236358641696", + "exact_val_ranking_score": "0.12152010605391074", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.07205944796033896", + "ranking_selected_val_pr_auc": "0.06887409447218568", + "scope": "top5", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7659067086974988", + "tolerated_ranking_score": "0.7763794772507261", + "tolerated_regret": 0.0, + "tolerated_test_gap": "-0.010472768553227318", + "tolerated_val_decision_score": "0.810906174819567", + "tolerated_val_gap": "0.009041372954765037", + "tolerated_val_ranking_score": "0.8018648018648019", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7659067086974988", + "union_ranking_score": "0.7763794772507261", + "union_regret": 0.0, + "union_test_gap": "-0.010472768553227318", + "union_val_decision_score": "0.810906174819567", + "union_val_gap": "0.009041372954765037", + "union_val_ranking_score": "0.8018648018648019" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.08151658767772511", + "exact_ranking_score": "0.07757857459052678", + "exact_regret": 0.003938013087198336, + "exact_test_gap": "0.003938013087198336", + "exact_val_decision_score": "0.07882782007806002", + "exact_val_gap": "0.003017081109426334", + "exact_val_ranking_score": "0.07581073896863369", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.0420642409494585", + "ranking_selected_val_pr_auc": "0.037860183103839665", + "scope": "top10", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.6658250522289358", + "tolerated_ranking_score": "0.707219799470722", + "tolerated_regret": 0.0, + "tolerated_test_gap": "-0.041394747241786156", + "tolerated_val_decision_score": "0.7266319786024423", + "tolerated_val_gap": "0.020155767669137514", + "tolerated_val_ranking_score": "0.7064762109333048", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.6658250522289358", + "union_ranking_score": "0.707219799470722", + "union_regret": 0.0, + "union_test_gap": "-0.041394747241786156", + "union_val_decision_score": "0.7266319786024423", + "union_val_gap": "0.020155767669137514", + "union_val_ranking_score": "0.7064762109333048" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04603830266616599", + "exact_ranking_score": "0.04603830266616599", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04428715218026559", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04428715218026559", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.020363596550511454", + "ranking_selected_val_pr_auc": "0.020674507603092498", + "scope": "top20", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7054843984273774", + "tolerated_ranking_score": "0.7054843984273774", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.72648949860566", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.72648949860566", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7054843984273774", + "union_ranking_score": "0.7054843984273774", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.72648949860566", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.72648949860566" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0008450613619815959", + "exact_ranking_score": "0.0008450613619815959", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0009152735719220856", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0009152735719220856", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00037118462783325537", + "ranking_selected_val_pr_auc": "0.0004157684867215522", + "scope": "global", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.3062654921252336", + "tolerated_ranking_score": "0.3062654921252336", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.30677382687191324", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.30677382687191324", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.3062654921252336", + "union_ranking_score": "0.3062654921252336", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.30677382687191324", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.30677382687191324" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.12366683595733877", + "exact_ranking_score": "0.12257405515832481", + "exact_regret": 0.0010927807990139538, + "exact_test_gap": "0.0010927807990139538", + "exact_val_decision_score": "0.1239751552795031", + "exact_val_gap": "0.004866804386744986", + "exact_val_ranking_score": "0.11910835089275812", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.06705127569812482", + "ranking_selected_val_pr_auc": "0.0609692021164841", + "scope": "top5", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7746207358980806", + "tolerated_ranking_score": "0.77137519758586", + "tolerated_regret": 0.0032455383122205195, + "tolerated_test_gap": "0.0032455383122205195", + "tolerated_val_decision_score": "0.8054623862002874", + "tolerated_val_gap": "0.007989731808005218", + "tolerated_val_ranking_score": "0.7974726543922822", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7746207358980806", + "union_ranking_score": "0.77137519758586", + "union_regret": 0.0032455383122205195, + "union_test_gap": "0.0032455383122205195", + "union_val_decision_score": "0.8054623862002874", + "union_val_gap": "0.007989731808005218", + "union_val_ranking_score": "0.7974726543922822" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07886239443616493", + "exact_ranking_score": "0.07886239443616493", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.07953855494839102", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.07953855494839102", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.03317855540638228", + "ranking_selected_val_pr_auc": "0.0369116762896455", + "scope": "top10", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7025612895058863", + "tolerated_ranking_score": "0.7025612895058863", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.710349352140397", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.710349352140397", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7025612895058863", + "union_ranking_score": "0.7025612895058863", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.710349352140397", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.710349352140397" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.044572342126298965", + "exact_ranking_score": "0.044572342126298965", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04500911929271383", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04500911929271383", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.018717347905238046", + "ranking_selected_val_pr_auc": "0.020626878606552698", + "scope": "top20", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7235687421646468", + "tolerated_ranking_score": "0.7235687421646468", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6980281690140845", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6980281690140845", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7235687421646468", + "union_ranking_score": "0.7235687421646468", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6980281690140845", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6980281690140845" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0008606592649969877", + "exact_ranking_score": "0.0008606592649969877", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.000883817480228723", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.000883817480228723", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0003867174887645146", + "ranking_selected_val_pr_auc": "0.00040688996624133387", + "scope": "global", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.3109717285865725", + "tolerated_ranking_score": "0.3109717285865725", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.31099917685286793", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.31099917685286793", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.3109717285865725", + "union_ranking_score": "0.3109717285865725", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.31099917685286793", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.31099917685286793" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.12165729369907036", + "exact_ranking_score": "0.11908268462023987", + "exact_regret": 0.0025746090788304893, + "exact_test_gap": "0.0025746090788304893", + "exact_val_decision_score": "0.12040952592922323", + "exact_val_gap": "0.0009159610029911419", + "exact_val_ranking_score": "0.11949356492623209", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.0679936659760212", + "ranking_selected_val_pr_auc": "0.08132317939286735", + "scope": "top5", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7764511526620109", + "tolerated_ranking_score": "0.7741670094611272", + "tolerated_regret": 0.0022841432008837703, + "tolerated_test_gap": "0.0022841432008837703", + "tolerated_val_decision_score": "0.8023909485519105", + "tolerated_val_gap": "0.004531416779335218", + "tolerated_val_ranking_score": "0.7978595317725753", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7764511526620109", + "union_ranking_score": "0.7741670094611272", + "union_regret": 0.0022841432008837703, + "union_test_gap": "0.0022841432008837703", + "union_val_decision_score": "0.8023909485519105", + "union_val_gap": "0.004531416779335218", + "union_val_ranking_score": "0.7978595317725753" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.06876734046769718", + "exact_ranking_score": "0.07384553133377168", + "exact_regret": 0.0, + "exact_test_gap": "-0.0050781908660745", + "exact_val_decision_score": "0.07681263460157932", + "exact_val_gap": "0.0019560091865476192", + "exact_val_ranking_score": "0.0748566254150317", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.03972371884586323", + "ranking_selected_val_pr_auc": "0.04922131117303804", + "scope": "top10", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7026029306512976", + "tolerated_ranking_score": "0.7027966072827605", + "tolerated_regret": 0.0, + "tolerated_test_gap": "-0.00019367663146285796", + "tolerated_val_decision_score": "0.7072442664725155", + "tolerated_val_gap": "0.0042447748609242275", + "tolerated_val_ranking_score": "0.7029994916115913", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7026029306512976", + "union_ranking_score": "0.7027966072827605", + "union_regret": 0.0, + "union_test_gap": "-0.00019367663146285796", + "union_val_decision_score": "0.7072442664725155", + "union_val_gap": "0.0042447748609242275", + "union_val_ranking_score": "0.7029994916115913" + }, + { + "exact_decision_head": "pixel MLP head", + "exact_decision_head_arch": "pixel_mlp", + "exact_decision_score": "0.027261462205700127", + "exact_ranking_score": "0.04233736419209311", + "exact_regret": 0.0, + "exact_test_gap": "-0.01507590198639298", + "exact_val_decision_score": "0.04932875429285045", + "exact_val_gap": "0.006512143511318333", + "exact_val_ranking_score": "0.042816610781532115", + "family": "FuXi", + "model_tag": "fuxi", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.026062499558490423", + "ranking_selected_val_pr_auc": "0.02771096205491839", + "scope": "top20", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7272787942491131", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.008746323525994693, + "tolerated_test_gap": "0.008746323525994693", + "tolerated_val_decision_score": "0.6966977214053203", + "tolerated_val_gap": "0.006022308057706649", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7272787942491131", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.008746323525994693, + "union_test_gap": "0.008746323525994693", + "union_val_decision_score": "0.6966977214053203", + "union_val_gap": "0.006022308057706649", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.004253509145044662", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.004253509145044662", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00042881385578730365", + "ranking_selected_val_pr_auc": "0.0004901027948814223", + "scope": "global", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.32066974948800614", + "tolerated_ranking_score": "0.32066974948800614", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.3120952399779768", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.3120952399779768", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.32066974948800614", + "union_ranking_score": "0.32066974948800614", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.3120952399779768", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.3120952399779768" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.12675350701402807", + "exact_ranking_score": "0.12675350701402807", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.1289902280130293", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.1289902280130293", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.07239587203738174", + "ranking_selected_val_pr_auc": "0.06555234277501942", + "scope": "top5", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.776857122271057", + "tolerated_ranking_score": "0.776857122271057", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.8022016303211871", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.8022016303211871", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.776857122271057", + "union_ranking_score": "0.776857122271057", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.8022016303211871", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.8022016303211871" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07904849039341263", + "exact_ranking_score": "0.07904849039341263", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.08153057396523697", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.08153057396523697", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.04315307069319409", + "ranking_selected_val_pr_auc": "0.03843818318005278", + "scope": "top10", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7050669731030272", + "tolerated_ranking_score": "0.7050669731030272", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7088338471351122", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7088338471351122", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7050669731030272", + "union_ranking_score": "0.7050669731030272", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7088338471351122", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7088338471351122" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04418397717161179", + "exact_ranking_score": "0.04418397717161179", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04535556523362161", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04535556523362161", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.02328829578760066", + "ranking_selected_val_pr_auc": "0.02010310006243177", + "scope": "top20", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7231388400090598", + "tolerated_ranking_score": "0.7231388400090598", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.698733466652633", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.698733466652633", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7231388400090598", + "union_ranking_score": "0.7231388400090598", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.698733466652633", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.698733466652633" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0042863266180882984", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0042863266180882984", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00043738577276574255", + "ranking_selected_val_pr_auc": "0.0004554894460132563", + "scope": "global", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.31812983009681495", + "tolerated_ranking_score": "0.31812983009681495", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.3113263265858816", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.3113263265858816", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.31812983009681495", + "union_ranking_score": "0.31812983009681495", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.3113263265858816", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.3113263265858816" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.11929903392496069", + "exact_ranking_score": "0.11833577252770228", + "exact_regret": 0.0009632613972584136, + "exact_test_gap": "0.0009632613972584136", + "exact_val_decision_score": "0.12107419208010925", + "exact_val_gap": "0.0017554166125510057", + "exact_val_ranking_score": "0.11931877546755824", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.07483645613667965", + "ranking_selected_val_pr_auc": "0.08124531938595873", + "scope": "top5", + "seed": 123, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7856782652546647", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 0.014263344771421216, + "tolerated_test_gap": "0.014263344771421216", + "tolerated_val_decision_score": "0.8139394851723619", + "tolerated_val_gap": "0.016746033785793313", + "tolerated_val_ranking_score": "0.7971934513865686", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7856782652546647", + "union_ranking_score": "0.7714149204832434", + "union_regret": 0.014263344771421216, + "union_test_gap": "0.014263344771421216", + "union_val_decision_score": "0.8139394851723619", + "union_val_gap": "0.016746033785793313", + "union_val_ranking_score": "0.7971934513865686" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07468382032272133", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.0014274229318533005, + "exact_test_gap": "0.0014274229318533005", + "exact_val_decision_score": "0.07596458629358399", + "exact_val_gap": "0.001355681359769309", + "exact_val_ranking_score": "0.07460890493381468", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.04354779125348941", + "ranking_selected_val_pr_auc": "0.04714252503011615", + "scope": "top10", + "seed": 123, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.711332312404288", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.011660780793438352, + "tolerated_test_gap": "0.011660780793438352", + "tolerated_val_decision_score": "0.7182744065368083", + "tolerated_val_gap": "0.01675612973026175", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.711332312404288", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.011660780793438352, + "union_test_gap": "0.011660780793438352", + "union_val_decision_score": "0.7182744065368083", + "union_val_gap": "0.01675612973026175", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04194314737445728", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0009306032446593365, + "exact_test_gap": "0.0009306032446593365", + "exact_val_decision_score": "0.04314380355334118", + "exact_val_gap": "0.0006119046291480426", + "exact_val_ranking_score": "0.04253189892419314", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.02276041840297606", + "ranking_selected_val_pr_auc": "0.02572965797764821", + "scope": "top20", + "seed": 123, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7261721555350364", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.007639684811918013, + "tolerated_test_gap": "0.007639684811918013", + "tolerated_val_decision_score": "0.7019436364687074", + "tolerated_val_gap": "0.011268223121093768", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7261721555350364", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.007639684811918013, + "union_test_gap": "0.007639684811918013", + "union_val_decision_score": "0.7019436364687074", + "union_val_gap": "0.011268223121093768", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0024802356223841263", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0024802356223841263", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00040651309469793043", + "ranking_selected_val_pr_auc": "0.0004045310999550776", + "scope": "global", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.32154249424354786", + "tolerated_ranking_score": "0.32154249424354786", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.31489341288382094", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.31489341288382094", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.32154249424354786", + "union_ranking_score": "0.32154249424354786", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.31489341288382094", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.31489341288382094" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.12204234122042343", + "exact_ranking_score": "0.11810512654120699", + "exact_regret": 0.003937214679216441, + "exact_test_gap": "0.003937214679216441", + "exact_val_decision_score": "0.12285777876697085", + "exact_val_gap": "0.0046720888180781794", + "exact_val_ranking_score": "0.11818568994889267", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.08436045538757507", + "ranking_selected_val_pr_auc": "0.06648389466400928", + "scope": "top5", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7898933615140744", + "tolerated_ranking_score": "0.7701027275947573", + "tolerated_regret": 0.019790633919317124, + "tolerated_test_gap": "0.019790633919317124", + "tolerated_val_decision_score": "0.8039800995024876", + "tolerated_val_gap": "0.005996860420931505", + "tolerated_val_ranking_score": "0.7979832390815561", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7898933615140744", + "union_ranking_score": "0.7701027275947573", + "union_regret": 0.019790633919317124, + "union_test_gap": "0.019790633919317124", + "union_val_decision_score": "0.8039800995024876", + "union_val_gap": "0.005996860420931505", + "union_val_ranking_score": "0.7979832390815561" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07639117550214027", + "exact_ranking_score": "0.07639117550214027", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.07748859672658975", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.07748859672658975", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.03994690077965589", + "ranking_selected_val_pr_auc": "0.03757162574160602", + "scope": "top10", + "seed": 42, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.5502983042755504", + "tolerated_ranking_score": "0.7194503995322549", + "tolerated_regret": 0.0, + "tolerated_test_gap": "-0.1691520952567045", + "tolerated_val_decision_score": "0.7221556788281226", + "tolerated_val_gap": "0.013579036492356145", + "tolerated_val_ranking_score": "0.7085766423357664", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.5502983042755504", + "union_ranking_score": "0.7194503995322549", + "union_regret": 0.0, + "union_test_gap": "-0.1691520952567045", + "union_val_decision_score": "0.7221556788281226", + "union_val_gap": "0.013579036492356145", + "union_val_ranking_score": "0.7085766423357664" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0430065972893146", + "exact_ranking_score": "0.0430065972893146", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.044111302847282145", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.044111302847282145", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.021420904714594617", + "ranking_selected_val_pr_auc": "0.019667925605377917", + "scope": "top20", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7241279907754397", + "tolerated_ranking_score": "0.7241279907754397", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6965685921876124", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6965685921876124", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7241279907754397", + "union_ranking_score": "0.7241279907754397", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6965685921876124", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6965685921876124" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.003894080996884735", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.003894080996884735", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00038395193539280824", + "ranking_selected_val_pr_auc": "0.0005470853500668261", + "scope": "global", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.31120670593952293", + "tolerated_ranking_score": "0.31120670593952293", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.30691963220059004", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.30691963220059004", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.31120670593952293", + "union_ranking_score": "0.31120670593952293", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.30691963220059004", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.30691963220059004" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.12325789540979415", + "exact_ranking_score": "0.11754630125254256", + "exact_regret": 0.005711594157251587, + "exact_test_gap": "0.005711594157251587", + "exact_val_decision_score": "0.12198694899453988", + "exact_val_gap": "0.004674396340790948", + "exact_val_ranking_score": "0.11731255265374893", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.05616248338975538", + "ranking_selected_val_pr_auc": "0.0679796765173011", + "scope": "top5", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7753490339867437", + "tolerated_ranking_score": "0.7714385719285964", + "tolerated_regret": 0.003910462058147268, + "tolerated_test_gap": "0.003910462058147268", + "tolerated_val_decision_score": "0.8010033444816053", + "tolerated_val_gap": "0.0033474102232191605", + "tolerated_val_ranking_score": "0.7976559342583861", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7753490339867437", + "union_ranking_score": "0.7714385719285964", + "union_regret": 0.003910462058147268, + "union_test_gap": "0.003910462058147268", + "union_val_decision_score": "0.8010033444816053", + "union_val_gap": "0.0033474102232191605", + "union_val_ranking_score": "0.7976559342583861" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07779171894604768", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.004535321555179647, + "exact_test_gap": "0.004535321555179647", + "exact_val_decision_score": "0.07823677257201779", + "exact_val_gap": "0.0036278676382031083", + "exact_val_ranking_score": "0.07460890493381468", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.03227259772140749", + "ranking_selected_val_pr_auc": "0.04023132643529763", + "scope": "top10", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7032262609185687", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.003554729307719051, + "tolerated_test_gap": "0.003554729307719051", + "tolerated_val_decision_score": "0.7063110230649498", + "tolerated_val_gap": "0.004792746258403202", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7032262609185687", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.003554729307719051, + "union_test_gap": "0.003554729307719051", + "union_val_decision_score": "0.7063110230649498", + "union_val_gap": "0.004792746258403202", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04372215277109978", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0027096086413018403, + "exact_test_gap": "0.0027096086413018403", + "exact_val_decision_score": "0.043845404352062356", + "exact_val_gap": "0.0013135054278692188", + "exact_val_ranking_score": "0.04253189892419314", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.018664183428008234", + "ranking_selected_val_pr_auc": "0.024661788918260808", + "scope": "top20", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.719461376864001", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0009289061408825905, + "tolerated_test_gap": "0.0009289061408825905", + "tolerated_val_decision_score": "0.6947253251451462", + "tolerated_val_gap": "0.004049911797532557", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.719461376864001", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0009289061408825905, + "union_test_gap": "0.0009289061408825905", + "union_val_decision_score": "0.6947253251451462", + "union_val_gap": "0.004049911797532557", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0", + "exact_ranking_score": "0.0", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.003532320734722713", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.003532320734722713", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.00044306685015413984", + "ranking_selected_val_pr_auc": "0.0004923922514041874", + "scope": "global", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.32045281275439075", + "tolerated_ranking_score": "0.32045281275439075", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.3031747624303057", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.3031747624303057", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.32045281275439075", + "union_ranking_score": "0.32045281275439075", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.3031747624303057", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.3031747624303057" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.12431310294297229", + "exact_ranking_score": "0.11915180074049142", + "exact_regret": 0.005161302202480872, + "exact_test_gap": "0.005161302202480872", + "exact_val_decision_score": "0.12584327970939285", + "exact_val_gap": "0.005682776510173682", + "exact_val_ranking_score": "0.12016050319921917", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.0711724674009212", + "ranking_selected_val_pr_auc": "0.08911464580470933", + "scope": "top5", + "seed": 99, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7714385719285964", + "tolerated_ranking_score": "0.7714385719285964", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7976559342583861", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7976559342583861", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7714385719285964", + "union_ranking_score": "0.7714385719285964", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7976559342583861", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7976559342583861" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07693219535324798", + "exact_ranking_score": "0.07422212618841832", + "exact_regret": 0.0027100691648296593, + "exact_test_gap": "0.0027100691648296593", + "exact_val_decision_score": "0.07818598599838919", + "exact_val_gap": "0.0029847395688903794", + "exact_val_ranking_score": "0.07520124642949881", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.045241782115254044", + "ranking_selected_val_pr_auc": "0.05810049257853764", + "scope": "top10", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.6996952251947172", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 2.369358386755671e-05, + "tolerated_test_gap": "2.369358386755671e-05", + "tolerated_val_decision_score": "0.7016606351675854", + "tolerated_val_gap": "0.00014235836103881905", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.6996952251947172", + "union_ranking_score": "0.6996715316108496", + "union_regret": 2.369358386755671e-05, + "union_test_gap": "2.369358386755671e-05", + "union_val_decision_score": "0.7016606351675854", + "union_val_gap": "0.00014235836103881905", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "linear probe", + "exact_decision_head_arch": "linear", + "exact_decision_score": "0.0044004400440044", + "exact_ranking_score": "0.0044004400440044", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.06142857142857142", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.06142857142857142", + "family": "Pangu-Weather", + "model_tag": "pangu6", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.024740444457605402", + "ranking_selected_val_pr_auc": "0.036676155849720546", + "scope": "top20", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7191095957774646", + "tolerated_ranking_score": "0.7183382629739177", + "tolerated_regret": 0.0007713328035469091, + "tolerated_test_gap": "0.0007713328035469091", + "tolerated_val_decision_score": "0.6915201983814487", + "tolerated_val_gap": "0.0008234265830721066", + "tolerated_val_ranking_score": "0.6906967717983766", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7191095957774646", + "union_ranking_score": "0.7183382629739177", + "union_regret": 0.0007713328035469091, + "union_test_gap": "0.0007713328035469091", + "union_val_decision_score": "0.6915201983814487", + "union_val_gap": "0.0008234265830721066", + "union_val_ranking_score": "0.6906967717983766" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.0004925501476206198", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0004727121748000204", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0004727121748000204", + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0002463357401629007", + "ranking_selected_val_pr_auc": "0.00023641196480702655", + "scope": "global", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.240793572992212", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.2179368807313983", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.240793572992212", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.2179368807313983" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.11833577252770228", + "exact_ranking_score": "0.11833577252770228", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.11931877546755824", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.11931877546755824", + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.06288888888888888", + "ranking_selected_val_pr_auc": "0.06344444444444444", + "scope": "top5", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7714149204832434", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7971934513865686", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7971934513865686", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7714149204832434", + "union_ranking_score": "0.7714149204832434", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7971934513865686", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7971934513865686" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07325639739086803", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.07460890493381468", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.07460890493381468", + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.03802083333333333", + "ranking_selected_val_pr_auc": "0.03875", + "scope": "top10", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.6996715316108496", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7015182768065465", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.6996715316108496", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7015182768065465", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.02093558282208589", + "ranking_selected_val_pr_auc": "0.0217280163599182", + "scope": "top20", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.0004925501476206198", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0004727121748000204", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0004727121748000204", + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0002463357401629007", + "ranking_selected_val_pr_auc": "0.00023641196480702655", + "scope": "global", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.240793572992212", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.2179368807313983", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.240793572992212", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.2179368807313983" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.11833577252770228", + "exact_ranking_score": "0.11833577252770228", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.11931877546755824", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.11931877546755824", + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.06288888888888888", + "ranking_selected_val_pr_auc": "0.06344444444444444", + "scope": "top5", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7714149204832434", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7971934513865686", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7971934513865686", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7714149204832434", + "union_ranking_score": "0.7714149204832434", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7971934513865686", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7971934513865686" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07325639739086803", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.07460890493381468", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.07460890493381468", + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.03802083333333333", + "ranking_selected_val_pr_auc": "0.03875", + "scope": "top10", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.6996715316108496", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7015182768065465", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.6996715316108496", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7015182768065465", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.02093558282208589", + "ranking_selected_val_pr_auc": "0.0217280163599182", + "scope": "top20", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.0004925501476206198", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0004727121748000204", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0004727121748000204", + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0002463357401629007", + "ranking_selected_val_pr_auc": "0.00023641196480702655", + "scope": "global", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.240793572992212", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.2179368807313983", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.240793572992212", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.2179368807313983" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.11833577252770228", + "exact_ranking_score": "0.11833577252770228", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.11931877546755824", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.11931877546755824", + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.06288888888888888", + "ranking_selected_val_pr_auc": "0.06344444444444444", + "scope": "top5", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7714149204832434", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7971934513865686", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7971934513865686", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7714149204832434", + "union_ranking_score": "0.7714149204832434", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7971934513865686", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7971934513865686" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07325639739086803", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.07460890493381468", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.07460890493381468", + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.03802083333333333", + "ranking_selected_val_pr_auc": "0.03875", + "scope": "top10", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.6996715316108496", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7015182768065465", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.6996715316108496", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7015182768065465", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.02093558282208589", + "ranking_selected_val_pr_auc": "0.0217280163599182", + "scope": "top20", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.0004925501476206198", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0004727121748000204", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0004727121748000204", + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0002463357401629007", + "ranking_selected_val_pr_auc": "0.00023641196480702655", + "scope": "global", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.240793572992212", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.2179368807313983", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.240793572992212", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.2179368807313983" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.11833577252770228", + "exact_ranking_score": "0.11833577252770228", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.11931877546755824", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.11931877546755824", + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.06288888888888888", + "ranking_selected_val_pr_auc": "0.06344444444444444", + "scope": "top5", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7714149204832434", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7971934513865686", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7971934513865686", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7714149204832434", + "union_ranking_score": "0.7714149204832434", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7971934513865686", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7971934513865686" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07325639739086803", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.07460890493381468", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.07460890493381468", + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.03802083333333333", + "ranking_selected_val_pr_auc": "0.03875", + "scope": "top10", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.6996715316108496", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7015182768065465", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.6996715316108496", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7015182768065465", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.02093558282208589", + "ranking_selected_val_pr_auc": "0.0217280163599182", + "scope": "top20", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.00048497822606044473", + "exact_ranking_score": "0.00048497822606044473", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.00046544514931712863", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.00046544514931712863", + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.000244434472614863", + "ranking_selected_val_pr_auc": "0.00023458729090319847", + "scope": "global", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.23755358049655212", + "tolerated_ranking_score": "0.23755358049655212", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.2149459035878704", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.2149459035878704", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.23755358049655212", + "union_ranking_score": "0.23755358049655212", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.2149459035878704", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.2149459035878704" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.11833577252770228", + "exact_ranking_score": "0.11833577252770228", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.11931877546755824", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.11931877546755824", + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.06288888888888888", + "ranking_selected_val_pr_auc": "0.06344444444444444", + "scope": "top5", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7714149204832434", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7971934513865686", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7971934513865686", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7714149204832434", + "union_ranking_score": "0.7714149204832434", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7971934513865686", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7971934513865686" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07325639739086803", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.07460890493381468", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.07460890493381468", + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.03802083333333333", + "ranking_selected_val_pr_auc": "0.03875", + "scope": "top10", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.6996715316108496", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7015182768065465", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.6996715316108496", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7015182768065465", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.02093558282208589", + "ranking_selected_val_pr_auc": "0.0217280163599182", + "scope": "top20", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.019669096378572257", + "exact_ranking_score": "0.019669096378572257", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.016877637130801686", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.016877637130801686", + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0027275598484246552", + "ranking_selected_val_pr_auc": "0.0024772994278327914", + "scope": "global", + "seed": 1, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7286109603326707", + "tolerated_ranking_score": "0.6293736539482548", + "tolerated_regret": 0.09923730638441586, + "tolerated_test_gap": "0.09923730638441586", + "tolerated_val_decision_score": "0.7800653928063089", + "tolerated_val_gap": "0.02211882039338331", + "tolerated_val_ranking_score": "0.7579465724129256", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7286109603326707", + "union_ranking_score": "0.6293736539482548", + "union_regret": 0.09923730638441586, + "union_test_gap": "0.09923730638441586", + "union_val_decision_score": "0.7800653928063089", + "union_val_gap": "0.02211882039338331", + "union_val_ranking_score": "0.7579465724129256" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.24975320829220138", + "exact_ranking_score": "0.24975320829220138", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.2308050255932992", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.2308050255932992", + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.1876883857001677", + "ranking_selected_val_pr_auc": "0.15408804654455288", + "scope": "top5", + "seed": 1, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.9093400335144248", + "tolerated_ranking_score": "0.8936632713133041", + "tolerated_regret": 0.015676762201120686, + "tolerated_test_gap": "0.015676762201120686", + "tolerated_val_decision_score": "0.8904903385864028", + "tolerated_val_gap": "0.029999615649296207", + "tolerated_val_ranking_score": "0.8604907229371066", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.9093400335144248", + "union_ranking_score": "0.8936632713133041", + "union_regret": 0.015676762201120686, + "union_test_gap": "0.015676762201120686", + "union_val_decision_score": "0.8904903385864028", + "union_val_gap": "0.029999615649296207", + "union_val_ranking_score": "0.8604907229371066" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.22468179302711677", + "exact_ranking_score": "0.22468179302711677", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.20189905047476264", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.20189905047476264", + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.1514864942696933", + "ranking_selected_val_pr_auc": "0.12361527601973549", + "scope": "top10", + "seed": 1, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.8764832842263638", + "tolerated_ranking_score": "0.8527949979229895", + "tolerated_regret": 0.023688286303374273, + "tolerated_test_gap": "0.023688286303374273", + "tolerated_val_decision_score": "0.8461060598607641", + "tolerated_val_gap": "0.02668959040704999", + "tolerated_val_ranking_score": "0.8194164694537142", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.8764832842263638", + "union_ranking_score": "0.8527949979229895", + "union_regret": 0.023688286303374273, + "union_test_gap": "0.023688286303374273", + "union_val_decision_score": "0.8461060598607641", + "union_val_gap": "0.02668959040704999", + "union_val_ranking_score": "0.8194164694537142" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.18739352640545145", + "exact_ranking_score": "0.18739352640545145", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.16846229187071501", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.16846229187071501", + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.10815899441738173", + "ranking_selected_val_pr_auc": "0.08968210725224908", + "scope": "top20", + "seed": 1, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.8685988450931266", + "tolerated_ranking_score": "0.8118474430575734", + "tolerated_regret": 0.05675140203555318, + "tolerated_test_gap": "0.05675140203555318", + "tolerated_val_decision_score": "0.8546299882836719", + "tolerated_val_gap": "0.06584679531636395", + "tolerated_val_ranking_score": "0.7887831929673079", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.8685988450931266", + "union_ranking_score": "0.8118474430575734", + "union_regret": 0.05675140203555318, + "union_test_gap": "0.05675140203555318", + "union_val_decision_score": "0.8546299882836719", + "union_val_gap": "0.06584679531636395", + "union_val_ranking_score": "0.7887831929673079" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.02046783625730994", + "exact_ranking_score": "0.02046783625730994", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.022231722958529286", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.022231722958529286", + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0027479122488527847", + "ranking_selected_val_pr_auc": "0.002960918884295192", + "scope": "global", + "seed": 123, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.7342945764100982", + "tolerated_ranking_score": "0.5593234977538011", + "tolerated_regret": 0.17497107865629713, + "tolerated_test_gap": "0.17497107865629713", + "tolerated_val_decision_score": "0.7498273537587483", + "tolerated_val_gap": "0.059135931802591135", + "tolerated_val_ranking_score": "0.6906914219561572", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.7342945764100982", + "union_ranking_score": "0.5593234977538011", + "union_regret": 0.17497107865629713, + "union_test_gap": "0.17497107865629713", + "union_val_decision_score": "0.7498273537587483", + "union_val_gap": "0.059135931802591135", + "union_val_ranking_score": "0.6906914219561572" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.30023455824863177", + "exact_ranking_score": "0.30023455824863177", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.32366412213740453", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.32366412213740453", + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.22871793267947133", + "ranking_selected_val_pr_auc": "0.2090698081382707", + "scope": "top5", + "seed": 123, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.879307317963807", + "tolerated_ranking_score": "0.8766654895872674", + "tolerated_regret": 0.00264182837653959, + "tolerated_test_gap": "0.00264182837653959", + "tolerated_val_decision_score": "0.8667860857895108", + "tolerated_val_gap": "0.027700659869665634", + "tolerated_val_ranking_score": "0.8390854259198451", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.879307317963807", + "union_ranking_score": "0.8766654895872674", + "union_regret": 0.00264182837653959, + "union_test_gap": "0.00264182837653959", + "union_val_decision_score": "0.8667860857895108", + "union_val_gap": "0.027700659869665634", + "union_val_ranking_score": "0.8390854259198451" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.26836581709145424", + "exact_ranking_score": "0.26836581709145424", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.2748815165876777", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.2748815165876777", + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.18388043920625555", + "ranking_selected_val_pr_auc": "0.1622717982352778", + "scope": "top10", + "seed": 123, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.8621715199854308", + "tolerated_ranking_score": "0.8511911675536367", + "tolerated_regret": 0.010980352431794183, + "tolerated_test_gap": "0.010980352431794183", + "tolerated_val_decision_score": "0.8240905796307538", + "tolerated_val_gap": "0.034993802443587296", + "tolerated_val_ranking_score": "0.7890967771871665", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.8621715199854308", + "union_ranking_score": "0.8511911675536367", + "union_regret": 0.010980352431794183, + "union_test_gap": "0.010980352431794183", + "union_val_decision_score": "0.8240905796307538", + "union_val_gap": "0.034993802443587296", + "union_val_ranking_score": "0.7890967771871665" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.2156398104265403", + "exact_ranking_score": "0.2156398104265403", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.20444444444444446", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.20444444444444446", + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.12031424704824639", + "ranking_selected_val_pr_auc": "0.10479806700502603", + "scope": "top20", + "seed": 123, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.8587104723511633", + "tolerated_ranking_score": "0.8190550527106161", + "tolerated_regret": 0.039655419640547196, + "tolerated_test_gap": "0.039655419640547196", + "tolerated_val_decision_score": "0.8349865778850624", + "tolerated_val_gap": "0.06746718455313094", + "tolerated_val_ranking_score": "0.7675193933319314", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.8587104723511633", + "union_ranking_score": "0.8190550527106161", + "union_regret": 0.039655419640547196, + "union_test_gap": "0.039655419640547196", + "union_val_decision_score": "0.8349865778850624", + "union_val_gap": "0.06746718455313094", + "union_val_ranking_score": "0.7675193933319314" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.02939068100358423", + "exact_ranking_score": "0.02939068100358423", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0233226297110022", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0233226297110022", + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0036593166775145225", + "ranking_selected_val_pr_auc": "0.0030153084181398305", + "scope": "global", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.6375251353613843", + "tolerated_ranking_score": "0.6375251353613843", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7530848622062031", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7530848622062031", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.6375251353613843", + "union_ranking_score": "0.6375251353613843", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7530848622062031", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7530848622062031" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.2930207778369739", + "exact_ranking_score": "0.2930207778369739", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.28806133625410735", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.28806133625410735", + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.2392007486709378", + "ranking_selected_val_pr_auc": "0.18866470031812624", + "scope": "top5", + "seed": 42, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.8978895622772368", + "tolerated_ranking_score": "0.898795526537839", + "tolerated_regret": 0.0, + "tolerated_test_gap": "-0.0009059642606021878", + "tolerated_val_decision_score": "0.8718139484243548", + "tolerated_val_gap": "0.02207407287605323", + "tolerated_val_ranking_score": "0.8497398755483015", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.8978895622772368", + "union_ranking_score": "0.898795526537839", + "union_regret": 0.0, + "union_test_gap": "-0.0009059642606021878", + "union_val_decision_score": "0.8718139484243548", + "union_val_gap": "0.02207407287605323", + "union_val_ranking_score": "0.8497398755483015" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.2800697269029634", + "exact_ranking_score": "0.2800697269029634", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.23774647887323946", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.23774647887323946", + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.19246520724380978", + "ranking_selected_val_pr_auc": "0.14576604274214022", + "scope": "top10", + "seed": 42, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.8187209263802845", + "tolerated_ranking_score": "0.8592617140093417", + "tolerated_regret": 0.0, + "tolerated_test_gap": "-0.04054078762905722", + "tolerated_val_decision_score": "0.796582029663564", + "tolerated_val_gap": "0.006808162632569004", + "tolerated_val_ranking_score": "0.789773867030995", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.8187209263802845", + "union_ranking_score": "0.8592617140093417", + "union_regret": 0.0, + "union_test_gap": "-0.04054078762905722", + "union_val_decision_score": "0.796582029663564", + "union_val_gap": "0.006808162632569004", + "union_val_ranking_score": "0.789773867030995" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.22105263157894736", + "exact_ranking_score": "0.22105263157894736", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.17796610169491525", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.17796610169491525", + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.12467047355946634", + "ranking_selected_val_pr_auc": "0.0921769344676084", + "scope": "top20", + "seed": 42, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.8018567656897366", + "tolerated_ranking_score": "0.8307715292671506", + "tolerated_regret": 0.0, + "tolerated_test_gap": "-0.02891476357741396", + "tolerated_val_decision_score": "0.7802052162248743", + "tolerated_val_gap": "0.02094585104470381", + "tolerated_val_ranking_score": "0.7592593651801705", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.8018567656897366", + "union_ranking_score": "0.8307715292671506", + "union_regret": 0.0, + "union_test_gap": "-0.02891476357741396", + "union_val_decision_score": "0.7802052162248743", + "union_val_gap": "0.02094585104470381", + "union_val_ranking_score": "0.7592593651801705" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.022511431586352448", + "exact_ranking_score": "0.022511431586352448", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.02669537136706136", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.02669537136706136", + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0030782058188584395", + "ranking_selected_val_pr_auc": "0.0034856171692621155", + "scope": "global", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.4897984500499446", + "tolerated_ranking_score": "0.4897984500499446", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6888590116997952", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6888590116997952", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.4897984500499446", + "union_ranking_score": "0.4897984500499446", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6888590116997952", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6888590116997952" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.34154929577464793", + "exact_ranking_score": "0.34154929577464793", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.3481654957064793", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.3481654957064793", + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.26178619067169895", + "ranking_selected_val_pr_auc": "0.22178697667275007", + "scope": "top5", + "seed": 7, + "tolerated_decision_head": "pixel MLP head", + "tolerated_decision_head_arch": "pixel_mlp", + "tolerated_decision_score": "0.8388961892247042", + "tolerated_ranking_score": "0.8740083557882952", + "tolerated_regret": 0.0, + "tolerated_test_gap": "-0.03511216656359095", + "tolerated_val_decision_score": "0.8463626765874013", + "tolerated_val_gap": "0.020898995996049274", + "tolerated_val_ranking_score": "0.825463680591352", + "union_decision_head": "pixel MLP head", + "union_decision_head_arch": "pixel_mlp", + "union_decision_score": "0.8388961892247042", + "union_ranking_score": "0.8740083557882952", + "union_regret": 0.0, + "union_test_gap": "-0.03511216656359095", + "union_val_decision_score": "0.8463626765874013", + "union_val_gap": "0.020898995996049274", + "union_val_ranking_score": "0.825463680591352" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.305226480836237", + "exact_ranking_score": "0.305226480836237", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.29323758228605623", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.29323758228605623", + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.21329613245127957", + "ranking_selected_val_pr_auc": "0.17698631108606733", + "scope": "top10", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.8407348467432524", + "tolerated_ranking_score": "0.8407348467432524", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7878499830095744", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7878499830095744", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.8407348467432524", + "union_ranking_score": "0.8407348467432524", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7878499830095744", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7878499830095744" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.23666210670314639", + "exact_ranking_score": "0.23666210670314639", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.22416812609457093", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.22416812609457093", + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.14100646513486956", + "ranking_selected_val_pr_auc": "0.11693412738760335", + "scope": "top20", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.8085758039816233", + "tolerated_ranking_score": "0.8085758039816233", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7682832316376906", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7682832316376906", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.8085758039816233", + "union_ranking_score": "0.8085758039816233", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7682832316376906", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7682832316376906" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.021876258220373104", + "exact_ranking_score": "0.021876258220373104", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.017964421114027414", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.017964421114027414", + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0031953098868323193", + "ranking_selected_val_pr_auc": "0.0026642660544025525", + "scope": "global", + "seed": 99, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.7647466876509988", + "tolerated_ranking_score": "0.6698005926442897", + "tolerated_regret": 0.09494609500670914, + "tolerated_test_gap": "0.09494609500670914", + "tolerated_val_decision_score": "0.7870299254989238", + "tolerated_val_gap": "0.010195461444427734", + "tolerated_val_ranking_score": "0.7768344640544961", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.7647466876509988", + "union_ranking_score": "0.6698005926442897", + "union_regret": 0.09494609500670914, + "union_test_gap": "0.09494609500670914", + "union_val_decision_score": "0.7870299254989238", + "union_val_gap": "0.010195461444427734", + "union_val_ranking_score": "0.7768344640544961" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.2789445341949381", + "exact_ranking_score": "0.2789445341949381", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.2726772952171523", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.2726772952171523", + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.22946688569711154", + "ranking_selected_val_pr_auc": "0.17172613901536987", + "scope": "top5", + "seed": 99, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.9093512887929417", + "tolerated_ranking_score": "0.9158278502929932", + "tolerated_regret": 0.0, + "tolerated_test_gap": "-0.006476561500051492", + "tolerated_val_decision_score": "0.8688807502119548", + "tolerated_val_gap": "0.0021751625260553187", + "tolerated_val_ranking_score": "0.8667055876858994", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.9093512887929417", + "union_ranking_score": "0.9158278502929932", + "union_regret": 0.0, + "union_test_gap": "-0.006476561500051492", + "union_val_decision_score": "0.8688807502119548", + "union_val_gap": "0.0021751625260553187", + "union_val_ranking_score": "0.8667055876858994" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.24522698345354263", + "exact_ranking_score": "0.24522698345354263", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.22014622258326566", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.22014622258326566", + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.1797901995298074", + "ranking_selected_val_pr_auc": "0.12829763032218214", + "scope": "top10", + "seed": 99, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.9129217910828317", + "tolerated_ranking_score": "0.8862129818575714", + "tolerated_regret": 0.02670880922526031, + "tolerated_test_gap": "0.02670880922526031", + "tolerated_val_decision_score": "0.8661975382820603", + "tolerated_val_gap": "0.05189228302437687", + "tolerated_val_ranking_score": "0.8143052552576834", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.9129217910828317", + "union_ranking_score": "0.8862129818575714", + "union_regret": 0.02670880922526031, + "union_test_gap": "0.02670880922526031", + "union_val_decision_score": "0.8661975382820603", + "union_val_gap": "0.05189228302437687", + "union_val_ranking_score": "0.8143052552576834" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.19477124183006533", + "exact_ranking_score": "0.19477124183006533", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.1633352370074243", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.1633352370074243", + "family": "Reference", + "model_tag": "reference", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.11947246238005743", + "ranking_selected_val_pr_auc": "0.08253093816555805", + "scope": "top20", + "seed": 99, + "tolerated_decision_head": "linear probe", + "tolerated_decision_head_arch": "linear", + "tolerated_decision_score": "0.9039923296574045", + "tolerated_ranking_score": "0.8534759193943048", + "tolerated_regret": 0.05051641026309972, + "tolerated_test_gap": "0.05051641026309972", + "tolerated_val_decision_score": "0.8730039040005804", + "tolerated_val_gap": "0.08884125994391634", + "tolerated_val_ranking_score": "0.784162644056664", + "union_decision_head": "linear probe", + "union_decision_head_arch": "linear", + "union_decision_score": "0.9039923296574045", + "union_ranking_score": "0.8534759193943048", + "union_regret": 0.05051641026309972, + "union_test_gap": "0.05051641026309972", + "union_val_decision_score": "0.8730039040005804", + "union_val_gap": "0.08884125994391634", + "union_val_ranking_score": "0.784162644056664" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.0004925501476206198", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0004727121748000204", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0004727121748000204", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0002463357401629007", + "ranking_selected_val_pr_auc": "0.00023641196480702655", + "scope": "global", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.240793572992212", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.2179368807313983", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.240793572992212", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.2179368807313983" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.11833577252770228", + "exact_ranking_score": "0.11833577252770228", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.11931877546755824", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.11931877546755824", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.06288888888888888", + "ranking_selected_val_pr_auc": "0.06344444444444444", + "scope": "top5", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7714149204832434", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7971934513865686", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7971934513865686", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7714149204832434", + "union_ranking_score": "0.7714149204832434", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7971934513865686", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7971934513865686" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07325639739086803", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.07460890493381468", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.07460890493381468", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.03802083333333333", + "ranking_selected_val_pr_auc": "0.03875", + "scope": "top10", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.6996715316108496", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7015182768065465", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.6996715316108496", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7015182768065465", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.02093558282208589", + "ranking_selected_val_pr_auc": "0.0217280163599182", + "scope": "top20", + "seed": 1, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.0004925501476206198", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0004727121748000204", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0004727121748000204", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0002463357401629007", + "ranking_selected_val_pr_auc": "0.00023641196480702655", + "scope": "global", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.240793572992212", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.2179368807313983", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.240793572992212", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.2179368807313983" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.11833577252770228", + "exact_ranking_score": "0.11833577252770228", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.11931877546755824", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.11931877546755824", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.06288888888888888", + "ranking_selected_val_pr_auc": "0.06344444444444444", + "scope": "top5", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7714149204832434", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7971934513865686", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7971934513865686", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7714149204832434", + "union_ranking_score": "0.7714149204832434", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7971934513865686", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7971934513865686" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07325639739086803", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.07460890493381468", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.07460890493381468", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.03802083333333333", + "ranking_selected_val_pr_auc": "0.03875", + "scope": "top10", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.6996715316108496", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7015182768065465", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.6996715316108496", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7015182768065465", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.02093558282208589", + "ranking_selected_val_pr_auc": "0.0217280163599182", + "scope": "top20", + "seed": 123, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.0004925501476206198", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0004727121748000204", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0004727121748000204", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0002463357401629007", + "ranking_selected_val_pr_auc": "0.00023641196480702655", + "scope": "global", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.240793572992212", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.2179368807313983", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.240793572992212", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.2179368807313983" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.11833577252770228", + "exact_ranking_score": "0.11833577252770228", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.11931877546755824", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.11931877546755824", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.06288888888888888", + "ranking_selected_val_pr_auc": "0.06344444444444444", + "scope": "top5", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7714149204832434", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7971934513865686", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7971934513865686", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7714149204832434", + "union_ranking_score": "0.7714149204832434", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7971934513865686", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7971934513865686" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07325639739086803", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.07460890493381468", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.07460890493381468", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.03802083333333333", + "ranking_selected_val_pr_auc": "0.03875", + "scope": "top10", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.6996715316108496", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7015182768065465", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.6996715316108496", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7015182768065465", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.02093558282208589", + "ranking_selected_val_pr_auc": "0.0217280163599182", + "scope": "top20", + "seed": 42, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004925501476206198", + "exact_ranking_score": "0.0004925501476206198", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.0004727121748000204", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.0004727121748000204", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0002463357401629007", + "ranking_selected_val_pr_auc": "0.00023641196480702655", + "scope": "global", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.240793572992212", + "tolerated_ranking_score": "0.240793572992212", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.2179368807313983", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.2179368807313983", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.240793572992212", + "union_ranking_score": "0.240793572992212", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.2179368807313983", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.2179368807313983" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.11833577252770228", + "exact_ranking_score": "0.11833577252770228", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.11931877546755824", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.11931877546755824", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.06288888888888888", + "ranking_selected_val_pr_auc": "0.06344444444444444", + "scope": "top5", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7714149204832434", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7971934513865686", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7971934513865686", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7714149204832434", + "union_ranking_score": "0.7714149204832434", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7971934513865686", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7971934513865686" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07325639739086803", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.07460890493381468", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.07460890493381468", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.03802083333333333", + "ranking_selected_val_pr_auc": "0.03875", + "scope": "top10", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.6996715316108496", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7015182768065465", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.6996715316108496", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7015182768065465", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "linear probe", + "ranking_selected_head_arch": "linear", + "ranking_selected_test_pr_auc": "0.02093558282208589", + "ranking_selected_val_pr_auc": "0.0217280163599182", + "scope": "top20", + "seed": 7, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.0004849854226143435", + "exact_ranking_score": "0.0004849854226143435", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.00046545205608870404", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.00046545205608870404", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "shallow spatial adapter", + "ranking_selected_head_arch": "shallow", + "ranking_selected_test_pr_auc": "0.0002463357401629007", + "ranking_selected_val_pr_auc": "0.00023641196480702655", + "scope": "global", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.23755668759067805", + "tolerated_ranking_score": "0.23755668759067805", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.21494875104722905", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.21494875104722905", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.23755668759067805", + "union_ranking_score": "0.23755668759067805", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.21494875104722905", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.21494875104722905" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.11833577252770228", + "exact_ranking_score": "0.11833577252770228", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.11931877546755824", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.11931877546755824", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.06288888888888888", + "ranking_selected_val_pr_auc": "0.06344444444444444", + "scope": "top5", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7714149204832434", + "tolerated_ranking_score": "0.7714149204832434", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7971934513865686", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7971934513865686", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7714149204832434", + "union_ranking_score": "0.7714149204832434", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7971934513865686", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7971934513865686" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.07325639739086803", + "exact_ranking_score": "0.07325639739086803", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.07460890493381468", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.07460890493381468", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.03802083333333333", + "ranking_selected_val_pr_auc": "0.03875", + "scope": "top10", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.6996715316108496", + "tolerated_ranking_score": "0.6996715316108496", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.7015182768065465", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.7015182768065465", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.6996715316108496", + "union_ranking_score": "0.6996715316108496", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.7015182768065465", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.7015182768065465" + }, + { + "exact_decision_head": "shallow spatial adapter", + "exact_decision_head_arch": "shallow", + "exact_decision_score": "0.04101254412979794", + "exact_ranking_score": "0.04101254412979794", + "exact_regret": 0.0, + "exact_test_gap": "0.0", + "exact_val_decision_score": "0.04253189892419314", + "exact_val_gap": "0.0", + "exact_val_ranking_score": "0.04253189892419314", + "family": "StormCast", + "model_tag": "stormcast", + "path": "REDACTED_LOCAL_RESULT_PATH", + "ranking_selected_head": "pixel MLP head", + "ranking_selected_head_arch": "pixel_mlp", + "ranking_selected_test_pr_auc": "0.02093558282208589", + "ranking_selected_val_pr_auc": "0.0217280163599182", + "scope": "top20", + "seed": 99, + "tolerated_decision_head": "shallow spatial adapter", + "tolerated_decision_head_arch": "shallow", + "tolerated_decision_score": "0.7185324707231184", + "tolerated_ranking_score": "0.7185324707231184", + "tolerated_regret": 0.0, + "tolerated_test_gap": "0.0", + "tolerated_val_decision_score": "0.6906754133476136", + "tolerated_val_gap": "0.0", + "tolerated_val_ranking_score": "0.6906754133476136", + "union_decision_head": "shallow spatial adapter", + "union_decision_head_arch": "shallow", + "union_decision_score": "0.7185324707231184", + "union_ranking_score": "0.7185324707231184", + "union_regret": 0.0, + "union_test_gap": "0.0", + "union_val_decision_score": "0.6906754133476136", + "union_val_gap": "0.0", + "union_val_ranking_score": "0.6906754133476136" + } + ], + "summary": [ + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "FireWx-FM ref.", + "model_tag": "reference", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "global", + "scope_label": "\\(\\Omega=\\)global", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.17497107865629713, + "mean": 0.07383089600948442, + "min": 0.0, + "n": 5, + "std": 0.07453636071372995 + }, + "union_regret": { + "max": 0.17497107865629713, + "mean": 0.07383089600948442, + "min": 0.0, + "n": 5, + "std": 0.07453636071372995 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "FireWx-FM ref.", + "model_tag": "reference", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top5", + "scope_label": "\\(\\Omega=\\)top 5\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.015676762201120686, + "mean": 0.003663718115532055, + "min": 0.0, + "n": 5, + "std": 0.006812231244812292 + }, + "union_regret": { + "max": 0.015676762201120686, + "mean": 0.003663718115532055, + "min": 0.0, + "n": 5, + "std": 0.006812231244812292 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "FireWx-FM ref.", + "model_tag": "reference", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top10", + "scope_label": "\\(\\Omega=\\)top 10\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.02670880922526031, + "mean": 0.012275489592085752, + "min": 0.0, + "n": 5, + "std": 0.012665162001740834 + }, + "union_regret": { + "max": 0.02670880922526031, + "mean": 0.012275489592085752, + "min": 0.0, + "n": 5, + "std": 0.012665162001740834 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "FireWx-FM ref.", + "model_tag": "reference", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top20", + "scope_label": "\\(\\Omega=\\)top 20\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.05675140203555318, + "mean": 0.029384646387840017, + "min": 0.0, + "n": 5, + "std": 0.02751315335001922 + }, + "union_regret": { + "max": 0.05675140203555318, + "mean": 0.029384646387840017, + "min": 0.0, + "n": 5, + "std": 0.02751315335001922 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "global", + "scope_label": "\\(\\Omega=\\)global", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "union_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top5", + "scope_label": "\\(\\Omega=\\)top 5\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "union_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top10", + "scope_label": "\\(\\Omega=\\)top 10\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "union_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "Prithvi-WxC", + "model_tag": "prithvi_wxc", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top20", + "scope_label": "\\(\\Omega=\\)top 20\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "union_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + } + }, + { + "exact_regret": { + "max": 0.0004925501476206198, + "mean": 0.00010153879814819402, + "min": 0.0, + "n": 5, + "std": 0.00021861477435572763 + }, + "label": "Aurora", + "model_tag": "aurora", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "global", + "scope_label": "\\(\\Omega=\\)global", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.240793572992212, + "mean": 0.04945471159670635, + "min": 0.0, + "n": 5, + "std": 0.10697394238964528 + }, + "union_regret": { + "max": 0.240793572992212, + "mean": 0.04945471159670635, + "min": 0.0, + "n": 5, + "std": 0.10697394238964528 + } + }, + { + "exact_regret": { + "max": 0.11833577252770228, + "mean": 0.023667154505540456, + "min": 0.0, + "n": 5, + "std": 0.05292136630837888 + }, + "label": "Aurora", + "model_tag": "aurora", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top5", + "scope_label": "\\(\\Omega=\\)top 5\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.7714149204832434, + "mean": 0.1542829840966487, + "min": 0.0, + "n": 5, + "std": 0.34498724021162547 + }, + "union_regret": { + "max": 0.7714149204832434, + "mean": 0.1542829840966487, + "min": 0.0, + "n": 5, + "std": 0.34498724021162547 + } + }, + { + "exact_regret": { + "max": 0.07325639739086803, + "mean": 0.014651279478173606, + "min": 0.0, + "n": 5, + "std": 0.032761256870543834 + }, + "label": "Aurora", + "model_tag": "aurora", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top10", + "scope_label": "\\(\\Omega=\\)top 10\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.6996715316108496, + "mean": 0.1399343063221699, + "min": 0.0, + "n": 5, + "std": 0.31290262132065055 + }, + "union_regret": { + "max": 0.6996715316108496, + "mean": 0.1399343063221699, + "min": 0.0, + "n": 5, + "std": 0.31290262132065055 + } + }, + { + "exact_regret": { + "max": 0.04101254412979794, + "mean": 0.008202508825959588, + "min": 0.0, + "n": 5, + "std": 0.01834136732088763 + }, + "label": "Aurora", + "model_tag": "aurora", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top20", + "scope_label": "\\(\\Omega=\\)top 20\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.7185324707231184, + "mean": 0.1437064941446237, + "min": 0.0, + "n": 5, + "std": 0.32133748971555404 + }, + "union_regret": { + "max": 0.7185324707231184, + "mean": 0.1437064941446237, + "min": 0.0, + "n": 5, + "std": 0.32133748971555404 + } + }, + { + "exact_regret": { + "max": 7.571921560175121e-06, + "mean": 3.0287686240700486e-06, + "min": 0.0, + "n": 5, + "std": 4.147312242167625e-06 + }, + "label": "ClimaX", + "model_tag": "climax", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "global", + "scope_label": "\\(\\Omega=\\)global", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0032399924956598714, + "mean": 0.0012959969982639485, + "min": 0.0, + "n": 5, + "std": 0.0017746169760203706 + }, + "union_regret": { + "max": 0.0032399924956598714, + "mean": 0.0012959969982639485, + "min": 0.0, + "n": 5, + "std": 0.0017746169760203706 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "ClimaX", + "model_tag": "climax", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top5", + "scope_label": "\\(\\Omega=\\)top 5\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "union_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "ClimaX", + "model_tag": "climax", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top10", + "scope_label": "\\(\\Omega=\\)top 10\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "union_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "ClimaX", + "model_tag": "climax", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top20", + "scope_label": "\\(\\Omega=\\)top 20\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "union_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "StormCast", + "model_tag": "stormcast", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "global", + "scope_label": "\\(\\Omega=\\)global", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "union_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "StormCast", + "model_tag": "stormcast", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top5", + "scope_label": "\\(\\Omega=\\)top 5\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "union_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "StormCast", + "model_tag": "stormcast", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top10", + "scope_label": "\\(\\Omega=\\)top 10\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "union_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "StormCast", + "model_tag": "stormcast", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top20", + "scope_label": "\\(\\Omega=\\)top 20\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "union_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "DLWP", + "model_tag": "dlwp", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "global", + "scope_label": "\\(\\Omega=\\)global", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "union_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + } + }, + { + "exact_regret": { + "max": 0.015203005078871956, + "mean": 0.0048037709215293075, + "min": 0.0, + "n": 5, + "std": 0.006217185202152866 + }, + "label": "DLWP", + "model_tag": "dlwp", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top5", + "scope_label": "\\(\\Omega=\\)top 5\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.03305057342744666, + "mean": 0.016716228534155796, + "min": 0.0, + "n": 5, + "std": 0.016079313546074458 + }, + "union_regret": { + "max": 0.03305057342744666, + "mean": 0.016716228534155796, + "min": 0.0, + "n": 5, + "std": 0.016079313546074458 + } + }, + { + "exact_regret": { + "max": 0.005523780499856246, + "mean": 0.0017281632798742507, + "min": 0.0, + "n": 5, + "std": 0.002514722758075371 + }, + "label": "DLWP", + "model_tag": "dlwp", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top10", + "scope_label": "\\(\\Omega=\\)top 10\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.053927677500854476, + "mean": 0.02846514801700826, + "min": 0.0, + "n": 5, + "std": 0.026938012702643194 + }, + "union_regret": { + "max": 0.053927677500854476, + "mean": 0.02846514801700826, + "min": 0.0, + "n": 5, + "std": 0.026938012702643194 + } + }, + { + "exact_regret": { + "max": 0.0023651634514294945, + "mean": 0.0007702319787454587, + "min": 0.0, + "n": 5, + "std": 0.0010995336594539604 + }, + "label": "DLWP", + "model_tag": "dlwp", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top20", + "scope_label": "\\(\\Omega=\\)top 20\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.11215006616165157, + "mean": 0.04463354681768479, + "min": 0.0, + "n": 5, + "std": 0.04356064433532197 + }, + "union_regret": { + "max": 0.11215006616165157, + "mean": 0.04463354681768479, + "min": 0.0, + "n": 5, + "std": 0.04356064433532197 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "FCN", + "model_tag": "fcn", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "global", + "scope_label": "\\(\\Omega=\\)global", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "union_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + } + }, + { + "exact_regret": { + "max": 0.002257520679520411, + "mean": 0.0006342898232943345, + "min": 0.0, + "n": 5, + "std": 0.0009899554165032742 + }, + "label": "FCN", + "model_tag": "fcn", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top5", + "scope_label": "\\(\\Omega=\\)top 5\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.022524473456150496, + "mean": 0.004509624980300697, + "min": 0.0, + "n": 5, + "std": 0.010070611656609236 + }, + "union_regret": { + "max": 0.022524473456150496, + "mean": 0.004509624980300697, + "min": 0.0, + "n": 5, + "std": 0.010070611656609236 + } + }, + { + "exact_regret": { + "max": 0.0010578427408801938, + "mean": 0.00021156854817603877, + "min": 0.0, + "n": 5, + "std": 0.0004730816556225618 + }, + "label": "FCN", + "model_tag": "fcn", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top10", + "scope_label": "\\(\\Omega=\\)top 10\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.020997685254088072, + "mean": 0.004199537050817615, + "min": 0.0, + "n": 5, + "std": 0.009390450319657174 + }, + "union_regret": { + "max": 0.020997685254088072, + "mean": 0.004199537050817615, + "min": 0.0, + "n": 5, + "std": 0.009390450319657174 + } + }, + { + "exact_regret": { + "max": 2.877280037168889e-05, + "mean": 5.754560074337778e-06, + "min": 0.0, + "n": 5, + "std": 1.2867587506825515e-05 + }, + "label": "FCN", + "model_tag": "fcn", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top20", + "scope_label": "\\(\\Omega=\\)top 20\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.045880503237594294, + "mean": 0.011679805987441694, + "min": 0.0, + "n": 5, + "std": 0.019872372458657642 + }, + "union_regret": { + "max": 0.045880503237594294, + "mean": 0.011679805987441694, + "min": 0.0, + "n": 5, + "std": 0.019872372458657642 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "FengWu", + "model_tag": "fengwu", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "global", + "scope_label": "\\(\\Omega=\\)global", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "union_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + } + }, + { + "exact_regret": { + "max": 0.0018628094907809783, + "mean": 0.0005029843170376968, + "min": 0.0, + "n": 5, + "std": 0.0008109166521114917 + }, + "label": "FengWu", + "model_tag": "fengwu", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top5", + "scope_label": "\\(\\Omega=\\)top 5\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.01484136735033148, + "mean": 0.008795951947678215, + "min": 0.0, + "n": 5, + "std": 0.005532321338017505 + }, + "union_regret": { + "max": 0.01484136735033148, + "mean": 0.008795951947678215, + "min": 0.0, + "n": 5, + "std": 0.005532321338017505 + } + }, + { + "exact_regret": { + "max": 0.0016387212062008855, + "mean": 0.000495228089292582, + "min": 0.0, + "n": 5, + "std": 0.0007349190216431337 + }, + "label": "FengWu", + "model_tag": "fengwu", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top10", + "scope_label": "\\(\\Omega=\\)top 10\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.010273937098576491, + "mean": 0.00402300475984525, + "min": 0.0, + "n": 5, + "std": 0.005510851442075993 + }, + "union_regret": { + "max": 0.010273937098576491, + "mean": 0.00402300475984525, + "min": 0.0, + "n": 5, + "std": 0.005510851442075993 + } + }, + { + "exact_regret": { + "max": 0.0027505832409660327, + "mean": 0.0006908222234409067, + "min": 0.0, + "n": 5, + "std": 0.0011910586589384115 + }, + "label": "FengWu", + "model_tag": "fengwu", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top20", + "scope_label": "\\(\\Omega=\\)top 20\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.015150501067724198, + "mean": 0.005222389249812243, + "min": 0.0, + "n": 5, + "std": 0.0062394095558402415 + }, + "union_regret": { + "max": 0.015150501067724198, + "mean": 0.005222389249812243, + "min": 0.0, + "n": 5, + "std": 0.0062394095558402415 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "FuXi", + "model_tag": "fuxi", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "global", + "scope_label": "\\(\\Omega=\\)global", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "union_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + } + }, + { + "exact_regret": { + "max": 0.007024214143542151, + "mean": 0.002973545331200933, + "min": 0.0010927807990139538, + "n": 5, + "std": 0.0023946274991058026 + }, + "label": "FuXi", + "model_tag": "fuxi", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top5", + "scope_label": "\\(\\Omega=\\)top 5\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.050156261654859424, + "mean": 0.013545122545609134, + "min": 0.0, + "n": 5, + "std": 0.02097023683418404 + }, + "union_regret": { + "max": 0.050156261654859424, + "mean": 0.013545122545609134, + "min": 0.0, + "n": 5, + "std": 0.02097023683418404 + } + }, + { + "exact_regret": { + "max": 0.003938013087198336, + "mean": 0.001383793743586542, + "min": 0.0, + "n": 5, + "std": 0.0019248128430711165 + }, + "label": "FuXi", + "model_tag": "fuxi", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top10", + "scope_label": "\\(\\Omega=\\)top 10\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.008279917485013666, + "mean": 0.0016559834970027332, + "min": 0.0, + "n": 5, + "std": 0.0037028916689159307 + }, + "union_regret": { + "max": 0.008279917485013666, + "mean": 0.0016559834970027332, + "min": 0.0, + "n": 5, + "std": 0.0037028916689159307 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "FuXi", + "model_tag": "fuxi", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top20", + "scope_label": "\\(\\Omega=\\)top 20\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.008746323525994693, + "mean": 0.00283318355751887, + "min": 0.0, + "n": 5, + "std": 0.0036808289681375247 + }, + "union_regret": { + "max": 0.008746323525994693, + "mean": 0.00283318355751887, + "min": 0.0, + "n": 5, + "std": 0.0036808289681375247 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "global", + "scope_label": "\\(\\Omega=\\)global", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "union_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + } + }, + { + "exact_regret": { + "max": 0.005711594157251587, + "mean": 0.003154674487241463, + "min": 0.0, + "n": 5, + "std": 0.002546125713211599 + }, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top5", + "scope_label": "\\(\\Omega=\\)top 5\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.019790633919317124, + "mean": 0.007592888149777122, + "min": 0.0, + "n": 5, + "std": 0.00897418737588444 + }, + "union_regret": { + "max": 0.019790633919317124, + "mean": 0.007592888149777122, + "min": 0.0, + "n": 5, + "std": 0.00897418737588444 + } + }, + { + "exact_regret": { + "max": 0.004535321555179647, + "mean": 0.0017345627303725214, + "min": 0.0, + "n": 5, + "std": 0.0019305189318827886 + }, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top10", + "scope_label": "\\(\\Omega=\\)top 10\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.011660780793438352, + "mean": 0.003047840737004992, + "min": 0.0, + "n": 5, + "std": 0.005053805614558161 + }, + "union_regret": { + "max": 0.011660780793438352, + "mean": 0.003047840737004992, + "min": 0.0, + "n": 5, + "std": 0.005053805614558161 + } + }, + { + "exact_regret": { + "max": 0.0027096086413018403, + "mean": 0.0007280423771922354, + "min": 0.0, + "n": 5, + "std": 0.001178746460551365 + }, + "label": "Pangu-Weather", + "model_tag": "pangu6", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top20", + "scope_label": "\\(\\Omega=\\)top 20\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.007639684811918013, + "mean": 0.0018679847512695024, + "min": 0.0, + "n": 5, + "std": 0.0032548337047755126 + }, + "union_regret": { + "max": 0.007639684811918013, + "mean": 0.0018679847512695024, + "min": 0.0, + "n": 5, + "std": 0.0032548337047755126 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "global", + "scope_label": "\\(\\Omega=\\)global", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.272468682523694, + "mean": 0.1722171037486726, + "min": 0.050655329891645895, + "n": 5, + "std": 0.08849214830495522 + }, + "union_regret": { + "max": 0.272468682523694, + "mean": 0.1722171037486726, + "min": 0.050655329891645895, + "n": 5, + "std": 0.08849214830495522 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top5", + "scope_label": "\\(\\Omega=\\)top 5\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.1365277562230962, + "mean": 0.06384618090125256, + "min": 0.0, + "n": 5, + "std": 0.04965276403138872 + }, + "union_regret": { + "max": 0.1365277562230962, + "mean": 0.06384618090125256, + "min": 0.0, + "n": 5, + "std": 0.04965276403138872 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top10", + "scope_label": "\\(\\Omega=\\)top 10\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.1615566566666954, + "mean": 0.06573776411084173, + "min": 0.0, + "n": 5, + "std": 0.06897015340160571 + }, + "union_regret": { + "max": 0.1615566566666954, + "mean": 0.06573776411084173, + "min": 0.0, + "n": 5, + "std": 0.06897015340160571 + } + }, + { + "exact_regret": { + "max": 0.0, + "mean": 0.0, + "min": 0.0, + "n": 5, + "std": 0.0 + }, + "label": "AlphaEarth", + "model_tag": "alphaearth", + "n": 5, + "paths": [ + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH", + "REDACTED_LOCAL_RESULT_PATH" + ], + "scope": "top20", + "scope_label": "\\(\\Omega=\\)top 20\\%", + "seeds": [ + 1, + 7, + 42, + 99, + 123 + ], + "tolerated_regret": { + "max": 0.13482302181308625, + "mean": 0.038803552655092256, + "min": 0.0, + "n": 5, + "std": 0.0594825313313219 + }, + "union_regret": { + "max": 0.13482302181308625, + "mean": 0.038803552655092256, + "min": 0.0, + "n": 5, + "std": 0.0594825313313219 + } + } + ] +} diff --git a/artifacts/results/selection_regret_summary.csv b/artifacts/results/selection_regret_summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..ea8a02982a8c66171b1344716be24df46cd272c8 --- /dev/null +++ b/artifacts/results/selection_regret_summary.csv @@ -0,0 +1,25 @@ +model_tag,label,scope,n,seeds,exact_regret_mean,exact_regret_std,tolerated_regret_mean,tolerated_regret_std,union_regret_mean,union_regret_std +reference,Reference,global,5,1 7 42 99 123,0.0,0.0,0.08783024981138902,0.09670495645481135,0.08783024981138902,0.09670495645481135 +reference,Reference,fire_prone,5,1 7 42 99 123,0.0,0.0,0.03402707057672223,0.032044658643147844,0.03402707057672223,0.032044658643147844 +prithvi_wxc,Prithvi-WxC,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 +prithvi_wxc,Prithvi-WxC,fire_prone,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 +aurora,Aurora,global,5,1 7 42 99 123,0.00020004882767231798,0.00026703384456332115,0.09851983041506818,0.1298781980037557,0.09851983041506818,0.1298781980037557 +aurora,Aurora,fire_prone,5,1 7 42 99 123,0.008202508825959588,0.01834136732088763,0.14391889430974364,0.32121904665016227,0.14391889430974364,0.32121904665016227 +climax,ClimaX,global,5,1 7 42 99 123,3.0287686240700486e-06,4.147312242167625e-06,0.0012959969982639485,0.0017746169760203706,0.0012959969982639485,0.0017746169760203706 +climax,ClimaX,fire_prone,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 +stormcast,StormCast,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 +stormcast,StormCast,fire_prone,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 +pangu_weather,Pangu-Weather,global,5,1 7 42 99 123,0.00013033979247265275,0.0002685372203690466,0.048806713097574374,0.10733308684741971,0.048806713097574374,0.10733308684741971 +pangu_weather,Pangu-Weather,fire_prone,5,1 7 42 99 123,0.027875386332505546,0.02348779386900393,0.43111948243387105,0.39355644251497235,0.43111948243387105,0.39355644251497235 +dlwp,DLWP,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 +dlwp,DLWP,fire_prone,5,1 7 42 99 123,0.0007702319787454587,0.0010995336594539604,0.043265915053601556,0.04332331365579739,0.043265915053601556,0.04332331365579739 +fcn,FCN,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 +fcn,FCN,fire_prone,5,1 7 42 99 123,5.960229415004348e-06,1.3327478133443526e-05,0.011679805987441694,0.019872372458657642,0.011679805987441694,0.019872372458657642 +fengwu,FengWu,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 +fengwu,FengWu,fire_prone,5,1 7 42 99 123,0.0006908222234409067,0.0011910586589384115,0.005222389249812243,0.0062394095558402415,0.005222389249812243,0.0062394095558402415 +fuxi,FuXi,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 +fuxi,FuXi,fire_prone,5,1 7 42 99 123,0.0,0.0,0.0010839188523199318,0.0017288780545672386,0.0010839188523199318,0.0017288780545672386 +pangu6,Pangu-Weather,global,5,1 7 42 99 123,0.0,0.0,0.0,0.0,0.0,0.0 +pangu6,Pangu-Weather,fire_prone,5,1 7 42 99 123,0.0007280423771922354,0.001178746460551365,0.0018491271881979853,0.0032630386057089294,0.0018491271881979853,0.0032630386057089294 +alphaearth,AlphaEarth,global,5,1 7 42 99 123,0.0,0.0,0.1722171037486726,0.08849214830495522,0.1722171037486726,0.08849214830495522 +alphaearth,AlphaEarth,fire_prone,5,1 7 42 99 123,0.0,0.0,0.038803552655092256,0.0594825313313219,0.038803552655092256,0.0594825313313219 diff --git a/artifacts/results/selection_regret_tolerance_family_table.generated.tex b/artifacts/results/selection_regret_tolerance_family_table.generated.tex new file mode 100644 index 0000000000000000000000000000000000000000..57c52f58f9e80d6485a18d8b31869cd25913c3e1 --- /dev/null +++ b/artifacts/results/selection_regret_tolerance_family_table.generated.tex @@ -0,0 +1,2 @@ +% Replaced by the all-backbone value table in sections/appendix.tex +% (Table~\ref{tab:appendix_selection_regret_tolerance}). diff --git a/data_sources/DATA_SOURCES.md b/data_sources/DATA_SOURCES.md new file mode 100644 index 0000000000000000000000000000000000000000..17673b341375e3eca9ec38a8b56f8d9c922c604e --- /dev/null +++ b/data_sources/DATA_SOURCES.md @@ -0,0 +1,27 @@ +# Data Sources + +This repository does not redistribute raw data. The table below records the +resources used by the paper, their role in the experiments, and public access +entry points. Users must obtain each source under its own terms. + +| Source | Role in paper | Public access entry point | +|---|---|---| +| NOAA High-Resolution Rapid Refresh (HRRR) | Dynamic weather fields for the California regional gridded occupancy inputs. | NOAA/NCEI HRRR product page: ; AWS Open Data archive: . | +| NASA FIRMS active-fire detections | Active-fire detections used to derive gridded occupancy labels. | FIRMS download and API services: and . | +| LANDFIRE 40 Fire Behavior Fuel Models | Static fuel layer used in the FireWx-FM gridded input. | LANDFIRE data access portal: . | +| LANDFIRE Forest Canopy Cover | Static canopy layer used in the FireWx-FM gridded input. | LANDFIRE data access portal: . | +| Wildfire Risk to Communities housing-unit density | Static exposure layer used in the FireWx-FM gridded input. | Wildfire Risk to Communities data access: . | +| LandScan Global 2024 | Static population layer used in the FireWx-FM gridded input. | Oak Ridge National Laboratory LandScan access: . | +| WFIGS incident/perimeter attributes | Event-level incident metadata for supporting burned-area and analog tasks. | NIFC Open Data portal for WFIGS layers: . | +| MTBS burned area and burn severity | Event-scale burned-area and burn-severity records for supporting tasks. | MTBS data access and direct download pages: and . | +| Earth-FM/backbone sources | Frozen feature sources for transferred Earth-FM comparisons. | Original model providers and their terms. Examples include Hugging Face model cards, model-provider GitHub repositories, and provider-hosted model files. | + +## Notes + +- The paper places gridded resources on a projected 5 km EPSG:5070 grid. +- The bundled artifacts contain summary values only. They are not a substitute + for the original data. +- Full raw-data reruns require users to obtain each source independently and to + construct the intermediate grids/features described in the paper. +- Access mechanisms and licensing can change. The links above are entry points, + not redistributed copies. diff --git a/docs/artifact_map.md b/docs/artifact_map.md new file mode 100644 index 0000000000000000000000000000000000000000..15fab155d31a5803302b88f19d705438283584b6 --- /dev/null +++ b/docs/artifact_map.md @@ -0,0 +1,56 @@ +# Paper Artifact Map + +This map links every table and figure label in the current manuscript to the +public release artifact and its provenance. Final output checksums are stored in +`artifacts/manifests/paper_outputs.sha256`. + +## Figures + +| Paper label | Release file | Provenance | +|---|---|---| +| `fig:toy_occupancy_contract` | `paper_outputs/figures/matching.pdf` | Static vector schematic used by the manuscript. | +| `fig:task_contract_tiles` | `paper_outputs/figures/fig_task_contract_tiles.pdf` | Static contract-map figure used by the manuscript. | +| `fig:selection_regret_diagnostic` | `paper_outputs/figures/fig_selection_regret_rq2.tikz` | Rebuilt by `scripts/build_selection_regret_rq2_figure.py` from `artifacts/results/selection_regret_scope_sweep_20260505.csv`. | +| `fig:fireprone_contract_progression` | `paper_outputs/figures/fig_fireprone_contract_progression_compact.pdf` | Rebuilt by `scripts/build_fireprone_contract_progression_figure.py` from `artifacts/results/fireprone_contract_progression_summary.json`. | +| `fig:task_comparator_normalized_map` | `paper_outputs/figures/fig_task_rank_map.pdf` | Rebuilt by `scripts/build_task_rank_map.py` from `tab_primary_results.tex` and `tab_supporting_results.tex`. | + +## Main Tables + +| Paper label | Release file | Provenance | +|---|---|---| +| `tab:primary_results` | `paper_outputs/tables/tab_primary_results.tex` | Frozen paper-output TeX extracted from the current manuscript source and verified by checksum. Raw reruns require the task scripts and non-redistributed feature caches. | +| `tab:supporting_results` | `paper_outputs/tables/tab_supporting_results.tex` | Frozen paper-output TeX extracted from the current manuscript source and verified by checksum. Raw reruns require the task scripts and non-redistributed feature caches. | + +## Appendix Tables + +| Paper label | Release file | Provenance | +|---|---|---| +| `tab:app_matching_rule_params` | `paper_outputs/tables/tab_app_matching_rule_params.tex` | Contract parameter table from manuscript source, verified by checksum. | +| `tab:app_contract_params_full` | `paper_outputs/tables/tab_app_contract_params_full.tex` | Contract parameter table from manuscript source, verified by checksum. | +| `tab:app_scope_params` | `paper_outputs/tables/tab_app_scope_params.tex` | Scope parameter table from manuscript source, verified by checksum. | +| `tab:fireprone_contract_progression` | `paper_outputs/tables/tab_fireprone_contract_progression.tex` | Values from `artifacts/results/fireprone_contract_progression_summary.json`. | +| `tab:appendix_selection_regret_tolerance` | `paper_outputs/tables/tab_appendix_selection_regret_tolerance.tex` | Values from selection-regret summary artifacts. | +| `tab:app_occupancy_ppr_scope` | `paper_outputs/tables/tab_app_occupancy_ppr_scope.tex` | Values from `artifacts/results/fireprone_contract_progression_summary.json`. | +| `tab:app_spread_ap_by_scope` | `paper_outputs/tables/tab_app_spread_ap_by_scope.tex` | Frozen paper-output TeX extracted from current manuscript source, verified by checksum. | +| `tab:app_burned_area_median_acre` | `paper_outputs/tables/tab_app_burned_area_median_acre.tex` | Frozen paper-output TeX extracted from current manuscript source, verified by checksum. | +| `tab:app_analog_rank_depth` | `paper_outputs/tables/tab_app_analog_rank_depth.tex` | Frozen paper-output TeX extracted from current manuscript source, verified by checksum. | +| `tab:app_smoke_high_event` | `paper_outputs/tables/tab_app_smoke_high_event.tex` | Frozen paper-output TeX extracted from current manuscript source, verified by checksum. | +| `tab:app_heat_event_pr` | `paper_outputs/tables/tab_app_heat_event_pr.tex` | Frozen paper-output TeX extracted from current manuscript source, verified by checksum. | +| `tab:app_seed_robustness` | `paper_outputs/tables/tab_app_seed_robustness.tex` | Seed summary table from manuscript source, verified by checksum. | +| `tab:app_head_architectures` | `paper_outputs/tables/tab_app_head_architectures.tex` | Architecture description table from manuscript source, verified by checksum. | + +## Reproduction Commands + +```bash +python3 scripts/reproduce_paper_outputs.py +``` + +This command rebuilds the outputs that depend only on released summary files, +checks all final paper-output hashes, and runs the release audit. + +## Raw Rerun Boundary + +Some tables depend on raw gridded data, event data, or backbone feature caches +that are not redistributed. For public release, we provide the compact summary +artifacts used to reproduce the displayed paper values and document the raw data +sources separately. diff --git a/docs/huggingface_release_design.md b/docs/huggingface_release_design.md new file mode 100644 index 0000000000000000000000000000000000000000..d246caab01b727c9cd9d8745c9ceeb05899f617c --- /dev/null +++ b/docs/huggingface_release_design.md @@ -0,0 +1,16 @@ +# Hugging Face Release Design + +This release follows the common Hugging Face pattern for research artifacts: + +- `README.md` is the public card. It contains YAML metadata, intended use, + limitations, data provenance, reproduction commands, and citation text. +- `paper_outputs/` stores the final TeX, TikZ, and PDF artifacts used by the + manuscript. +- `artifacts/results/` stores compact CSV/JSON summaries that can be public. +- `artifacts/manifests/` maps paper labels to files and records output hashes. +- `data_sources/` documents external data resources without redistributing them. +- `experiments/` contains raw-rerun reference scripts and Slurm templates. + +The repository is intentionally a paper-artifact release rather than a dataset +mirror or model-weight release. Full raw-data reruns require separately obtained +source data and local feature caches. diff --git a/experiments/README.md b/experiments/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e81750c78cb189d8e03591e2d37b8d877a1e8e89 --- /dev/null +++ b/experiments/README.md @@ -0,0 +1,25 @@ +# Raw Rerun Notes + +This directory documents the raw rerun boundary. The public artifact release does +not include local Slurm scripts with machine paths, raw wildfire inputs, or local +feature caches. Full raw reruns require users to obtain the source data listed in +`data_sources/DATA_SOURCES.md` and adapt the templates below to their own cluster. + +The bundled paper-output reproduction path does not require these raw reruns. + +## Reference Scripts + +The scripts under `raw_reference/` are sanitized references for the task-level +runs used in the paper. They preserve the command-line interfaces and evaluation +logic, but they require user-provided data tables, feature caches, and model +dependencies. + +If a script imports local project modules from an external preprocessing tree, +set `WILDFIRE_FM_EXTRA_PYTHONPATH` before running it: + +```bash +export WILDFIRE_FM_EXTRA_PYTHONPATH=/path/to/your/project/src:/path/to/extra/site-packages +``` + +The Slurm file in `slurm/` is a template only. Replace all placeholder paths +before submitting jobs on your own cluster. diff --git a/experiments/raw_reference/run_selection_regret_scope_sweep_20260505.py b/experiments/raw_reference/run_selection_regret_scope_sweep_20260505.py new file mode 100644 index 0000000000000000000000000000000000000000..0af05d0e0856d347867971bc1cc648b4fd19b47c --- /dev/null +++ b/experiments/raw_reference/run_selection_regret_scope_sweep_20260505.py @@ -0,0 +1,335 @@ +#!/usr/bin/env python3 +"""Run fixed-feature head-selection regret for global and top-k fire-prone scopes.""" + +from __future__ import annotations + +import argparse +import csv +import importlib.util +import json +import math +from pathlib import Path +from typing import Any + +import numpy as np + + +BASE_RUNNER = Path(__file__).resolve().parent / "task_scripts" / "run_all_backbone_selection_regret_20260504.py" +spec = importlib.util.spec_from_file_location("selection_regret_base_20260504", BASE_RUNNER) +if spec is None or spec.loader is None: + raise RuntimeError(f"Cannot import base runner: {BASE_RUNNER}") +base = importlib.util.module_from_spec(spec) +spec.loader.exec_module(base) + +head_control = base.head_control + +SCOPE_FRACS = (0.05, 0.10, 0.20) +SCOPE_ORDER = ("global", "top5", "top10", "top20") +SCOPE_LABELS = { + "global": "global", + "top5": "top 5%", + "top10": "top 10%", + "top20": "top 20%", +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Selection-regret scope sweep.") + parser.add_argument("--source-kind", choices=("reference", "attached", "spatial", "alphaearth"), required=True) + parser.add_argument("--feature-root", type=Path, required=True) + parser.add_argument("--daily-rows-csv", type=Path) + parser.add_argument("--support-dir", type=Path) + parser.add_argument("--alphaearth-cache-root", type=Path) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--fm-family", type=str, required=True) + parser.add_argument("--model-tag", type=str, required=True) + parser.add_argument("--seed", type=int, required=True) + parser.add_argument("--heads", nargs="+", choices=base.HEADS, default=["linear", "pixel_mlp", "shallow"]) + parser.add_argument("--batch-size", type=int, default=8) + parser.add_argument("--epochs", type=int, default=2) + parser.add_argument("--learning-rate", type=float, default=8e-4) + parser.add_argument("--weight-decay", type=float, default=1e-5) + parser.add_argument("--pos-weight-cap", type=float, default=150.0) + parser.add_argument("--device", choices=("cpu", "cuda", "auto"), default="cpu") + parser.add_argument( + "--metric-thresholds", + nargs="+", + type=float, + default=[ + 1e-5, + 2e-5, + 5e-5, + 1e-4, + 2e-4, + 5e-4, + 1e-3, + 2e-3, + 5e-3, + 1e-2, + 2e-2, + 5e-2, + 8e-2, + 1e-1, + 1.5e-1, + 2e-1, + 3e-1, + 5e-1, + ], + ) + parser.add_argument("--variants", nargs="+", default=["identity"]) + parser.add_argument("--fire-prone-top-fracs", nargs="+", type=float, default=list(SCOPE_FRACS)) + parser.add_argument("--temporal-steps", type=int, default=3) + parser.add_argument("--spatial-radius", type=int, default=8) + parser.add_argument("--buffer-radius", type=int, default=8) + parser.add_argument("--boundary-radius", type=int, default=8) + parser.add_argument("--coarse-factor", type=int, default=8) + parser.add_argument("--time-step-hours", type=int, default=6) + return parser.parse_args() + + +def scope_name(top_frac: float) -> str: + pct = int(round(float(top_frac) * 100.0)) + return f"top{pct}" + + +def scope_label(top_frac: float) -> str: + pct = int(round(float(top_frac) * 100.0)) + return f"top {pct}%" + + +def build_scope_masks( + split_rows: dict[str, list[dict[str, str]]], + store: Any, + top_fracs: list[float], +) -> tuple[dict[str, np.ndarray | None], dict[str, dict[str, Any]]]: + masks: dict[str, np.ndarray | None] = {"global": None} + meta: dict[str, dict[str, Any]] = { + "global": { + "scope_name": "global", + "reported_as": "global", + "top_fraction": None, + } + } + for frac in top_fracs: + name = scope_name(frac) + mask, mask_meta = head_control.build_fire_prone_mask(split_rows["train"], store, float(frac)) + masks[name] = mask + meta[name] = { + "scope_name": name, + "reported_as": scope_label(frac), + **mask_meta, + } + return masks, meta + + +def build_posthoc_rows_for_scopes( + probs: np.ndarray, + targets: np.ndarray, + sample_times: np.ndarray, + split: str, + scope_masks: dict[str, np.ndarray | None], + args: argparse.Namespace, +) -> list[dict[str, object]]: + rows_out: list[dict[str, object]] = [] + for threshold in [float(v) for v in args.metric_thresholds]: + base_binary = probs >= threshold + for variant in args.variants: + binary = head_control.apply_variant(base_binary, variant) + tensors = head_control.evaluate_threshold_variant( + binary_np=binary, + target_np=targets, + sample_times=sample_times, + time_step_hours=args.time_step_hours, + temporal_steps=args.temporal_steps, + spatial_radius=args.spatial_radius, + buffer_radius=args.buffer_radius, + boundary_radius=args.boundary_radius, + coarse_factor=args.coarse_factor, + tolerance_hours=args.temporal_steps * args.time_step_hours, + ) + for scope, region_mask in scope_masks.items(): + row: dict[str, object] = { + "split": split, + "scope": scope, + "threshold": float(threshold), + "variant": variant, + "time_step_hours": int(args.time_step_hours), + "temporal_steps": int(args.temporal_steps), + "tolerance_hours": int(args.temporal_steps * args.time_step_hours), + "spatial_radius": int(args.spatial_radius), + "buffer_radius": int(args.buffer_radius), + "boundary_radius": int(args.boundary_radius), + "coarse_factor": int(args.coarse_factor), + } + row.update(head_control.metrics_for_scope(tensors, region_mask)) + rows_out.append(row) + return rows_out + + +def read_csv(path: Path) -> list[dict[str, str]]: + with path.open("r", encoding="utf-8", newline="") as fh: + return list(csv.DictReader(fh)) + + +def load_head_summary( + head_dir: Path, + head_arch: str, + scopes: tuple[str, ...], +) -> tuple[list[dict[str, object]], dict[str, dict[str, float]], dict[str, object]] | None: + posthoc_path = head_dir / "posthoc_rows.csv" + summary_path = head_dir / "summary.json" + if not posthoc_path.exists() or not summary_path.exists(): + return None + rows = [dict(row) for row in read_csv(posthoc_path)] + if not rows: + return None + try: + summary = json.loads(summary_path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return None + if str(summary.get("head_arch")) != str(head_arch): + return None + raw_pr_auc = summary.get("raw_pr_auc") + if not isinstance(raw_pr_auc, dict): + return None + try: + parsed_pr_auc = { + split: {scope: float(raw_pr_auc[split][scope]) for scope in scopes} + for split in ("val", "test") + } + except Exception: + return None + return rows, parsed_pr_auc, summary + + +def finite_json(value: Any) -> Any: + if isinstance(value, float): + return value if math.isfinite(value) else None + if isinstance(value, dict): + return {key: finite_json(val) for key, val in value.items()} + if isinstance(value, list): + return [finite_json(val) for val in value] + return value + + +def main() -> None: + args = parse_args() + args.output_dir.mkdir(parents=True, exist_ok=True) + base.set_seed(int(args.seed)) + device = base.choose_device(args.device) + + top_fracs = sorted({float(v) for v in args.fire_prone_top_fracs}) + scope_order = ("global",) + tuple(scope_name(frac) for frac in top_fracs) + base.SCOPE_ORDER = scope_order + + split_rows = { + split: base.read_rows(args.feature_root / "splits" / f"{split}.csv") + for split in ("train", "val", "test") + } + if args.source_kind == "reference": + store = base.build_reference_store(split_rows) + elif args.source_kind == "attached": + store = base.build_attached_store(args, split_rows) + elif args.source_kind == "spatial": + store = base.build_spatial_store(args, split_rows) + else: + store = base.build_alphaearth_store(args, split_rows) + + loaders = base.make_loaders(split_rows, store, int(args.batch_size), device, int(args.seed)) + first = next(iter(loaders["train"])) + in_ch = int(first["x"].shape[1]) + prior_prob = base.total_positive_rate(split_rows["train"]) + scope_masks, scope_meta = build_scope_masks(split_rows, store, top_fracs) + + head_metrics: list[dict[str, object]] = [] + head_artifacts: dict[str, str] = {} + for head_index, head_arch in enumerate(args.heads): + head_dir = args.output_dir / head_arch + head_dir.mkdir(parents=True, exist_ok=True) + cached = load_head_summary(head_dir, head_arch, scope_order) + if cached is not None: + posthoc_rows, raw_pr_auc, _ = cached + print(f"[scope-sweep] reuse {args.fm_family} seed={args.seed} head={head_arch}", flush=True) + else: + print(f"[scope-sweep] training {args.fm_family} seed={args.seed} head={head_arch}", flush=True) + model, history = base.train_one_head( + head_arch=head_arch, + in_ch=in_ch, + prior_prob=prior_prob, + loaders=loaders, + args=args, + device=device, + seed_offset=1009 * (head_index + 1), + ) + posthoc_rows = [] + raw_pr_auc: dict[str, dict[str, float]] = {} + for split in ("val", "test"): + probs, targets = base.collect_predictions(model, loaders[split], device) + sample_times = base.build_sample_times(split_rows[split]) + raw_pr_auc[split] = { + scope: head_control._masked_average_precision(probs, targets, region_mask=mask) + for scope, mask in scope_masks.items() + } + posthoc_rows.extend( + build_posthoc_rows_for_scopes( + probs=probs, + targets=targets, + sample_times=sample_times, + split=split, + scope_masks=scope_masks, + args=args, + ) + ) + base.write_csv(posthoc_rows, head_dir / "posthoc_rows.csv") + head_summary = { + "head_arch": head_arch, + "head_label": head_control.HEAD_LABELS[head_arch], + "history": history, + "raw_pr_auc": raw_pr_auc, + "scope_meta": scope_meta, + "posthoc_rows_csv": str(head_dir / "posthoc_rows.csv"), + } + (head_dir / "summary.json").write_text(json.dumps(finite_json(head_summary), indent=2), encoding="utf-8") + head_artifacts[head_arch] = str(head_dir / "summary.json") + base.append_head_metrics(head_metrics, posthoc_rows, raw_pr_auc, head_arch, args) + + selection_rows = base.summarize_head_scores(head_metrics) + for row in selection_rows: + row["model_tag"] = args.model_tag + row["family"] = args.fm_family + row["seed"] = int(args.seed) + + base.write_csv(head_metrics, args.output_dir / "head_metrics.csv") + base.write_csv(selection_rows, args.output_dir / "selection_rows.csv") + summary = { + "experiment": "fixed-feature head-selection regret scope sweep", + "task": "wildfire_occupancy", + "model_tag": args.model_tag, + "fm_family": args.fm_family, + "source_kind": args.source_kind, + "seed": int(args.seed), + "feature_root": str(args.feature_root), + "daily_rows_csv": str(args.daily_rows_csv) if args.daily_rows_csv else None, + "support_dir": str(args.support_dir) if args.support_dir else None, + "alphaearth_cache_root": str(args.alphaearth_cache_root) if args.alphaearth_cache_root else None, + "device": str(device), + "heads": list(args.heads), + "scope_order": list(scope_order), + "scope_meta": scope_meta, + "input_channels": int(in_ch), + "prior_prob": float(prior_prob), + "metrics": base.METRICS, + "head_metrics": head_metrics, + "selection_rows": selection_rows, + "head_artifacts": head_artifacts, + "artifacts": { + "head_metrics_csv": str(args.output_dir / "head_metrics.csv"), + "selection_rows_csv": str(args.output_dir / "selection_rows.csv"), + }, + } + (args.output_dir / "summary.json").write_text(json.dumps(finite_json(summary), indent=2), encoding="utf-8") + print(json.dumps(finite_json(summary), indent=2), flush=True) + + +if __name__ == "__main__": + main() diff --git a/experiments/raw_reference/task_scripts/run_all_backbone_selection_regret_20260504.py b/experiments/raw_reference/task_scripts/run_all_backbone_selection_regret_20260504.py new file mode 100644 index 0000000000000000000000000000000000000000..82d79574a216700acccc0f6a9139de33c7c499ea --- /dev/null +++ b/experiments/raw_reference/task_scripts/run_all_backbone_selection_regret_20260504.py @@ -0,0 +1,656 @@ +#!/usr/bin/env python3 +"""Run one fixed-feature head-selection regret job for one backbone and seed.""" + +from __future__ import annotations + +import argparse +import csv +import json +import math +import random +import sys +from pathlib import Path +from typing import Any + +import numpy as np +import torch +import torch.nn as nn +from torch.utils.data import DataLoader, Dataset + + +import os + +for _p in os.environ.get("WILDFIRE_FM_EXTRA_PYTHONPATH", "").split(os.pathsep): + if _p and _p not in sys.path: + sys.path.insert(0, _p) + +import run_alphaearth_occupancy_benchmark as alpha_runner # noqa: E402 +import run_attached_daily_occupancy_head_control as head_control # noqa: E402 + + +HEADS = ("constant", "linear", "pixel_mlp", "shallow", "shallow_wide") +METRICS = { + "exact": "strict_f1", + "tolerated": "ts_f1", + "union": "comprehensive_union_f1", +} +SCOPE_ORDER = ("global", "fire_prone") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="All-backbone fixed-feature head-selection regret.") + parser.add_argument("--source-kind", choices=("reference", "attached", "spatial", "alphaearth"), required=True) + parser.add_argument("--feature-root", type=Path, required=True) + parser.add_argument("--daily-rows-csv", type=Path) + parser.add_argument("--support-dir", type=Path) + parser.add_argument("--alphaearth-cache-root", type=Path) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--fm-family", type=str, required=True) + parser.add_argument("--model-tag", type=str, required=True) + parser.add_argument("--seed", type=int, required=True) + parser.add_argument("--heads", nargs="+", choices=HEADS, default=list(HEADS)) + parser.add_argument("--batch-size", type=int, default=4) + parser.add_argument("--epochs", type=int, default=4) + parser.add_argument("--learning-rate", type=float, default=8e-4) + parser.add_argument("--weight-decay", type=float, default=1e-5) + parser.add_argument("--pos-weight-cap", type=float, default=150.0) + parser.add_argument("--device", choices=("cpu", "cuda", "auto"), default="auto") + parser.add_argument( + "--metric-thresholds", + nargs="+", + type=float, + default=[ + 1e-5, + 2e-5, + 5e-5, + 1e-4, + 2e-4, + 5e-4, + 1e-3, + 2e-3, + 5e-3, + 1e-2, + 2e-2, + 5e-2, + 8e-2, + 1e-1, + 1.5e-1, + 2e-1, + 3e-1, + 5e-1, + ], + ) + parser.add_argument( + "--variants", + nargs="+", + default=["identity", "erode_r1", "close_r1"], + ) + parser.add_argument("--fire-prone-top-frac", type=float, default=0.20) + parser.add_argument("--temporal-steps", type=int, default=3) + parser.add_argument("--spatial-radius", type=int, default=8) + parser.add_argument("--buffer-radius", type=int, default=8) + parser.add_argument("--boundary-radius", type=int, default=8) + parser.add_argument("--coarse-factor", type=int, default=8) + parser.add_argument("--time-step-hours", type=int, default=6) + return parser.parse_args() + + +def read_rows(path: Path) -> list[dict[str, str]]: + with path.open("r", encoding="utf-8", newline="") as fh: + return list(csv.DictReader(fh)) + + +def choose_device(value: str) -> torch.device: + if value == "auto": + return torch.device("cuda" if torch.cuda.is_available() else "cpu") + device = torch.device(value) + if device.type == "cuda" and not torch.cuda.is_available(): + raise RuntimeError("CUDA requested but not available.") + return device + + +def set_seed(seed: int) -> None: + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + +def total_positive_rate(rows: list[dict[str, str]]) -> float: + pos = float(sum(int(row["pos_cells"]) for row in rows)) + arr = np.load(rows[0]["feature_path"], allow_pickle=True) + try: + total = float(len(rows) * np.squeeze(arr["y_occ"]).size) + finally: + arr.close() + return float(pos / total) if total > 0 else 0.0 + + +class Normalizer: + def __init__(self, mean: np.ndarray, std: np.ndarray) -> None: + self.mean = mean.astype(np.float32) + self.std = np.maximum(std.astype(np.float32), 1e-6) + + def apply(self, x: np.ndarray) -> np.ndarray: + return (x.astype(np.float32) - self.mean[:, None, None]) / self.std[:, None, None] + + +def compute_map_normalizer(rows: list[dict[str, str]], map_path_fn: Any) -> Normalizer: + sum_c: np.ndarray | None = None + sumsq_c: np.ndarray | None = None + count = 0 + for row in rows: + arr = np.load(map_path_fn(row), allow_pickle=True) + try: + x = np.nan_to_num(arr["features"].astype(np.float32), nan=0.0, posinf=0.0, neginf=0.0) + finally: + arr.close() + flat = x.reshape(x.shape[0], -1) + if sum_c is None: + sum_c = np.zeros(x.shape[0], dtype=np.float64) + sumsq_c = np.zeros(x.shape[0], dtype=np.float64) + sum_c += flat.sum(axis=1) + sumsq_c += np.square(flat, dtype=np.float64).sum(axis=1) + count += flat.shape[1] + if sum_c is None or sumsq_c is None or count == 0: + raise RuntimeError("Cannot compute feature normalizer.") + mean = sum_c / float(count) + var = np.maximum(sumsq_c / float(count) - mean * mean, 1e-12) + return Normalizer(mean=mean.astype(np.float32), std=np.sqrt(var).astype(np.float32)) + + +def load_support_manifest(path: Path) -> dict[str, dict[str, str]]: + rows = read_rows(path) + support: dict[str, dict[str, str]] = {} + for row in rows: + support_path = Path(row["support_path"]) + if row.get("status") in {"generated", "existing"} and support_path.exists(): + support[str(row["sample_id"])] = row + return support + + +class ReferenceFeatureStore: + def __init__(self, rows: list[dict[str, str]], normalizer: Normalizer) -> None: + self.rows_by_id = {str(row["sample_id"]): row for row in rows} + self.normalizer = normalizer + + def get(self, sample_id: str) -> dict[str, np.ndarray | str]: + row = self.rows_by_id[str(sample_id)] + arr = np.load(row["feature_path"], allow_pickle=True) + try: + x = np.nan_to_num(arr["features"].astype(np.float32), nan=0.0, posinf=0.0, neginf=0.0) + y = np.nan_to_num(arr["y_occ"].astype(np.float32), nan=0.0, posinf=0.0, neginf=0.0) + finally: + arr.close() + return {"x": self.normalizer.apply(x), "y_occ": y, "target_timestamp": row["target_timestamp"]} + + +class SpatialSupportStore: + def __init__( + self, + rows: list[dict[str, str]], + support: dict[str, dict[str, str]], + normalizer: Normalizer, + ) -> None: + self.rows_by_id = {str(row["sample_id"]): row for row in rows} + self.support = support + self.normalizer = normalizer + + def get(self, sample_id: str) -> dict[str, np.ndarray | str]: + sid = str(sample_id) + row = self.rows_by_id[sid] + sarr = np.load(self.support[sid]["support_path"], allow_pickle=True) + farr = np.load(row["feature_path"], allow_pickle=True) + try: + x = np.nan_to_num(sarr["features"].astype(np.float32), nan=0.0, posinf=0.0, neginf=0.0) + y = np.nan_to_num(farr["y_occ"].astype(np.float32), nan=0.0, posinf=0.0, neginf=0.0) + finally: + sarr.close() + farr.close() + return {"x": self.normalizer.apply(x), "y_occ": y, "target_timestamp": row["target_timestamp"]} + + +class FullMapDataset(Dataset): + def __init__(self, rows: list[dict[str, str]], store: Any) -> None: + self.rows = rows + self.store = store + + def __len__(self) -> int: + return len(self.rows) + + def __getitem__(self, idx: int) -> dict[str, Any]: + row = self.rows[idx] + sample = self.store.get(str(row["sample_id"])) + return { + "x": torch.from_numpy(np.asarray(sample["x"], dtype=np.float32)), + "y": torch.from_numpy(np.asarray(sample["y_occ"], dtype=np.float32)), + "sample_id": str(row["sample_id"]), + "target_timestamp": str(sample["target_timestamp"]), + } + + +def make_loaders( + split_rows: dict[str, list[dict[str, str]]], + store: Any, + batch_size: int, + device: torch.device, + seed: int, +) -> dict[str, DataLoader]: + loaders: dict[str, DataLoader] = {} + for split, rows in split_rows.items(): + kwargs: dict[str, Any] = {} + if split == "train": + kwargs["generator"] = torch.Generator().manual_seed(int(seed)) + loaders[split] = DataLoader( + FullMapDataset(rows, store), + batch_size=int(batch_size), + shuffle=(split == "train"), + num_workers=0, + pin_memory=device.type == "cuda", + **kwargs, + ) + return loaders + + +def build_attached_store(args: argparse.Namespace, split_rows: dict[str, list[dict[str, str]]]) -> Any: + if args.daily_rows_csv is None: + raise ValueError("--daily-rows-csv is required for attached source.") + daily_lookup, ordered_times, ordered_features = head_control.build_daily_lookup(args.daily_rows_csv) + return head_control.FeatureStore( + split_rows["train"] + split_rows["val"] + split_rows["test"], + daily_lookup, + ordered_times, + ordered_features, + ) + + +def build_alphaearth_store(args: argparse.Namespace, split_rows: dict[str, list[dict[str, str]]]) -> Any: + if args.alphaearth_cache_root is None: + raise ValueError("--alphaearth-cache-root is required for AlphaEarth source.") + grid_cache = alpha_runner.GridCache(args.alphaearth_cache_root) + return alpha_runner.FeatureStore(split_rows["train"] + split_rows["val"] + split_rows["test"], grid_cache) + + +def build_spatial_store(args: argparse.Namespace, split_rows: dict[str, list[dict[str, str]]]) -> SpatialSupportStore: + if args.support_dir is None: + raise ValueError("--support-dir is required for spatial source.") + support = load_support_manifest(args.support_dir / "support_manifest.csv") + missing = [ + row["sample_id"] + for rows in split_rows.values() + for row in rows + if str(row["sample_id"]) not in support + ] + if missing: + raise RuntimeError(f"Missing spatial support maps for {len(missing)} samples; first={missing[:5]}") + normalizer = compute_map_normalizer(split_rows["train"], lambda row: support[str(row["sample_id"])]["support_path"]) + return SpatialSupportStore(split_rows["train"] + split_rows["val"] + split_rows["test"], support, normalizer) + + +def build_reference_store(split_rows: dict[str, list[dict[str, str]]]) -> ReferenceFeatureStore: + normalizer = compute_map_normalizer(split_rows["train"], lambda row: row["feature_path"]) + return ReferenceFeatureStore(split_rows["train"] + split_rows["val"] + split_rows["test"], normalizer) + + +def build_head(head_arch: str, in_ch: int, prior_prob: float) -> nn.Module: + if head_arch == "constant": + return head_control.ConstantHead(prior_prob=prior_prob) + if head_arch == "linear": + return head_control.LinearHead(in_ch=in_ch, prior_prob=prior_prob) + if head_arch == "pixel_mlp": + return head_control.PixelMLPHead(in_ch=in_ch, hidden=16, dropout=0.05, prior_prob=prior_prob) + if head_arch == "shallow_wide": + return head_control.WildfireHead( + in_ch=in_ch, + hidden=64, + dropout=0.10, + norm_type="group", + norm_groups=8, + prior_prob=prior_prob, + ) + return head_control.WildfireHead( + in_ch=in_ch, + hidden=32, + dropout=0.05, + norm_type="group", + norm_groups=8, + prior_prob=prior_prob, + ) + + +@torch.no_grad() +def collect_predictions(model: nn.Module, loader: DataLoader, device: torch.device) -> tuple[np.ndarray, np.ndarray]: + model.eval() + probs: list[np.ndarray] = [] + targets: list[np.ndarray] = [] + for batch in loader: + x = batch["x"].to(device, non_blocking=True) + y = batch["y"].to(device, non_blocking=True) + logits = model(x) + probs.append(np.nan_to_num(torch.sigmoid(logits).detach().cpu().numpy()[:, 0], nan=0.0, posinf=1.0, neginf=0.0)) + targets.append(np.nan_to_num(y.detach().cpu().numpy()[:, 0], nan=0.0, posinf=0.0, neginf=0.0)) + return np.concatenate(probs, axis=0), np.concatenate(targets, axis=0) + + +def train_one_head( + head_arch: str, + in_ch: int, + prior_prob: float, + loaders: dict[str, DataLoader], + args: argparse.Namespace, + device: torch.device, + seed_offset: int, +) -> tuple[nn.Module, list[dict[str, float]]]: + set_seed(int(args.seed) + int(seed_offset)) + model = build_head(head_arch, in_ch=in_ch, prior_prob=prior_prob).to(device) + optimizer = torch.optim.AdamW(model.parameters(), lr=float(args.learning_rate), weight_decay=float(args.weight_decay)) + raw_weight = (1.0 - float(prior_prob)) / max(float(prior_prob), 1e-9) + pos_weight = float(min(float(args.pos_weight_cap), raw_weight)) + criterion = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([pos_weight], dtype=torch.float32, device=device)) + history: list[dict[str, float]] = [] + for epoch in range(1, int(args.epochs) + 1): + model.train() + losses: list[float] = [] + for batch in loaders["train"]: + x = batch["x"].to(device, non_blocking=True) + y = batch["y"].to(device, non_blocking=True) + optimizer.zero_grad(set_to_none=True) + logits = model(x) + loss = criterion(logits, y) + if not torch.isfinite(loss): + raise RuntimeError(f"Non-finite loss for head={head_arch}") + loss.backward() + optimizer.step() + losses.append(float(loss.item())) + history.append({"epoch": epoch, "train_bce": float(np.mean(losses)), "pos_weight": pos_weight}) + return model, history + + +def build_sample_times(rows: list[dict[str, str]]) -> np.ndarray: + return np.array([row["target_timestamp"] for row in rows], dtype="datetime64[h]") + + +def select_val_posthoc( + rows: list[dict[str, object]], + scope: str, + metric: str, +) -> dict[str, object]: + prefix = metric.rsplit("_", 1)[0] + precision_key = f"{prefix}_precision" + recall_key = f"{prefix}_recall" + selected = [row for row in rows if row["split"] == "val" and row["scope"] == scope] + if not selected: + raise RuntimeError(f"No validation rows for scope={scope}") + return max( + selected, + key=lambda row: ( + float(row.get(metric, 0.0)), + float(row.get(precision_key, 0.0)), + float(row.get(recall_key, 0.0)), + -abs(float(row["threshold"]) - 0.5), + ), + ) + + +def matching_test_row(rows: list[dict[str, object]], scope: str, selected: dict[str, object]) -> dict[str, object]: + threshold = float(selected["threshold"]) + variant = str(selected["variant"]) + for row in rows: + if ( + row["split"] == "test" + and row["scope"] == scope + and str(row["variant"]) == variant + and abs(float(row["threshold"]) - threshold) < 1e-12 + ): + return row + raise RuntimeError(f"No matching test row for scope={scope}, threshold={threshold}, variant={variant}") + + +def summarize_head_scores( + head_metrics: list[dict[str, object]], +) -> list[dict[str, object]]: + selection_rows: list[dict[str, object]] = [] + for scope in SCOPE_ORDER: + candidates = [row for row in head_metrics if row["scope"] == scope] + if not candidates: + continue + ranking_selected = max( + candidates, + key=lambda row: ( + float(row["val_pr_auc"]), + float(row["val_union_f1"]), + float(row["val_tolerated_f1"]), + float(row["val_exact_f1"]), + ), + ) + out: dict[str, object] = { + "scope": scope, + "ranking_selected_head": ranking_selected["head_label"], + "ranking_selected_head_arch": ranking_selected["head_arch"], + "ranking_selected_val_pr_auc": float(ranking_selected["val_pr_auc"]), + "ranking_selected_test_pr_auc": float(ranking_selected["test_pr_auc"]), + } + for short, column in (("exact", "exact_f1"), ("tolerated", "tolerated_f1"), ("union", "union_f1")): + val_column = f"val_{column}" + test_column = f"test_{column}" + decision_selected = max( + candidates, + key=lambda row: ( + float(row[val_column]), + float(row["val_pr_auc"]), + str(row["head_arch"]), + ), + ) + val_gap = float(decision_selected[val_column]) - float(ranking_selected[val_column]) + test_gap = float(decision_selected[test_column]) - float(ranking_selected[test_column]) + out[f"{short}_val_ranking_score"] = float(ranking_selected[val_column]) + out[f"{short}_val_decision_score"] = float(decision_selected[val_column]) + out[f"{short}_val_gap"] = float(max(0.0, val_gap)) + out[f"{short}_ranking_score"] = float(ranking_selected[test_column]) + out[f"{short}_decision_score"] = float(decision_selected[test_column]) + out[f"{short}_test_gap"] = float(test_gap) + out[f"{short}_regret"] = float(max(0.0, test_gap)) + out[f"{short}_decision_head"] = decision_selected["head_label"] + out[f"{short}_decision_head_arch"] = decision_selected["head_arch"] + selection_rows.append(out) + return selection_rows + + +def write_csv(rows: list[dict[str, object]], path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fieldnames = sorted({key for row in rows for key in row}) + with path.open("w", newline="", encoding="utf-8") as fh: + writer = csv.DictWriter(fh, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + +def load_head_summary(head_dir: Path, head_arch: str) -> tuple[list[dict[str, object]], dict[str, dict[str, float]], dict[str, object]] | None: + posthoc_path = head_dir / "posthoc_rows.csv" + summary_path = head_dir / "summary.json" + if not posthoc_path.exists() or not summary_path.exists(): + return None + rows = [dict(row) for row in read_rows(posthoc_path)] + if not rows: + return None + try: + summary = json.loads(summary_path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return None + if str(summary.get("head_arch")) != str(head_arch): + return None + raw_pr_auc = summary.get("raw_pr_auc") + if not isinstance(raw_pr_auc, dict): + return None + try: + parsed_pr_auc = { + split: { + scope: float(raw_pr_auc[split][scope]) + for scope in SCOPE_ORDER + } + for split in ("val", "test") + } + except Exception: + return None + return rows, parsed_pr_auc, summary + + +def append_head_metrics( + head_metrics: list[dict[str, object]], + posthoc_rows: list[dict[str, object]], + raw_pr_auc: dict[str, dict[str, float]], + head_arch: str, + args: argparse.Namespace, +) -> None: + for scope in SCOPE_ORDER: + metric_scores: dict[str, float] = {} + selected_thresholds: dict[str, float] = {} + selected_variants: dict[str, str] = {} + for short, metric in METRICS.items(): + selected = select_val_posthoc(posthoc_rows, scope, metric) + test_row = matching_test_row(posthoc_rows, scope, selected) + metric_scores[f"val_{short}_f1"] = float(selected[metric]) + metric_scores[f"test_{short}_f1"] = float(test_row[metric]) + selected_thresholds[short] = float(selected["threshold"]) + selected_variants[short] = str(selected["variant"]) + head_metrics.append( + { + "model_tag": args.model_tag, + "family": args.fm_family, + "seed": int(args.seed), + "scope": scope, + "head_arch": head_arch, + "head_label": head_control.HEAD_LABELS[head_arch], + "val_pr_auc": float(raw_pr_auc["val"][scope]), + "test_pr_auc": float(raw_pr_auc["test"][scope]), + **metric_scores, + "selected_thresholds": selected_thresholds, + "selected_variants": selected_variants, + } + ) + + +def main() -> None: + args = parse_args() + args.output_dir.mkdir(parents=True, exist_ok=True) + set_seed(int(args.seed)) + device = choose_device(args.device) + + split_rows = { + split: read_rows(args.feature_root / "splits" / f"{split}.csv") + for split in ("train", "val", "test") + } + if args.source_kind == "reference": + store = build_reference_store(split_rows) + elif args.source_kind == "attached": + store = build_attached_store(args, split_rows) + elif args.source_kind == "spatial": + store = build_spatial_store(args, split_rows) + else: + store = build_alphaearth_store(args, split_rows) + + loaders = make_loaders(split_rows, store, int(args.batch_size), device, int(args.seed)) + first = next(iter(loaders["train"])) + in_ch = int(first["x"].shape[1]) + prior_prob = total_positive_rate(split_rows["train"]) + fire_prone_mask, fire_prone_meta = head_control.build_fire_prone_mask( + split_rows["train"], + store, + float(args.fire_prone_top_frac), + ) + + head_metrics: list[dict[str, object]] = [] + head_artifacts: dict[str, str] = {} + for head_index, head_arch in enumerate(args.heads): + head_dir = args.output_dir / head_arch + head_dir.mkdir(parents=True, exist_ok=True) + cached = load_head_summary(head_dir, head_arch) + if cached is not None: + posthoc_rows, raw_pr_auc, _ = cached + print(f"[selection-regret] reuse {args.fm_family} seed={args.seed} head={head_arch}", flush=True) + else: + print(f"[selection-regret] training {args.fm_family} seed={args.seed} head={head_arch}", flush=True) + model, history = train_one_head( + head_arch=head_arch, + in_ch=in_ch, + prior_prob=prior_prob, + loaders=loaders, + args=args, + device=device, + seed_offset=1009 * (head_index + 1), + ) + posthoc_rows = [] + raw_pr_auc = {} + for split in ("val", "test"): + probs, targets = collect_predictions(model, loaders[split], device) + sample_times = build_sample_times(split_rows[split]) + raw_pr_auc[split] = { + "global": head_control._masked_average_precision(probs, targets, region_mask=None), + "fire_prone": head_control._masked_average_precision(probs, targets, region_mask=fire_prone_mask), + } + posthoc_rows.extend( + head_control.build_posthoc_rows( + probs=probs, + targets=targets, + sample_times=sample_times, + split=split, + fire_prone_mask=fire_prone_mask, + args=args, + ) + ) + + write_csv(posthoc_rows, head_dir / "posthoc_rows.csv") + head_summary = { + "head_arch": head_arch, + "head_label": head_control.HEAD_LABELS[head_arch], + "history": history, + "raw_pr_auc": raw_pr_auc, + "posthoc_rows_csv": str(head_dir / "posthoc_rows.csv"), + } + (head_dir / "summary.json").write_text(json.dumps(head_summary, indent=2), encoding="utf-8") + head_artifacts[head_arch] = str(head_dir / "summary.json") + append_head_metrics(head_metrics, posthoc_rows, raw_pr_auc, head_arch, args) + + selection_rows = summarize_head_scores(head_metrics) + for row in selection_rows: + row["model_tag"] = args.model_tag + row["family"] = args.fm_family + row["seed"] = int(args.seed) + + write_csv(head_metrics, args.output_dir / "head_metrics.csv") + write_csv(selection_rows, args.output_dir / "selection_rows.csv") + summary = { + "experiment": "all-backbone fixed-feature head-selection regret", + "task": "wildfire_occupancy", + "model_tag": args.model_tag, + "fm_family": args.fm_family, + "source_kind": args.source_kind, + "seed": int(args.seed), + "feature_root": str(args.feature_root), + "daily_rows_csv": str(args.daily_rows_csv) if args.daily_rows_csv else None, + "support_dir": str(args.support_dir) if args.support_dir else None, + "alphaearth_cache_root": str(args.alphaearth_cache_root) if args.alphaearth_cache_root else None, + "device": str(device), + "heads": list(args.heads), + "input_channels": int(in_ch), + "prior_prob": float(prior_prob), + "fire_prone_scope": { + "scope_name": "fire_prone", + "reported_as": "top 20%", + **fire_prone_meta, + }, + "metrics": METRICS, + "head_metrics": head_metrics, + "selection_rows": selection_rows, + "head_artifacts": head_artifacts, + "artifacts": { + "head_metrics_csv": str(args.output_dir / "head_metrics.csv"), + "selection_rows_csv": str(args.output_dir / "selection_rows.csv"), + }, + } + (args.output_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") + print(json.dumps(summary, indent=2), flush=True) + + +if __name__ == "__main__": + main() diff --git a/experiments/raw_reference/task_scripts/run_analog_extended_retrieval_sweep_seeded.py b/experiments/raw_reference/task_scripts/run_analog_extended_retrieval_sweep_seeded.py new file mode 100644 index 0000000000000000000000000000000000000000..c68640647d8478e5c6250495a9d06402bb1b3c0c --- /dev/null +++ b/experiments/raw_reference/task_scripts/run_analog_extended_retrieval_sweep_seeded.py @@ -0,0 +1,333 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Dict, List, Tuple + + +import os + +for _p in os.environ.get("WILDFIRE_FM_EXTRA_PYTHONPATH", "").split(os.pathsep): + if _p and _p not in sys.path: + sys.path.insert(0, _p) + +import numpy as np +import pandas as pd +from sklearn.compose import ColumnTransformer +from sklearn.impute import SimpleImputer +from sklearn.linear_model import ElasticNet, Ridge +from sklearn.metrics.pairwise import cosine_similarity +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import OneHotEncoder, StandardScaler + + +DROP_COLUMNS = { + "Event_ID", + "Incid_Name", + "incident_name_norm", + "wfigs_name", + "Ig_Date", + "weather_date", + "BurnBndAc", + "target_log_burn_acres", +} +CATEGORICAL_COLUMNS = ["Incid_Type", "state_abbr", "county_name", "wfigs_match_type"] + + +def build_splits(df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: + ordered = df.sort_values("Ig_Date").reset_index(drop=True) + n = len(ordered) + train_end = max(int(round(n * 0.6)), 1) + val_end = max(int(round(n * 0.8)), train_end + 1) + val_end = min(val_end, n - 1) if n >= 3 else n + train = ordered.iloc[:train_end].copy() + val = ordered.iloc[train_end:val_end].copy() + test = ordered.iloc[val_end:].copy() + if len(val) == 0 and len(test) > 1: + val = test.iloc[:1].copy() + test = test.iloc[1:].copy() + return train, val, test + + +def block_columns(df: pd.DataFrame, exclude: set[str]) -> Dict[str, List[str]]: + numeric = [ + c + for c in df.columns + if c not in DROP_COLUMNS + and c not in CATEGORICAL_COLUMNS + and c not in exclude + and pd.api.types.is_numeric_dtype(df[c]) + ] + return { + "weather": [c for c in numeric if c.startswith("weather_")], + "geo_fire": [ + c + for c in numeric + if c.startswith("firms_") + or c.startswith("landfire_") + or c in {"BurnBndLat", "BurnBndLon", "lat", "lon", "wfigs_acres", "wfigs_date_diff_days", "wfigs_dist_km", "is_conus_static"} + ], + "categorical": [c for c in CATEGORICAL_COLUMNS if c in df.columns and c not in exclude], + } + + +def make_block_matrix(train: pd.DataFrame, val: pd.DataFrame, test: pd.DataFrame, cols: List[str], categorical: bool) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + if not cols: + n_train, n_val, n_test = len(train), len(val), len(test) + return np.zeros((n_train, 0), dtype=np.float32), np.zeros((n_val, 0), dtype=np.float32), np.zeros((n_test, 0), dtype=np.float32) + if categorical: + transformer = ColumnTransformer( + [("cat", Pipeline([("impute", SimpleImputer(strategy="most_frequent")), ("onehot", OneHotEncoder(handle_unknown="ignore"))]), cols)], + remainder="drop", + ) + else: + transformer = ColumnTransformer( + [("num", Pipeline([("impute", SimpleImputer(strategy="median")), ("scale", StandardScaler())]), cols)], + remainder="drop", + ) + train_x = transformer.fit_transform(train[cols]) + val_x = transformer.transform(val[cols]) + test_x = transformer.transform(test[cols]) + if hasattr(train_x, "toarray"): + train_x = train_x.toarray() + val_x = val_x.toarray() + test_x = test_x.toarray() + return train_x.astype(np.float32), val_x.astype(np.float32), test_x.astype(np.float32) + + +def graded_relevance(query_target: float, retrieved_targets: np.ndarray) -> np.ndarray: + delta = np.abs(np.asarray(retrieved_targets, dtype=np.float64) - float(query_target)) + return np.select([delta <= 0.5, delta <= 1.0, delta <= 1.5], [3.0, 2.0, 1.0], default=0.0) + + +def dcg(relevance: np.ndarray) -> float: + rel = np.asarray(relevance, dtype=np.float64) + discounts = 1.0 / np.log2(np.arange(rel.size, dtype=np.float64) + 2.0) + return float(np.sum(rel * discounts)) + + +def ndcg_at_k(relevance: np.ndarray, ideal_relevance: np.ndarray, k: int) -> float: + denom = dcg(ideal_relevance[:k]) + return float(dcg(relevance[:k]) / denom) if denom > 0 else 0.0 + + +def rmse(y_true: np.ndarray, y_pred: np.ndarray) -> float: + return float(np.sqrt(np.mean((np.asarray(y_true) - np.asarray(y_pred)) ** 2))) + + +def spearman_corr(y_true: np.ndarray, y_pred: np.ndarray) -> float: + value = pd.Series(y_true).corr(pd.Series(y_pred), method="spearman") + return float(value) if pd.notna(value) else 0.0 + + +def target_weight_vectors(train_vec: np.ndarray, val_vec: np.ndarray, test_vec: np.ndarray, target: np.ndarray, power: float, floor: float) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + if train_vec.shape[1] == 0: + return train_vec, val_vec, test_vec + x = np.asarray(train_vec, dtype=np.float64) + y = np.asarray(target, dtype=np.float64) + y = y - y.mean() + x_centered = x - x.mean(axis=0, keepdims=True) + denom = np.clip(np.sqrt(np.sum(x_centered**2, axis=0)) * np.sqrt(np.sum(y**2)), 1e-12, None) + corr = np.abs(np.sum(x_centered * y[:, None], axis=0) / denom) + corr = np.nan_to_num(corr, nan=0.0, posinf=0.0, neginf=0.0) + if float(corr.max()) > 0: + corr = corr / float(corr.max()) + weights = (floor + np.power(corr, power)).astype(np.float32) + return train_vec * weights, val_vec * weights, test_vec * weights + + +def score_vectors(query_vec: np.ndarray, library_vec: np.ndarray, query_df: pd.DataFrame, library_df: pd.DataFrame, k: int, mode: str) -> Dict[str, float]: + k_eff = min(k, library_vec.shape[0]) + lib_norm = library_vec / np.clip(np.linalg.norm(library_vec, axis=1, keepdims=True), 1e-12, None) + query_norm = query_vec / np.clip(np.linalg.norm(query_vec, axis=1, keepdims=True), 1e-12, None) + sim_all = cosine_similarity(query_norm, lib_norm) + knn_idx = np.argsort(-sim_all, axis=1)[:, :k_eff] + knn_sim = np.take_along_axis(sim_all, knn_idx, axis=1) + target_lib = library_df["target_log_burn_acres"].to_numpy(dtype=np.float64) + preds = [] + ndcg5 = [] + ndcg10 = [] + hit1 = [] + hit5 = [] + hit10 = [] + best_abs = [] + for i in range(query_df.shape[0]): + idx = knn_idx[i] + sims = knn_sim[i] + top_targets = target_lib[idx] + true = float(query_df.iloc[i]["target_log_burn_acres"]) + relevance = graded_relevance(true, top_targets) + ideal = np.sort(graded_relevance(true, target_lib))[::-1] + ndcg5.append(ndcg_at_k(relevance, ideal, 5)) + ndcg10.append(ndcg_at_k(relevance, ideal, 10)) + hit1.append(float(relevance[:1].max() >= 2.0)) + hit5.append(float(relevance[: min(5, k_eff)].max() >= 2.0)) + hit10.append(float(relevance[: min(10, k_eff)].max() >= 2.0)) + best_abs.append(float(np.min(np.abs(top_targets - true)))) + if mode == "weighted": + weights = np.maximum((sims + 1.0) / 2.0, 1e-6) + preds.append(float(np.sum(weights * top_targets) / np.sum(weights))) + else: + preds.append(float(np.mean(top_targets))) + pred = np.asarray(preds, dtype=np.float64) + true_log = query_df["target_log_burn_acres"].to_numpy(dtype=np.float64) + return { + "count": int(len(query_df)), + "log_mae": float(np.mean(np.abs(true_log - pred))), + "log_rmse": rmse(true_log, pred), + "log_spearman": spearman_corr(true_log, pred), + "ndcg_at_5": float(np.mean(ndcg5)), + "ndcg_at_10": float(np.mean(ndcg10)), + "hit_at_1_tol1": float(np.mean(hit1)), + "hit_at_5_tol1": float(np.mean(hit5)), + "hit_at_10_tol1": float(np.mean(hit10)), + "mean_best_abs_log_delta_at_k": float(np.mean(best_abs)), + } + + +def append_supervised_scalar( + train_vec: np.ndarray, + val_vec: np.ndarray, + test_vec: np.ndarray, + train_df: pd.DataFrame, + model_name: str, + weight: float, + seed: int, +) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + y = train_df["target_log_burn_acres"].to_numpy(dtype=np.float64) + model = Ridge(alpha=1.0) if model_name == "ridge" else ElasticNet(alpha=0.01, l1_ratio=0.2, random_state=seed, max_iter=10000) + model.fit(train_vec, y) + train_pred = model.predict(train_vec) + val_pred = model.predict(val_vec) + test_pred = model.predict(test_vec) + mean = float(np.mean(train_pred)) + std = float(np.std(train_pred)) or 1.0 + def _append(x: np.ndarray, pred: np.ndarray) -> np.ndarray: + scalar = ((pred - mean) / std).reshape(-1, 1).astype(np.float32) * float(weight) + return np.concatenate([x, scalar], axis=1) + return _append(train_vec, train_pred), _append(val_vec, val_pred), _append(test_vec, test_pred) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--event-table", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--exclude-columns", nargs="*", default=[]) + parser.add_argument("--seed", type=int, default=7) + args = parser.parse_args() + + df = pd.read_csv(args.event_table) + df["Ig_Date"] = pd.to_datetime(df["Ig_Date"]) + train_df, val_df, test_df = build_splits(df) + exclude = set(args.exclude_columns) + blocks = block_columns(df, exclude) + matrices = { + "weather": make_block_matrix(train_df, val_df, test_df, blocks["weather"], categorical=False), + "geo_fire": make_block_matrix(train_df, val_df, test_df, blocks["geo_fire"], categorical=False), + "categorical": make_block_matrix(train_df, val_df, test_df, blocks["categorical"], categorical=True), + } + + candidate_rows: List[Dict[str, object]] = [] + best = None + best_score = None + best_test = None + block_weight_grid = [ + {"weather": 1.0, "geo_fire": 1.0, "categorical": 1.0}, + {"weather": 0.5, "geo_fire": 1.5, "categorical": 1.0}, + {"weather": 0.25, "geo_fire": 2.0, "categorical": 1.0}, + {"weather": 1.5, "geo_fire": 1.0, "categorical": 0.5}, + {"weather": 0.0, "geo_fire": 2.0, "categorical": 1.0}, + {"weather": 2.0, "geo_fire": 0.5, "categorical": 0.5}, + ] + target_weight_settings = [(0.0, 1.0), (0.5, 0.25), (1.0, 0.25), (2.0, 0.10)] + scalar_settings = [("none", 0.0), ("ridge", 0.5), ("ridge", 1.0), ("ridge", 2.0), ("enet", 1.0), ("enet", 2.0)] + + for bw in block_weight_grid: + base_train = np.concatenate([matrices[name][0] * bw[name] for name in ["weather", "geo_fire", "categorical"]], axis=1) + base_val = np.concatenate([matrices[name][1] * bw[name] for name in ["weather", "geo_fire", "categorical"]], axis=1) + base_test = np.concatenate([matrices[name][2] * bw[name] for name in ["weather", "geo_fire", "categorical"]], axis=1) + for power, floor in target_weight_settings: + tw_train, tw_val, tw_test = target_weight_vectors( + base_train, + base_val, + base_test, + train_df["target_log_burn_acres"].to_numpy(dtype=np.float64), + power=power, + floor=floor, + ) + for scalar_model, scalar_weight in scalar_settings: + if scalar_model == "none": + train_vec, val_vec, test_vec = tw_train, tw_val, tw_test + else: + train_vec, val_vec, test_vec = append_supervised_scalar( + tw_train, + tw_val, + tw_test, + train_df, + scalar_model, + scalar_weight, + args.seed, + ) + for k in [3, 5, 10, 15, 20]: + for mode in ["mean", "weighted"]: + val_metrics = score_vectors(val_vec, train_vec, val_df, train_df, k=k, mode=mode) + test_metrics = score_vectors(test_vec, train_vec, test_df, train_df, k=k, mode=mode) + row = { + "block_weights": bw, + "target_weight_power": power, + "target_weight_floor": floor, + "supervised_scalar": scalar_model, + "supervised_scalar_weight": scalar_weight, + "k": k, + "mode": mode, + "val_metrics": val_metrics, + "test_metrics": test_metrics, + } + candidate_rows.append(row) + score = float(val_metrics["ndcg_at_10"]) + if best_score is None or score > best_score: + best_score = score + best = row + best_test = test_metrics + + args.output_dir.mkdir(parents=True, exist_ok=True) + candidate_df = pd.DataFrame( + [ + { + "val_ndcg_at_10": r["val_metrics"]["ndcg_at_10"], + "val_log_mae": r["val_metrics"]["log_mae"], + "test_ndcg_at_10": r["test_metrics"]["ndcg_at_10"], + "test_log_mae": r["test_metrics"]["log_mae"], + "k": r["k"], + "mode": r["mode"], + "target_weight_power": r["target_weight_power"], + "target_weight_floor": r["target_weight_floor"], + "supervised_scalar": r["supervised_scalar"], + "supervised_scalar_weight": r["supervised_scalar_weight"], + **{f"block_{k}": v for k, v in r["block_weights"].items()}, + } + for r in candidate_rows + ] + ) + candidate_df.to_csv(args.output_dir / "candidate_grid.csv", index=False) + summary = { + "task_id": "wildfire_analog_retrieval_extended_hybrid_sweep", + "event_table": str(args.event_table), + "seed": int(args.seed), + "excluded_columns": sorted(exclude), + "split_sizes": {"train": len(train_df), "val": len(val_df), "test": len(test_df)}, + "feature_blocks": blocks, + "selection_metric": "val_ndcg_at_10", + "selected_retrieval": best, + "test_metrics": best_test, + "candidate_count": len(candidate_rows), + } + (args.output_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/experiments/raw_reference/task_scripts/run_event_analog_taskmodel_seeded.py b/experiments/raw_reference/task_scripts/run_event_analog_taskmodel_seeded.py new file mode 100644 index 0000000000000000000000000000000000000000..2562933f222a6605c9552152bed9701824e2c83a --- /dev/null +++ b/experiments/raw_reference/task_scripts/run_event_analog_taskmodel_seeded.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Dict, List, Tuple + + +import os + +for _p in os.environ.get("WILDFIRE_FM_EXTRA_PYTHONPATH", "").split(os.pathsep): + if _p and _p not in sys.path: + sys.path.insert(0, _p) + +import faiss +import hnswlib +import numpy as np +import pandas as pd +from sklearn.compose import ColumnTransformer +from sklearn.impute import SimpleImputer +from sklearn.metrics.pairwise import cosine_similarity +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import OneHotEncoder, StandardScaler + + +DROP_COLUMNS = { + "Event_ID", + "Incid_Name", + "incident_name_norm", + "wfigs_name", + "Ig_Date", + "weather_date", + "BurnBndAc", + "target_log_burn_acres", +} +CATEGORICAL_COLUMNS = ["Incid_Type", "state_abbr", "county_name", "wfigs_match_type"] + + +def rmse(y_true: np.ndarray, y_pred: np.ndarray) -> float: + return float(np.sqrt(np.mean((np.asarray(y_true) - np.asarray(y_pred)) ** 2))) + + +def mape(y_true: np.ndarray, y_pred: np.ndarray) -> float: + denom = np.clip(np.asarray(y_true, dtype=np.float64), 1e-6, None) + frac = np.abs(np.asarray(y_true, dtype=np.float64) - np.asarray(y_pred, dtype=np.float64)) / denom + return float(np.mean(frac)) + + +def r2_score_manual(y_true: np.ndarray, y_pred: np.ndarray) -> float: + y_true = np.asarray(y_true, dtype=np.float64) + y_pred = np.asarray(y_pred, dtype=np.float64) + ss_res = float(np.sum((y_true - y_pred) ** 2)) + ss_tot = float(np.sum((y_true - y_true.mean()) ** 2)) + return float(1.0 - ss_res / ss_tot) if ss_tot > 0 else 0.0 + + +def spearman_corr(y_true: np.ndarray, y_pred: np.ndarray) -> float: + a = pd.Series(np.asarray(y_true, dtype=np.float64)) + b = pd.Series(np.asarray(y_pred, dtype=np.float64)) + value = a.corr(b, method="spearman") + return float(value) if pd.notna(value) else 0.0 + + +def build_splits(df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: + ordered = df.sort_values("Ig_Date").reset_index(drop=True) + n = len(ordered) + train_end = max(int(round(n * 0.6)), 1) + val_end = max(int(round(n * 0.8)), train_end + 1) + val_end = min(val_end, n - 1) if n >= 3 else n + train = ordered.iloc[:train_end].copy() + val = ordered.iloc[train_end:val_end].copy() + test = ordered.iloc[val_end:].copy() + if len(val) == 0 and len(test) > 1: + val = test.iloc[:1].copy() + test = test.iloc[1:].copy() + return train, val, test + + +def feature_columns(df: pd.DataFrame, feature_profile: str = "all") -> Tuple[List[str], List[str]]: + categorical = [c for c in CATEGORICAL_COLUMNS if c in df.columns] + numeric = [] + for col in df.columns: + if col in DROP_COLUMNS or col in categorical: + continue + if pd.api.types.is_numeric_dtype(df[col]): + numeric.append(col) + if feature_profile == "weather_fm": + numeric = [c for c in numeric if c.startswith("weather_")] + categorical = [] + return numeric, categorical + + +def make_preprocessor(numeric_cols: List[str], categorical_cols: List[str]) -> ColumnTransformer: + return ColumnTransformer( + transformers=[ + ( + "num", + Pipeline( + steps=[ + ("impute", SimpleImputer(strategy="median")), + ("scale", StandardScaler()), + ] + ), + numeric_cols, + ), + ( + "cat", + Pipeline( + steps=[ + ("impute", SimpleImputer(strategy="most_frequent")), + ("onehot", OneHotEncoder(handle_unknown="ignore")), + ] + ), + categorical_cols, + ), + ], + remainder="drop", + ) + + +def to_dense_float32(x) -> np.ndarray: + if hasattr(x, "toarray"): + x = x.toarray() + return np.asarray(x, dtype=np.float32) + + +def weighted_prediction(sim: np.ndarray, targets: np.ndarray) -> float: + weights = np.maximum((np.asarray(sim, dtype=np.float64) + 1.0) / 2.0, 1e-6) + return float(np.sum(weights * targets) / np.sum(weights)) + + +def graded_relevance(query_target: float, retrieved_targets: np.ndarray) -> np.ndarray: + delta = np.abs(np.asarray(retrieved_targets, dtype=np.float64) - float(query_target)) + return np.select([delta <= 0.5, delta <= 1.0, delta <= 1.5], [3.0, 2.0, 1.0], default=0.0) + + +def dcg(relevance: np.ndarray) -> float: + rel = np.asarray(relevance, dtype=np.float64) + if rel.size == 0: + return 0.0 + discounts = 1.0 / np.log2(np.arange(rel.size, dtype=np.float64) + 2.0) + return float(np.sum(rel * discounts)) + + +def ndcg_at_k(relevance: np.ndarray, ideal_relevance: np.ndarray, k: int) -> float: + rel = np.asarray(relevance, dtype=np.float64)[:k] + ideal = np.asarray(ideal_relevance, dtype=np.float64)[:k] + denom = dcg(ideal) + return float(dcg(rel) / denom) if denom > 0 else 0.0 + + +def score_backend( + name: str, + query_vec: np.ndarray, + library_vec: np.ndarray, + query_df: pd.DataFrame, + library_df: pd.DataFrame, + k: int, + mode: str, +) -> Tuple[Dict[str, float], pd.DataFrame]: + target_lib = library_df["target_log_burn_acres"].to_numpy(dtype=np.float64) + rows = [] + preds = [] + ndcg5 = [] + ndcg10 = [] + hit1 = [] + hit5 = [] + hit10 = [] + best_abs_delta = [] + + k_eff = min(int(k), int(library_vec.shape[0])) + if name == "cosine_exact": + sim_all = cosine_similarity(query_vec, library_vec) + knn_idx = np.argsort(-sim_all, axis=1)[:, :k_eff] + knn_sim = np.take_along_axis(sim_all, knn_idx, axis=1) + else: + library_norm = library_vec / np.clip(np.linalg.norm(library_vec, axis=1, keepdims=True), 1e-12, None) + query_norm = query_vec / np.clip(np.linalg.norm(query_vec, axis=1, keepdims=True), 1e-12, None) + if name == "faiss_flat_ip": + index = faiss.IndexFlatIP(library_norm.shape[1]) + index.add(library_norm.astype(np.float32)) + knn_sim, knn_idx = index.search(query_norm.astype(np.float32), k_eff) + elif name == "hnsw_cosine": + index = hnswlib.Index(space="cosine", dim=library_norm.shape[1]) + index.init_index(max_elements=library_norm.shape[0], ef_construction=100, M=16) + index.add_items(library_norm.astype(np.float32), np.arange(library_norm.shape[0])) + index.set_ef(max(50, k_eff)) + knn_idx, dist = index.knn_query(query_norm.astype(np.float32), k=k_eff) + knn_sim = 1.0 - dist + else: + raise ValueError(name) + + for i in range(query_df.shape[0]): + order = knn_idx[i] + top_sim = knn_sim[i] + top_targets = target_lib[order] + query_target = float(query_df.iloc[i]["target_log_burn_acres"]) + relevance = graded_relevance(query_target, top_targets) + ideal_relevance = np.sort(graded_relevance(query_target, target_lib))[::-1] + abs_delta = np.abs(top_targets - float(query_df.iloc[i]["target_log_burn_acres"])) + ndcg5.append(ndcg_at_k(relevance, ideal_relevance, 5)) + ndcg10.append(ndcg_at_k(relevance, ideal_relevance, 10)) + hit1.append(float(relevance[:1].max() >= 2.0)) + hit5.append(float(relevance[: min(5, k_eff)].max() >= 2.0)) + hit10.append(float(relevance[: min(10, k_eff)].max() >= 2.0)) + best_abs_delta.append(float(abs_delta.min())) + pred = float(np.mean(top_targets)) if mode == "mean" else weighted_prediction(top_sim, top_targets) + preds.append(pred) + rows.append( + { + "query_event_id": query_df.iloc[i]["Event_ID"], + "true_log_burn_acres": float(query_df.iloc[i]["target_log_burn_acres"]), + "pred_log_burn_acres": pred, + "backend": name, + "k": k, + "effective_k": k_eff, + "mode": mode, + "top_relevance": relevance.tolist(), + "best_abs_log_delta": float(abs_delta.min()), + } + ) + + pred_arr = np.asarray(preds, dtype=np.float64) + true_log = query_df["target_log_burn_acres"].to_numpy(dtype=np.float64) + true_acres = query_df["BurnBndAc"].to_numpy(dtype=np.float64) + pred_acres = np.exp(pred_arr) + metrics = { + "count": int(len(query_df)), + "log_mae": float(np.mean(np.abs(true_log - pred_arr))), + "log_rmse": rmse(true_log, pred_arr), + "log_r2": r2_score_manual(true_log, pred_arr), + "log_spearman": spearman_corr(true_log, pred_arr), + "log_median_ae": float(np.median(np.abs(true_log - pred_arr))), + "acres_mae": float(np.mean(np.abs(true_acres - pred_acres))), + "acres_rmse": rmse(true_acres, pred_acres), + "acres_median_ae": float(np.median(np.abs(true_acres - pred_acres))), + "acres_mape": mape(true_acres, pred_acres), + "ndcg_at_5": float(np.mean(ndcg5)) if ndcg5 else 0.0, + "ndcg_at_10": float(np.mean(ndcg10)) if ndcg10 else 0.0, + "hit_at_1_tol1": float(np.mean(hit1)) if hit1 else 0.0, + "hit_at_5_tol1": float(np.mean(hit5)) if hit5 else 0.0, + "hit_at_10_tol1": float(np.mean(hit10)) if hit10 else 0.0, + "mean_best_abs_log_delta_at_k": float(np.mean(best_abs_delta)) if best_abs_delta else 0.0, + } + return metrics, pd.DataFrame(rows) + + +def target_weight_vectors(train_vec: np.ndarray, val_vec: np.ndarray, test_vec: np.ndarray, target: np.ndarray) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + x = np.asarray(train_vec, dtype=np.float64) + y = np.asarray(target, dtype=np.float64) + y = y - y.mean() + x_centered = x - x.mean(axis=0, keepdims=True) + denom = np.clip(np.sqrt(np.sum(x_centered**2, axis=0)) * np.sqrt(np.sum(y**2)), 1e-12, None) + corr = np.abs(np.sum(x_centered * y[:, None], axis=0) / denom) + corr = np.nan_to_num(corr, nan=0.0, posinf=0.0, neginf=0.0) + if float(corr.max()) > 0: + corr = corr / float(corr.max()) + weights = (0.25 + corr).astype(np.float32) + return train_vec * weights, val_vec * weights, test_vec * weights + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--event-table", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--selection-metric", choices=("log_mae", "ndcg_at_10"), default="ndcg_at_10") + parser.add_argument("--feature-profile", choices=("all", "weather_fm"), default="all") + parser.add_argument("--fm-family", type=str, default="") + parser.add_argument("--seed", type=int, default=7) + args = parser.parse_args() + + df = pd.read_csv(args.event_table) + df["Ig_Date"] = pd.to_datetime(df["Ig_Date"]) + train_df, val_df, test_df = build_splits(df) + numeric_cols, categorical_cols = feature_columns(df, feature_profile=args.feature_profile) + if not numeric_cols and not categorical_cols: + raise SystemExit(f"No usable features found for profile={args.feature_profile}") + x_cols = numeric_cols + categorical_cols + pre = make_preprocessor(numeric_cols, categorical_cols) + train_vec = to_dense_float32(pre.fit_transform(train_df[x_cols])) + val_vec = to_dense_float32(pre.transform(val_df[x_cols])) + test_vec = to_dense_float32(pre.transform(test_df[x_cols])) + weighted_train_vec, weighted_val_vec, weighted_test_vec = target_weight_vectors( + train_vec, + val_vec, + test_vec, + train_df["target_log_burn_acres"].to_numpy(dtype=np.float64), + ) + vector_variants = { + "standard": (train_vec, val_vec, test_vec), + "target_weighted": (weighted_train_vec, weighted_val_vec, weighted_test_vec), + } + + candidate_validation: List[Dict[str, object]] = [] + best = None + best_score = None + best_val_rows = None + best_test_rows = None + for variant, (lib_vec, v_vec, _) in vector_variants.items(): + for backend in ["cosine_exact", "faiss_flat_ip", "hnsw_cosine"]: + for k in [1, 3, 5, 10]: + for mode in ["mean", "weighted"]: + val_metrics, val_rows = score_backend(backend, v_vec, lib_vec, val_df, train_df, k, mode) + candidate_validation.append({"variant": variant, "backend": backend, "k": k, "mode": mode, "val_metrics": val_metrics}) + score = float(val_metrics[args.selection_metric]) + better = score > best_score if args.selection_metric == "ndcg_at_10" and best_score is not None else score < best_score if best_score is not None else True + if better: + best_score = score + best = {"variant": variant, "backend": backend, "k": k, "mode": mode} + best_val_rows = val_rows + + assert best is not None + best_train_vec, _, best_test_vec = vector_variants[str(best["variant"])] + test_metrics, test_rows = score_backend(best["backend"], best_test_vec, best_train_vec, test_df, train_df, int(best["k"]), str(best["mode"])) + best_test_rows = test_rows + + args.output_dir.mkdir(parents=True, exist_ok=True) + if best_val_rows is not None: + best_val_rows.to_csv(args.output_dir / "val_retrieval_examples.csv", index=False) + if best_test_rows is not None: + best_test_rows.to_csv(args.output_dir / "test_retrieval_examples.csv", index=False) + + summary = { + "task_id": "wildfire_analog_retrieval_taskmodels", + "task_form": "event_level_retrieval_with_induced_outcome_error", + "event_table": str(args.event_table), + "output_dir": str(args.output_dir), + "feature_profile": args.feature_profile, + "seed": int(args.seed), + "split_sizes": { + "train": int(len(train_df)), + "val": int(len(val_df)), + "test": int(len(test_df)), + }, + "feature_columns": {"numeric": numeric_cols, "categorical": categorical_cols}, + "candidate_validation": candidate_validation, + "selected_retrieval": best, + "selection_metric": args.selection_metric, + "test_metrics": test_metrics, + "model_family": "popular_open_source_retrieval_backends_with_train_only_target_weighting", + "fm_family": (args.fm_family or "weather_fm_derived_features") if args.feature_profile == "weather_fm" else None, + } + (args.output_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/experiments/raw_reference/task_scripts/run_extreme_heat_alphaearth_suite_seeded.py b/experiments/raw_reference/task_scripts/run_extreme_heat_alphaearth_suite_seeded.py new file mode 100644 index 0000000000000000000000000000000000000000..9e551666cd3ab7d11b1ea885be3d61e95f9b5eaf --- /dev/null +++ b/experiments/raw_reference/task_scripts/run_extreme_heat_alphaearth_suite_seeded.py @@ -0,0 +1,344 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import math +import re +import sys +from pathlib import Path +from typing import Dict, Iterable, List, Optional, Tuple + + +import os + +for _p in os.environ.get("WILDFIRE_FM_EXTRA_PYTHONPATH", "").split(os.pathsep): + if _p and _p not in sys.path: + sys.path.insert(0, _p) + +import numpy as np +import pandas as pd +from catboost import CatBoostRegressor +from lightgbm import LGBMRegressor +from netCDF4 import Dataset +from sklearn.linear_model import ElasticNet, Ridge +from sklearn.metrics import mean_absolute_error, mean_squared_error +from xgboost import XGBRegressor + + +PRED_RE = re.compile(r"pred_(\d{8})_(\d{2})\.nc$") +WEATHER_VARS = ["T2M", "QV2M", "TQV", "U10M", "V10M", "TS"] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--pred-root", type=Path, action="append", required=True) + parser.add_argument("--merra-root", type=Path, required=True) + parser.add_argument("--alphaearth-year-csv", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--model-family", choices=("full", "lite"), default="full") + parser.add_argument("--alphaearth-prefix", type=str, default="alphaearth_") + parser.add_argument("--lat-min", type=float, default=24.0) + parser.add_argument("--lat-max", type=float, default=50.0) + parser.add_argument("--lon-min", type=float, default=-125.0) + parser.add_argument("--lon-max", type=float, default=-66.0) + parser.add_argument("--seed", type=int, default=7) + return parser.parse_args() + + +def choose_split(path: Path) -> Optional[str]: + name = path.name + if name == "output2022": + return "train" + if name == "output2024": + return "val" + if name == "output2025": + return "test" + return None + + +def parse_pred_timestamp(path: Path) -> Tuple[str, int]: + match = PRED_RE.match(path.name) + if not match: + raise ValueError(f"Unexpected prediction filename: {path}") + return match.group(1), int(match.group(2)) + + +def nearest_indices(src: np.ndarray, dst: np.ndarray) -> np.ndarray: + idx = np.searchsorted(dst, src) + idx = np.clip(idx, 0, len(dst) - 1) + prev_idx = np.clip(idx - 1, 0, len(dst) - 1) + choose_prev = np.abs(dst[prev_idx] - src) <= np.abs(dst[idx] - src) + return np.where(choose_prev, prev_idx, idx).astype(np.int64) + + +def build_grid_alignment(sample_pred: Path, sample_merra: Path, lat_min: float, lat_max: float, lon_min: float, lon_max: float) -> Dict[str, np.ndarray]: + with Dataset(sample_pred) as pred_ds, Dataset(sample_merra) as merra_ds: + pred_lat = np.asarray(pred_ds.variables["lat"][:], dtype=np.float64) + pred_lon = np.asarray(pred_ds.variables["lon"][:], dtype=np.float64) + merra_lat = np.asarray(merra_ds.variables["lat"][:], dtype=np.float64) + merra_lon = np.asarray(merra_ds.variables["lon"][:], dtype=np.float64) + + lat_mask = (pred_lat >= lat_min) & (pred_lat <= lat_max) + lon_mask = (pred_lon >= lon_min) & (pred_lon <= lon_max) + pred_lat_idx = np.flatnonzero(lat_mask) + pred_lon_idx = np.flatnonzero(lon_mask) + pred_lat_sel = pred_lat[pred_lat_idx] + pred_lon_sel = pred_lon[pred_lon_idx] + merra_lat_idx = nearest_indices(pred_lat_sel, merra_lat) + merra_lon_idx = nearest_indices(pred_lon_sel, merra_lon) + + return { + "pred_lat_idx": pred_lat_idx, + "pred_lon_idx": pred_lon_idx, + "merra_lat_idx": merra_lat_idx, + "merra_lon_idx": merra_lon_idx, + } + + +def feature_stats(arr: np.ndarray) -> Dict[str, float]: + return {"mean": float(np.mean(arr)), "max": float(np.max(arr)), "std": float(np.std(arr))} + + +def build_rows(pred_roots: Iterable[Path], merra_root: Path, alignment: Dict[str, np.ndarray]) -> pd.DataFrame: + rows: List[Dict[str, float]] = [] + for root in pred_roots: + split = choose_split(root) + if split is None: + continue + for path in sorted(root.glob("pred_*.nc")): + day, hour = parse_pred_timestamp(path) + if hour % 3 != 0: + continue + merra_path = merra_root / f"MERRA2_sfc_{day}.nc" + if not merra_path.exists(): + continue + time_index = hour // 3 + with Dataset(path) as pred_ds, Dataset(merra_path) as merra_ds: + date = pd.Timestamp(day) + record: Dict[str, float] = {"split": split, "hour": float(hour), "year": float(date.year), "date": day} + record["doy"] = float(date.dayofyear) + record["month"] = float(date.month) + for var in WEATHER_VARS: + pred_arr = np.asarray( + pred_ds.variables[var][0, alignment["pred_lat_idx"], alignment["pred_lon_idx"]], + dtype=np.float64, + ) + stats = feature_stats(pred_arr) + record[f"pred_{var.lower()}_mean"] = stats["mean"] + record[f"pred_{var.lower()}_max"] = stats["max"] + record[f"pred_{var.lower()}_std"] = stats["std"] + record["pred_wind_mean"] = float( + np.mean( + np.sqrt( + np.square(pred_ds.variables["U10M"][0, alignment["pred_lat_idx"], alignment["pred_lon_idx"]]) + + np.square(pred_ds.variables["V10M"][0, alignment["pred_lat_idx"], alignment["pred_lon_idx"]]) + ) + ) + ) + truth_t2m = np.asarray(merra_ds.variables["T2M"][time_index], dtype=np.float64)[ + np.ix_(alignment["merra_lat_idx"], alignment["merra_lon_idx"]) + ] + truth_ts = np.asarray(merra_ds.variables["TS"][time_index], dtype=np.float64)[ + np.ix_(alignment["merra_lat_idx"], alignment["merra_lon_idx"]) + ] + record["target_t2m_mean_c"] = float(np.mean(truth_t2m) - 273.15) + record["target_t2m_max_c"] = float(np.max(truth_t2m) - 273.15) + record["target_ts_mean_c"] = float(np.mean(truth_ts) - 273.15) + rows.append(record) + + if not rows: + raise SystemExit("No extreme-heat rows were built from the provided roots.") + df = pd.DataFrame(rows) + angle_day = 2.0 * np.pi * df["doy"].to_numpy(dtype=np.float64) / 366.0 + angle_hour = 2.0 * np.pi * df["hour"].to_numpy(dtype=np.float64) / 24.0 + df["doy_sin"] = np.sin(angle_day) + df["doy_cos"] = np.cos(angle_day) + df["hour_sin"] = np.sin(angle_hour) + df["hour_cos"] = np.cos(angle_hour) + return df + + +def drop_nonfinite_rows(df: pd.DataFrame, columns: List[str]) -> pd.DataFrame: + mask = np.ones(len(df), dtype=bool) + for col in columns: + mask &= np.isfinite(pd.to_numeric(df[col], errors="coerce").to_numpy(dtype=np.float64)) + return df.loc[mask].reset_index(drop=True) + + +def rmse(y_true: np.ndarray, y_pred: np.ndarray) -> float: + return float(math.sqrt(mean_squared_error(y_true, y_pred))) + + +def pearson_corr(y_true: np.ndarray, y_pred: np.ndarray) -> float: + a = np.asarray(y_true, dtype=np.float64) + b = np.asarray(y_pred, dtype=np.float64) + if a.size < 2 or np.allclose(a, a[0]) or np.allclose(b, b[0]): + return 0.0 + value = float(np.corrcoef(a, b)[0, 1]) + return value if np.isfinite(value) else 0.0 + + +def prf(y_true: np.ndarray, y_pred: np.ndarray, threshold: float) -> Dict[str, float]: + truth = np.asarray(y_true >= threshold) + pred = np.asarray(y_pred >= threshold) + tp = int(np.logical_and(pred, truth).sum()) + fp = int(np.logical_and(pred, ~truth).sum()) + fn = int(np.logical_and(~pred, truth).sum()) + precision = float(tp / (tp + fp)) if (tp + fp) else 0.0 + recall = float(tp / (tp + fn)) if (tp + fn) else 0.0 + f1 = float((2.0 * precision * recall) / (precision + recall)) if (precision + recall) else 0.0 + return {"precision": precision, "recall": recall, "f1": f1} + + +def evaluate(y_true: np.ndarray, y_pred: np.ndarray) -> Dict[str, float]: + return { + "count": int(y_true.shape[0]), + "rmse_c": rmse(y_true, y_pred), + "mae_c": float(mean_absolute_error(y_true, y_pred)), + "pearson_r": pearson_corr(y_true, y_pred), + } + + +def main() -> None: + args = parse_args() + pred_files = [path for root in args.pred_root for path in root.glob("pred_*.nc")] + if not pred_files: + raise SystemExit("No prediction files found.") + sample_pred = sorted(pred_files)[0] + sample_day, _ = parse_pred_timestamp(sample_pred) + sample_merra = args.merra_root / f"MERRA2_sfc_{sample_day}.nc" + if not sample_merra.exists(): + raise SystemExit(f"Sample MERRA file missing: {sample_merra}") + + alignment = build_grid_alignment( + sample_pred, + sample_merra, + lat_min=args.lat_min, + lat_max=args.lat_max, + lon_min=args.lon_min, + lon_max=args.lon_max, + ) + df = build_rows(args.pred_root, args.merra_root, alignment) + + alpha = pd.read_csv(args.alphaearth_year_csv) + alpha["source_year_key"] = pd.to_numeric(alpha["alphaearth_source_year"], errors="coerce").astype("Int64") + df["source_year_key"] = pd.to_numeric(df["year"], errors="coerce").clip(lower=2017, upper=2024).astype("Int64") + df = df.merge(alpha.drop(columns=[c for c in ["requested_year"] if c in alpha.columns]), on="source_year_key", how="left") + + feature_cols = [c for c in df.columns if c.startswith("pred_") or c in {"month", "doy_sin", "doy_cos", "hour_sin", "hour_cos"}] + feature_cols.extend(sorted([c for c in df.columns if c.startswith(args.alphaearth_prefix)])) + finite_cols = feature_cols + ["target_t2m_mean_c"] + df = drop_nonfinite_rows(df, finite_cols) + if df.empty: + raise SystemExit("Extreme-heat AlphaEarth suite has no finite rows after filtering.") + + train = df[df["split"] == "train"].copy() + val = df[df["split"] == "val"].copy() + test = df[df["split"] == "test"].copy() + if len(train) == 0 or len(val) == 0 or len(test) == 0: + raise SystemExit("Extreme-heat AlphaEarth suite is missing one of train/val/test.") + + x_train = train[feature_cols].to_numpy(dtype=np.float64) + x_val = val[feature_cols].to_numpy(dtype=np.float64) + x_test = test[feature_cols].to_numpy(dtype=np.float64) + y_train = train["target_t2m_mean_c"].to_numpy(dtype=np.float64) + y_val = val["target_t2m_mean_c"].to_numpy(dtype=np.float64) + y_test = test["target_t2m_mean_c"].to_numpy(dtype=np.float64) + + candidates: Dict[str, object] = { + "ridge": Ridge(alpha=1.0, random_state=args.seed), + "enet": ElasticNet(alpha=0.01, l1_ratio=0.2, random_state=args.seed, max_iter=10000), + } + if args.model_family == "full": + candidates.update( + { + "xgboost": XGBRegressor( + n_estimators=300, + max_depth=6, + learning_rate=0.05, + subsample=0.8, + colsample_bytree=0.8, + objective="reg:squarederror", + tree_method="hist", + random_state=args.seed, + n_jobs=8, + ), + "lightgbm": LGBMRegressor( + n_estimators=300, + learning_rate=0.05, + num_leaves=63, + subsample=0.8, + colsample_bytree=0.8, + random_state=args.seed, + n_jobs=8, + verbose=-1, + ), + "catboost": CatBoostRegressor( + iterations=400, + depth=8, + learning_rate=0.05, + loss_function="RMSE", + eval_metric="RMSE", + random_seed=args.seed, + verbose=False, + ), + } + ) + + candidate_rows = [] + best_name = None + best_model = None + best_rmse = None + for name, model in candidates.items(): + model.fit(x_train, y_train) + val_pred = np.asarray(model.predict(x_val), dtype=np.float64) + val_metrics = evaluate(y_val, val_pred) + candidate_rows.append({"model": name, "validation": val_metrics}) + if best_rmse is None or val_metrics["rmse_c"] < best_rmse: + best_name = name + best_model = model + best_rmse = val_metrics["rmse_c"] + + assert best_model is not None and best_name is not None + val_pred = np.asarray(best_model.predict(x_val), dtype=np.float64) + test_pred = np.asarray(best_model.predict(x_test), dtype=np.float64) + + thresholds = [27.0, 30.0, 33.0] + val_events = [{"threshold_c": t, **prf(y_val, val_pred, t)} for t in thresholds] + val_events = sorted(val_events, key=lambda row: (-row["f1"], -row["recall"], -row["precision"], row["threshold_c"])) + selected_event = val_events[0] + test_event = {"threshold_c": selected_event["threshold_c"], **prf(y_test, test_pred, selected_event["threshold_c"])} + + summary = { + "task_id": "extreme_heat_alphaearth", + "core_line": "extreme_heat", + "task_form": "continuous_temperature_forecast_with_secondary_exceedance_view", + "seed": int(args.seed), + "pred_roots": [str(path) for path in args.pred_root], + "merra_root": str(args.merra_root), + "alphaearth_year_csv": str(args.alphaearth_year_csv), + "model_family": args.model_family, + "feature_columns": feature_cols, + "alphaearth_feature_count": int(sum(c.startswith(args.alphaearth_prefix) for c in feature_cols)), + "candidate_validation": candidate_rows, + "selected_model": best_name, + "validation_metrics": evaluate(y_val, val_pred), + "test_metrics": evaluate(y_test, test_pred), + "selected_event_candidate": selected_event, + "selected_event_candidate_test": test_event, + "selection_rule": "same heat benchmark; choose regressor by validation RMSE and choose exceedance threshold by validation F1", + "tmt_policy": { + "task": "extreme_heat", + "metric": "continuous RMSE/MAE with thresholded exceedance as a secondary event policy", + "tolerance": "none for continuous headline; event threshold only for operational view" + }, + } + + args.output_dir.mkdir(parents=True, exist_ok=True) + (args.output_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/experiments/raw_reference/task_scripts/run_final_area_taskmodel_seeded.py b/experiments/raw_reference/task_scripts/run_final_area_taskmodel_seeded.py new file mode 100644 index 0000000000000000000000000000000000000000..2709fa33720640d948dff9b6a65a96ebc23a0fc1 --- /dev/null +++ b/experiments/raw_reference/task_scripts/run_final_area_taskmodel_seeded.py @@ -0,0 +1,353 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Dict, List, Tuple + + +import os + +for _p in os.environ.get("WILDFIRE_FM_EXTRA_PYTHONPATH", "").split(os.pathsep): + if _p and _p not in sys.path: + sys.path.insert(0, _p) + +import numpy as np +import pandas as pd +from catboost import CatBoostRegressor +from lightgbm import LGBMRegressor +from sklearn.compose import ColumnTransformer +from sklearn.impute import SimpleImputer +from sklearn.linear_model import ElasticNet +from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import OneHotEncoder, StandardScaler +from xgboost import XGBRegressor + + +DROP_COLUMNS = { + "Event_ID", + "Incid_Name", + "incident_name_norm", + "wfigs_name", + "Ig_Date", + "weather_date", + "BurnBndAc", + "target_log_burn_acres", +} + +CATEGORICAL_COLUMNS = [ + "Incid_Type", + "state_abbr", + "county_name", + "wfigs_match_type", +] + + +def rmse(y_true: np.ndarray, y_pred: np.ndarray) -> float: + return float(np.sqrt(mean_squared_error(y_true, y_pred))) + + +def mape(y_true: np.ndarray, y_pred: np.ndarray) -> float: + denom = np.clip(np.asarray(y_true, dtype=np.float64), 1e-6, None) + frac = np.abs(np.asarray(y_true, dtype=np.float64) - np.asarray(y_pred, dtype=np.float64)) / denom + return float(np.mean(frac)) + + +def spearman_corr(y_true: np.ndarray, y_pred: np.ndarray) -> float: + a = pd.Series(np.asarray(y_true, dtype=np.float64)) + b = pd.Series(np.asarray(y_pred, dtype=np.float64)) + value = a.corr(b, method="spearman") + return float(value) if pd.notna(value) else 0.0 + + +def build_splits(df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: + ordered = df.sort_values("Ig_Date").reset_index(drop=True) + n = len(ordered) + train_end = max(int(round(n * 0.6)), 1) + val_end = max(int(round(n * 0.8)), train_end + 1) + val_end = min(val_end, n - 1) if n >= 3 else n + train = ordered.iloc[:train_end].copy() + val = ordered.iloc[train_end:val_end].copy() + test = ordered.iloc[val_end:].copy() + if len(val) == 0 and len(test) > 1: + val = test.iloc[:1].copy() + test = test.iloc[1:].copy() + return train, val, test + + +def feature_columns(df: pd.DataFrame, feature_profile: str = "all") -> Tuple[List[str], List[str]]: + categorical = [c for c in CATEGORICAL_COLUMNS if c in df.columns] + numeric = [] + for col in df.columns: + if col in DROP_COLUMNS or col in categorical: + continue + if pd.api.types.is_numeric_dtype(df[col]): + numeric.append(col) + if feature_profile == "weather_fm": + numeric = [c for c in numeric if c.startswith("weather_")] + categorical = [] + return numeric, categorical + + +def make_sparse_preprocessor(numeric_cols: List[str], categorical_cols: List[str]) -> ColumnTransformer: + return ColumnTransformer( + transformers=[ + ( + "num", + Pipeline( + steps=[ + ("impute", SimpleImputer(strategy="median")), + ("scale", StandardScaler()), + ] + ), + numeric_cols, + ), + ( + "cat", + Pipeline( + steps=[ + ("impute", SimpleImputer(strategy="most_frequent")), + ("onehot", OneHotEncoder(handle_unknown="ignore")), + ] + ), + categorical_cols, + ), + ], + remainder="drop", + ) + + +def prepare_catboost_frames( + train_df: pd.DataFrame, + val_df: pd.DataFrame, + test_df: pd.DataFrame, + numeric_cols: List[str], + categorical_cols: List[str], +) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: + medians = {c: float(train_df[c].median()) for c in numeric_cols} + modes = { + c: str(train_df[c].mode(dropna=True).iloc[0]) if not train_df[c].mode(dropna=True).empty else "missing" + for c in categorical_cols + } + + def _prep(frame: pd.DataFrame) -> pd.DataFrame: + out = frame[numeric_cols + categorical_cols].copy() + for col in numeric_cols: + out[col] = pd.to_numeric(out[col], errors="coerce").fillna(medians[col]) + for col in categorical_cols: + out[col] = out[col].astype("string").fillna(modes[col]).astype(str) + return out + + return _prep(train_df), _prep(val_df), _prep(test_df) + + +def evaluate_split(frame: pd.DataFrame, pred_log: np.ndarray) -> Dict[str, float]: + true_log = frame["target_log_burn_acres"].to_numpy(dtype=np.float64) + true_acres = frame["BurnBndAc"].to_numpy(dtype=np.float64) + pred_log = np.asarray(pred_log, dtype=np.float64) + pred_acres = np.exp(pred_log) + return { + "count": int(len(frame)), + "log_mae": float(mean_absolute_error(true_log, pred_log)), + "log_rmse": rmse(true_log, pred_log), + "log_r2": float(r2_score(true_log, pred_log)) if len(frame) > 1 else 0.0, + "log_spearman": spearman_corr(true_log, pred_log), + "log_median_ae": float(np.median(np.abs(true_log - pred_log))), + "acres_mae": float(mean_absolute_error(true_acres, pred_acres)), + "acres_rmse": rmse(true_acres, pred_acres), + "acres_median_ae": float(np.median(np.abs(true_acres - pred_acres))), + "acres_mape": mape(true_acres, pred_acres), + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--event-table", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--feature-profile", choices=("all", "weather_fm"), default="all") + parser.add_argument("--model-family", choices=("full", "lite"), default="full") + parser.add_argument("--fm-family", type=str, default="") + parser.add_argument("--seed", type=int, default=7) + args = parser.parse_args() + + df = pd.read_csv(args.event_table) + df["Ig_Date"] = pd.to_datetime(df["Ig_Date"]) + train_df, val_df, test_df = build_splits(df) + numeric_cols, categorical_cols = feature_columns(df, feature_profile=args.feature_profile) + if not numeric_cols and not categorical_cols: + raise SystemExit(f"No usable features found for profile={args.feature_profile}") + x_cols = numeric_cols + categorical_cols + + pre = make_sparse_preprocessor(numeric_cols, categorical_cols) + x_train = pre.fit_transform(train_df[x_cols]) + x_val = pre.transform(val_df[x_cols]) + x_test = pre.transform(test_df[x_cols]) + y_train = train_df["target_log_burn_acres"].to_numpy(dtype=np.float64) + + cat_train, cat_val, cat_test = prepare_catboost_frames(train_df, val_df, test_df, numeric_cols, categorical_cols) + cat_feature_idx = list(range(len(numeric_cols), len(numeric_cols) + len(categorical_cols))) + + candidates: List[Tuple[str, object, str]] = [ + ( + "enet", + ElasticNet(alpha=0.01, l1_ratio=0.2, random_state=args.seed, max_iter=10000), + "sparse", + ), + ] + if args.model_family == "full": + candidates.extend( + [ + ( + "xgboost", + XGBRegressor( + n_estimators=400, + max_depth=6, + learning_rate=0.05, + subsample=0.8, + colsample_bytree=0.8, + reg_lambda=1.0, + objective="reg:squarederror", + tree_method="hist", + random_state=args.seed, + n_jobs=8, + ), + "sparse", + ), + ( + "lightgbm", + LGBMRegressor( + n_estimators=400, + learning_rate=0.05, + num_leaves=63, + subsample=0.8, + colsample_bytree=0.8, + reg_lambda=1.0, + random_state=args.seed, + n_jobs=8, + verbose=-1, + ), + "sparse", + ), + ( + "catboost", + CatBoostRegressor( + iterations=500, + depth=8, + learning_rate=0.05, + loss_function="RMSE", + eval_metric="RMSE", + random_seed=args.seed, + verbose=False, + ), + "cat", + ), + ] + ) + + candidate_validation: List[Dict[str, object]] = [] + best_name = None + best_kind = None + best_model = None + best_score = None + + for name, model, kind in candidates: + if kind == "sparse": + model.fit(x_train, y_train) + val_pred = model.predict(x_val) + else: + model.fit(cat_train, y_train, cat_features=cat_feature_idx, eval_set=(cat_val, val_df["target_log_burn_acres"]), use_best_model=False) + val_pred = model.predict(cat_val) + val_metrics = evaluate_split(val_df, val_pred) + candidate_validation.append({"model_name": name, "val_metrics": val_metrics}) + score = float(val_metrics["log_mae"]) + if best_score is None or score < best_score: + best_score = score + best_name = name + best_kind = kind + best_model = model + + assert best_model is not None and best_name is not None and best_kind is not None + + combined_train = pd.concat([train_df, val_df], ignore_index=True) + if best_kind == "sparse": + x_combined = pre.fit_transform(combined_train[x_cols]) + x_train_final = pre.transform(train_df[x_cols]) + x_val_final = pre.transform(val_df[x_cols]) + x_test_final = pre.transform(test_df[x_cols]) + best_model.fit(x_combined, combined_train["target_log_burn_acres"].to_numpy(dtype=np.float64)) + train_pred = best_model.predict(x_train_final) + val_pred = best_model.predict(x_val_final) + test_pred = best_model.predict(x_test_final) + else: + cat_combined, cat_train_final, cat_test_final = prepare_catboost_frames( + combined_train, train_df, test_df, numeric_cols, categorical_cols + ) + cat_val_final = prepare_catboost_frames(val_df, val_df, val_df, numeric_cols, categorical_cols)[0] + best_model.fit( + cat_combined, + combined_train["target_log_burn_acres"].to_numpy(dtype=np.float64), + cat_features=cat_feature_idx, + use_best_model=False, + ) + train_pred = best_model.predict(cat_train_final) + val_pred = best_model.predict(cat_val_final) + test_pred = best_model.predict(cat_test_final) + + args.output_dir.mkdir(parents=True, exist_ok=True) + pred_df = pd.concat( + [ + train_df.assign(split="train", pred_log_burn_acres=train_pred, pred_burn_acres=np.exp(train_pred)), + val_df.assign(split="val", pred_log_burn_acres=val_pred, pred_burn_acres=np.exp(val_pred)), + test_df.assign(split="test", pred_log_burn_acres=test_pred, pred_burn_acres=np.exp(test_pred)), + ], + axis=0, + ignore_index=True, + ) + pred_path = args.output_dir / "predictions.csv" + pred_df.to_csv(pred_path, index=False) + + summary = { + "task_id": "wildfire_final_area_scalar_taskmodels", + "task_form": "event_level_regression", + "event_table": str(args.event_table), + "output_dir": str(args.output_dir), + "feature_profile": args.feature_profile, + "seed": int(args.seed), + "benchmark_protocol": "fm_lite_protocol" if args.feature_profile == "weather_fm" and args.model_family == "lite" else "standard_protocol", + "split_sizes": { + "train": int(len(train_df)), + "val": int(len(val_df)), + "test": int(len(test_df)), + }, + "feature_columns": { + "numeric": numeric_cols, + "categorical": categorical_cols, + }, + "candidate_validation": candidate_validation, + "selected_model": best_name, + "train_metrics": evaluate_split(train_df, train_pred), + "val_metrics": evaluate_split(val_df, val_pred), + "test_metrics": evaluate_split(test_df, test_pred), + "headline_metrics": { + "log_mae": float(evaluate_split(test_df, test_pred)["log_mae"]), + "log_rmse": float(evaluate_split(test_df, test_pred)["log_rmse"]), + "log_spearman": float(evaluate_split(test_df, test_pred)["log_spearman"]), + }, + "predictions_path": str(pred_path), + "model_family": "lightweight_linear_task_heads" if args.model_family == "lite" else "popular_open_source_task_models", + "fm_family": (args.fm_family or "weather_fm_derived_features") if args.feature_profile == "weather_fm" else None, + "tmt_policy": { + "task": "final_burned_area", + "metric": "log-area regression error with rank agreement", + "tolerance": "secondary magnitude-band interpretation only", + }, + } + (args.output_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/experiments/raw_reference/task_scripts/run_smoke_pm25_alphaearth_suite_seeded.py b/experiments/raw_reference/task_scripts/run_smoke_pm25_alphaearth_suite_seeded.py new file mode 100644 index 0000000000000000000000000000000000000000..cd91da61e9891feab6438eee9c052fa286b545eb --- /dev/null +++ b/experiments/raw_reference/task_scripts/run_smoke_pm25_alphaearth_suite_seeded.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import math +import sys +from pathlib import Path +from typing import Dict, List, Tuple + + +import os + +for _p in os.environ.get("WILDFIRE_FM_EXTRA_PYTHONPATH", "").split(os.pathsep): + if _p and _p not in sys.path: + sys.path.insert(0, _p) + +import numpy as np +import pandas as pd +from catboost import CatBoostRegressor +from lightgbm import LGBMRegressor +from sklearn.metrics import mean_absolute_error, mean_squared_error +from xgboost import XGBRegressor + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--aqs-daily", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--exceedance-threshold", type=float, default=35.0) + parser.add_argument("--alphaearth-prefix", type=str, default="alphaearth_") + parser.add_argument("--seed", type=int, default=7) + return parser.parse_args() + + +def assign_split(ts: pd.Timestamp) -> str: + year = int(ts.year) + if year <= 2023: + return "train" + if year == 2024: + return "val" + if year == 2025: + return "test" + return "other" + + +def rmse(y_true: np.ndarray, y_pred: np.ndarray) -> float: + return float(math.sqrt(mean_squared_error(y_true, y_pred))) + + +def pearson_corr(y_true: np.ndarray, y_pred: np.ndarray) -> float: + a = np.asarray(y_true, dtype=np.float64) + b = np.asarray(y_pred, dtype=np.float64) + if a.size < 2 or np.allclose(a, a[0]) or np.allclose(b, b[0]): + return 0.0 + value = float(np.corrcoef(a, b)[0, 1]) + return value if np.isfinite(value) else 0.0 + + +def prf(y_true: np.ndarray, y_pred: np.ndarray, threshold: float) -> Dict[str, float]: + truth = np.asarray(y_true >= threshold) + pred = np.asarray(y_pred >= threshold) + tp = int(np.logical_and(pred, truth).sum()) + fp = int(np.logical_and(pred, ~truth).sum()) + fn = int(np.logical_and(~pred, truth).sum()) + precision = float(tp / (tp + fp)) if (tp + fp) else 0.0 + recall = float(tp / (tp + fn)) if (tp + fn) else 0.0 + f1 = float((2.0 * precision * recall) / (precision + recall)) if (precision + recall) else 0.0 + return {"precision": precision, "recall": recall, "f1": f1} + + +def evaluate_frame(frame: pd.DataFrame, pred_col: str, threshold: float) -> Dict[str, float]: + y_true = frame["pm25_mean"].to_numpy(dtype=np.float64) + y_pred = frame[pred_col].to_numpy(dtype=np.float64) + event = prf(y_true, y_pred, threshold) + bias = np.asarray(y_pred - y_true, dtype=np.float64) + denom = float(np.sum(y_true)) + return { + "count": int(len(frame)), + "rmse": rmse(y_true, y_pred), + "mae": float(mean_absolute_error(y_true, y_pred)), + "mean_bias": float(np.mean(bias)), + "normalized_mean_bias": float(np.sum(bias) / denom) if abs(denom) > 1e-12 else 0.0, + "pearson_r": pearson_corr(y_true, y_pred), + "event_precision": event["precision"], + "event_recall": event["recall"], + "event_f1": event["f1"], + } + + +def tune_event_shift(val_frame: pd.DataFrame, pred_col: str, threshold: float) -> Dict[str, float]: + best = None + for delta in np.linspace(-5.0, 15.0, 161): + shifted = val_frame.copy() + shifted["_shifted_pred"] = shifted[pred_col] + float(delta) + metrics = evaluate_frame(shifted, "_shifted_pred", threshold) + score = (metrics["event_f1"], metrics["event_recall"], -abs(float(delta))) + if best is None or score > best["score"]: + best = {"delta": float(delta), "metrics": metrics, "score": score} + assert best is not None + return {"delta": best["delta"], "val_event_calibrated_metrics": best["metrics"]} + + +def build_features(df: pd.DataFrame) -> pd.DataFrame: + df = df.sort_values(["site_key", "date"]).reset_index(drop=True).copy() + df["doy"] = df["date"].dt.dayofyear.astype(np.int32) + df["month"] = df["date"].dt.month.astype(np.int32) + df["doy_sin"] = np.sin(2.0 * np.pi * df["doy"] / 366.0) + df["doy_cos"] = np.cos(2.0 * np.pi * df["doy"] / 366.0) + grp = df.groupby("site_key", sort=False) + for lag in [1, 2, 3, 7]: + df[f"lag{lag}_pm25"] = grp["pm25_mean"].shift(lag) + df["roll3_prev"] = grp["pm25_mean"].rolling(3, min_periods=1).mean().reset_index(level=0, drop=True).shift(1) + df["roll7_prev"] = grp["pm25_mean"].rolling(7, min_periods=1).mean().reset_index(level=0, drop=True).shift(1) + return df + + +def prepare_frames(df: pd.DataFrame, alphaearth_prefix: str) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, List[str]]: + df = build_features(df) + train = df[df["split"] == "train"].copy() + val = df[df["split"] == "val"].copy() + test = df[df["split"] == "test"].copy() + + site_mean = train.groupby("site_key")["pm25_mean"].mean() + global_mean = float(train["pm25_mean"].mean()) + for frame in [train, val, test]: + frame["site_climo"] = frame["site_key"].map(site_mean).fillna(global_mean) + for frame in [train, val, test]: + for col in ["lag1_pm25", "lag2_pm25", "lag3_pm25", "lag7_pm25", "roll3_prev", "roll7_prev"]: + frame[col] = pd.to_numeric(frame[col], errors="coerce").fillna(frame["site_climo"]) + + feature_cols = [ + "latitude", + "longitude", + "obs_count", + "site_climo", + "lag1_pm25", + "lag2_pm25", + "lag3_pm25", + "lag7_pm25", + "roll3_prev", + "roll7_prev", + "doy_sin", + "doy_cos", + "month", + ] + alpha_cols = [ + c for c in df.columns + if c.startswith(alphaearth_prefix) and pd.api.types.is_numeric_dtype(df[c]) + ] + feature_cols.extend(sorted(alpha_cols)) + medians = train[feature_cols].median(numeric_only=True).fillna(0.0) + for frame in [train, val, test]: + frame.loc[:, feature_cols] = frame[feature_cols].fillna(medians) + frame.loc[:, feature_cols] = frame[feature_cols].fillna(0.0) + return train, val, test, feature_cols + + +def main() -> None: + args = parse_args() + df = pd.read_csv(args.aqs_daily, compression="infer", low_memory=False) + df["date"] = pd.to_datetime(df["date_gmt"], errors="coerce") + df["pm25_mean"] = pd.to_numeric(df["pm25_mean"], errors="coerce") + df["pm25_max"] = pd.to_numeric(df["pm25_max"], errors="coerce") + df["obs_count"] = pd.to_numeric(df["obs_count"], errors="coerce") + df = df.dropna(subset=["date", "site_key", "pm25_mean"]).copy() + df = ( + df.groupby(["date", "site_key"], as_index=False) + .agg( + latitude=("Latitude", "first"), + longitude=("Longitude", "first"), + pm25_mean=("pm25_mean", "mean"), + pm25_max=("pm25_max", "max"), + obs_count=("obs_count", "sum"), + **{c: (c, "first") for c in df.columns if c.startswith(args.alphaearth_prefix)}, + ) + .sort_values(["site_key", "date"]) + .reset_index(drop=True) + ) + df["split"] = df["date"].map(assign_split) + df = df[df["split"].isin(["train", "val", "test"])].copy() + + train, val, test, feature_cols = prepare_frames(df, alphaearth_prefix=args.alphaearth_prefix) + y_train = train["pm25_mean"].to_numpy(dtype=np.float64) + + candidates = { + "xgboost": XGBRegressor( + n_estimators=300, + max_depth=8, + learning_rate=0.05, + subsample=0.8, + colsample_bytree=0.8, + objective="reg:squarederror", + tree_method="hist", + random_state=args.seed, + n_jobs=8, + ), + "lightgbm": LGBMRegressor( + n_estimators=300, + learning_rate=0.05, + num_leaves=127, + subsample=0.8, + colsample_bytree=0.8, + random_state=args.seed, + n_jobs=8, + verbose=-1, + ), + "catboost": CatBoostRegressor( + iterations=400, + depth=8, + learning_rate=0.05, + loss_function="RMSE", + eval_metric="RMSE", + random_seed=args.seed, + verbose=False, + ), + } + + candidate_validation: List[Dict[str, object]] = [] + best_name = None + best_model = None + best_score = None + for name, model in candidates.items(): + if name == "catboost": + model.fit(train[feature_cols], y_train, use_best_model=False) + val_pred = model.predict(val[feature_cols]) + else: + model.fit(train[feature_cols].to_numpy(dtype=np.float32), y_train) + val_pred = model.predict(val[feature_cols].to_numpy(dtype=np.float32)) + val_frame = val.copy() + val_frame["pred"] = val_pred + metrics = evaluate_frame(val_frame, "pred", args.exceedance_threshold) + candidate_validation.append({"candidate": name, "val_metrics": metrics}) + score = float(metrics["rmse"]) + if best_score is None or score < best_score: + best_score = score + best_name = name + best_model = model + + assert best_name is not None and best_model is not None + combined = pd.concat([train, val], ignore_index=True) + if best_name == "catboost": + best_model.fit(combined[feature_cols], combined["pm25_mean"].to_numpy(dtype=np.float64), use_best_model=False) + train_pred = best_model.predict(train[feature_cols]) + val_pred = best_model.predict(val[feature_cols]) + test_pred = best_model.predict(test[feature_cols]) + else: + best_model.fit(combined[feature_cols].to_numpy(dtype=np.float32), combined["pm25_mean"].to_numpy(dtype=np.float64)) + train_pred = best_model.predict(train[feature_cols].to_numpy(dtype=np.float32)) + val_pred = best_model.predict(val[feature_cols].to_numpy(dtype=np.float32)) + test_pred = best_model.predict(test[feature_cols].to_numpy(dtype=np.float32)) + + args.output_dir.mkdir(parents=True, exist_ok=True) + pred_df = pd.concat( + [ + train.assign(pred_pm25=train_pred), + val.assign(pred_pm25=val_pred), + test.assign(pred_pm25=test_pred), + ], + ignore_index=True, + ) + pred_path = args.output_dir / "predictions.csv.gz" + pred_df.to_csv(pred_path, index=False, compression="gzip") + + train_eval = train.assign(pred=train_pred) + val_eval = val.assign(pred=val_pred) + test_eval = test.assign(pred=test_pred) + event_shift = tune_event_shift(val_eval, "pred", args.exceedance_threshold) + delta = event_shift["delta"] + train_event_eval = train_eval.assign(pred_event_calibrated=train_eval["pred"] + delta) + val_event_eval = val_eval.assign(pred_event_calibrated=val_eval["pred"] + delta) + test_event_eval = test_eval.assign(pred_event_calibrated=test_eval["pred"] + delta) + + summary = { + "task_id": "smoke_pm25_alphaearth", + "task_form": "station_daily_regression", + "aqs_daily": str(args.aqs_daily), + "output_dir": str(args.output_dir), + "seed": int(args.seed), + "feature_columns": feature_cols, + "alphaearth_feature_count": int(sum(c.startswith(args.alphaearth_prefix) for c in feature_cols)), + "split_sizes": {"train": int(len(train)), "val": int(len(val)), "test": int(len(test))}, + "candidate_validation": candidate_validation, + "selected_model": best_name, + "train_metrics": evaluate_frame(train_eval, "pred", args.exceedance_threshold), + "val_metrics": evaluate_frame(val_eval, "pred", args.exceedance_threshold), + "test_metrics": evaluate_frame(test_eval, "pred", args.exceedance_threshold), + "event_calibration": { + "delta": float(delta), + "val_metrics": event_shift["val_event_calibrated_metrics"], + "test_metrics": evaluate_frame(test_event_eval, "pred_event_calibrated", args.exceedance_threshold), + }, + "predictions_path": str(pred_path), + "selection_rule": "same smoke benchmark; choose task-specific regressor by validation RMSE, then calibrate exceedance on validation only", + "tmt_policy": { + "task": "smoke_pm25", + "metric": "continuous RMSE/MAE with thresholded exceedance PRF", + "tolerance": "secondary event policy only", + }, + } + (args.output_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/experiments/raw_reference/task_scripts/run_smoke_pm25_attached_fm_suite_seeded.py b/experiments/raw_reference/task_scripts/run_smoke_pm25_attached_fm_suite_seeded.py new file mode 100644 index 0000000000000000000000000000000000000000..9ecc6057a28b69b7701ffe991b46b967d0a638b6 --- /dev/null +++ b/experiments/raw_reference/task_scripts/run_smoke_pm25_attached_fm_suite_seeded.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import math +import sys +from pathlib import Path +from typing import Dict, List + + +import os + +for _p in os.environ.get("WILDFIRE_FM_EXTRA_PYTHONPATH", "").split(os.pathsep): + if _p and _p not in sys.path: + sys.path.insert(0, _p) + +import numpy as np +import pandas as pd +from catboost import CatBoostRegressor +from lightgbm import LGBMRegressor +from sklearn.linear_model import ElasticNet, Ridge +from sklearn.metrics import mean_absolute_error, mean_squared_error +from xgboost import XGBRegressor + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--attached-csv", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--fm-prefix", type=str, required=True) + parser.add_argument("--fm-family", type=str, required=True) + parser.add_argument("--model-family", choices=("full", "lite"), default="lite") + parser.add_argument("--exceedance-threshold", type=float, default=35.0) + parser.add_argument("--seed", type=int, default=7) + return parser.parse_args() + + +def rmse(y_true: np.ndarray, y_pred: np.ndarray) -> float: + return float(math.sqrt(mean_squared_error(y_true, y_pred))) + + +def pearson_corr(y_true: np.ndarray, y_pred: np.ndarray) -> float: + a = np.asarray(y_true, dtype=np.float64) + b = np.asarray(y_pred, dtype=np.float64) + if a.size < 2 or np.allclose(a, a[0]) or np.allclose(b, b[0]): + return 0.0 + value = float(np.corrcoef(a, b)[0, 1]) + return value if np.isfinite(value) else 0.0 + + +def prf(y_true: np.ndarray, y_pred: np.ndarray, threshold: float) -> Dict[str, float]: + truth = np.asarray(y_true >= threshold) + pred = np.asarray(y_pred >= threshold) + tp = int(np.logical_and(pred, truth).sum()) + fp = int(np.logical_and(pred, ~truth).sum()) + fn = int(np.logical_and(~pred, truth).sum()) + precision = float(tp / (tp + fp)) if (tp + fp) else 0.0 + recall = float(tp / (tp + fn)) if (tp + fn) else 0.0 + f1 = float((2.0 * precision * recall) / (precision + recall)) if (precision + recall) else 0.0 + return {"precision": precision, "recall": recall, "f1": f1} + + +def evaluate_frame(frame: pd.DataFrame, pred_col: str, threshold: float) -> Dict[str, float]: + y_true = frame["pm25_mean"].to_numpy(dtype=np.float64) + y_pred = frame[pred_col].to_numpy(dtype=np.float64) + event = prf(y_true, y_pred, threshold) + bias = np.asarray(y_pred - y_true, dtype=np.float64) + denom = float(np.sum(y_true)) + return { + "count": int(len(frame)), + "rmse": rmse(y_true, y_pred), + "mae": float(mean_absolute_error(y_true, y_pred)), + "mean_bias": float(np.mean(bias)), + "normalized_mean_bias": float(np.sum(bias) / denom) if abs(denom) > 1e-12 else 0.0, + "pearson_r": pearson_corr(y_true, y_pred), + "event_precision": event["precision"], + "event_recall": event["recall"], + "event_f1": event["f1"], + } + + +def main() -> None: + args = parse_args() + df = pd.read_csv(args.attached_csv) + df["date"] = pd.to_datetime(df["date_gmt"], errors="coerce") + df["pm25_mean"] = pd.to_numeric(df["pm25_mean"], errors="coerce") + df = df.dropna(subset=["date", "pm25_mean"]).copy() + + feature_cols = [c for c in df.columns if c.startswith(args.fm_prefix)] + feature_cols = [c for c in feature_cols if pd.api.types.is_numeric_dtype(df[c])] + if not feature_cols: + raise SystemExit(f"No numeric FM feature columns found with prefix {args.fm_prefix}") + + split_map = {"2020": "train", "2021": "train", "2022": "train", "2024": "val", "2025": "test"} + df["split"] = df["date"].dt.year.astype(str).map(split_map) + df = df[df["split"].isin(["train", "val", "test"])].copy() + train = df[df["split"] == "train"].copy() + val = df[df["split"] == "val"].copy() + test = df[df["split"] == "test"].copy() + if len(train) == 0 or len(val) == 0 or len(test) == 0: + raise SystemExit("Attached FM smoke table is missing one of train/val/test.") + + medians = train[feature_cols].median(numeric_only=True).fillna(0.0) + train.loc[:, feature_cols] = train[feature_cols].fillna(medians) + val.loc[:, feature_cols] = val[feature_cols].fillna(medians) + test.loc[:, feature_cols] = test[feature_cols].fillna(medians) + train.loc[:, feature_cols] = train[feature_cols].fillna(0.0) + val.loc[:, feature_cols] = val[feature_cols].fillna(0.0) + test.loc[:, feature_cols] = test[feature_cols].fillna(0.0) + + y_train = train["pm25_mean"].to_numpy(dtype=np.float64) + candidates: Dict[str, object] = { + "ridge": Ridge(alpha=1.0, random_state=args.seed), + "enet": ElasticNet(alpha=0.01, l1_ratio=0.2, random_state=args.seed, max_iter=10000), + } + if args.model_family == "full": + candidates.update( + { + "xgboost": XGBRegressor( + n_estimators=300, + max_depth=8, + learning_rate=0.05, + subsample=0.8, + colsample_bytree=0.8, + objective="reg:squarederror", + tree_method="hist", + random_state=args.seed, + n_jobs=8, + ), + "lightgbm": LGBMRegressor( + n_estimators=300, + learning_rate=0.05, + num_leaves=127, + subsample=0.8, + colsample_bytree=0.8, + random_state=args.seed, + n_jobs=8, + verbose=-1, + ), + "catboost": CatBoostRegressor( + iterations=400, + depth=8, + learning_rate=0.05, + loss_function="RMSE", + eval_metric="RMSE", + random_seed=args.seed, + verbose=False, + ), + } + ) + + candidate_validation: List[Dict[str, object]] = [] + best_name = None + best_model = None + best_score = None + for name, model in candidates.items(): + if name == "catboost": + model.fit(train[feature_cols], y_train, use_best_model=False) + val_pred = model.predict(val[feature_cols]) + else: + model.fit(train[feature_cols].to_numpy(dtype=np.float32), y_train) + val_pred = model.predict(val[feature_cols].to_numpy(dtype=np.float32)) + val_frame = val.copy() + val_frame["pred"] = val_pred + metrics = evaluate_frame(val_frame, "pred", args.exceedance_threshold) + candidate_validation.append({"candidate": name, "val_metrics": metrics}) + score = float(metrics["rmse"]) + if best_score is None or score < best_score: + best_score = score + best_name = name + best_model = model + + assert best_name is not None and best_model is not None + combined = pd.concat([train, val], ignore_index=True) + if best_name == "catboost": + best_model.fit(combined[feature_cols], combined["pm25_mean"].to_numpy(dtype=np.float64), use_best_model=False) + train_pred = best_model.predict(train[feature_cols]) + val_pred = best_model.predict(val[feature_cols]) + test_pred = best_model.predict(test[feature_cols]) + else: + best_model.fit(combined[feature_cols].to_numpy(dtype=np.float32), combined["pm25_mean"].to_numpy(dtype=np.float64)) + train_pred = best_model.predict(train[feature_cols].to_numpy(dtype=np.float32)) + val_pred = best_model.predict(val[feature_cols].to_numpy(dtype=np.float32)) + test_pred = best_model.predict(test[feature_cols].to_numpy(dtype=np.float32)) + + args.output_dir.mkdir(parents=True, exist_ok=True) + pred_df = pd.concat( + [ + train.assign(pred_pm25=train_pred), + val.assign(pred_pm25=val_pred), + test.assign(pred_pm25=test_pred), + ], + ignore_index=True, + ) + pred_path = args.output_dir / "predictions.csv.gz" + pred_df.to_csv(pred_path, index=False, compression="gzip") + + train_eval = train.assign(pred=train_pred) + val_eval = val.assign(pred=val_pred) + test_eval = test.assign(pred=test_pred) + summary = { + "task_id": "smoke_pm25_named_fm", + "task_form": "station_daily_regression", + "attached_csv": str(args.attached_csv), + "output_dir": str(args.output_dir), + "seed": int(args.seed), + "feature_columns": feature_cols, + "split_sizes": {"train": int(len(train)), "val": int(len(val)), "test": int(len(test))}, + "candidate_validation": candidate_validation, + "selected_model": best_name, + "train_metrics": evaluate_frame(train_eval, "pred", args.exceedance_threshold), + "val_metrics": evaluate_frame(val_eval, "pred", args.exceedance_threshold), + "test_metrics": evaluate_frame(test_eval, "pred", args.exceedance_threshold), + "predictions_path": str(pred_path), + "model_family": "lightweight_linear_task_heads" if args.model_family == "lite" else "popular_open_source_task_models", + "fm_family": args.fm_family, + "benchmark_protocol": "fm_lite_protocol" if args.model_family == "lite" else "standard_protocol", + "selection_rule": "choose model by validation RMSE on named-FM attached rows; report on held-out test dates", + "tmt_policy": { + "task": "smoke_pm25", + "metric": "continuous RMSE/MAE with thresholded exceedance PRF", + "tolerance": "secondary event policy only", + }, + } + (args.output_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/experiments/raw_reference/task_scripts/summarize_forced_meanstd_20260429.py b/experiments/raw_reference/task_scripts/summarize_forced_meanstd_20260429.py new file mode 100644 index 0000000000000000000000000000000000000000..f2d6ceeda56d884dd050cd52df6efd7d6095d429 --- /dev/null +++ b/experiments/raw_reference/task_scripts/summarize_forced_meanstd_20260429.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import math +import re +import statistics +from pathlib import Path +from typing import Any + + +SLUG_LABELS = { + "reference": "Reference", + "prithvi_wxc": "Prithvi-WxC", + "stormcast": "StormCast", + "aurora": "Aurora", + "climax": "ClimaX", + "alphaearth": "AlphaEarth", +} + + +def load(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def stats(values: list[float]) -> dict[str, float | int]: + values = [float(v) for v in values if math.isfinite(float(v))] + if not values: + return {"n": 0, "mean": math.nan, "std": math.nan} + return { + "n": len(values), + "mean": float(statistics.fmean(values)), + "std": float(statistics.stdev(values)) if len(values) > 1 else 0.0, + } + + +def seed_from_path(path: Path) -> int | None: + match = re.search(r"_seed_(\d+)", str(path)) + return int(match.group(1)) if match else None + + +def label_from_seed_dir(path: Path, prefix: str) -> str: + for part in path.parts: + if part.startswith(prefix) and "_seed_" in part: + slug = part[len(prefix) :].split("_seed_", 1)[0] + return SLUG_LABELS.get(slug, slug) + return "unknown" + + +def dedupe_rows(rows: list[dict[str, Any]], keys: tuple[str, ...]) -> list[dict[str, Any]]: + selected: dict[tuple[Any, ...], dict[str, Any]] = {} + for row in rows: + key = tuple(row.get(name) for name in keys) + old = selected.get(key) + if old is None: + selected[key] = row + continue + old_mtime = Path(str(old["path"])).stat().st_mtime + new_mtime = Path(str(row["path"])).stat().st_mtime + if new_mtime >= old_mtime: + selected[key] = row + return list(selected.values()) + + +def best_val_threshold(data: dict[str, Any]) -> str: + entries = data["splits"]["val"]["threshold_metrics"] + return max(entries, key=lambda key: (float(entries[key]["f1"]), -float(entries[key]["threshold"]))) + + +def collect_occupancy(run_root: Path) -> dict[str, Any]: + rows: list[dict[str, Any]] = [] + for path in sorted(run_root.glob("table3_occupancy_*_seed_*/run_*/summary.json")): + data = load(path) + threshold_key = best_val_threshold(data) + test = data["splits"]["test"] + rows.append( + { + "label": data.get("fm_family") or label_from_seed_dir(path, "table3_occupancy_"), + "seed": seed_from_path(path), + "strict_f1": float(test["threshold_metrics"][threshold_key]["f1"]), + "tolerant_f1": float(test["tolerant_threshold_metrics"]["t0_s3"][threshold_key]["f1"]), + "union_f1": float(test["tolerant_threshold_metrics"]["t3_s3"][threshold_key]["f1"]), + "path": str(path), + } + ) + return group(rows, ["strict_f1", "tolerant_f1", "union_f1"]) + + +def collect_headcontrol(run_root: Path) -> dict[str, Any]: + rows: list[dict[str, Any]] = [] + for path in sorted(run_root.glob("table2_prithvi_wxc_headcontrol_seed_*/run_*/summary.json")): + data = load(path) + seed = seed_from_path(path) + for row in data.get("selection_summary", {}).get("rows", []): + rows.append( + { + "label": "Prithvi-WxC", + "scope": row["scope"], + "seed": seed, + "ranking_selected_union_f1": float(row["ranking_selected_union_f1"]), + "decision_selected_union_f1": float(row["decision_selected_union_f1"]), + "decision_regret_union_f1": float(row["decision_regret_union_f1"]), + "selection_failure": bool(row.get("selection_failure", False)), + "path": str(path), + } + ) + grouped: dict[str, Any] = {} + rows = dedupe_rows(rows, ("label", "scope", "seed")) + for scope in sorted({str(row["scope"]) for row in rows}): + selected = [row for row in rows if row["scope"] == scope] + grouped[scope] = { + "n": len(selected), + "failure_count": int(sum(1 for row in selected if row["selection_failure"])), + "ranking_selected_union_f1": stats([row["ranking_selected_union_f1"] for row in selected]), + "decision_selected_union_f1": stats([row["decision_selected_union_f1"] for row in selected]), + "decision_regret_union_f1": stats([row["decision_regret_union_f1"] for row in selected]), + } + return {"rows": rows, "summary": grouped} + + +def collect_spread(run_root: Path) -> dict[str, Any]: + rows: list[dict[str, Any]] = [] + for pattern, prefix in [ + ("table3_spread_*_seed_*/run_*/summary.json", "table3_spread_"), + ("table3_reference_spread_seed_*/run_*/summary.json", "table3_reference_spread_"), + ]: + for path in sorted(run_root.glob(pattern)): + data = load(path) + headline = data["headline_metrics"] + label = data.get("fm_family") or ("Reference" if "reference_spread" in str(path) else label_from_seed_dir(path, prefix)) + rows.append( + { + "label": label, + "seed": seed_from_path(path), + "strict_f1": float(headline["strict_f1"]), + "spatial_f1": float(headline["same_sample_spatial_tolerance_f1"]["s4"]), + "ap": float(headline["strict_AP"]), + "path": str(path), + } + ) + return group(rows, ["strict_f1", "spatial_f1", "ap"]) + + +def collect_task(run_root: Path, glob_pattern: str, prefix: str, metrics_path: list[str], metric_keys: list[str]) -> dict[str, Any]: + rows: list[dict[str, Any]] = [] + for path in sorted(run_root.glob(glob_pattern)): + data = load(path) + label = data.get("fm_family") or label_from_seed_dir(path, prefix) + node: Any = data + for key in metrics_path: + node = node[key] + row = {"label": label, "seed": seed_from_path(path), "path": str(path)} + for key in metric_keys: + row[key] = float(node[key]) + rows.append(row) + return group(rows, metric_keys) + + +def group(rows: list[dict[str, Any]], metric_keys: list[str]) -> dict[str, Any]: + if rows and "seed" in rows[0]: + rows = dedupe_rows(rows, ("label", "seed")) + summary: dict[str, Any] = {} + for label in sorted({str(row["label"]) for row in rows}): + selected = [row for row in rows if row["label"] == label] + summary[label] = {"n": len(selected)} + for key in metric_keys: + summary[label][key] = stats([row[key] for row in selected]) + return {"rows": rows, "summary": summary} + + +def fmt(value: dict[str, Any], scale: float = 1.0, digits: int = 2) -> str: + if int(value["n"]) == 0: + return "missing" + return f"{float(value['mean']) * scale:.{digits}f} +/- {float(value['std']) * scale:.{digits}f} (n={int(value['n'])})" + + +def write_markdown(out: Path, summary: dict[str, Any]) -> None: + lines = ["# Forced Mean/Std Gap-Fill Summary", ""] + for section in [ + "table2_headcontrol", + "table3_occupancy", + "table3_spread", + "table4_final_area", + "table4_analog", + "table4_smoke", + "table4_heat", + ]: + lines += [f"## {section}", ""] + sec = summary.get(section, {}).get("summary", {}) + if section == "table2_headcontrol": + for scope, row in sec.items(): + lines.append( + f"- {scope}: regret {fmt(row['decision_regret_union_f1'], 100.0)}; " + f"ranking union {fmt(row['ranking_selected_union_f1'], 100.0)}; " + f"decision union {fmt(row['decision_selected_union_f1'], 100.0)}; " + f"failures {row['failure_count']}/{row['n']}" + ) + else: + for label, row in sec.items(): + pieces = [f"{key} {fmt(val, 100.0 if key.endswith('_f1') or key == 'ap' else 1.0)}" for key, val in row.items() if isinstance(val, dict)] + lines.append(f"- {label}: " + "; ".join(pieces)) + lines.append("") + out.write_text("\n".join(lines), encoding="utf-8") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--run-root", type=Path, default=Path("${RUN_ROOT}")) + parser.add_argument("--out-json", type=Path, default=Path("${OUT_JSON}")) + parser.add_argument("--out-md", type=Path, default=Path("${OUT_MD}")) + args = parser.parse_args() + + summary = { + "run_root": str(args.run_root), + "table2_headcontrol": collect_headcontrol(args.run_root), + "table3_occupancy": collect_occupancy(args.run_root), + "table3_spread": collect_spread(args.run_root), + "table4_final_area": collect_task(args.run_root, "table4_final_area_*_seed_*/run_*/summary.json", "table4_final_area_", ["headline_metrics"], ["log_rmse", "log_mae", "log_spearman"]), + "table4_analog": collect_task(args.run_root, "table4_analog_*_seed_*/run_*/summary.json", "table4_analog_", ["test_metrics"], ["ndcg_at_10", "log_rmse", "log_mae"]), + "table4_smoke": collect_task(args.run_root, "table4_smoke_*_seed_*/run_*/summary.json", "table4_smoke_", ["test_metrics"], ["rmse", "mae", "pearson_r"]), + "table4_heat": collect_task(args.run_root, "table4_heat_*_seed_*/run_*/summary.json", "table4_heat_", ["test_metrics"], ["rmse_c", "mae_c", "pearson_r"]), + } + args.out_json.parent.mkdir(parents=True, exist_ok=True) + args.out_json.write_text(json.dumps(summary, indent=2), encoding="utf-8") + write_markdown(args.out_md, summary) + print(f"wrote={args.out_json}") + print(f"wrote={args.out_md}") + + +if __name__ == "__main__": + main() diff --git a/experiments/slurm/submit_template.sbatch b/experiments/slurm/submit_template.sbatch new file mode 100644 index 0000000000000000000000000000000000000000..f9f84b341aa80e4447726f020b5767986ca4e85f --- /dev/null +++ b/experiments/slurm/submit_template.sbatch @@ -0,0 +1,13 @@ +#!/bin/bash +#SBATCH --job-name=wildfire-contract-rerun +#SBATCH --cpus-per-task=4 +#SBATCH --mem=24G +#SBATCH --time=02:00:00 + +# Template only. Set these paths for your environment after obtaining data. +PROJECT_ROOT=/path/to/this/repository +DATA_ROOT=/path/to/raw/or/processed/data +OUTPUT_ROOT=/path/to/output + +cd "$PROJECT_ROOT" +python3 scripts/reproduce_paper_outputs.py diff --git a/paper_outputs/figures/fig_fireprone_contract_progression_compact.pdf b/paper_outputs/figures/fig_fireprone_contract_progression_compact.pdf new file mode 100644 index 0000000000000000000000000000000000000000..0e685407b7b36b947363ae871a81d496391c859c --- /dev/null +++ b/paper_outputs/figures/fig_fireprone_contract_progression_compact.pdf @@ -0,0 +1,262 @@ +%PDF-1.4 +%âãÏÓ +1 0 obj +<< /Type /Catalog /Pages 2 0 R >> +endobj +2 0 obj +<< /Type /Pages /Kids [3 0 R] /Count 1 >> +endobj +3 0 obj +<< /Type /Page /Parent 2 0 R /MediaBox [0 0 1320 470] /Resources << /Font << /F1 4 0 R /F2 5 0 R >> >> /Contents 6 0 R >> +endobj +4 0 obj +<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> +endobj +5 0 obj +<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica-Bold >> +endobj +6 0 obj +<< /Length 19373 >> +stream +1.0000 1.0000 1.0000 rg 0.00 0.00 1320.00 470.00 re f +0.80 w 0.1500 0.1500 0.1500 RG 72.00 132.00 m 72.00 400.00 l S +0.80 w 0.1500 0.1500 0.1500 RG 72.00 132.00 m 1266.00 132.00 l S +0.45 w 0.8600 0.8600 0.8600 RG 68.00 132.00 m 1266.00 132.00 l S +BT /F1 7.00 Tf 0.2500 0.2500 0.2500 rg 1 0 0 1 60.36 129.00 Tm (0) Tj ET +0.45 w 0.8600 0.8600 0.8600 RG 68.00 199.00 m 1266.00 199.00 l S +BT /F1 7.00 Tf 0.2500 0.2500 0.2500 rg 1 0 0 1 56.72 196.00 Tm (20) Tj ET +0.45 w 0.8600 0.8600 0.8600 RG 68.00 266.00 m 1266.00 266.00 l S +BT /F1 7.00 Tf 0.2500 0.2500 0.2500 rg 1 0 0 1 56.72 263.00 Tm (40) Tj ET +0.45 w 0.8600 0.8600 0.8600 RG 68.00 333.00 m 1266.00 333.00 l S +BT /F1 7.00 Tf 0.2500 0.2500 0.2500 rg 1 0 0 1 56.72 330.00 Tm (60) Tj ET +0.45 w 0.8600 0.8600 0.8600 RG 68.00 400.00 m 1266.00 400.00 l S +BT /F1 7.00 Tf 0.2500 0.2500 0.2500 rg 1 0 0 1 56.72 397.00 Tm (80) Tj ET +BT /F2 8.00 Tf 0.1500 0.1500 0.1500 rg 1 0 0 1 34.00 408.00 Tm (F1 \(%\)) Tj ET +BT /F2 15.00 Tf 0.0000 0.0000 0.0000 rg 1 0 0 1 202.93 417.00 Tm (global) Tj ET +0.75 w 0.4200 0.4400 0.4600 RG 380.50 126.00 m 380.50 133.00 l S +0.75 w 0.4200 0.4400 0.4600 RG 380.50 138.00 m 380.50 145.00 l S +0.75 w 0.4200 0.4400 0.4600 RG 380.50 150.00 m 380.50 157.00 l S +0.75 w 0.4200 0.4400 0.4600 RG 380.50 162.00 m 380.50 169.00 l S +0.75 w 0.4200 0.4400 0.4600 RG 380.50 174.00 m 380.50 181.00 l S +0.75 w 0.4200 0.4400 0.4600 RG 380.50 186.00 m 380.50 193.00 l S +0.75 w 0.4200 0.4400 0.4600 RG 380.50 198.00 m 380.50 205.00 l S +0.75 w 0.4200 0.4400 0.4600 RG 380.50 210.00 m 380.50 217.00 l S +0.75 w 0.4200 0.4400 0.4600 RG 380.50 222.00 m 380.50 229.00 l S +0.75 w 0.4200 0.4400 0.4600 RG 380.50 234.00 m 380.50 241.00 l S +0.75 w 0.4200 0.4400 0.4600 RG 380.50 246.00 m 380.50 253.00 l S +0.75 w 0.4200 0.4400 0.4600 RG 380.50 258.00 m 380.50 265.00 l S +0.75 w 0.4200 0.4400 0.4600 RG 380.50 270.00 m 380.50 277.00 l S +0.75 w 0.4200 0.4400 0.4600 RG 380.50 282.00 m 380.50 289.00 l S +0.75 w 0.4200 0.4400 0.4600 RG 380.50 294.00 m 380.50 301.00 l S +0.75 w 0.4200 0.4400 0.4600 RG 380.50 306.00 m 380.50 313.00 l S +0.75 w 0.4200 0.4400 0.4600 RG 380.50 318.00 m 380.50 325.00 l S +0.75 w 0.4200 0.4400 0.4600 RG 380.50 330.00 m 380.50 337.00 l S +0.75 w 0.4200 0.4400 0.4600 RG 380.50 342.00 m 380.50 349.00 l S +0.75 w 0.4200 0.4400 0.4600 RG 380.50 354.00 m 380.50 361.00 l S +0.75 w 0.4200 0.4400 0.4600 RG 380.50 366.00 m 380.50 373.00 l S +0.75 w 0.4200 0.4400 0.4600 RG 380.50 378.00 m 380.50 385.00 l S +0.75 w 0.4200 0.4400 0.4600 RG 380.50 390.00 m 380.50 397.00 l S +0.75 w 0.4200 0.4400 0.4600 RG 380.50 402.00 m 380.50 409.00 l S +0.75 w 0.4200 0.4400 0.4600 RG 380.50 414.00 m 380.50 416.00 l S +BT /F2 15.00 Tf 0.0000 0.0000 0.0000 rg 1 0 0 1 515.07 417.00 Tm (top 5%) Tj ET +BT /F2 15.00 Tf 0.0000 0.0000 0.0000 rg 1 0 0 1 807.48 417.00 Tm (top 10%) Tj ET +BT /F2 15.00 Tf 0.0000 0.0000 0.0000 rg 1 0 0 1 1103.97 417.00 Tm (top 20%) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 84.02 132.00 18.00 1.52 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 84.02 133.52 18.00 98.13 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 84.02 231.66 18.00 98.21 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 76.52 69.51 Tm (Ref.) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 110.07 132.00 18.00 0.19 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 110.07 132.19 18.00 23.82 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 110.07 156.00 18.00 43.62 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 102.21 69.86 Tm (WxC) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 136.11 132.00 18.00 0.22 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 136.11 132.22 18.00 28.26 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 136.11 160.48 18.00 48.92 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 119.34 78.77 Tm (Aurora) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 162.16 132.00 18.00 1.17 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 162.16 133.17 18.00 98.51 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 162.16 231.67 18.00 101.83 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 146.38 77.78 Tm (ClimaX) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 188.20 132.00 18.00 0.21 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 188.20 132.21 18.00 27.24 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 188.20 159.45 18.00 47.52 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 172.99 77.21 Tm (Storm) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 214.25 132.00 18.00 0.57 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 214.25 132.57 18.00 49.40 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 214.25 181.96 18.00 44.47 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 201.30 74.95 Tm (DLWP) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 240.30 132.00 18.00 0.95 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 240.30 132.95 18.00 64.40 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 240.30 197.35 18.00 68.86 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 233.14 69.15 Tm (FCN) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 266.34 132.00 18.00 0.88 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 266.34 132.88 18.00 39.34 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 266.34 172.22 18.00 40.53 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 247.45 80.89 Tm (FengWu) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 292.39 132.00 18.00 1.26 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 292.39 133.26 18.00 69.19 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 292.39 202.46 18.00 54.46 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 284.17 70.21 Tm (FuXi) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 318.43 132.00 18.00 0.92 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 318.43 132.92 18.00 56.33 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 318.43 189.25 18.00 62.13 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 295.86 84.57 Tm (Pangu-W) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 344.48 132.00 18.00 6.90 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 344.48 138.90 18.00 91.75 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 344.48 230.65 18.00 26.74 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 333.29 73.18 Tm (Alpha) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 398.52 132.00 18.00 11.93 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 398.52 143.93 18.00 119.60 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 398.52 263.53 18.00 112.45 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 391.02 69.51 Tm (Ref.) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 424.57 132.00 18.00 4.73 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 424.57 136.73 18.00 59.80 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 424.57 196.53 18.00 78.11 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 416.71 69.86 Tm (WxC) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 450.61 132.00 18.00 3.30 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 450.61 135.30 18.00 47.40 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 450.61 182.70 18.00 68.17 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 433.84 78.77 Tm (Aurora) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 476.66 132.00 18.00 4.33 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 476.66 136.33 18.00 111.51 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 476.66 247.84 18.00 116.04 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 460.88 77.78 Tm (ClimaX) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 502.70 132.00 18.00 3.21 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 502.70 135.21 18.00 48.12 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 502.70 183.33 18.00 69.89 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 487.49 77.21 Tm (Storm) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 528.75 132.00 18.00 6.05 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 528.75 138.05 18.00 100.22 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 528.75 238.27 18.00 79.52 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 515.80 74.95 Tm (DLWP) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 554.80 132.00 18.00 5.44 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 554.80 137.44 18.00 92.98 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 554.80 230.41 18.00 83.50 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 547.64 69.15 Tm (FCN) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 580.84 132.00 18.00 5.26 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 580.84 137.26 18.00 49.27 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 580.84 186.53 18.00 46.33 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 561.95 80.89 Tm (FengWu) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 606.89 132.00 18.00 6.80 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 606.89 138.80 18.00 100.04 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 606.89 238.85 18.00 73.82 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 598.67 70.21 Tm (FuXi) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 632.93 132.00 18.00 4.57 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 632.93 136.57 18.00 69.87 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 632.93 206.44 18.00 71.02 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 610.36 84.57 Tm (Pangu-W) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 658.98 132.00 18.00 23.16 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 658.98 155.16 18.00 120.48 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 658.98 275.64 18.00 29.70 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 647.79 73.18 Tm (Alpha) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 695.02 132.00 18.00 11.92 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 695.02 143.92 18.00 119.29 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 695.02 263.21 18.00 111.74 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 687.52 69.51 Tm (Ref.) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 721.07 132.00 18.00 4.15 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 721.07 136.15 18.00 45.70 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 721.07 181.84 18.00 59.67 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 713.21 69.86 Tm (WxC) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 747.11 132.00 18.00 2.61 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 747.11 134.61 18.00 40.06 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 747.11 174.67 18.00 59.60 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 730.34 78.77 Tm (Aurora) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 773.16 132.00 18.00 4.19 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 773.16 136.19 18.00 110.82 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 773.16 247.02 18.00 114.69 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 757.38 77.78 Tm (ClimaX) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 799.20 132.00 18.00 2.44 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 799.20 134.44 18.00 39.99 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 799.20 174.43 18.00 59.66 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 783.99 77.21 Tm (Storm) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 825.25 132.00 18.00 5.40 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 825.25 137.40 18.00 87.26 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 825.25 224.65 18.00 65.22 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 812.30 74.95 Tm (DLWP) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 851.30 132.00 18.00 3.95 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 851.30 135.95 18.00 71.17 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 851.30 207.11 18.00 70.45 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 844.14 69.15 Tm (FCN) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 877.34 132.00 18.00 4.16 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 877.34 136.16 18.00 39.22 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 877.34 175.38 18.00 37.64 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 858.45 80.89 Tm (FengWu) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 903.39 132.00 18.00 5.54 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 903.39 137.54 18.00 74.90 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 903.39 212.44 18.00 54.27 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 895.17 70.21 Tm (FuXi) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 929.43 132.00 18.00 3.66 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 929.43 135.66 18.00 59.77 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 929.43 195.43 18.00 65.66 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 906.86 84.57 Tm (Pangu-W) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 955.48 132.00 18.00 22.23 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 955.48 154.23 18.00 118.13 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 955.48 272.36 18.00 29.05 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 944.29 73.18 Tm (Alpha) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 991.52 132.00 18.00 11.83 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 991.52 143.83 18.00 116.43 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 991.52 260.25 18.00 105.32 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 984.02 69.51 Tm (Ref.) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 1017.57 132.00 18.00 3.86 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 1017.57 135.86 18.00 40.20 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 1017.57 176.06 18.00 50.19 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 1009.71 69.86 Tm (WxC) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 1043.61 132.00 18.00 2.23 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 1043.61 134.23 18.00 33.05 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 1043.61 167.28 18.00 48.29 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 1026.84 78.77 Tm (Aurora) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 1069.66 132.00 18.00 3.45 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 1069.66 135.45 18.00 97.77 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 1069.66 233.22 18.00 100.00 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 1053.88 77.78 Tm (ClimaX) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 1095.70 132.00 18.00 1.94 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 1095.70 133.94 18.00 32.95 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 1095.70 166.89 18.00 47.72 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 1080.49 77.21 Tm (Storm) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 1121.75 132.00 18.00 5.11 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 1121.75 137.11 18.00 65.04 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 1121.75 202.15 18.00 46.87 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 1108.80 74.95 Tm (DLWP) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 1147.80 132.00 18.00 3.34 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 1147.80 135.34 18.00 53.54 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 1147.80 188.88 18.00 57.31 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 1140.64 69.15 Tm (FCN) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 1173.84 132.00 18.00 3.75 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 1173.84 135.75 18.00 36.29 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 1173.84 172.04 18.00 36.30 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 1154.95 80.89 Tm (FengWu) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 1199.89 132.00 18.00 4.57 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 1199.89 136.57 18.00 68.98 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 1199.89 205.55 18.00 49.50 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 1191.67 70.21 Tm (FuXi) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 1225.93 132.00 18.00 2.96 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 1225.93 134.96 18.00 54.04 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 1225.93 189.01 18.00 58.80 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 1203.36 84.57 Tm (Pangu-W) Tj ET +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 1251.98 132.00 18.00 20.74 re B +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 1251.98 152.74 18.00 109.35 re B +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 1251.98 262.09 18.00 25.30 re B +BT /F1 10.00 Tf 0.0000 0.0000 0.0000 rg 0.70711 -0.70711 0.70711 0.70711 1240.79 73.18 Tm (Alpha) Tj ET +0.45 w 0.9800 0.9800 0.9600 rg 0.7800 0.8000 0.7800 RG 77.00 362.00 304.00 23.00 re B +0.35 w 0.0900 0.2200 0.3700 rg 1.0000 1.0000 1.0000 RG 90.00 371.00 24.00 9.00 re B +BT /F1 8.00 Tf 0.0000 0.0000 0.0000 rg 1 0 0 1 121.00 373.00 Tm (Strict) Tj ET +0.35 w 0.3100 0.5500 0.8000 rg 1.0000 1.0000 1.0000 RG 188.00 371.00 24.00 9.00 re B +BT /F1 8.00 Tf 0.0000 0.0000 0.0000 rg 1 0 0 1 219.00 373.00 Tm (Tolerance) Tj ET +0.35 w 0.7500 0.8400 0.9400 rg 1.0000 1.0000 1.0000 RG 286.00 371.00 24.00 9.00 re B +BT /F1 8.00 Tf 0.0000 0.0000 0.0000 rg 1 0 0 1 317.00 373.00 Tm (Union) Tj ET +endstream +endobj +xref +0 7 +0000000000 65535 f +0000000015 00000 n +0000000064 00000 n +0000000121 00000 n +0000000258 00000 n +0000000328 00000 n +0000000403 00000 n +trailer +<< /Size 7 /Root 1 0 R >> +startxref +19829 +%%EOF diff --git a/paper_outputs/figures/fig_selection_regret_rq2.tikz b/paper_outputs/figures/fig_selection_regret_rq2.tikz new file mode 100644 index 0000000000000000000000000000000000000000..97f504539dfc01f3b573208069c592ed58e7062f --- /dev/null +++ b/paper_outputs/figures/fig_selection_regret_rq2.tikz @@ -0,0 +1,120 @@ +% Auto-generated by scripts/build_selection_regret_rq2_figure.py. +\begin{tikzpicture}[x=1cm,y=1cm] +\footnotesize +\draw[black!12, line width=0.35pt] (2.450,-0.350) -- (2.450,4.530); +\node[anchor=north, font=\scriptsize, text=black!70] at (2.450,-0.410) {-20}; +\draw[black!12, line width=0.35pt] (3.243,-0.350) -- (3.243,4.530); +\node[anchor=north, font=\scriptsize, text=black!70] at (3.243,-0.410) {-10}; +\draw[wfgray, line width=0.55pt] (4.036,-0.350) -- (4.036,4.530); +\node[anchor=north, font=\scriptsize, text=black!70] at (4.036,-0.410) {0}; +\draw[black!12, line width=0.35pt] (4.829,-0.350) -- (4.829,4.530); +\node[anchor=north, font=\scriptsize, text=black!70] at (4.829,-0.410) {10}; +\draw[black!12, line width=0.35pt] (5.621,-0.350) -- (5.621,4.530); +\node[anchor=north, font=\scriptsize, text=black!70] at (5.621,-0.410) {20}; +\draw[black!12, line width=0.35pt] (6.414,-0.350) -- (6.414,4.530); +\node[anchor=north, font=\scriptsize, text=black!70] at (6.414,-0.410) {30}; +\draw[black!12, line width=0.35pt] (7.207,-0.350) -- (7.207,4.530); +\node[anchor=north, font=\scriptsize, text=black!70] at (7.207,-0.410) {40}; +\draw[black!12, line width=0.35pt] (8.000,-0.350) -- (8.000,4.530); +\node[anchor=north, font=\scriptsize, text=black!70] at (8.000,-0.410) {50}; +\draw[black!45, line width=0.4pt] (2.450,-0.350) -- (8.000,-0.350); +\node[anchor=east, font=\scriptsize, text=black!82] at (2.320,4.350) {\textcolor{wfblue}{\textbf{FireWx-FM ref.}}}; +\draw[wfslate, line width=0.72pt] (4.030,4.220) -- (5.212,4.220); +\draw[wfslate, line width=0.72pt] (4.030,4.185) -- (4.030,4.255); +\draw[wfslate, line width=0.72pt] (5.212,4.185) -- (5.212,4.255); +\filldraw[wfslate] (4.621,4.220) circle[radius=0.045]; +\draw[wforange, line width=0.72pt] (4.051,4.480) -- (4.487,4.480); +\draw[wforange, line width=0.72pt] (4.051,4.445) -- (4.051,4.515); +\draw[wforange, line width=0.72pt] (4.487,4.445) -- (4.487,4.515); +\filldraw[wforange] (4.224,4.435) rectangle (4.314,4.525); +\node[anchor=east, font=\scriptsize, text=black!82] at (2.320,3.940) {Prithvi-WxC}; +\draw[wfslate, line width=0.72pt] (4.036,3.810) -- (4.036,3.810); +\draw[wfslate, line width=0.72pt] (4.036,3.775) -- (4.036,3.845); +\draw[wfslate, line width=0.72pt] (4.036,3.775) -- (4.036,3.845); +\filldraw[wfslate] (4.036,3.810) circle[radius=0.045]; +\draw[wforange, line width=0.72pt] (4.036,4.070) -- (4.036,4.070); +\draw[wforange, line width=0.72pt] (4.036,4.035) -- (4.036,4.105); +\draw[wforange, line width=0.72pt] (4.036,4.035) -- (4.036,4.105); +\filldraw[wforange] (3.991,4.025) rectangle (4.081,4.115); +\node[anchor=east, font=\scriptsize, text=black!82] at (2.320,3.530) {Aurora}; +\draw[wfslate, line width=0.72pt] (3.580,3.400) -- (5.276,3.400); +\draw[wfslate, line width=0.72pt] (3.580,3.365) -- (3.580,3.435); +\draw[wfslate, line width=0.72pt] (5.276,3.365) -- (5.276,3.435); +\filldraw[wfslate] (4.428,3.400) circle[radius=0.045]; +\draw[wforange, line width=0.72pt] (2.627,3.660) -- (7.723,3.660); +\draw[wforange, line width=0.72pt] (2.627,3.625) -- (2.627,3.695); +\draw[wforange, line width=0.72pt] (7.723,3.625) -- (7.723,3.695); +\filldraw[wforange] (5.130,3.615) rectangle (5.220,3.705); +\node[anchor=east, font=\scriptsize, text=black!82] at (2.320,3.120) {ClimaX}; +\draw[wfslate, line width=0.72pt] (4.032,2.990) -- (4.060,2.990); +\draw[wfslate, line width=0.72pt] (4.032,2.955) -- (4.032,3.025); +\draw[wfslate, line width=0.72pt] (4.060,2.955) -- (4.060,3.025); +\filldraw[wfslate] (4.046,2.990) circle[radius=0.045]; +\draw[wforange, line width=0.72pt] (4.036,3.250) -- (4.036,3.250); +\draw[wforange, line width=0.72pt] (4.036,3.215) -- (4.036,3.285); +\draw[wforange, line width=0.72pt] (4.036,3.215) -- (4.036,3.285); +\filldraw[wforange] (3.991,3.205) rectangle (4.081,3.295); +\node[anchor=east, font=\scriptsize, text=black!82] at (2.320,2.710) {StormCast}; +\draw[wfslate, line width=0.72pt] (4.036,2.580) -- (4.036,2.580); +\draw[wfslate, line width=0.72pt] (4.036,2.545) -- (4.036,2.615); +\draw[wfslate, line width=0.72pt] (4.036,2.545) -- (4.036,2.615); +\filldraw[wfslate] (4.036,2.580) circle[radius=0.045]; +\draw[wforange, line width=0.72pt] (4.036,2.840) -- (4.036,2.840); +\draw[wforange, line width=0.72pt] (4.036,2.805) -- (4.036,2.875); +\draw[wforange, line width=0.72pt] (4.036,2.805) -- (4.036,2.875); +\filldraw[wforange] (3.991,2.795) rectangle (4.081,2.885); +\node[anchor=east, font=\scriptsize, text=black!82] at (2.320,2.300) {DLWP}; +\draw[wfslate, line width=0.72pt] (4.036,2.170) -- (4.036,2.170); +\draw[wfslate, line width=0.72pt] (4.036,2.135) -- (4.036,2.205); +\draw[wfslate, line width=0.72pt] (4.036,2.135) -- (4.036,2.205); +\filldraw[wfslate] (4.036,2.170) circle[radius=0.045]; +\draw[wforange, line width=0.72pt] (4.044,2.430) -- (4.735,2.430); +\draw[wforange, line width=0.72pt] (4.044,2.395) -- (4.044,2.465); +\draw[wforange, line width=0.72pt] (4.735,2.395) -- (4.735,2.465); +\filldraw[wforange] (4.345,2.385) rectangle (4.435,2.475); +\node[anchor=east, font=\scriptsize, text=black!82] at (2.320,1.890) {FCN}; +\draw[wfslate, line width=0.72pt] (4.036,1.760) -- (4.036,1.760); +\draw[wfslate, line width=0.72pt] (4.036,1.725) -- (4.036,1.795); +\draw[wfslate, line width=0.72pt] (4.036,1.725) -- (4.036,1.795); +\filldraw[wfslate] (4.036,1.760) circle[radius=0.045]; +\draw[wforange, line width=0.72pt] (3.971,2.020) -- (4.286,2.020); +\draw[wforange, line width=0.72pt] (3.971,1.985) -- (3.971,2.055); +\draw[wforange, line width=0.72pt] (4.286,1.985) -- (4.286,2.055); +\filldraw[wforange] (4.083,1.975) rectangle (4.173,2.065); +\node[anchor=east, font=\scriptsize, text=black!82] at (2.320,1.480) {FengWu}; +\draw[wfslate, line width=0.72pt] (4.036,1.350) -- (4.036,1.350); +\draw[wfslate, line width=0.72pt] (4.036,1.315) -- (4.036,1.385); +\draw[wfslate, line width=0.72pt] (4.036,1.315) -- (4.036,1.385); +\filldraw[wfslate] (4.036,1.350) circle[radius=0.045]; +\draw[wforange, line width=0.72pt] (4.028,1.610) -- (4.127,1.610); +\draw[wforange, line width=0.72pt] (4.028,1.575) -- (4.028,1.645); +\draw[wforange, line width=0.72pt] (4.127,1.575) -- (4.127,1.645); +\filldraw[wforange] (4.032,1.565) rectangle (4.122,1.655); +\node[anchor=east, font=\scriptsize, text=black!82] at (2.320,1.070) {FuXi}; +\draw[wfslate, line width=0.72pt] (4.036,0.940) -- (4.036,0.940); +\draw[wfslate, line width=0.72pt] (4.036,0.905) -- (4.036,0.975); +\draw[wfslate, line width=0.72pt] (4.036,0.905) -- (4.036,0.975); +\filldraw[wfslate] (4.036,0.940) circle[radius=0.045]; +\draw[wforange, line width=0.72pt] (4.029,1.200) -- (4.087,1.200); +\draw[wforange, line width=0.72pt] (4.029,1.165) -- (4.029,1.235); +\draw[wforange, line width=0.72pt] (4.087,1.165) -- (4.087,1.235); +\filldraw[wforange] (4.013,1.155) rectangle (4.103,1.245); +\node[anchor=east, font=\scriptsize, text=black!82] at (2.320,0.660) {Pangu-Weather}; +\draw[wfslate, line width=0.72pt] (4.036,0.530) -- (4.036,0.530); +\draw[wfslate, line width=0.72pt] (4.036,0.495) -- (4.036,0.565); +\draw[wfslate, line width=0.72pt] (4.036,0.495) -- (4.036,0.565); +\filldraw[wfslate] (4.036,0.530) circle[radius=0.045]; +\draw[wforange, line width=0.72pt] (4.025,0.790) -- (4.076,0.790); +\draw[wforange, line width=0.72pt] (4.025,0.755) -- (4.025,0.825); +\draw[wforange, line width=0.72pt] (4.076,0.755) -- (4.076,0.825); +\filldraw[wforange] (4.006,0.745) rectangle (4.096,0.835); +\node[anchor=east, font=\scriptsize, text=black!82] at (2.320,0.250) {AlphaEarth}; +\draw[wfslate, line width=0.72pt] (4.700,0.120) -- (6.103,0.120); +\draw[wfslate, line width=0.72pt] (4.700,0.085) -- (4.700,0.155); +\draw[wfslate, line width=0.72pt] (6.103,0.085) -- (6.103,0.155); +\filldraw[wfslate] (5.401,0.120) circle[radius=0.045]; +\draw[wforange, line width=0.72pt] (3.872,0.380) -- (4.815,0.380); +\draw[wforange, line width=0.72pt] (3.872,0.345) -- (3.872,0.415); +\draw[wforange, line width=0.72pt] (4.815,0.345) -- (4.815,0.415); +\filldraw[wforange] (4.298,0.335) rectangle (4.388,0.425); +\end{tikzpicture} diff --git a/paper_outputs/figures/fig_task_contract_tiles.pdf b/paper_outputs/figures/fig_task_contract_tiles.pdf new file mode 100644 index 0000000000000000000000000000000000000000..0f5a30ead43e185eccf1a996abeb9c8d82dd11bd Binary files /dev/null and b/paper_outputs/figures/fig_task_contract_tiles.pdf differ diff --git a/paper_outputs/figures/fig_task_rank_map.pdf b/paper_outputs/figures/fig_task_rank_map.pdf new file mode 100644 index 0000000000000000000000000000000000000000..4ae1bcef71271945bba47c8c34a53751155e9db9 --- /dev/null +++ b/paper_outputs/figures/fig_task_rank_map.pdf @@ -0,0 +1,348 @@ +%PDF-1.4 +%âãÏÓ +1 0 obj +<< /Type /Catalog /Pages 2 0 R >> +endobj +2 0 obj +<< /Type /Pages /Kids [3 0 R] /Count 1 >> +endobj +3 0 obj +<< /Type /Page /Parent 2 0 R /MediaBox [0 0 1120 430] /Resources << /Font << /F1 4 0 R /F2 5 0 R >> >> /Contents 6 0 R >> +endobj +4 0 obj +<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> +endobj +5 0 obj +<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica-Bold >> +endobj +6 0 obj +<< /Length 22799 >> +stream +1.0000 1.0000 1.0000 rg 0.00 0.00 1120.00 430.00 re f +BT /F2 8.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 117.66 376.00 Tm (FireWx-FM ref.) Tj ET +BT /F2 8.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 211.06 376.00 Tm (Prithvi-WxC) Tj ET +BT /F2 8.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 308.29 376.00 Tm (Aurora) Tj ET +BT /F2 8.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 394.93 376.00 Tm (ClimaX) Tj ET +BT /F2 8.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 471.34 376.00 Tm (StormCast) Tj ET +BT /F2 8.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 568.76 376.00 Tm (DLWP) Tj ET +BT /F2 8.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 658.50 376.00 Tm (FCN) Tj ET +BT /F2 8.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 736.92 376.00 Tm (FengWu) Tj ET +BT /F2 8.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 829.82 376.00 Tm (FuXi) Tj ET +BT /F2 8.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 892.30 376.00 Tm (Pangu-Weather) Tj ET +BT /F2 8.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 987.57 376.00 Tm (AlphaEarth) Tj ET +BT /F2 7.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 12.00 325.00 Tm (Occupancy) Tj ET +BT /F2 7.10 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 12.00 314.00 Tm (Union F1 \(%\)) Tj ET +BT /F1 6.40 Tf 0.4200 0.4400 0.4600 rg 1 0 0 1 12.00 303.00 Tm (higher better) Tj ET +0.80 w 0.1500 0.4760 0.4860 rg 1.0000 1.0000 1.0000 RG 108.00 300.00 86.00 42.00 re B +BT /F2 11.20 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 144.88 324.00 Tm (#2) Tj ET +BT /F1 7.00 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 142.84 309.00 Tm (59.07) Tj ET +0.80 w 0.9300 0.9500 0.9400 rg 1.0000 1.0000 1.0000 RG 194.00 300.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 227.83 324.00 Tm (#11) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 228.84 309.00 Tm (20.19) Tj ET +0.80 w 0.7780 0.8820 0.8640 rg 1.0000 1.0000 1.0000 RG 280.00 300.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 316.88 324.00 Tm (#9) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 314.85 309.00 Tm (23.10) Tj ET +0.80 w 0.0500 0.4000 0.4200 rg 1.0000 1.0000 1.0000 RG 366.00 300.00 86.00 42.00 re B +BT /F2 11.20 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 402.88 324.00 Tm (#1) Tj ET +BT /F1 7.00 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 400.85 309.00 Tm (60.15) Tj ET +0.80 w 0.8540 0.9160 0.9020 rg 1.0000 1.0000 1.0000 RG 452.00 300.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 485.83 324.00 Tm (#10) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 486.85 309.00 Tm (22.38) Tj ET +0.80 w 0.6260 0.8140 0.7880 rg 1.0000 1.0000 1.0000 RG 538.00 300.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 574.88 324.00 Tm (#7) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 572.85 309.00 Tm (28.19) Tj ET +0.80 w 0.2500 0.5520 0.5520 rg 1.0000 1.0000 1.0000 RG 624.00 300.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 660.88 324.00 Tm (#3) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 658.85 309.00 Tm (40.06) Tj ET +0.80 w 0.7020 0.8480 0.8260 rg 1.0000 1.0000 1.0000 RG 710.00 300.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 746.88 324.00 Tm (#8) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 744.85 309.00 Tm (24.10) Tj ET +0.80 w 0.4500 0.7040 0.6840 rg 1.0000 1.0000 1.0000 RG 796.00 300.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 832.88 324.00 Tm (#5) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 830.85 309.00 Tm (37.29) Tj ET +0.80 w 0.5500 0.7800 0.7500 rg 1.0000 1.0000 1.0000 RG 882.00 300.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 918.88 324.00 Tm (#6) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 916.85 309.00 Tm (35.64) Tj ET +0.80 w 0.3500 0.6280 0.6180 rg 1.0000 1.0000 1.0000 RG 968.00 300.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 1004.88 324.00 Tm (#4) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 1002.85 309.00 Tm (37.43) Tj ET +BT /F2 7.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 12.00 283.00 Tm (Fire spread) Tj ET +BT /F2 7.10 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 12.00 272.00 Tm (AP \(%\)) Tj ET +BT /F1 6.40 Tf 0.4200 0.4400 0.4600 rg 1 0 0 1 12.00 261.00 Tm (higher better) Tj ET +0.80 w 0.0500 0.4000 0.4200 rg 1.0000 1.0000 1.0000 RG 108.00 258.00 86.00 42.00 re B +BT /F2 11.20 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 144.88 282.00 Tm (#1) Tj ET +BT /F1 7.00 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 142.84 267.00 Tm (30.09) Tj ET +0.80 w 0.7780 0.8820 0.8640 rg 1.0000 1.0000 1.0000 RG 194.00 258.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 230.88 282.00 Tm (#9) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 230.66 267.00 Tm (5.00) Tj ET +0.80 w 0.1500 0.4760 0.4860 rg 1.0000 1.0000 1.0000 RG 280.00 258.00 86.00 42.00 re B +BT /F2 11.20 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 316.88 282.00 Tm (#2) Tj ET +BT /F1 7.00 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 314.85 267.00 Tm (16.62) Tj ET +0.80 w 0.6260 0.8140 0.7880 rg 1.0000 1.0000 1.0000 RG 366.00 258.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 402.88 282.00 Tm (#7) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 400.85 267.00 Tm (11.17) Tj ET +0.80 w 0.8540 0.9160 0.9020 rg 1.0000 1.0000 1.0000 RG 452.00 258.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 485.83 282.00 Tm (#10) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 488.67 267.00 Tm (2.81) Tj ET +0.80 w 0.7020 0.8480 0.8260 rg 1.0000 1.0000 1.0000 RG 538.00 258.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 574.88 282.00 Tm (#8) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 574.66 267.00 Tm (5.94) Tj ET +0.80 w 0.9300 0.9500 0.9400 rg 1.0000 1.0000 1.0000 RG 624.00 258.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 657.83 282.00 Tm (#11) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 660.66 267.00 Tm (2.39) Tj ET +0.80 w 0.3500 0.6280 0.6180 rg 1.0000 1.0000 1.0000 RG 710.00 258.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 746.88 282.00 Tm (#4) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 744.85 267.00 Tm (13.17) Tj ET +0.80 w 0.2500 0.5520 0.5520 rg 1.0000 1.0000 1.0000 RG 796.00 258.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 832.88 282.00 Tm (#3) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 830.85 267.00 Tm (14.35) Tj ET +0.80 w 0.4500 0.7040 0.6840 rg 1.0000 1.0000 1.0000 RG 882.00 258.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 918.88 282.00 Tm (#5) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 916.85 267.00 Tm (12.69) Tj ET +0.80 w 0.5500 0.7800 0.7500 rg 1.0000 1.0000 1.0000 RG 968.00 258.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 1004.88 282.00 Tm (#6) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 1002.85 267.00 Tm (11.83) Tj ET +BT /F2 7.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 12.00 241.00 Tm (Burned area) Tj ET +BT /F2 7.10 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 12.00 230.00 Tm (log-RMSE) Tj ET +BT /F1 6.40 Tf 0.4200 0.4400 0.4600 rg 1 0 0 1 12.00 219.00 Tm (lower better) Tj ET +0.80 w 0.0500 0.4000 0.4200 rg 1.0000 1.0000 1.0000 RG 108.00 216.00 86.00 42.00 re B +BT /F2 11.20 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 144.88 240.00 Tm (#1) Tj ET +BT /F1 7.00 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 144.66 225.00 Tm (1.17) Tj ET +0.80 w 0.3500 0.6280 0.6180 rg 1.0000 1.0000 1.0000 RG 194.00 216.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 230.88 240.00 Tm (#4) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 230.66 225.00 Tm (1.36) Tj ET +0.80 w 0.7780 0.8820 0.8640 rg 1.0000 1.0000 1.0000 RG 280.00 216.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 316.88 240.00 Tm (#9) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 316.67 225.00 Tm (1.87) Tj ET +0.80 w 0.8540 0.9160 0.9020 rg 1.0000 1.0000 1.0000 RG 366.00 216.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 399.83 240.00 Tm (#10) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 402.67 225.00 Tm (2.03) Tj ET +0.80 w 0.7020 0.8480 0.8260 rg 1.0000 1.0000 1.0000 RG 452.00 216.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 488.88 240.00 Tm (#8) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 488.67 225.00 Tm (1.67) Tj ET +0.80 w 0.1500 0.4760 0.4860 rg 1.0000 1.0000 1.0000 RG 538.00 216.00 86.00 42.00 re B +BT /F2 11.20 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 574.88 240.00 Tm (#2) Tj ET +BT /F1 7.00 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 574.66 225.00 Tm (1.31) Tj ET +0.80 w 0.4500 0.7040 0.6840 rg 1.0000 1.0000 1.0000 RG 624.00 216.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 660.88 240.00 Tm (#5) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 660.66 225.00 Tm (1.37) Tj ET +0.80 w 0.5500 0.7800 0.7500 rg 1.0000 1.0000 1.0000 RG 710.00 216.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 746.88 240.00 Tm (#6) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 746.66 225.00 Tm (1.37) Tj ET +0.80 w 0.6260 0.8140 0.7880 rg 1.0000 1.0000 1.0000 RG 796.00 216.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 832.88 240.00 Tm (#7) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 832.66 225.00 Tm (1.41) Tj ET +0.80 w 0.2500 0.5520 0.5520 rg 1.0000 1.0000 1.0000 RG 882.00 216.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 918.88 240.00 Tm (#3) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 918.66 225.00 Tm (1.33) Tj ET +0.80 w 0.9300 0.9500 0.9400 rg 1.0000 1.0000 1.0000 RG 968.00 216.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 1001.83 240.00 Tm (#11) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 1004.66 225.00 Tm (2.41) Tj ET +BT /F2 7.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 12.00 199.00 Tm (Analog retrieval) Tj ET +BT /F2 7.10 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 12.00 188.00 Tm (nDCG@10) Tj ET +BT /F1 6.40 Tf 0.4200 0.4400 0.4600 rg 1 0 0 1 12.00 177.00 Tm (higher better) Tj ET +0.80 w 0.0500 0.4000 0.4200 rg 1.0000 1.0000 1.0000 RG 108.00 174.00 86.00 42.00 re B +BT /F2 11.20 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 144.88 198.00 Tm (#1) Tj ET +BT /F1 7.00 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 142.84 183.00 Tm (0.510) Tj ET +0.80 w 0.9300 0.9500 0.9400 rg 1.0000 1.0000 1.0000 RG 194.00 174.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 227.83 198.00 Tm (#11) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 228.84 183.00 Tm (0.386) Tj ET +0.80 w 0.7020 0.8480 0.8260 rg 1.0000 1.0000 1.0000 RG 280.00 174.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 316.88 198.00 Tm (#8) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 314.85 183.00 Tm (0.405) Tj ET +0.80 w 0.5500 0.7800 0.7500 rg 1.0000 1.0000 1.0000 RG 366.00 174.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 402.88 198.00 Tm (#6) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 400.85 183.00 Tm (0.414) Tj ET +0.80 w 0.6260 0.8140 0.7880 rg 1.0000 1.0000 1.0000 RG 452.00 174.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 488.88 198.00 Tm (#7) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 486.85 183.00 Tm (0.408) Tj ET +0.80 w 0.8540 0.9160 0.9020 rg 1.0000 1.0000 1.0000 RG 538.00 174.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 571.83 198.00 Tm (#10) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 572.85 183.00 Tm (0.397) Tj ET +0.80 w 0.2500 0.5520 0.5520 rg 1.0000 1.0000 1.0000 RG 624.00 174.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 660.88 198.00 Tm (#3) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 658.85 183.00 Tm (0.432) Tj ET +0.80 w 0.4500 0.7040 0.6840 rg 1.0000 1.0000 1.0000 RG 710.00 174.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 746.88 198.00 Tm (#5) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 744.85 183.00 Tm (0.425) Tj ET +0.80 w 0.3500 0.6280 0.6180 rg 1.0000 1.0000 1.0000 RG 796.00 174.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 832.88 198.00 Tm (#4) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 830.85 183.00 Tm (0.428) Tj ET +0.80 w 0.7780 0.8820 0.8640 rg 1.0000 1.0000 1.0000 RG 882.00 174.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 918.88 198.00 Tm (#9) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 916.85 183.00 Tm (0.402) Tj ET +0.80 w 0.1500 0.4760 0.4860 rg 1.0000 1.0000 1.0000 RG 968.00 174.00 86.00 42.00 re B +BT /F2 11.20 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 1004.88 198.00 Tm (#2) Tj ET +BT /F1 7.00 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 1002.85 183.00 Tm (0.509) Tj ET +BT /F2 7.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 12.00 157.00 Tm (Smoke PM2.5) Tj ET +BT /F2 7.10 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 12.00 146.00 Tm (RMSE) Tj ET +BT /F1 6.40 Tf 0.4200 0.4400 0.4600 rg 1 0 0 1 12.00 135.00 Tm (lower better) Tj ET +0.80 w 0.1500 0.4760 0.4860 rg 1.0000 1.0000 1.0000 RG 108.00 132.00 86.00 42.00 re B +BT /F2 11.20 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 144.88 156.00 Tm (#2) Tj ET +BT /F1 7.00 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 144.66 141.00 Tm (4.46) Tj ET +0.80 w 0.7020 0.8480 0.8260 rg 1.0000 1.0000 1.0000 RG 194.00 132.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 230.88 156.00 Tm (#8) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 230.66 141.00 Tm (6.04) Tj ET +0.80 w 0.7780 0.8820 0.8640 rg 1.0000 1.0000 1.0000 RG 280.00 132.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 316.88 156.00 Tm (#9) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 316.67 141.00 Tm (6.04) Tj ET +0.80 w 0.8540 0.9160 0.9020 rg 1.0000 1.0000 1.0000 RG 366.00 132.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 399.83 156.00 Tm (#10) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 402.67 141.00 Tm (6.04) Tj ET +0.80 w 0.9300 0.9500 0.9400 rg 1.0000 1.0000 1.0000 RG 452.00 132.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 485.83 156.00 Tm (#11) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 488.67 141.00 Tm (6.12) Tj ET +0.80 w 0.4500 0.7040 0.6840 rg 1.0000 1.0000 1.0000 RG 538.00 132.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 574.88 156.00 Tm (#5) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 574.66 141.00 Tm (5.93) Tj ET +0.80 w 0.3500 0.6280 0.6180 rg 1.0000 1.0000 1.0000 RG 624.00 132.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 660.88 156.00 Tm (#4) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 660.66 141.00 Tm (5.93) Tj ET +0.80 w 0.5500 0.7800 0.7500 rg 1.0000 1.0000 1.0000 RG 710.00 132.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 746.88 156.00 Tm (#6) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 746.66 141.00 Tm (5.93) Tj ET +0.80 w 0.6260 0.8140 0.7880 rg 1.0000 1.0000 1.0000 RG 796.00 132.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 832.88 156.00 Tm (#7) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 832.66 141.00 Tm (5.93) Tj ET +0.80 w 0.2500 0.5520 0.5520 rg 1.0000 1.0000 1.0000 RG 882.00 132.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 918.88 156.00 Tm (#3) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 918.66 141.00 Tm (5.93) Tj ET +0.80 w 0.0500 0.4000 0.4200 rg 1.0000 1.0000 1.0000 RG 968.00 132.00 86.00 42.00 re B +BT /F2 11.20 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 1004.88 156.00 Tm (#1) Tj ET +BT /F1 7.00 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 1004.66 141.00 Tm (4.44) Tj ET +BT /F2 7.70 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 12.00 115.00 Tm (Extreme heat) Tj ET +BT /F2 7.10 Tf 0.1200 0.1400 0.1600 rg 1 0 0 1 12.00 104.00 Tm (RMSE-C) Tj ET +BT /F1 6.40 Tf 0.4200 0.4400 0.4600 rg 1 0 0 1 12.00 93.00 Tm (lower better) Tj ET +0.80 w 0.0500 0.4000 0.4200 rg 1.0000 1.0000 1.0000 RG 108.00 90.00 86.00 42.00 re B +BT /F2 11.20 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 144.88 114.00 Tm (#1) Tj ET +BT /F1 7.00 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 142.84 99.00 Tm (0.218) Tj ET +0.80 w 0.7780 0.8820 0.8640 rg 1.0000 1.0000 1.0000 RG 194.00 90.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 230.88 114.00 Tm (#9) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 230.66 99.00 Tm (4.62) Tj ET +0.80 w 0.9300 0.9500 0.9400 rg 1.0000 1.0000 1.0000 RG 280.00 90.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 313.83 114.00 Tm (#11) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 314.85 99.00 Tm (18.05) Tj ET +0.80 w 0.8540 0.9160 0.9020 rg 1.0000 1.0000 1.0000 RG 366.00 90.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 399.83 114.00 Tm (#10) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 400.85 99.00 Tm (17.65) Tj ET +0.80 w 0.2500 0.5520 0.5520 rg 1.0000 1.0000 1.0000 RG 452.00 90.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 488.88 114.00 Tm (#3) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 488.67 99.00 Tm (1.77) Tj ET +0.80 w 0.7020 0.8480 0.8260 rg 1.0000 1.0000 1.0000 RG 538.00 90.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 574.88 114.00 Tm (#8) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 574.66 99.00 Tm (2.27) Tj ET +0.80 w 0.5500 0.7800 0.7500 rg 1.0000 1.0000 1.0000 RG 624.00 90.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 660.88 114.00 Tm (#6) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 660.66 99.00 Tm (2.17) Tj ET +0.80 w 0.3500 0.6280 0.6180 rg 1.0000 1.0000 1.0000 RG 710.00 90.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 746.88 114.00 Tm (#4) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 746.66 99.00 Tm (2.13) Tj ET +0.80 w 0.4500 0.7040 0.6840 rg 1.0000 1.0000 1.0000 RG 796.00 90.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 832.88 114.00 Tm (#5) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 832.66 99.00 Tm (2.13) Tj ET +0.80 w 0.6260 0.8140 0.7880 rg 1.0000 1.0000 1.0000 RG 882.00 90.00 86.00 42.00 re B +BT /F2 11.20 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 918.88 114.00 Tm (#7) Tj ET +BT /F1 7.00 Tf 0.0700 0.0900 0.1100 rg 1 0 0 1 918.66 99.00 Tm (2.20) Tj ET +0.80 w 0.1500 0.4760 0.4860 rg 1.0000 1.0000 1.0000 RG 968.00 90.00 86.00 42.00 re B +BT /F2 11.20 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 1004.88 114.00 Tm (#2) Tj ET +BT /F1 7.00 Tf 1.0000 1.0000 1.0000 rg 1 0 0 1 1002.85 99.00 Tm (0.219) Tj ET +0.80 w 0.2000 0.2200 0.2400 RG 108.00 90.00 946.00 252.00 re S +BT /F2 9.00 Tf 0.2400 0.2500 0.2600 rg 1 0 0 1 908.00 63.00 Tm (within-row rank) Tj ET +0.9300 0.9500 0.9400 rg 834.00 46.00 2.60 10.00 re f +0.9300 0.9500 0.9400 rg 836.50 46.00 2.60 10.00 re f +0.9300 0.9500 0.9400 rg 839.00 46.00 2.60 10.00 re f +0.9300 0.9500 0.9400 rg 841.50 46.00 2.60 10.00 re f +0.9300 0.9500 0.9400 rg 844.00 46.00 2.60 10.00 re f +0.9300 0.9500 0.9400 rg 846.50 46.00 2.60 10.00 re f +0.9300 0.9500 0.9400 rg 849.00 46.00 2.60 10.00 re f +0.9300 0.9500 0.9400 rg 851.50 46.00 2.60 10.00 re f +0.8540 0.9160 0.9020 rg 854.00 46.00 2.60 10.00 re f +0.8540 0.9160 0.9020 rg 856.50 46.00 2.60 10.00 re f +0.8540 0.9160 0.9020 rg 859.00 46.00 2.60 10.00 re f +0.8540 0.9160 0.9020 rg 861.50 46.00 2.60 10.00 re f +0.8540 0.9160 0.9020 rg 864.00 46.00 2.60 10.00 re f +0.8540 0.9160 0.9020 rg 866.50 46.00 2.60 10.00 re f +0.8540 0.9160 0.9020 rg 869.00 46.00 2.60 10.00 re f +0.8540 0.9160 0.9020 rg 871.50 46.00 2.60 10.00 re f +0.7780 0.8820 0.8640 rg 874.00 46.00 2.60 10.00 re f +0.7780 0.8820 0.8640 rg 876.50 46.00 2.60 10.00 re f +0.7780 0.8820 0.8640 rg 879.00 46.00 2.60 10.00 re f +0.7780 0.8820 0.8640 rg 881.50 46.00 2.60 10.00 re f +0.7780 0.8820 0.8640 rg 884.00 46.00 2.60 10.00 re f +0.7780 0.8820 0.8640 rg 886.50 46.00 2.60 10.00 re f +0.7780 0.8820 0.8640 rg 889.00 46.00 2.60 10.00 re f +0.7780 0.8820 0.8640 rg 891.50 46.00 2.60 10.00 re f +0.7020 0.8480 0.8260 rg 894.00 46.00 2.60 10.00 re f +0.7020 0.8480 0.8260 rg 896.50 46.00 2.60 10.00 re f +0.7020 0.8480 0.8260 rg 899.00 46.00 2.60 10.00 re f +0.7020 0.8480 0.8260 rg 901.50 46.00 2.60 10.00 re f +0.7020 0.8480 0.8260 rg 904.00 46.00 2.60 10.00 re f +0.7020 0.8480 0.8260 rg 906.50 46.00 2.60 10.00 re f +0.7020 0.8480 0.8260 rg 909.00 46.00 2.60 10.00 re f +0.7020 0.8480 0.8260 rg 911.50 46.00 2.60 10.00 re f +0.6260 0.8140 0.7880 rg 914.00 46.00 2.60 10.00 re f +0.6260 0.8140 0.7880 rg 916.50 46.00 2.60 10.00 re f +0.6260 0.8140 0.7880 rg 919.00 46.00 2.60 10.00 re f +0.6260 0.8140 0.7880 rg 921.50 46.00 2.60 10.00 re f +0.6260 0.8140 0.7880 rg 924.00 46.00 2.60 10.00 re f +0.6260 0.8140 0.7880 rg 926.50 46.00 2.60 10.00 re f +0.6260 0.8140 0.7880 rg 929.00 46.00 2.60 10.00 re f +0.6260 0.8140 0.7880 rg 931.50 46.00 2.60 10.00 re f +0.5500 0.7800 0.7500 rg 934.00 46.00 2.60 10.00 re f +0.5500 0.7800 0.7500 rg 936.50 46.00 2.60 10.00 re f +0.5500 0.7800 0.7500 rg 939.00 46.00 2.60 10.00 re f +0.5500 0.7800 0.7500 rg 941.50 46.00 2.60 10.00 re f +0.5500 0.7800 0.7500 rg 944.00 46.00 2.60 10.00 re f +0.5500 0.7800 0.7500 rg 946.50 46.00 2.60 10.00 re f +0.5500 0.7800 0.7500 rg 949.00 46.00 2.60 10.00 re f +0.5500 0.7800 0.7500 rg 951.50 46.00 2.60 10.00 re f +0.4500 0.7040 0.6840 rg 954.00 46.00 2.60 10.00 re f +0.4500 0.7040 0.6840 rg 956.50 46.00 2.60 10.00 re f +0.4500 0.7040 0.6840 rg 959.00 46.00 2.60 10.00 re f +0.4500 0.7040 0.6840 rg 961.50 46.00 2.60 10.00 re f +0.4500 0.7040 0.6840 rg 964.00 46.00 2.60 10.00 re f +0.4500 0.7040 0.6840 rg 966.50 46.00 2.60 10.00 re f +0.4500 0.7040 0.6840 rg 969.00 46.00 2.60 10.00 re f +0.4500 0.7040 0.6840 rg 971.50 46.00 2.60 10.00 re f +0.3500 0.6280 0.6180 rg 974.00 46.00 2.60 10.00 re f +0.3500 0.6280 0.6180 rg 976.50 46.00 2.60 10.00 re f +0.3500 0.6280 0.6180 rg 979.00 46.00 2.60 10.00 re f +0.3500 0.6280 0.6180 rg 981.50 46.00 2.60 10.00 re f +0.3500 0.6280 0.6180 rg 984.00 46.00 2.60 10.00 re f +0.3500 0.6280 0.6180 rg 986.50 46.00 2.60 10.00 re f +0.3500 0.6280 0.6180 rg 989.00 46.00 2.60 10.00 re f +0.3500 0.6280 0.6180 rg 991.50 46.00 2.60 10.00 re f +0.2500 0.5520 0.5520 rg 994.00 46.00 2.60 10.00 re f +0.2500 0.5520 0.5520 rg 996.50 46.00 2.60 10.00 re f +0.2500 0.5520 0.5520 rg 999.00 46.00 2.60 10.00 re f +0.2500 0.5520 0.5520 rg 1001.50 46.00 2.60 10.00 re f +0.2500 0.5520 0.5520 rg 1004.00 46.00 2.60 10.00 re f +0.2500 0.5520 0.5520 rg 1006.50 46.00 2.60 10.00 re f +0.2500 0.5520 0.5520 rg 1009.00 46.00 2.60 10.00 re f +0.2500 0.5520 0.5520 rg 1011.50 46.00 2.60 10.00 re f +0.1500 0.4760 0.4860 rg 1014.00 46.00 2.60 10.00 re f +0.1500 0.4760 0.4860 rg 1016.50 46.00 2.60 10.00 re f +0.1500 0.4760 0.4860 rg 1019.00 46.00 2.60 10.00 re f +0.1500 0.4760 0.4860 rg 1021.50 46.00 2.60 10.00 re f +0.1500 0.4760 0.4860 rg 1024.00 46.00 2.60 10.00 re f +0.1500 0.4760 0.4860 rg 1026.50 46.00 2.60 10.00 re f +0.1500 0.4760 0.4860 rg 1029.00 46.00 2.60 10.00 re f +0.1500 0.4760 0.4860 rg 1031.50 46.00 2.60 10.00 re f +BT /F1 7.00 Tf 0.2500 0.2600 0.2700 rg 1 0 0 1 834.00 34.00 Tm (rank 11) Tj ET +BT /F1 7.00 Tf 0.2500 0.2600 0.2700 rg 1 0 0 1 1013.84 34.00 Tm (rank 1) Tj ET +endstream +endobj +xref +0 7 +0000000000 65535 f +0000000015 00000 n +0000000064 00000 n +0000000121 00000 n +0000000258 00000 n +0000000328 00000 n +0000000403 00000 n +trailer +<< /Size 7 /Root 1 0 R >> +startxref +23255 +%%EOF diff --git a/paper_outputs/figures/matching.pdf b/paper_outputs/figures/matching.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ae95d5c35d95128e04b3ce3950ed8e6d626f2511 Binary files /dev/null and b/paper_outputs/figures/matching.pdf differ diff --git a/paper_outputs/tables/tab_app_analog_rank_depth.tex b/paper_outputs/tables/tab_app_analog_rank_depth.tex new file mode 100644 index 0000000000000000000000000000000000000000..1b389d1880e02acad242c83f63b5433441431950 --- /dev/null +++ b/paper_outputs/tables/tab_app_analog_rank_depth.tex @@ -0,0 +1,24 @@ +\begin{table*}[t] +\centering +\scriptsize +\setlength{\tabcolsep}{3pt} +\caption{For fixed retrieval \(\mathcal{T}\) and \(\Omega\), this table reports nDCG@5, best log gap, and rank \(\rho\) in addition to the main nDCG@10/log-error metrics. Cells report mean with small std.} +\label{tab:app_analog_rank_depth} +\begin{tabular}{lccc} +\toprule +Backbone & nDCG@5 & best log gap & rank $\rho$ \\ +\midrule +FireWx-FM ref. & \ms{0.5175}{0.0445} & \ms{0.1868}{0.0285} & \ms{0.6019}{0.1460} \\ +Prithvi-WxC & \ms{0.3591}{0.0107} & \ms{0.2151}{0.0594} & \ms{0.1514}{0.1489} \\ +Aurora & \ms{0.4423}{0.0210} & \ms{0.1551}{0.0437} & \ms{0.2162}{0.1856} \\ +ClimaX & \ms{0.4151}{0.0293} & \ms{0.2129}{0.0653} & \ms{0.1587}{0.2831} \\ +StormCast & \ms{0.3960}{0.0240} & \ms{0.1714}{0.0310} & \ms{0.1258}{0.1625} \\ +DLWP & \ms{0.3795}{0.0274} & \ms{0.1944}{0.0807} & \ms{-0.3865}{0.2802} \\ +FCN & \ms{0.4250}{0.0112} & \ms{0.1856}{0.0846} & \ms{-0.1357}{0.2571} \\ +FengWu & \ms{0.4228}{0.0310} & \ms{0.1870}{0.0858} & \ms{-0.1926}{0.2194} \\ +FuXi & \ms{0.4544}{0.0356} & \ms{0.2171}{0.0806} & \ms{-0.1367}{0.2885} \\ +Pangu-Weather & \ms{0.3988}{0.0506} & \ms{0.1901}{0.0838} & \ms{-0.1970}{0.2216} \\ +AlphaEarth & \ms{0.5276}{0.0531} & \ms{0.1782}{0.0454} & \ms{0.4639}{0.2802} \\ +\bottomrule +\end{tabular} +\end{table*} diff --git a/paper_outputs/tables/tab_app_burned_area_median_acre.tex b/paper_outputs/tables/tab_app_burned_area_median_acre.tex new file mode 100644 index 0000000000000000000000000000000000000000..0b87f9a2aaed34d3c000d97aa5415c0455be1d20 --- /dev/null +++ b/paper_outputs/tables/tab_app_burned_area_median_acre.tex @@ -0,0 +1,24 @@ +\begin{table*}[t] +\centering +\scriptsize +\setlength{\tabcolsep}{3pt} +\caption{For fixed final-area \(\mathcal{T}\) and \(\Omega\), this table reports median log error and acre-scale errors in addition to the main log-RMSE/log-MAE/Spearman metrics. Cells report mean with small std.} +\label{tab:app_burned_area_median_acre} +\begin{tabular}{lccc} +\toprule +Backbone & log median AE & acre median AE & acre MAPE \\ +\midrule +FireWx-FM ref. & \ms{1.0235}{0.0982} & \ms{4504.0692}{459.0483} & \ms{1.4525}{0.0254} \\ +Prithvi-WxC & \ms{1.2184}{0.2107} & \ms{5375.8770}{788.7906} & \ms{1.9517}{0.2875} \\ +Aurora & \ms{1.4547}{0.0301} & \ms{9904.9483}{457.4260} & \ms{6.8728}{3.0026} \\ +ClimaX & \ms{1.6841}{0.1818} & \ms{18130.4820}{3248.3873} & \ms{8.2373}{2.8540} \\ +StormCast & \ms{1.4522}{0.1519} & \ms{11155.7881}{2020.8656} & \ms{4.6142}{1.1500} \\ +DLWP & \ms{1.0952}{0.1306} & \ms{4406.9315}{303.0944} & \ms{1.7357}{0.3625} \\ +FCN & \ms{1.1688}{0.1139} & \ms{5166.9993}{213.0333} & \ms{2.0800}{0.4004} \\ +FengWu & \ms{1.1589}{0.1772} & \ms{5137.2822}{628.7543} & \ms{2.0944}{0.4545} \\ +FuXi & \ms{1.1855}{0.0612} & \ms{5697.7117}{796.8785} & \ms{2.4411}{0.5567} \\ +Pangu-Weather & \ms{1.1221}{0.1470} & \ms{5092.3621}{483.8243} & \ms{1.9571}{0.3113} \\ +AlphaEarth & \ms{1.7459}{0.6057} & \ms{15110.7573}{7106.3417} & \ms{9.7398}{2.7425} \\ +\bottomrule +\end{tabular} +\end{table*} diff --git a/paper_outputs/tables/tab_app_contract_params_full.tex b/paper_outputs/tables/tab_app_contract_params_full.tex new file mode 100644 index 0000000000000000000000000000000000000000..457c4f7ee7762ae290b434314769956702251a96 --- /dev/null +++ b/paper_outputs/tables/tab_app_contract_params_full.tex @@ -0,0 +1,22 @@ +\begin{table}[h] +\centering +\scriptsize +\setlength{\tabcolsep}{3.5pt} +\renewcommand{\arraystretch}{1.2} +\caption{Fixed scoring values used by each task-form contract.} +\label{tab:app_contract_params_full} +\begin{adjustbox}{max width=\textwidth} +\begin{tabular}{llll} +\toprule +\textbf{\(\mathcal{T}\)} & \textbf{Scoring} & \textbf{Validation} & \textbf{\(\Omega\)} \\ +\midrule +Occupancy & \(k=8,\Delta t=3\); exact/tol./union \(F_1\) & val. strict \(F_1\) & global; top-5/10/20\% fire-prone \\ +Fire spread & \(k=4,\Delta t=0\); exact/spatial \(F_1\), AP & val. spatial \(F_1\) & spread-region cells \\ +Final burned area & log-RMSE, log-MAE, Spearman \(\rho\) & val. log-RMSE & test events \\ +Analog retrieval & nDCG@10; retrieved-event log error & val. nDCG@10 & test events \\ +Smoke PM\(_{2.5}\) & RMSE, MAE, Pearson \(r\); exceedance 35 & val. RMSE & test stations \\ +Extreme heat & RMSE-C, MAE-C, exceedance \(F_1\) & val. threshold 27/30/33\(^{\circ}\)C & heat-region stations \\ +\bottomrule +\end{tabular} +\end{adjustbox} +\end{table} diff --git a/paper_outputs/tables/tab_app_head_architectures.tex b/paper_outputs/tables/tab_app_head_architectures.tex new file mode 100644 index 0000000000000000000000000000000000000000..9b990275e153260932258f22ebe1188e1e2bda9a --- /dev/null +++ b/paper_outputs/tables/tab_app_head_architectures.tex @@ -0,0 +1,36 @@ +\begin{table}[h] +\centering +\small +\setlength{\tabcolsep}{5pt} +\renewcommand{\arraystretch}{1.3} +\caption{Lightweight head architectures used in the fixed-contract transfer comparisons. +All heads are trained from random initialisation on the frozen backbone features. +Parameter counts are approximate and depend on the feature dimensionality of each backbone.} +\label{tab:app_head_architectures} +\begin{tabular}{p{0.15\textwidth}p{0.30\textwidth}p{0.12\textwidth}p{0.33\textwidth}} +\toprule +\textbf{$\mathcal{A}$ head} & \textbf{Architecture} & \textbf{Approx.\ params} & \textbf{Notes} \\ +\midrule +Constant prior & + Outputs a fixed bias vector, ignoring input features. & + Output dimension only & + Provides a degenerate baseline; selected when backbone features carry no useful signal. \\ +Linear probe & + Single linear layer mapping backbone features to output. No nonlinearity. & + $d\times c + c$ & + Standard frozen-representation baseline. \\ +Pixel MLP & + Two-layer MLP applied independently per spatial unit. & + $d\times h + h\times c$ & + Captures per-pixel nonlinearity; ignores spatial context. \\ +Shallow adapter & + Two-layer MLP with a spatial context window; uses $3\times3$ convolution before the linear output. & + $9dh + hc$ & + Balances local spatial context with parameter efficiency. \\ +Wide adapter & + Shallow adapter with wider hidden dimension. & + $9dH + Hc$ & + Higher capacity variant; can overfit on small fire-event sets. \\ +\bottomrule +\end{tabular} +\end{table} diff --git a/paper_outputs/tables/tab_app_heat_event_pr.tex b/paper_outputs/tables/tab_app_heat_event_pr.tex new file mode 100644 index 0000000000000000000000000000000000000000..2ba79fb92d15e7523682f3ef7f9de59591d1654c --- /dev/null +++ b/paper_outputs/tables/tab_app_heat_event_pr.tex @@ -0,0 +1,24 @@ +\begin{table*}[t] +\centering +\scriptsize +\setlength{\tabcolsep}{3pt} +\caption{For fixed heat \(\mathcal{T}\) and heat-region \(\Omega\), this table reports precision and recall for the exceedance label used by the main \(F_1\). Cells report mean with small std.} +\label{tab:app_heat_event_pr} +\begin{tabular}{lcc} +\toprule +Backbone & precision & recall \\ +\midrule +FireWx-FM ref. & \ms{0.9767}{0.0117} & \ms{0.9330}{0.0299} \\ +Prithvi-WxC & \ms{0.8260}{0.0030} & \ms{0.9173}{0.0033} \\ +Aurora & \ms{0.5920}{0.0347} & \ms{0.0517}{0.0020} \\ +ClimaX & \ms{0.7397}{0.0099} & \ms{0.7994}{0.0051} \\ +StormCast & \ms{0.8840}{0.0237} & \ms{0.9320}{0.0165} \\ +DLWP & \ms{0.9429}{0.0085} & \ms{0.8899}{0.0167} \\ +FCN & \ms{0.9408}{0.0097} & \ms{0.9111}{0.0127} \\ +FengWu & \ms{0.3808}{0.2719} & \ms{0.0266}{0.0267} \\ +FuXi & \ms{0.3262}{0.1262} & \ms{0.1810}{0.0481} \\ +Pangu-Weather & \ms{0.1159}{0.0743} & \ms{0.0112}{0.0032} \\ +AlphaEarth & \ms{0.9824}{0.0040} & \ms{0.9278}{0.0178} \\ +\bottomrule +\end{tabular} +\end{table*} diff --git a/paper_outputs/tables/tab_app_matching_rule_params.tex b/paper_outputs/tables/tab_app_matching_rule_params.tex new file mode 100644 index 0000000000000000000000000000000000000000..92383ae8e5ece2db3b4229318cc8fadffc9c29cf --- /dev/null +++ b/paper_outputs/tables/tab_app_matching_rule_params.tex @@ -0,0 +1,17 @@ +\begin{table}[h] +\centering +\small +\setlength{\tabcolsep}{10pt} +\renewcommand{\arraystretch}{1.2} +\caption{Matching-rule values used in the evaluation contracts.} +\label{tab:app_matching_rule_params} +\begin{tabular}{lll} +\toprule +\textbf{Parameter} & \textbf{Occupancy} & \textbf{Fire spread} \\ +\midrule +\(k\) & 8 cells & 4 cells \\ +\(\Delta t\) & 3 for union; 0 spatial-only & 0 \\ +\(\tau\) & val. strict \(F_1\) & val. spatial \(F_1\) \\ +\bottomrule +\end{tabular} +\end{table} diff --git a/paper_outputs/tables/tab_app_occupancy_ppr_scope.tex b/paper_outputs/tables/tab_app_occupancy_ppr_scope.tex new file mode 100644 index 0000000000000000000000000000000000000000..8de962689d8f7b91093fb9250bed986088166cf3 --- /dev/null +++ b/paper_outputs/tables/tab_app_occupancy_ppr_scope.tex @@ -0,0 +1,27 @@ +\begin{table*}[t] +\centering +\small +\setlength{\tabcolsep}{4pt} +\renewcommand{\arraystretch}{1.18} +\caption{For fixed occupancy \(\mathcal{T}\), this table reports predicted-positive rate. +Values are percentages under the same validation-selected strict threshold. +Scopes \(\Omega\) are fixed before test scoring; cells report five-seed mean with std in small type.} +\label{tab:app_occupancy_ppr_scope} +\begin{tabular}{lcccc} +\toprule +\textbf{Backbone} & \textbf{\(\Omega=\)global} & \textbf{\(\Omega=\)top 5\%} & \textbf{\(\Omega=\)top 10\%} & \textbf{\(\Omega=\)top 20\%} \\ +\midrule +FireWx-FM ref. & \ms{1.6808}{0.3684} & \ms{3.0619}{1.0925} & \ms{1.5310}{0.5463} & \ms{0.7655}{0.2732} \\ +Prithvi-WxC & \ms{61.9711}{30.9101} & \ms{57.4117}{47.8987} & \ms{58.4565}{51.0897} & \ms{58.9788}{52.6991} \\ +Aurora & \ms{55.5849}{19.7524} & \ms{57.2238}{35.3400} & \ms{68.7942}{37.6958} & \ms{67.2891}{38.3991} \\ +ClimaX & \ms{5.6763}{3.9261} & \ms{24.0091}{9.2816} & \ms{11.8450}{4.5067} & \ms{5.7442}{4.1341} \\ +StormCast & \ms{60.6507}{17.4895} & \ms{57.6017}{35.2921} & \ms{68.0766}{37.3899} & \ms{67.8397}{39.2410} \\ +DLWP & \ms{4.3221}{1.5619} & \ms{9.4001}{5.0807} & \ms{4.9700}{3.6849} & \ms{1.9198}{1.4678} \\ +FCN & \ms{1.5202}{1.3446} & \ms{4.7856}{2.9409} & \ms{2.7257}{1.6353} & \ms{0.8368}{0.2358} \\ +FengWu & \ms{0.4277}{0.4830} & \ms{0.6004}{0.3041} & \ms{0.2609}{0.1935} & \ms{0.1501}{0.1206} \\ +FuXi & \ms{0.4505}{0.2773} & \ms{2.9315}{2.6392} & \ms{0.5197}{0.6074} & \ms{0.3621}{0.4346} \\ +Pangu-Weather & \ms{1.0801}{1.1308} & \ms{2.0549}{2.1893} & \ms{1.4029}{1.4739} & \ms{1.0103}{1.1084} \\ +AlphaEarth & \ms{0.0691}{0.0499} & \ms{0.2826}{0.1497} & \ms{0.1524}{0.0770} & \ms{0.0656}{0.0414} \\ +\bottomrule +\end{tabular} +\end{table*} diff --git a/paper_outputs/tables/tab_app_scope_params.tex b/paper_outputs/tables/tab_app_scope_params.tex new file mode 100644 index 0000000000000000000000000000000000000000..d0aa091b60e643a61a7ab074096d4db3b6cc9482 --- /dev/null +++ b/paper_outputs/tables/tab_app_scope_params.tex @@ -0,0 +1,19 @@ +\begin{table}[h] +\centering +\small +\setlength{\tabcolsep}{8pt} +\renewcommand{\arraystretch}{1.2} +\caption{Scope values used in the evaluation contracts.} +\label{tab:app_scope_params} +\begin{tabular}{lcc} +\toprule +\textbf{\(\Omega\)} & \textbf{Definition} & \textbf{Units} \\ +\midrule +Global & full domain & 8,085,000 test cells \\ +Fire-prone top-5\% & top 5\% by training-period fire frequency & 404,280 test cells \\ +Fire-prone top-10\% & top 10\% by training-period fire frequency & 808,560 test cells \\ +Fire-prone top-20\% & top 20\% by training-period fire frequency & 1,617,000 test cells \\ +Spread region & union of \(\widehat{B}\) and \(B\) & event-specific cells \\ +\bottomrule +\end{tabular} +\end{table} diff --git a/paper_outputs/tables/tab_app_seed_robustness.tex b/paper_outputs/tables/tab_app_seed_robustness.tex new file mode 100644 index 0000000000000000000000000000000000000000..0cf1a2f914fb989e71600e6eeab661e138b00969 --- /dev/null +++ b/paper_outputs/tables/tab_app_seed_robustness.tex @@ -0,0 +1,36 @@ +\begin{table}[h] +\centering +\small +\setlength{\tabcolsep}{5pt} +\renewcommand{\arraystretch}{1.2} +\caption{Seed summaries for stochastic checks. Values report mean with small std over completed seeds.} +\label{tab:app_seed_robustness} +\begin{adjustbox}{max width=\textwidth} +\begin{tabular}{p{0.28\textwidth}cllp{0.18\textwidth}} +\toprule +\textbf{\(\mathcal{T}\) check} & \textbf{Seeds} & \textbf{Primary value} & \textbf{Other value(s)} & \textbf{Reading} \\ +\midrule +Final burned area & +5 & log-RMSE \ms{1.1657}{0.0126} & +log-MAE \ms{1.0423}{0.0081}; Spear.\ \ms{0.6298}{0.0338} & +stable across seeds \\ +Smoke PM\(_{2.5}\) & +5 & RMSE \ms{4.4646}{0.0060} & +MAE \ms{2.4108}{0.0016}; \(r\) \ms{0.6368}{0.0013} & +stable at table precision \\ +Extreme heat & +5 & RMSE-C \ms{0.2179}{0.0043} & +MAE-C \ms{0.1787}{0.0018}; exceed.\ \(F_1\) \ms{0.9541}{0.0164} & +stable across seeds \\ +Fire spread & +5 & exact \(F_1\) \ms{37.6700}{0.9800} & +spatial \(F_1\) \ms{80.9700}{2.0200}; AP \ms{30.0900}{1.2500} & +stable across seeds \\ +Aurora paired-head check & +5 & fire-prone score diff.\ \ms{6.3500}{13.2800} & +PR-AUC and union choices differ in 2/5 seeds & +variable across seeds \\ +\bottomrule +\end{tabular} +\end{adjustbox} +\end{table} diff --git a/paper_outputs/tables/tab_app_smoke_high_event.tex b/paper_outputs/tables/tab_app_smoke_high_event.tex new file mode 100644 index 0000000000000000000000000000000000000000..26fb78693bfa64e1603c07c7c57c3fc2851d208e --- /dev/null +++ b/paper_outputs/tables/tab_app_smoke_high_event.tex @@ -0,0 +1,24 @@ +\begin{table*}[t] +\centering +\scriptsize +\setlength{\tabcolsep}{3pt} +\caption{For fixed smoke \(\mathcal{T}\) and station \(\Omega\), this table reports RMSE, MAE, and 90th-percentile absolute error on test rows with observed PM$_{2.5}\ge35$; std uses a row bootstrap over those rows. Cells report mean with small std.} +\label{tab:app_smoke_high_event} +\begin{tabular}{lccc} +\toprule +Backbone & high-smoke RMSE & high-smoke MAE & high-smoke 90th AE \\ +\midrule +FireWx-FM ref. & \ms{47.4870}{0.6346} & \ms{34.3954}{0.7654} & \ms{65.6213}{3.8778} \\ +Prithvi-WxC & \ms{57.2224}{1.7268} & \ms{47.3871}{0.3153} & \ms{74.9666}{3.2381} \\ +Aurora & \ms{57.2752}{1.7248} & \ms{47.4368}{0.3149} & \ms{75.0755}{3.1074} \\ +ClimaX & \ms{57.2828}{1.7239} & \ms{47.4407}{0.3140} & \ms{75.1012}{3.0777} \\ +StormCast & \ms{56.6512}{1.7517} & \ms{46.7914}{0.3281} & \ms{74.0794}{3.4707} \\ +DLWP & \ms{57.0075}{1.7359} & \ms{47.1971}{0.3198} & \ms{74.4936}{3.3826} \\ +FCN & \ms{57.0582}{1.7339} & \ms{47.2401}{0.3187} & \ms{74.6431}{3.1982} \\ +FengWu & \ms{57.0158}{1.7357} & \ms{47.1957}{0.3194} & \ms{74.5652}{3.2871} \\ +FuXi & \ms{56.9622}{1.7371} & \ms{47.1508}{0.3201} & \ms{74.3278}{3.4435} \\ +Pangu-Weather & \ms{57.1282}{1.7307} & \ms{47.3050}{0.3170} & \ms{74.6830}{3.2375} \\ +AlphaEarth & \ms{48.0665}{0.7904} & \ms{35.6088}{0.7341} & \ms{66.7613}{3.9235} \\ +\bottomrule +\end{tabular} +\end{table*} diff --git a/paper_outputs/tables/tab_app_spread_ap_by_scope.tex b/paper_outputs/tables/tab_app_spread_ap_by_scope.tex new file mode 100644 index 0000000000000000000000000000000000000000..4e25fdfec6c12e671d0c5065bc40185243d8aaef --- /dev/null +++ b/paper_outputs/tables/tab_app_spread_ap_by_scope.tex @@ -0,0 +1,24 @@ +\begin{table*}[t] +\centering +\scriptsize +\setlength{\tabcolsep}{3pt} +\caption{For fixed spread \(\mathcal{T}\) and strict \(\Lambda\), this table reports AP under three \(\Omega\) scopes: full test, top-5\% train-fire area, and top-10\% train-fire area. Values are percentages; cells report mean with small std.} +\label{tab:app_spread_ap_by_scope} +\begin{tabular}{lccc} +\toprule +Backbone & full \(\Omega\) AP & top-5\% \(\Omega\) AP & top-10\% \(\Omega\) AP \\ +\midrule +FireWx-FM ref. & \ms{30.0197}{1.5651} & \ms{40.7452}{2.0542} & \ms{37.4096}{1.8731} \\ +Prithvi-WxC & \ms{4.8319}{0.1731} & \ms{12.6086}{0.4468} & \ms{8.7051}{0.1889} \\ +Aurora & \ms{17.7723}{0.4293} & \ms{30.3106}{0.9404} & \ms{26.4732}{0.6932} \\ +ClimaX & \ms{11.1726}{0.2337} & \ms{25.7871}{1.2896} & \ms{19.9977}{1.2217} \\ +StormCast & \ms{8.1147}{1.1569} & \ms{18.5461}{1.1727} & \ms{14.1286}{1.2956} \\ +DLWP & \ms{9.2142}{2.6587} & \ms{19.3346}{2.3922} & \ms{14.9788}{2.6696} \\ +FCN & \ms{6.6774}{1.3001} & \ms{16.7396}{3.2955} & \ms{11.9308}{2.3881} \\ +FengWu & \ms{11.0046}{2.7092} & \ms{21.1506}{1.2163} & \ms{17.0113}{1.5778} \\ +FuXi & \ms{13.5507}{0.3840} & \ms{22.5434}{0.4100} & \ms{19.1964}{0.3943} \\ +Pangu-Weather & \ms{10.6250}{1.4643} & \ms{19.8294}{1.3044} & \ms{15.8013}{1.1602} \\ +AlphaEarth & \ms{12.2847}{1.3562} & \ms{22.8692}{0.4915} & \ms{18.2992}{1.2110} \\ +\bottomrule +\end{tabular} +\end{table*} diff --git a/paper_outputs/tables/tab_appendix_selection_regret_tolerance.tex b/paper_outputs/tables/tab_appendix_selection_regret_tolerance.tex new file mode 100644 index 0000000000000000000000000000000000000000..fba91a2110efd4f106cc89644798e25f84a76b3a --- /dev/null +++ b/paper_outputs/tables/tab_appendix_selection_regret_tolerance.tex @@ -0,0 +1,37 @@ +\begin{table*}[!t] + \centering + \scriptsize + \setlength{\tabcolsep}{4pt} + \caption{Selection-regret values under exact, tolerated, and union matching. Values are percentage-point regret from selecting \(h_R\) by PR-AUC instead of \(h_D\) by the decision metric. Rows report mean with small std over five seeds; \(0.0000\) denotes exact zero regret.} + \label{tab:appendix_selection_regret_tolerance} + \begin{adjustbox}{max width=\textwidth} + \begin{tabular}{llccc} + \toprule + \textbf{Feature} & \textbf{\(\Omega\)} & \textbf{Exact regret} & \textbf{Tolerated regret} & \textbf{Union regret} \\ + \midrule + FireWx-FM ref. & global & 0.0000 & \ms{8.7830}{9.6705} & \ms{8.7830}{9.6705} \\ + FireWx-FM ref. & fire-prone & 0.0000 & \ms{3.4027}{3.2045} & \ms{3.4027}{3.2045} \\ + Prithvi-WxC & global & 0.0000 & 0.0000 & 0.0000 \\ + Prithvi-WxC & fire-prone & 0.0000 & 0.0000 & 0.0000 \\ + Aurora & global & \ms{0.0200}{0.0267} & \ms{9.8520}{12.9878} & \ms{9.8520}{12.9878} \\ + Aurora & fire-prone & \ms{0.8203}{1.8341} & \ms{14.3919}{32.1219} & \ms{14.3919}{32.1219} \\ + ClimaX & global & \ms{0.0003}{0.0004} & \ms{0.1296}{0.1775} & \ms{0.1296}{0.1775} \\ + ClimaX & fire-prone & 0.0000 & 0.0000 & 0.0000 \\ + StormCast & global & 0.0000 & 0.0000 & 0.0000 \\ + StormCast & fire-prone & 0.0000 & 0.0000 & 0.0000 \\ + DLWP & global & 0.0000 & 0.0000 & 0.0000 \\ + DLWP & fire-prone & \ms{0.0770}{0.1100} & \ms{4.3266}{4.3323} & \ms{4.3266}{4.3323} \\ + FCN & global & 0.0000 & 0.0000 & 0.0000 \\ + FCN & fire-prone & \ms{0.0006}{0.0013} & \ms{1.1680}{1.9872} & \ms{1.1680}{1.9872} \\ + FengWu & global & 0.0000 & 0.0000 & 0.0000 \\ + FengWu & fire-prone & \ms{0.0691}{0.1191} & \ms{0.5222}{0.6239} & \ms{0.5222}{0.6239} \\ + FuXi & global & 0.0000 & 0.0000 & 0.0000 \\ + FuXi & fire-prone & 0.0000 & \ms{0.1084}{0.1729} & \ms{0.1084}{0.1729} \\ + Pangu-Weather & global & 0.0000 & 0.0000 & 0.0000 \\ + Pangu-Weather & fire-prone & \ms{0.0728}{0.1179} & \ms{0.1849}{0.3263} & \ms{0.1849}{0.3263} \\ + AlphaEarth & global & 0.0000 & \ms{17.2217}{8.8492} & \ms{17.2217}{8.8492} \\ + AlphaEarth & fire-prone & 0.0000 & \ms{3.8804}{5.9483} & \ms{3.8804}{5.9483} \\ + \bottomrule + \end{tabular} + \end{adjustbox} +\end{table*} diff --git a/paper_outputs/tables/tab_fireprone_contract_progression.tex b/paper_outputs/tables/tab_fireprone_contract_progression.tex new file mode 100644 index 0000000000000000000000000000000000000000..21e14d68fc8067d2995b15d2a77df55d7ba0158d --- /dev/null +++ b/paper_outputs/tables/tab_fireprone_contract_progression.tex @@ -0,0 +1,67 @@ +\begin{table*}[t] + \centering + \scriptsize + \setlength{\tabcolsep}{4pt} + \caption{Occupancy \(F_1\) scores across global and fire-prone scopes. Global uses the full validation/test domain; top-\(k\) rows use train-defined fire-prone masks from historical fire frequency. Values are percentages from the same validation-selected strict threshold. Tolerance is spatial-only; union adds temporal and spatial matching. \(\Delta\) is union minus strict. Cells report five-seed mean with std in small type.} + \label{tab:fireprone_contract_progression} + \begin{tabular}{@{}llcccc@{}} + \toprule + Backbone & \(\Omega\) & Strict \(F_1\uparrow\) & Tol.\ \(F_1\uparrow\) & Union \(F_1\uparrow\) & \(\Delta\) \(\uparrow\) \\ + \midrule + FireWx-FM ref. & global & \ms{0.4546}{0.1412} & \ms{29.7484}{1.2868} & \ms{59.0656}{2.7372} & \ms{58.6109}{2.6945} \\ + & top 5\% & \ms{3.5604}{0.8809} & \ms{39.2617}{1.4011} & \ms{72.8280}{2.5784} & \ms{69.2676}{1.9960} \\ + & top 10\% & \ms{3.5575}{0.8799} & \ms{39.1665}{1.3906} & \ms{72.5204}{2.5670} & \ms{68.9629}{1.9888} \\ + & top 20\% & \ms{3.5300}{0.8700} & \ms{38.2849}{1.2952} & \ms{69.7228}{2.4664} & \ms{66.1928}{1.9273} \\ + \addlinespace[1pt] + Prithvi-WxC & global & \ms{0.0552}{0.0039} & \ms{7.1649}{0.6557} & \ms{20.1853}{1.8299} & \ms{20.1301}{1.8297} \\ + & top 5\% & \ms{1.4119}{1.1635} & \ms{19.2636}{4.5019} & \ms{42.5793}{4.5495} & \ms{41.1674}{3.4846} \\ + & top 10\% & \ms{1.2376}{1.3201} & \ms{14.8780}{8.4429} & \ms{32.6913}{13.2085} & \ms{31.4536}{11.9053} \\ + & top 20\% & \ms{1.1520}{1.3770} & \ms{13.1512}{9.4556} & \ms{28.1319}{15.2866} & \ms{26.9800}{13.9224} \\ + \addlinespace[1pt] + Aurora & global & \ms{0.0656}{0.0094} & \ms{8.5009}{1.9594} & \ms{23.1037}{4.9418} & \ms{23.0382}{4.9325} \\ + & top 5\% & \ms{0.9859}{0.9299} & \ms{15.1337}{6.0821} & \ms{35.4834}{11.0192} & \ms{34.4975}{10.3728} \\ + & top 10\% & \ms{0.7790}{1.0453} & \ms{12.7381}{6.5558} & \ms{30.5305}{10.8842} & \ms{29.7515}{9.8656} \\ + & top 20\% & \ms{0.6655}{1.1043} & \ms{10.5304}{7.4309} & \ms{24.9444}{12.5844} & \ms{24.2790}{11.4943} \\ + \addlinespace[1pt] + ClimaX & global & \ms{0.3480}{0.0754} & \ms{29.7535}{3.6073} & \ms{60.1506}{7.5865} & \ms{59.8026}{7.5454} \\ + & top 5\% & \ms{1.2937}{0.1086} & \ms{34.5791}{2.3772} & \ms{69.2186}{5.7215} & \ms{67.9249}{5.7263} \\ + & top 10\% & \ms{1.2522}{0.1602} & \ms{34.3341}{2.2852} & \ms{68.5713}{5.5377} & \ms{67.3191}{5.5538} \\ + & top 20\% & \ms{1.0287}{0.2686} & \ms{30.2140}{4.2857} & \ms{60.0650}{7.5674} & \ms{59.0363}{7.5891} \\ + \addlinespace[1pt] + StormCast & global & \ms{0.0626}{0.0119} & \ms{8.1951}{2.1895} & \ms{22.3817}{5.4294} & \ms{22.3191}{5.4178} \\ + & top 5\% & \ms{0.9573}{0.8011} & \ms{15.3219}{5.5337} & \ms{36.1857}{9.7331} & \ms{35.2284}{9.1816} \\ + & top 10\% & \ms{0.7284}{0.9280} & \ms{12.6669}{6.3290} & \ms{30.4748}{10.6527} & \ms{29.7464}{9.7494} \\ + & top 20\% & \ms{0.5795}{0.9104} & \ms{10.4157}{7.3437} & \ms{24.6598}{12.3973} & \ms{24.0803}{11.4988} \\ + \addlinespace[1pt] + DLWP & global & \ms{0.1693}{0.0419} & \ms{14.9148}{3.2446} & \ms{28.1901}{6.9658} & \ms{28.0208}{6.9257} \\ + & top 5\% & \ms{1.8054}{0.4835} & \ms{31.7231}{3.2923} & \ms{55.4596}{5.2920} & \ms{53.6542}{5.4752} \\ + & top 10\% & \ms{1.6110}{0.5999} & \ms{27.6581}{5.9216} & \ms{47.1269}{8.0111} & \ms{45.5158}{7.7927} \\ + & top 20\% & \ms{1.5248}{0.8987} & \ms{20.9403}{4.7971} & \ms{34.9301}{7.8471} & \ms{33.4054}{7.8760} \\ + \addlinespace[1pt] + FCN & global & \ms{0.2829}{0.0839} & \ms{19.5061}{3.3412} & \ms{40.0604}{9.3701} & \ms{39.7775}{9.3423} \\ + & top 5\% & \ms{1.6231}{0.5064} & \ms{29.3769}{2.7626} & \ms{54.3033}{7.4089} & \ms{52.6801}{7.4389} \\ + & top 10\% & \ms{1.1777}{0.5118} & \ms{22.4217}{3.9803} & \ms{43.4510}{9.2513} & \ms{42.2734}{9.0251} \\ + & top 20\% & \ms{0.9962}{0.4315} & \ms{16.9792}{3.9371} & \ms{34.0859}{8.2616} & \ms{33.0897}{7.9275} \\ + \addlinespace[1pt] + FengWu & global & \ms{0.2613}{0.0757} & \ms{12.0050}{6.0239} & \ms{24.1022}{13.6293} & \ms{23.8410}{13.5736} \\ + & top 5\% & \ms{1.5695}{0.3592} & \ms{16.2763}{3.7024} & \ms{30.1055}{5.0103} & \ms{28.5360}{4.7696} \\ + & top 10\% & \ms{1.2427}{0.5333} & \ms{12.9503}{5.6052} & \ms{24.1854}{8.6854} & \ms{22.9427}{8.1863} \\ + & top 20\% & \ms{1.1192}{0.5023} & \ms{11.9508}{5.0745} & \ms{22.7860}{7.9115} & \ms{21.6668}{7.4438} \\ + \addlinespace[1pt] + FuXi & global & \ms{0.3774}{0.1212} & \ms{21.0323}{4.8211} & \ms{37.2888}{9.4470} & \ms{36.9114}{9.4327} \\ + & top 5\% & \ms{2.0307}{0.6800} & \ms{31.8944}{4.7331} & \ms{53.9308}{8.3822} & \ms{51.9001}{8.6878} \\ + & top 10\% & \ms{1.6542}{0.7316} & \ms{24.0128}{5.7784} & \ms{40.2140}{9.9307} & \ms{38.5597}{9.7744} \\ + & top 20\% & \ms{1.3646}{0.6773} & \ms{21.9548}{5.8601} & \ms{36.7314}{10.0289} & \ms{35.3668}{9.9223} \\ + \addlinespace[1pt] + Pangu-Weather & global & \ms{0.2755}{0.1089} & \ms{17.0909}{4.0477} & \ms{35.6386}{9.0327} & \ms{35.3630}{9.0774} \\ + & top 5\% & \ms{1.3656}{0.3064} & \ms{22.2222}{6.8613} & \ms{43.4234}{13.2383} & \ms{42.0578}{13.0599} \\ + & top 10\% & \ms{1.0931}{0.3535} & \ms{18.9337}{5.9329} & \ms{38.5325}{11.7221} & \ms{37.4394}{11.5261} \\ + & top 20\% & \ms{0.8844}{0.3601} & \ms{17.0172}{5.4859} & \ms{34.5688}{10.2932} & \ms{33.6844}{10.1334} \\ + \addlinespace[1pt] + AlphaEarth & global & \ms{2.0606}{0.4404} & \ms{29.4476}{6.0064} & \ms{37.4286}{9.9458} & \ms{35.3679}{10.0271} \\ + & top 5\% & \ms{6.9133}{0.8450} & \ms{42.8790}{4.6087} & \ms{51.7449}{8.7321} & \ms{44.8315}{9.0763} \\ + & top 10\% & \ms{6.6366}{0.9901} & \ms{41.8981}{5.9454} & \ms{50.5712}{10.0057} & \ms{43.9346}{9.9156} \\ + & top 20\% & \ms{6.1908}{1.1330} & \ms{38.8325}{7.4966} & \ms{46.3833}{12.1697} & \ms{40.1925}{11.6788} \\ + \bottomrule + \end{tabular} +\end{table*} diff --git a/paper_outputs/tables/tab_primary_results.tex b/paper_outputs/tables/tab_primary_results.tex new file mode 100644 index 0000000000000000000000000000000000000000..d6a35654eb966eaefad30f556bba7217b645dfec --- /dev/null +++ b/paper_outputs/tables/tab_primary_results.tex @@ -0,0 +1,62 @@ +\begin{table}[t] +\centering +\small +\setlength{\tabcolsep}{4pt} +\renewcommand{\arraystretch}{1.20} +\caption{% + \textbf{Primary fixed-contract transfer results (RQ3).} + Occupancy metrics: exact, tolerated, and union $F_1$ (\%). + Fire spread metrics: exact $F_1$, spatial $F_1$, and AP (\%). + Each block fixes $\mathcal{T}$, $\Lambda$, $\Omega$, $\mathcal{A}$. + \textbf{Bold} marks the best frozen backbone per metric. +} +\label{tab:primary_results} +\setlength{\arrayrulewidth}{0.4pt} +\resizebox{\textwidth}{!}{% +\begin{tabular}{lcccccc} +\toprule +& \multicolumn{3}{c}{\textbf{Occupancy}} +& \multicolumn{3}{c}{\textbf{Fire spread}} \\ +\cmidrule(lr){2-4}\cmidrule(lr){5-7} +\textbf{Comparator} +& \textbf{Exact $F_1\uparrow$} & \textbf{Tol.\ $F_1\uparrow$} & \textbf{Union $F_1\uparrow$} +& \textbf{Exact $F_1\uparrow$} & \textbf{Spatial $F_1\uparrow$} & \textbf{AP$\uparrow$} \\ +\midrule +\textcolor{blue}{FireWx-FM ref.} +& \ms{0.4546}{0.1412} & \ms{29.7484}{1.2868} & \ms{59.0656}{2.7372} +& \ms{37.6700}{0.9800} & \ms{80.9700}{2.0200} & \ms{30.0900}{1.2500} \\ +\midrule +Prithvi-WxC +& \ms{0.0552}{0.0039} & \ms{7.1649}{0.6557} & \ms{20.1853}{1.8299} +& \ms{22.3500}{3.4500} & \ms{65.2600}{1.0700} & \ms{5.0000}{0.3000} \\ +Aurora +& \ms{0.0656}{0.0094} & \ms{8.5009}{1.9594} & \ms{23.1037}{4.9418} +& \textbf{\ms{30.8757}{0.1343}} & \textbf{\ms{71.7329}{0.0141}} & \textbf{\ms{16.6221}{1.6965}} \\ +ClimaX +& \ms{0.3480}{0.0754} & \textbf{\ms{29.7535}{3.6073}} & \textbf{\ms{60.1506}{7.5865}} +& \ms{27.9853}{2.0532} & \ms{69.0634}{2.3832} & \ms{11.1726}{0.2337} \\ +StormCast +& \ms{0.0626}{0.0119} & \ms{8.1951}{2.1895} & \ms{22.3817}{5.4294} +& \ms{14.8387}{7.5791} & \ms{55.7568}{21.3003} & \ms{2.8114}{0.7377} \\ + DLWP +& \ms{0.1693}{0.0419} & \ms{14.9148}{3.2446} & \ms{28.1901}{6.9658} +& \ms{5.9335}{10.0712} & \ms{22.8587}{22.3750} & \ms{5.9435}{5.5194} \\ +FCN +& \ms{0.2829}{0.0839} & \ms{19.5061}{3.3412} & \ms{40.0604}{9.3701} +& \ms{3.1798}{2.6598} & \ms{15.6203}{12.4531} & \ms{2.3861}{1.2614} \\ +FengWu +& \ms{0.2613}{0.0757} & \ms{12.0050}{6.0239} & \ms{24.1022}{13.6293} +& \ms{5.5189}{9.0883} & \ms{18.4774}{22.4703} & \ms{13.1658}{1.3408} \\ +FuXi +& \ms{0.3774}{0.1212} & \ms{21.0323}{4.8211} & \ms{37.2888}{9.4470} +& \ms{19.9909}{2.1364} & \ms{56.1826}{3.0412} & \ms{14.3526}{0.3554} \\ +Pangu-Weather +& \ms{0.2755}{0.1089} & \ms{17.0909}{4.0477} & \ms{35.6386}{9.0327} +& \ms{11.2583}{11.0719} & \ms{32.5081}{25.4969} & \ms{12.6881}{1.6790} \\ +AlphaEarth +& \textbf{\ms{2.0606}{0.4404}} & \ms{29.4476}{6.0064} & \ms{37.4286}{9.9458} +& \ms{11.0995}{3.6088} & \ms{32.8316}{7.4634} & \ms{11.8343}{1.5050} \\ +\bottomrule +\end{tabular} +} +\end{table} diff --git a/paper_outputs/tables/tab_supporting_results.tex b/paper_outputs/tables/tab_supporting_results.tex new file mode 100644 index 0000000000000000000000000000000000000000..754b3ea97a0a5735e78352c7c74380b51bd18011 --- /dev/null +++ b/paper_outputs/tables/tab_supporting_results.tex @@ -0,0 +1,120 @@ +\begin{table}[t] +\centering +\small +\setlength{\tabcolsep}{3.5pt} +\renewcommand{\arraystretch}{1.18} +\caption{% + \textbf{Supporting task-metric matrix (RQ4).} + Top block: final burned area and analog retrieval. + Bottom block: smoke PM$_{2.5}$ and extreme heat. + Each block fixes $\mathcal{T}$, $\Lambda$, $\Omega$; backbone + column is shared across paired tasks. \textcolor{blue}{FireWx-FM reference row is} + separated by a rule as the empirical anchor. \textbf{Bold} marks + the largest atmospheric-FM heat error values. For error metrics + lower is better ($\downarrow$); for $F_1$, nDCG, and $r$ higher + is better ($\uparrow$). +} +\label{tab:supporting_results} +\resizebox{\textwidth}{!}{% +\begin{tabular}{lcccccc} +\toprule +& \multicolumn{3}{c}{\textbf{Burned area}} +& \multicolumn{3}{c}{\textbf{Analog retrieval}} \\ +\cmidrule(lr){2-4}\cmidrule(lr){5-7} +\textbf{Backbone} +& \textbf{log-RMSE$\downarrow$} & \textbf{log-MAE$\downarrow$} +& \textbf{Spearman$\uparrow$} +& \textbf{nDCG@10$\uparrow$} & \textbf{log-RMSE$\downarrow$} +& \textbf{log-MAE$\downarrow$} \\ +\midrule +\textcolor{blue}{FireWx-FM ref.} +& \ms{1.1657}{0.0126} & \ms{1.0423}{0.0081} & \ms{0.6298}{0.0338} +& \ms{0.5099}{0.0336} & \ms{1.1977}{0.1029} & \ms{1.0043}{0.0759} \\ +\midrule +Prithvi-WxC +& \ms{1.3630}{0.0681} & \ms{1.2435}{0.0668} & \ms{0.1799}{0.3002} +& \ms{0.3857}{0.0189} & \ms{1.3908}{0.0938} & \ms{1.2585}{0.0865} \\ +Aurora +& \ms{1.8658}{0.2009} & \ms{1.6717}{0.1245} & \ms{-0.1156}{0.2982} +& \ms{0.4046}{0.0144} & \ms{1.3659}{0.0792} & \ms{1.2596}{0.0968} \\ +ClimaX +& \ms{2.0300}{0.2103} & \ms{1.8443}{0.1528} & \ms{-0.2515}{0.2688} +& \ms{0.4143}{0.0191} & \ms{1.4526}{0.0926} & \ms{1.2441}{0.1446} \\ +StormCast +& \ms{1.6679}{0.1438} & \ms{1.4745}{0.1134} & \ms{0.1830}{0.1969} +& \ms{0.4076}{0.0094} & \ms{1.3663}{0.0781} & \ms{1.2371}{0.1078} \\ +DLWP +& \ms{1.3070}{0.0980} & \ms{1.1769}{0.0834} & \ms{0.4888}{0.1368} +& \ms{0.3972}{0.0146} & \ms{1.5351}{0.0802} & \ms{1.3196}{0.0781} \\ +FCN +& \ms{1.3693}{0.0885} & \ms{1.2599}{0.0723} & \ms{0.3484}{0.1662} +& \ms{0.4316}{0.0134} & \ms{1.4604}{0.1035} & \ms{1.2351}{0.0586} \\ +FengWu +& \ms{1.3715}{0.1011} & \ms{1.2604}{0.0820} & \ms{0.3221}{0.2004} +& \ms{0.4246}{0.0237} & \ms{1.4179}{0.0986} & \ms{1.2233}{0.0915} \\ +FuXi +& \ms{1.4068}{0.1011} & \ms{1.3023}{0.0789} & \ms{0.2663}{0.2561} +& \ms{0.4279}{0.0212} & \ms{1.4290}{0.0929} & \ms{1.2236}{0.0961} \\ +Pangu-Weather +& \ms{1.3280}{0.0735} & \ms{1.2081}{0.0607} & \ms{0.4141}{0.1573} +& \ms{0.4017}{0.0245} & \ms{1.4235}{0.0731} & \ms{1.2225}{0.0847} \\ +AlphaEarth +& \ms{2.4068}{0.2841} & \ms{2.0822}{0.2371} & \ms{-0.3428}{0.1716} +& \ms{0.5086}{0.0440} & \ms{1.2158}{0.1310} & \ms{1.0350}{0.1018} \\ +\bottomrule +\end{tabular} +} + +\vspace{4pt} + +\resizebox{\textwidth}{!}{% +\begin{tabular}{lcccccc} +\toprule +& \multicolumn{3}{c}{\textbf{Smoke PM$_{2.5}$}} +& \multicolumn{3}{c}{\textbf{Extreme heat}} \\ +\cmidrule(lr){2-4}\cmidrule(lr){5-7} +\textbf{Backbone} +& \textbf{RMSE$\downarrow$} & \textbf{MAE$\downarrow$} +& \textbf{Pearson $r\uparrow$} +& \textbf{RMSE-C$\downarrow$} & \textbf{MAE-C$\downarrow$} +& \textbf{Exceed.\ $F_1\uparrow$} \\ +\midrule +\textcolor{blue}{FireWx-FM ref.} +& \ms{4.4646}{0.0060} & \ms{2.4108}{0.0016} & \ms{0.6368}{0.0013} +& \ms{0.2179}{0.0043} & \ms{0.1787}{0.0018} & \ms{0.9541}{0.0164} \\ +\midrule +Prithvi-WxC +& \ms{6.0382}{0.0828} & \ms{3.7301}{0.0055} & \ms{0.0243}{0.0045} +& \ms{4.6225}{0.0192} & \ms{2.6315}{0.0128} & \ms{0.8693}{0.0023} \\ +Aurora +& \ms{6.0384}{0.0828} & \ms{3.7265}{0.0055} & \ms{0.0193}{0.0043} +& \textbf{\ms{18.0474}{0.0708}} & \textbf{\ms{15.3747}{0.0594}} +& \ms{0.0951}{0.0038} \\ +ClimaX +& \ms{6.0402}{0.0828} & \ms{3.7290}{0.0055} & \ms{0.0004}{0.0029} +& \ms{17.6492}{0.0347} & \ms{14.4938}{0.0319} & \ms{0.7684}{0.0068} \\ +StormCast +& \ms{6.1230}{0.0830} & \ms{3.8182}{0.0073} & \ms{0.0183}{0.0041} +& \ms{1.7671}{0.2145} & \ms{1.3507}{0.1576} & \ms{0.9073}{0.0189} \\ +DLWP +& \ms{5.9289}{0.1031} & \ms{3.7331}{0.0088} & \ms{0.0303}{0.0060} +& \ms{2.2662}{0.1106} & \ms{1.7153}{0.0748} & \ms{0.9156}{0.0112} \\ +FCN +& \ms{5.9277}{0.1033} & \ms{3.7345}{0.0088} & \ms{0.0312}{0.0062} +& \ms{2.1657}{0.1800} & \ms{1.6033}{0.1039} & \ms{0.9257}{0.0096} \\ +FengWu +& \ms{5.9297}{0.1032} & \ms{3.7395}{0.0088} & \ms{0.0304}{0.0063} +& \ms{2.1266}{0.1589} & \ms{1.5801}{0.1004} & \ms{0.0481}{0.0459} \\ +FuXi +& \ms{5.9319}{0.1029} & \ms{3.7398}{0.0088} & \ms{0.0299}{0.0061} +& \ms{2.1282}{0.0969} & \ms{1.5759}{0.0719} & \ms{0.2268}{0.0623} \\ +Pangu-Weather +& \ms{5.9270}{0.1036} & \ms{3.7320}{0.0088} & \ms{0.0301}{0.0060} +& \ms{2.2045}{0.1483} & \ms{1.6307}{0.0889} & \ms{0.0199}{0.0062} \\ +AlphaEarth +& \ms{4.4403}{0.0488} & \ms{2.3992}{0.0056} & \ms{0.6347}{0.0066} +& \ms{0.2194}{0.0039} & \ms{0.1800}{0.0014} & \ms{0.9542}{0.0107} \\ +\bottomrule +\end{tabular} +} +\end{table} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..10b87dfa80e125c916f4d87c85d69bebdd1eab03 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +# No runtime dependency is required for paper-output reproduction. diff --git a/scripts/audit_release.py b/scripts/audit_release.py new file mode 100644 index 0000000000000000000000000000000000000000..7a1011d1bea64c5267f3591514a70031f64d2e0a --- /dev/null +++ b/scripts/audit_release.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Audit the public artifact release before upload.""" + +from __future__ import annotations + +import re +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + +REQUIRED = [ + "README.md", + "LICENSE", + "requirements.txt", + "artifacts/manifests/paper_outputs.yml", + "artifacts/manifests/paper_outputs.sha256", + "docs/artifact_map.md", + "data_sources/DATA_SOURCES.md", + "paper_outputs/figures/matching.pdf", + "paper_outputs/figures/fig_task_contract_tiles.pdf", + "paper_outputs/figures/fig_selection_regret_rq2.tikz", + "paper_outputs/figures/fig_fireprone_contract_progression_compact.pdf", + "paper_outputs/figures/fig_task_rank_map.pdf", + "artifacts/results/fireprone_contract_progression_summary.json", + "artifacts/results/selection_regret_scope_sweep_20260505.csv", + "artifacts/results/selection_regret_rq2_figure_values.csv", + "scripts/check_paper_output_hashes.py", + "scripts/build_fireprone_contract_progression_figure.py", +] + +TABLE_LABELS = [ + "tab_primary_results.tex", + "tab_supporting_results.tex", + "tab_app_matching_rule_params.tex", + "tab_app_contract_params_full.tex", + "tab_app_scope_params.tex", + "tab_fireprone_contract_progression.tex", + "tab_appendix_selection_regret_tolerance.tex", + "tab_app_occupancy_ppr_scope.tex", + "tab_app_spread_ap_by_scope.tex", + "tab_app_burned_area_median_acre.tex", + "tab_app_analog_rank_depth.tex", + "tab_app_smoke_high_event.tex", + "tab_app_heat_event_pr.tex", + "tab_app_seed_robustness.tex", + "tab_app_head_architectures.tex", +] + +FORBIDDEN_TEXT = [ + "/home/yx21e", + "/blue/", + "/orange/", + "fsu-compsci", + "TBD", + "N/A", + "Pangu24", +] + +TEXT_SUFFIXES = {".md", ".py", ".sh", ".tex", ".csv", ".json", ".yml", ".yaml", ".txt"} + + +def iter_text_files() -> list[Path]: + out: list[Path] = [] + for path in ROOT.rglob("*"): + if ".git" in path.parts or "__pycache__" in path.parts: + continue + if path.name in {"audit_release.py", "build_selection_regret_rq2_figure.py"}: + continue + if path.is_file() and path.suffix in TEXT_SUFFIXES: + out.append(path) + return sorted(out) + + +def main() -> None: + issues: list[str] = [] + + for rel in REQUIRED: + if not (ROOT / rel).exists(): + issues.append(f"missing required file: {rel}") + for table in TABLE_LABELS: + if not (ROOT / "paper_outputs/tables" / table).exists(): + issues.append(f"missing paper table output: {table}") + + for path in iter_text_files(): + text = path.read_text(errors="ignore") + for token in FORBIDDEN_TEXT: + if token in text: + issues.append(f"{path.relative_to(ROOT)} contains forbidden token {token!r}") + + rq2 = ROOT / "paper_outputs/figures/fig_selection_regret_rq2.tikz" + if rq2.exists(): + text = rq2.read_text(errors="ignore") + if r"\begin{tikzpicture}" not in text: + issues.append("RQ2 TikZ output is missing tikzpicture") + if "Omega" in text or r"\Omega" in text: + issues.append("RQ2 TikZ still contains old legend text") + + for path in (ROOT / "paper_outputs/tables").glob("*.tex"): + text = path.read_text(errors="ignore") + if re.search(r"\\ms\{[^}]*\}\{0\.0000\}", text): + issues.append(f"{path.relative_to(ROOT)} displays zero std in an \\ms cell") + + checksum_manifest = ROOT / "artifacts/manifests/paper_outputs.sha256" + if checksum_manifest.exists(): + listed = [] + for line in checksum_manifest.read_text(errors="ignore").splitlines(): + if not line.strip(): + continue + parts = line.split(None, 1) + if len(parts) != 2: + issues.append(f"bad checksum manifest line: {line!r}") + continue + rel = parts[1].strip() + listed.append(rel) + if not (ROOT / rel).exists(): + issues.append(f"checksum manifest lists missing output: {rel}") + expected = sorted(str(p.relative_to(ROOT)) for p in (ROOT / "paper_outputs").rglob("*") if p.is_file()) + if sorted(listed) != expected: + missing = sorted(set(expected) - set(listed)) + extra = sorted(set(listed) - set(expected)) + if missing: + issues.append(f"checksum manifest missing outputs: {missing}") + if extra: + issues.append(f"checksum manifest has extra outputs: {extra}") + + if issues: + print("Release audit failed:") + for issue in issues: + print(f"- {issue}") + raise SystemExit(1) + print("Release audit passed.") + + +if __name__ == "__main__": + main() diff --git a/scripts/build_fireprone_contract_progression_figure.py b/scripts/build_fireprone_contract_progression_figure.py new file mode 100644 index 0000000000000000000000000000000000000000..864e0c13e9c9376d4d333bf51b9a857f7d11ec86 --- /dev/null +++ b/scripts/build_fireprone_contract_progression_figure.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Build the RQ1 fire-prone contract-progression figure from summary JSON.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from simple_pdf import PdfCanvas, draw_axes + + +ROOT = Path(__file__).resolve().parents[1] +IN_JSON = ROOT / "artifacts" / "results" / "fireprone_contract_progression_summary.json" +OUT = ROOT / "paper_outputs" / "figures" / "fig_fireprone_contract_progression_compact.pdf" + +MODEL_ORDER = [ + ("reference", "FireWx-FM ref.", "Ref."), + ("prithvi_wxc", "Prithvi-WxC", "WxC"), + ("aurora", "Aurora", "Aurora"), + ("climax", "ClimaX", "ClimaX"), + ("stormcast", "StormCast", "Storm"), + ("dlwp", "DLWP", "DLWP"), + ("fcn", "FCN", "FCN"), + ("fengwu", "FengWu", "FengWu"), + ("fuxi", "FuXi", "FuXi"), + ("pangu6", "Pangu-Weather", "Pangu-W"), + ("alphaearth", "AlphaEarth", "Alpha"), +] + +SCOPE_ORDER = [ + ("full_domain", "global"), + ("train_fire_top05pct", "top 5%"), + ("train_fire_top10pct", "top 10%"), + ("train_fire_top20pct", "top 20%"), +] + + +def dashed_vline(c: PdfCanvas, x: float, y_start: float, y_end: float) -> None: + dash, gap = 7.0, 5.0 + y = y_start + while y < y_end: + c.line([(x, y), (x, min(y + dash, y_end))], color=(0.42, 0.44, 0.46), lw=0.75) + y += dash + gap + + +def main() -> None: + data = json.loads(IN_JSON.read_text(encoding="utf-8")) + by_key = {(row["model_tag"], row["scope"]): row for row in data["summary"]} + + c = PdfCanvas(width=1320, height=470) + c.rect(0, 0, c.width, c.height, fill=(1, 1, 1)) + + x0, y0, plot_w, plot_h, ymax = 72, 132, 1194, 268, 80.0 + draw_axes(c, x0, y0, plot_w, plot_h, ymax, [0, 20, 40, 60, 80]) + c.text(x0 - 38, y0 + plot_h + 8, "F1 (%)", size=8, color=(0.15, 0.15, 0.15), bold=True) + + colors = { + "strict": (0.09, 0.22, 0.37), + "tolerance": (0.31, 0.55, 0.80), + "union": (0.75, 0.84, 0.94), + } + block_gap = 10.0 + fire_gap = 28.0 + block_w = (plot_w - fire_gap - 2 * block_gap) / len(SCOPE_ORDER) + bar_step = block_w / len(MODEL_ORDER) + bar_w = min(18.0, bar_step * 0.80) + + scope_lefts: dict[str, float] = {} + current_x = x0 + 8.0 + for scope_idx, (scope, scope_label) in enumerate(SCOPE_ORDER): + if scope_idx == 1: + dashed_vline(c, current_x - fire_gap / 2.0, y0 - 6, y0 + plot_h + 16) + scope_lefts[scope] = current_x + c.text(current_x + block_w / 2.0, y0 + plot_h + 17, scope_label, size=15.0, align="center", bold=True) + if scope_idx < len(SCOPE_ORDER) - 1: + current_x += block_w + (fire_gap if scope_idx == 0 else block_gap) + + for scope, _scope_label in SCOPE_ORDER: + block_x = scope_lefts[scope] + for idx, (model_tag, _label, short) in enumerate(MODEL_ORDER): + row = by_key[(model_tag, scope)] + strict = row["strict_f1"]["mean"] * 100.0 + tolerance = row["tolerance_f1"]["mean"] * 100.0 + union = row["union_f1"]["mean"] * 100.0 + bx = block_x + idx * bar_step + (bar_step - bar_w) / 2.0 + base = y0 + for segment, value in [ + ("strict", max(0.0, strict)), + ("tolerance", max(0.0, tolerance - strict)), + ("union", max(0.0, union - tolerance)), + ]: + height = plot_h * value / ymax + if height <= 0: + continue + c.rect(bx, base, bar_w, height, fill=colors[segment], stroke=(1, 1, 1), lw=0.35) + base += height + c.text_rotated(bx + bar_w / 2.0 - 3.0, y0 - 76, short, angle_deg=-45.0, size=10.0, align="right") + + legend_x, legend_y = x0 + 18, y0 + plot_h - 26 + c.rect(legend_x - 13, legend_y - 12, 304, 23, fill=(0.98, 0.98, 0.96), stroke=(0.78, 0.80, 0.78), lw=0.45) + for idx, (label, color) in enumerate([("Strict", colors["strict"]), ("Tolerance", colors["tolerance"]), ("Union", colors["union"])]): + x = legend_x + idx * 98 + c.rect(x, legend_y - 3, 24, 9, fill=color, stroke=(1, 1, 1), lw=0.35) + c.text(x + 31, legend_y - 1, label, size=8.0) + + OUT.parent.mkdir(parents=True, exist_ok=True) + c.save(OUT) + + +if __name__ == "__main__": + main() diff --git a/scripts/build_selection_regret_rq2_figure.py b/scripts/build_selection_regret_rq2_figure.py new file mode 100644 index 0000000000000000000000000000000000000000..523e505a463505e1aeeec5a45b960fbaecb6f755 --- /dev/null +++ b/scripts/build_selection_regret_rq2_figure.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +"""Build the compact RQ2 selection-regret interval figure. + +The output is TikZ rather than a raster/PDF plot so the manuscript does not +depend on a local matplotlib installation. +""" + +from __future__ import annotations + +import csv +import math +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +IN_CSV = ROOT / "artifacts" / "results" / "selection_regret_scope_sweep_20260505.csv" +OUT_TIKZ = ROOT / "paper_outputs" / "figures" / "fig_selection_regret_rq2.tikz" +OUT_VALUES = ROOT / "artifacts" / "results" / "selection_regret_rq2_figure_values.csv" + +ROW_ORDER = [ + "FireWx-FM ref.", + "Prithvi-WxC", + "Aurora", + "ClimaX", + "StormCast", + "DLWP", + "FCN", + "FengWu", + "FuXi", + "Pangu-Weather", + "AlphaEarth", +] + +SCOPES = { + "global": "global", + "fire_prone_top20": "top20", +} + + +def read_rows() -> dict[tuple[str, str], dict[str, str]]: + with IN_CSV.open("r", encoding="utf-8", newline="") as fh: + rows = list(csv.DictReader(fh)) + by_key = {(row["label"], row["scope"]): row for row in rows} + missing = [ + (label, scope) + for label in ROW_ORDER + for scope in SCOPES.values() + if (label, scope) not in by_key + ] + if missing: + raise SystemExit(f"Missing selection-regret rows: {missing}") + bad_labels = sorted({row["label"] for row in rows if "Pangu24" in row["label"]}) + if bad_labels: + raise SystemExit(f"Stale Pangu24 labels found in final CSV: {bad_labels}") + return by_key + + +def pct(row: dict[str, str], field: str) -> float: + return float(row[field]) * 100.0 + + +def escape_tex(text: str) -> str: + return text.replace("&", r"\&").replace("%", r"\%") + + +def nice_floor(value: float, step: float) -> float: + return math.floor(value / step) * step + + +def nice_ceil(value: float, step: float) -> float: + return math.ceil(value / step) * step + + +def write_tikz(records: list[dict[str, float | str]], x_min: float, x_max: float) -> None: + width = 5.55 + left = 2.45 + y_step = 0.41 + top = 4.35 + bottom = -0.35 + offset = 0.13 + x_span = x_max - x_min + + def x(value: float) -> float: + return left + (value - x_min) / x_span * width + + lines: list[str] = [ + r"% Auto-generated by scripts/build_selection_regret_rq2_figure.py.", + r"\begin{tikzpicture}[x=1cm,y=1cm]", + r"\footnotesize", + ] + + ticks = [tick for tick in range(int(x_min), int(x_max) + 1, 10) if tick % 10 == 0] + if 0 not in ticks: + ticks.append(0) + ticks = sorted(set(ticks)) + + for tick in ticks: + xt = x(float(tick)) + color = "wfgray" if tick == 0 else "black!12" + lw = "0.55pt" if tick == 0 else "0.35pt" + lines.append( + rf"\draw[{color}, line width={lw}] ({xt:.3f},{bottom:.3f}) -- ({xt:.3f},{top + 0.18:.3f});" + ) + lines.append(rf"\node[anchor=north, font=\scriptsize, text=black!70] at ({xt:.3f},{bottom - 0.06:.3f}) {{{tick}}};") + + axis_y = bottom + lines.append( + rf"\draw[black!45, line width=0.4pt] ({x(x_min):.3f},{axis_y:.3f}) -- ({x(x_max):.3f},{axis_y:.3f});" + ) + + for idx, record in enumerate(records): + y_base = top - idx * y_step + label = str(record["feature_source"]) + label_tex = escape_tex(label) + if label == "FireWx-FM ref.": + label_tex = rf"\textcolor{{wfblue}}{{\textbf{{{label_tex}}}}}" + lines.append(rf"\node[anchor=east, font=\scriptsize, text=black!82] at ({left - 0.13:.3f},{y_base:.3f}) {{{label_tex}}};") + + for scope_key, color, y_offset, marker in [ + ("global", "wfslate", -offset, "circle"), + ("top20", "wforange", offset, "square"), + ]: + mean = float(record[f"{scope_key}_mean_pp"]) + std = float(record[f"{scope_key}_std_pp"]) + y_val = y_base + y_offset + x_lo = x(mean - std) + x_hi = x(mean + std) + x_mid = x(mean) + lines.append(rf"\draw[{color}, line width=0.72pt] ({x_lo:.3f},{y_val:.3f}) -- ({x_hi:.3f},{y_val:.3f});") + lines.append(rf"\draw[{color}, line width=0.72pt] ({x_lo:.3f},{y_val - 0.035:.3f}) -- ({x_lo:.3f},{y_val + 0.035:.3f});") + lines.append(rf"\draw[{color}, line width=0.72pt] ({x_hi:.3f},{y_val - 0.035:.3f}) -- ({x_hi:.3f},{y_val + 0.035:.3f});") + if marker == "circle": + lines.append(rf"\filldraw[{color}] ({x_mid:.3f},{y_val:.3f}) circle[radius=0.045];") + else: + lines.append(rf"\filldraw[{color}] ({x_mid - 0.045:.3f},{y_val - 0.045:.3f}) rectangle ({x_mid + 0.045:.3f},{y_val + 0.045:.3f});") + + lines.append(r"\end{tikzpicture}") + OUT_TIKZ.parent.mkdir(parents=True, exist_ok=True) + OUT_TIKZ.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def main() -> None: + by_key = read_rows() + records: list[dict[str, float | str]] = [] + interval_edges: list[float] = [] + + for label in ROW_ORDER: + g = by_key[(label, SCOPES["global"])] + f = by_key[(label, SCOPES["fire_prone_top20"])] + g_mean = pct(g, "union_regret_mean") + g_std = pct(g, "union_regret_std") + f_mean = pct(f, "union_regret_mean") + f_std = pct(f, "union_regret_std") + interval_edges.extend([g_mean - g_std, g_mean + g_std, f_mean - f_std, f_mean + f_std]) + records.append( + { + "feature_source": label, + "global_mean_pp": g_mean, + "global_std_pp": g_std, + "top20_mean_pp": f_mean, + "top20_std_pp": f_std, + } + ) + + x_min = min(-5.0, nice_floor(min(interval_edges), 10.0)) + x_max = max(30.0, nice_ceil(max(interval_edges), 10.0)) + write_tikz(records, x_min, x_max) + + OUT_VALUES.parent.mkdir(parents=True, exist_ok=True) + with OUT_VALUES.open("w", encoding="utf-8", newline="") as fh: + fieldnames = ["feature_source", "global_mean_pp", "global_std_pp", "top20_mean_pp", "top20_std_pp"] + writer = csv.DictWriter(fh, fieldnames=fieldnames) + writer.writeheader() + for record in records: + writer.writerow( + { + "feature_source": record["feature_source"], + "global_mean_pp": f"{float(record['global_mean_pp']):.4f}", + "global_std_pp": f"{float(record['global_std_pp']):.4f}", + "top20_mean_pp": f"{float(record['top20_mean_pp']):.4f}", + "top20_std_pp": f"{float(record['top20_std_pp']):.4f}", + } + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/build_task_rank_map.py b/scripts/build_task_rank_map.py new file mode 100644 index 0000000000000000000000000000000000000000..65d65118cdb04485e0242539bb0790de9baa6bb7 --- /dev/null +++ b/scripts/build_task_rank_map.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Build the RQ4 task-rank map from released paper table TeX. + +This uses only the Python standard library plus the small PDF helper bundled in +this repository. It does not require raw data, feature caches, or matplotlib. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +from simple_pdf import PdfCanvas, mix + + +ROOT = Path(__file__).resolve().parents[1] +TABLE_DIR = ROOT / "paper_outputs" / "tables" +OUT = ROOT / "paper_outputs" / "figures" / "fig_task_rank_map.pdf" + +MODELS = [ + "FireWx-FM ref.", + "Prithvi-WxC", + "Aurora", + "ClimaX", + "StormCast", + "DLWP", + "FCN", + "FengWu", + "FuXi", + "Pangu-Weather", + "AlphaEarth", +] + +DISPLAY_MODELS = [ + "FireWx-FM ref.", + "Prithvi-WxC", + "Aurora", + "ClimaX", + "StormCast", + "DLWP", + "FCN", + "FengWu", + "FuXi", + "Pangu-Weather", + "AlphaEarth", +] + + +def means_from_row(block: str, label: str, occurrence: int = 0) -> list[float]: + starts = [m.start() for m in re.finditer(re.escape(label), block)] + if len(starts) <= occurrence: + raise ValueError(f"Missing row {label!r} occurrence {occurrence}") + start = starts[occurrence] + end = block.find(r"\\", start) + if end < 0: + raise ValueError(f"Missing row terminator after {label!r}") + row = block[start:end] + values = [float(x) for x in re.findall(r"\\ms\{(-?\d+(?:\.\d+)?)\}\{", row)] + if not values: + raise ValueError(f"No values found for row {label!r}") + return values + + +def rank_values(values: list[float], higher_is_better: bool) -> list[int]: + order = sorted(range(len(values)), key=lambda i: values[i], reverse=higher_is_better) + ranks = [0] * len(values) + for rank, idx in enumerate(order, start=1): + ranks[idx] = rank + return ranks + + +def rank_color(rank: int, n_cols: int) -> tuple[float, float, float]: + t = (n_cols - rank) / max(1, n_cols - 1) + if t <= 0.5: + return mix((0.93, 0.95, 0.94), (0.55, 0.78, 0.75), t / 0.5) + return mix((0.55, 0.78, 0.75), (0.05, 0.40, 0.42), (t - 0.5) / 0.5) + + +def fmt_value(value: float) -> str: + return f"{value:.2f}" if abs(value) >= 1 else f"{value:.3f}" + + +def main() -> None: + primary = (TABLE_DIR / "tab_primary_results.tex").read_text() + supporting = (TABLE_DIR / "tab_supporting_results.tex").read_text() + + primary_rows = {label: means_from_row(primary, label) for label in MODELS} + supporting_top = {label: means_from_row(supporting, label, occurrence=0) for label in MODELS} + supporting_bottom = {label: means_from_row(supporting, label, occurrence=1) for label in MODELS} + + tasks = [ + ("Occupancy", "Union F1 (%)", "higher better", [primary_rows[m][2] for m in MODELS], True), + ("Fire spread", "AP (%)", "higher better", [primary_rows[m][5] for m in MODELS], True), + ("Burned area", "log-RMSE", "lower better", [supporting_top[m][0] for m in MODELS], False), + ("Analog retrieval", "nDCG@10", "higher better", [supporting_top[m][3] for m in MODELS], True), + ("Smoke PM2.5", "RMSE", "lower better", [supporting_bottom[m][0] for m in MODELS], False), + ("Extreme heat", "RMSE-C", "lower better", [supporting_bottom[m][3] for m in MODELS], False), + ] + + n_rows = len(tasks) + n_cols = len(MODELS) + c = PdfCanvas(width=1120, height=430) + c.rect(0, 0, c.width, c.height, fill=(1, 1, 1)) + + x0, y0 = 108, 90 + cell_w, cell_h = 86, 42 + grid_top = y0 + n_rows * cell_h + + for j, model in enumerate(DISPLAY_MODELS): + c.text(x0 + j * cell_w + cell_w / 2, grid_top + 34, model, size=8.7, align="center", bold=True, color=(0.12, 0.14, 0.16)) + + for i, (task, metric, direction, values, higher) in enumerate(tasks): + y = grid_top - (i + 1) * cell_h + c.text(12, y + 25, task, size=7.7, bold=True, color=(0.12, 0.14, 0.16)) + c.text(12, y + 14, metric, size=7.1, bold=True, color=(0.12, 0.14, 0.16)) + c.text(12, y + 3, direction, size=6.4, color=(0.42, 0.44, 0.46)) + ranks = rank_values(values, higher) + for j, (rank, value) in enumerate(zip(ranks, values)): + x = x0 + j * cell_w + color = rank_color(rank, n_cols) + text_color = (1, 1, 1) if rank <= 2 else (0.07, 0.09, 0.11) + c.rect(x, y, cell_w, cell_h, fill=color, stroke=(1, 1, 1), lw=0.8) + c.text(x + cell_w / 2, y + 24, f"#{rank}", size=11.2, align="center", bold=True, color=text_color) + c.text(x + cell_w / 2, y + 9, fmt_value(value), size=7.0, align="center", color=text_color) + + c.rect(x0, y0, cell_w * n_cols, cell_h * n_rows, stroke=(0.20, 0.22, 0.24), lw=0.8) + + key_x, key_y = x0 + cell_w * n_cols - 220, 38 + c.text(key_x + 110, key_y + 25, "within-row rank", size=9.0, bold=True, align="center", color=(0.24, 0.25, 0.26)) + for i in range(80): + color = rank_color(n_cols - int(i / 80 * (n_cols - 1)), n_cols) + c.rect(key_x + i * 2.5, key_y + 8, 2.6, 10, fill=color) + c.text(key_x, key_y - 4, f"rank {n_cols}", size=7.0, color=(0.25, 0.26, 0.27)) + c.text(key_x + 200, key_y - 4, "rank 1", size=7.0, align="right", color=(0.25, 0.26, 0.27)) + + OUT.parent.mkdir(parents=True, exist_ok=True) + c.save(OUT) + + +if __name__ == "__main__": + main() diff --git a/scripts/check_paper_output_hashes.py b/scripts/check_paper_output_hashes.py new file mode 100644 index 0000000000000000000000000000000000000000..3014c0a41b5a6b07932708edb043b405ad11bd24 --- /dev/null +++ b/scripts/check_paper_output_hashes.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Verify final paper-output checksums.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +MANIFEST = ROOT / "artifacts" / "manifests" / "paper_outputs.sha256" + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as fh: + for chunk in iter(lambda: fh.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def main() -> None: + issues: list[str] = [] + for line_no, line in enumerate(MANIFEST.read_text(encoding="utf-8").splitlines(), start=1): + if not line.strip(): + continue + try: + expected, rel = line.split(None, 1) + except ValueError: + issues.append(f"bad checksum line {line_no}: {line!r}") + continue + path = ROOT / rel.strip() + if not path.exists(): + issues.append(f"missing output listed in checksum manifest: {rel}") + continue + actual = sha256(path) + if actual != expected: + issues.append(f"checksum mismatch for {rel}: expected {expected}, got {actual}") + + if issues: + print("Paper-output checksum check failed:") + for issue in issues: + print(f"- {issue}") + raise SystemExit(1) + print("Paper-output checksum check passed.") + + +if __name__ == "__main__": + main() diff --git a/scripts/reproduce_paper_outputs.py b/scripts/reproduce_paper_outputs.py new file mode 100644 index 0000000000000000000000000000000000000000..e50a17b3620c209743d36ef7f448835a89ae3522 --- /dev/null +++ b/scripts/reproduce_paper_outputs.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Rebuild release paper outputs from bundled lightweight artifacts. + +This release intentionally does not include raw HRRR, FIRMS, LANDFIRE, +WFIGS, MTBS, WRC, or LandScan data. The script rebuilds artifacts that can +be reproduced from the small summary files bundled in this repository. +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +def run(cmd: list[str]) -> None: + print("+", " ".join(cmd), flush=True) + subprocess.run(cmd, cwd=ROOT, check=True) + + +def main() -> None: + run([sys.executable, "scripts/build_fireprone_contract_progression_figure.py"]) + run([sys.executable, "scripts/build_selection_regret_rq2_figure.py"]) + run([sys.executable, "scripts/build_task_rank_map.py"]) + run([sys.executable, "scripts/check_paper_output_hashes.py"]) + run([sys.executable, "scripts/audit_release.py"]) + print("Rebuilt reproducible outputs and passed release audit.") + + +if __name__ == "__main__": + main() diff --git a/scripts/simple_pdf.py b/scripts/simple_pdf.py new file mode 100644 index 0000000000000000000000000000000000000000..9ba96a36742f873a6aea580f2745bb977de9261d --- /dev/null +++ b/scripts/simple_pdf.py @@ -0,0 +1,168 @@ +from __future__ import annotations + +import math +from pathlib import Path + + +def esc(text: object) -> str: + return str(text).replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)") + + +def clamp(value: float, lo: float = 0.0, hi: float = 1.0) -> float: + return max(lo, min(hi, value)) + + +def mix(c0: tuple[float, float, float], c1: tuple[float, float, float], t: float) -> tuple[float, float, float]: + t = clamp(t) + return tuple(c0[i] * (1.0 - t) + c1[i] * t for i in range(3)) + + +def text_width(text: str, size: float, bold: bool = False) -> float: + width = 0.0 + for ch in text: + if ch == " ": + width += 0.28 + elif ch in "ilI.,:;!|": + width += 0.25 + elif ch in "mwMW@": + width += 0.82 + elif ch.isupper(): + width += 0.62 + else: + width += 0.52 + if bold: + width *= 1.05 + return width * size + + +class PdfCanvas: + def __init__(self, width: float = 900, height: float = 560) -> None: + self.width = width + self.height = height + self.ops: list[str] = [] + + def _color(self, color: tuple[float, float, float], stroke: bool = False) -> str: + return f"{color[0]:.4f} {color[1]:.4f} {color[2]:.4f} {'RG' if stroke else 'rg'}" + + def rect( + self, + x: float, + y: float, + w: float, + h: float, + fill: tuple[float, float, float] | None = None, + stroke: tuple[float, float, float] | None = None, + lw: float = 0.7, + ) -> None: + if fill is not None and stroke is None: + self.ops.append(f"{self._color(fill)} {x:.2f} {y:.2f} {w:.2f} {h:.2f} re f") + elif fill is None and stroke is not None: + self.ops.append(f"{lw:.2f} w {self._color(stroke, True)} {x:.2f} {y:.2f} {w:.2f} {h:.2f} re S") + elif fill is not None and stroke is not None: + self.ops.append( + f"{lw:.2f} w {self._color(fill)} {self._color(stroke, True)} " + f"{x:.2f} {y:.2f} {w:.2f} {h:.2f} re B" + ) + + def line( + self, + points: list[tuple[float, float]], + color: tuple[float, float, float] = (0, 0, 0), + lw: float = 1.0, + ) -> None: + if len(points) < 2: + return + cmds = [f"{points[0][0]:.2f} {points[0][1]:.2f} m"] + cmds.extend(f"{x:.2f} {y:.2f} l" for x, y in points[1:]) + self.ops.append(f"{lw:.2f} w {self._color(color, True)} " + " ".join(cmds) + " S") + + def text( + self, + x: float, + y: float, + text: object, + size: float = 9, + color: tuple[float, float, float] = (0, 0, 0), + align: str = "left", + bold: bool = False, + ) -> None: + s = str(text) + w = text_width(s, size, bold) + if align == "center": + x -= w / 2.0 + elif align == "right": + x -= w + font = "F2" if bold else "F1" + self.ops.append( + f"BT /{font} {size:.2f} Tf {self._color(color)} " + f"1 0 0 1 {x:.2f} {y:.2f} Tm ({esc(s)}) Tj ET" + ) + + def text_rotated( + self, + x: float, + y: float, + text: object, + angle_deg: float, + size: float = 9, + color: tuple[float, float, float] = (0, 0, 0), + align: str = "left", + bold: bool = False, + ) -> None: + s = str(text) + w = text_width(s, size, bold) + angle = math.radians(angle_deg) + if align == "center": + x -= math.cos(angle) * w / 2.0 + y -= math.sin(angle) * w / 2.0 + elif align == "right": + x -= math.cos(angle) * w + y -= math.sin(angle) * w + a = math.cos(angle) + b = math.sin(angle) + c = -math.sin(angle) + d = math.cos(angle) + font = "F2" if bold else "F1" + self.ops.append( + f"BT /{font} {size:.2f} Tf {self._color(color)} " + f"{a:.5f} {b:.5f} {c:.5f} {d:.5f} {x:.2f} {y:.2f} Tm ({esc(s)}) Tj ET" + ) + + def save(self, path: Path) -> None: + content = "\n".join(self.ops).encode("latin-1", errors="replace") + objects: list[bytes] = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + ( + f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {self.width:.0f} {self.height:.0f}] " + f"/Resources << /Font << /F1 4 0 R /F2 5 0 R >> >> /Contents 6 0 R >>" + ).encode(), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica-Bold >>", + b"<< /Length " + str(len(content)).encode() + b" >>\nstream\n" + content + b"\nendstream", + ] + out = bytearray(b"%PDF-1.4\n%\xe2\xe3\xcf\xd3\n") + offsets = [0] + for i, obj in enumerate(objects, start=1): + offsets.append(len(out)) + out.extend(f"{i} 0 obj\n".encode()) + out.extend(obj) + out.extend(b"\nendobj\n") + xref_pos = len(out) + out.extend(f"xref\n0 {len(objects)+1}\n".encode()) + out.extend(b"0000000000 65535 f \n") + for offset in offsets[1:]: + out.extend(f"{offset:010d} 00000 n \n".encode()) + out.extend( + f"trailer\n<< /Size {len(objects)+1} /Root 1 0 R >>\nstartxref\n{xref_pos}\n%%EOF\n".encode() + ) + path.write_bytes(bytes(out)) + + +def draw_axes(c: PdfCanvas, x0: float, y0: float, w: float, h: float, ymax: float, ticks: list[float]) -> None: + c.line([(x0, y0), (x0, y0 + h)], color=(0.15, 0.15, 0.15), lw=0.8) + c.line([(x0, y0), (x0 + w, y0)], color=(0.15, 0.15, 0.15), lw=0.8) + for tick in ticks: + y = y0 + h * tick / ymax + c.line([(x0 - 4, y), (x0 + w, y)], color=(0.86, 0.86, 0.86), lw=0.45) + c.text(x0 - 8, y - 3, f"{tick:g}", size=7, color=(0.25, 0.25, 0.25), align="right") diff --git a/scripts/summarize_selection_regret_scope_sweep_20260505.py b/scripts/summarize_selection_regret_scope_sweep_20260505.py new file mode 100644 index 0000000000000000000000000000000000000000..983b462ad3b8bc1a5938b6a0faf2b2c20a400c78 --- /dev/null +++ b/scripts/summarize_selection_regret_scope_sweep_20260505.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +"""Summarize fixed-feature selection-regret scope sweep.""" + +from __future__ import annotations + +import argparse +import csv +import json +import math +import re +import statistics +from collections import defaultdict +from pathlib import Path +from typing import Any, Iterable + + +ROOT = Path(__file__).resolve().parents[1] +ROW_ORDER = [ + ("reference", "FireWx-FM ref."), + ("prithvi_wxc", "Prithvi-WxC"), + ("aurora", "Aurora"), + ("climax", "ClimaX"), + ("stormcast", "StormCast"), + ("dlwp", "DLWP"), + ("fcn", "FCN"), + ("fengwu", "FengWu"), + ("fuxi", "FuXi"), + ("pangu6", "Pangu-Weather"), + ("alphaearth", "AlphaEarth"), +] +SCOPE_ORDER = [ + ("global", r"\(\Omega=\)global"), + ("top5", r"\(\Omega=\)top 5\%"), + ("top10", r"\(\Omega=\)top 10\%"), + ("top20", r"\(\Omega=\)top 20\%"), +] +METRICS = [ + ("exact_regret", "Exact"), + ("tolerated_regret", "Tol."), + ("union_regret", "Union"), +] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Summarize selection-regret scope sweep.") + parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("--out-json", type=Path, default=ROOT / "generated" / "selection_regret_scope_sweep_20260505.json") + parser.add_argument("--out-csv", type=Path, default=ROOT / "generated" / "selection_regret_scope_sweep_20260505.csv") + parser.add_argument("--out-tex", type=Path, default=ROOT / "generated" / "selection_regret_scope_sweep_20260505.tex") + parser.add_argument("--out-audit", type=Path, default=ROOT / "generated" / "selection_regret_scope_sweep_20260505_audit.md") + parser.add_argument("--min-seeds", type=int, default=5) + return parser.parse_args() + + +def read_csv(path: Path) -> list[dict[str, str]]: + with path.open("r", encoding="utf-8", newline="") as fh: + return list(csv.DictReader(fh)) + + +def finite(values: Iterable[Any]) -> list[float]: + out: list[float] = [] + for value in values: + try: + number = float(value) + except Exception: + continue + if math.isfinite(number): + out.append(number) + return out + + +def stat(values: Iterable[Any]) -> dict[str, Any]: + vals = finite(values) + if not vals: + return {"n": 0, "mean": math.nan, "std": math.nan, "min": math.nan, "max": math.nan} + return { + "n": len(vals), + "mean": float(statistics.fmean(vals)), + "std": float(statistics.stdev(vals)) if len(vals) > 1 else 0.0, + "min": float(min(vals)), + "max": float(max(vals)), + } + + +def ms(summary: dict[str, Any], scale: float = 100.0) -> str: + n = int(summary.get("n", 0)) + if n < 2: + return "not_bundled" + mean = float(summary["mean"]) * scale + std = float(summary["std"]) * scale + if abs(mean) < 0.00005 and abs(std) < 0.00005: + return "0.0000" + return rf"\ms{{{mean:.4f}}}{{{std:.4f}}}" + + +def collect_rows(run_root: Path) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for path in sorted(run_root.glob("*/*/selection_rows.csv")): + for row in read_csv(path): + enriched: dict[str, Any] = dict(row) + enriched["path"] = str(path) + enriched["seed"] = int(float(row["seed"])) + for metric, _ in METRICS: + enriched[metric] = float(row[metric]) + rows.append(enriched) + return rows + + +def summarize(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + by_key: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + for row in rows: + by_key[(str(row["model_tag"]), str(row["scope"]))].append(row) + + summary: list[dict[str, Any]] = [] + for model_tag, label in ROW_ORDER: + for scope, scope_label in SCOPE_ORDER: + selected = by_key[(model_tag, scope)] + seeds = sorted({int(row["seed"]) for row in selected}) + item: dict[str, Any] = { + "model_tag": model_tag, + "label": label, + "scope": scope, + "scope_label": scope_label, + "n": len(selected), + "seeds": seeds, + "paths": sorted({str(row["path"]) for row in selected}), + } + for metric, _ in METRICS: + item[metric] = stat(row[metric] for row in selected) + summary.append(item) + return summary + + +def write_csv(rows: list[dict[str, Any]], path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + flat_rows: list[dict[str, Any]] = [] + for row in rows: + out = { + "model_tag": row["model_tag"], + "label": row["label"], + "scope": row["scope"], + "scope_label": row["scope_label"], + "n": row["n"], + "seeds": " ".join(str(seed) for seed in row["seeds"]), + } + for metric, _ in METRICS: + out[f"{metric}_mean"] = row[metric]["mean"] + out[f"{metric}_std"] = row[metric]["std"] + out[f"{metric}_min"] = row[metric]["min"] + out[f"{metric}_max"] = row[metric]["max"] + flat_rows.append(out) + fieldnames = list(flat_rows[0]) if flat_rows else [] + with path.open("w", newline="", encoding="utf-8") as fh: + writer = csv.DictWriter(fh, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(flat_rows) + + +def write_tex(rows: list[dict[str, Any]], path: Path) -> str: + by_key = {(row["model_tag"], row["scope"]): row for row in rows} + lines = [ + r"\begin{table*}[!t]", + r" \centering", + r" \small", + r" \setlength{\tabcolsep}{4pt}", + r" \caption{Fixed-feature selection-regret sweep across evaluation scopes. Values are percentage-point regret \(\delta = D(h_D)-D(h_R)\) under union-\(F_1\). Top-\(k\) scopes are train-defined fire-prone masks. Rows report mean with small std over five seeds.}", + r" \label{tab:selection_regret_scope_sweep}", + r" \begin{tabular}{lcccc}", + r" \toprule", + r" \textbf{Feature source} & \textbf{\(\Omega=\)global} & \textbf{\(\Omega=\)top 5\%} & \textbf{\(\Omega=\)top 10\%} & \textbf{\(\Omega=\)top 20\%} \\", + r" \midrule", + ] + for model_tag, label in ROW_ORDER: + cells = [ms(by_key[(model_tag, scope)]["union_regret"]) for scope, _ in SCOPE_ORDER] + if model_tag == "reference": + label = r"\textcolor{blue}{FireWx-FM ref.}" + lines.append(" " + label + " & " + " & ".join(cells) + r" \\") + lines.extend( + [ + r" \bottomrule", + r" \end{tabular}", + r"\end{table*}", + "", + ] + ) + tex = "\n".join(lines) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(tex, encoding="utf-8") + return tex + + +def audit(summary: list[dict[str, Any]], tex: str, min_seeds: int) -> tuple[list[str], list[str]]: + issues: list[str] = [] + notes: list[str] = [] + for row in summary: + if int(row["n"]) < int(min_seeds): + issues.append(f"{row['label']} {row['scope']} has n={row['n']}, expected >= {min_seeds}") + for metric, _ in METRICS: + values = row[metric] + if not math.isfinite(float(values["mean"])) or not math.isfinite(float(values["std"])): + issues.append(f"{row['label']} {row['scope']} {metric} is not finite") + if f"{float(values['std']) * 100.0:.4f}" == "0.0000": + notes.append(f"{row['label']} {row['scope']} {metric} has true zero displayed std") + cells = re.findall(r"\\ms\{([^}]*)\}\{([^}]*)\}", tex) + by_cell: dict[tuple[str, str], list[int]] = defaultdict(list) + for idx, (mean, std) in enumerate(cells, start=1): + by_cell[(mean, std)].append(idx) + if std == "0.0000": + issues.append(f"zero displayed std in cell {idx}: {mean} +/- {std}") + for cell, idxs in by_cell.items(): + if len(idxs) > 1 and cell != ("0.0000", "0.0000"): + issues.append(f"duplicate displayed cell {cell} at positions {idxs}") + forbidden = ["not_bundled", "not_applicable", "--", "nan", "NaN", "tied"] + for token in forbidden: + if token in tex: + issues.append(f"forbidden token in tex: {token}") + return issues, notes + + +def main() -> None: + args = parse_args() + rows = collect_rows(args.run_root) + summary = summarize(rows) + tex = write_tex(summary, args.out_tex) + write_csv(summary, args.out_csv) + issues, notes = audit(summary, tex, int(args.min_seeds)) + args.out_json.parent.mkdir(parents=True, exist_ok=True) + args.out_json.write_text( + json.dumps({"rows": rows, "summary": summary, "issues": issues, "notes": notes}, indent=2), + encoding="utf-8", + ) + if issues: + args.out_audit.write_text("FAIL\n" + "\n".join(f"- {issue}" for issue in issues) + "\n", encoding="utf-8") + raise SystemExit("Selection-regret scope-sweep audit failed; see " + str(args.out_audit)) + args.out_audit.write_text("PASS\n" + "\n".join(f"- {note}" for note in notes) + ("\n" if notes else ""), encoding="utf-8") + print(json.dumps({"rows": len(rows), "summary_rows": len(summary), "tex": str(args.out_tex), "audit": "PASS"}, indent=2)) + + +if __name__ == "__main__": + main()