| { |
| "purpose": "Quantify how much each feature group contributes to the headline XGBoost score. Identical architecture, same group-aware split, with one feature group dropped at a time.", |
| "full_model_metrics": { |
| "model": "xgboost", |
| "accuracy": 0.6898452783211808, |
| "macro_f1": 0.6751447018282526, |
| "weighted_f1": 0.6881356546405818, |
| "per_class_f1": { |
| "lone_actor": 0.6297297297297297, |
| "organised_syndicate": 0.7391393864525427, |
| "raas_affiliate": 0.6458906202260922, |
| "nation_state_nexus": 0.6858190709046454 |
| }, |
| "confusion_matrix": { |
| "labels": [ |
| "lone_actor", |
| "organised_syndicate", |
| "raas_affiliate", |
| "nation_state_nexus" |
| ], |
| "matrix": [ |
| [ |
| 466, |
| 67, |
| 216, |
| 1 |
| ], |
| [ |
| 83, |
| 1795, |
| 275, |
| 172 |
| ], |
| [ |
| 156, |
| 433, |
| 1057, |
| 79 |
| ], |
| [ |
| 25, |
| 237, |
| 0, |
| 561 |
| ] |
| ] |
| }, |
| "macro_roc_auc_ovr": 0.873606865711172 |
| }, |
| "ablations": { |
| "no_topology": { |
| "n_features": 35, |
| "dropped_count": 28, |
| "metrics": { |
| "model": "xgboost_no_topology", |
| "accuracy": 0.6146185310332563, |
| "macro_f1": 0.630244354214636, |
| "weighted_f1": 0.6146007963862242, |
| "per_class_f1": { |
| "lone_actor": 0.5802285146547441, |
| "organised_syndicate": 0.595659765527563, |
| "raas_affiliate": 0.5862656072644722, |
| "nation_state_nexus": 0.7588235294117647 |
| }, |
| "confusion_matrix": { |
| "labels": [ |
| "lone_actor", |
| "organised_syndicate", |
| "raas_affiliate", |
| "nation_state_nexus" |
| ], |
| "matrix": [ |
| [ |
| 584, |
| 41, |
| 110, |
| 15 |
| ], |
| [ |
| 308, |
| 1194, |
| 655, |
| 168 |
| ], |
| [ |
| 273, |
| 370, |
| 1033, |
| 49 |
| ], |
| [ |
| 98, |
| 79, |
| 1, |
| 645 |
| ] |
| ] |
| }, |
| "macro_roc_auc_ovr": 0.8706652220620055 |
| }, |
| "delta_accuracy": 0.07522674728792456, |
| "delta_macro_f1": 0.04490034761361661 |
| }, |
| "no_behavioural": { |
| "n_features": 36, |
| "dropped_count": 27, |
| "metrics": { |
| "model": "xgboost_no_behavioural", |
| "accuracy": 0.5673128223368309, |
| "macro_f1": 0.5213632789864133, |
| "weighted_f1": 0.5706324884542183, |
| "per_class_f1": { |
| "lone_actor": 0.44366608289550497, |
| "organised_syndicate": 0.6739977090492555, |
| "raas_affiliate": 0.5680505911465493, |
| "nation_state_nexus": 0.3997387328543436 |
| }, |
| "confusion_matrix": { |
| "labels": [ |
| "lone_actor", |
| "organised_syndicate", |
| "raas_affiliate", |
| "nation_state_nexus" |
| ], |
| "matrix": [ |
| [ |
| 380, |
| 45, |
| 306, |
| 19 |
| ], |
| [ |
| 101, |
| 1471, |
| 498, |
| 255 |
| ], |
| [ |
| 319, |
| 245, |
| 1033, |
| 128 |
| ], |
| [ |
| 163, |
| 279, |
| 75, |
| 306 |
| ] |
| ] |
| }, |
| "macro_roc_auc_ovr": 0.8106558391572862 |
| }, |
| "delta_accuracy": 0.12253245598434992, |
| "delta_macro_f1": 0.15378142284183927 |
| }, |
| "no_timestep": { |
| "n_features": 62, |
| "dropped_count": 1, |
| "metrics": { |
| "model": "xgboost_no_timestep", |
| "accuracy": 0.6717054952872132, |
| "macro_f1": 0.6417349625987673, |
| "weighted_f1": 0.6719572046072043, |
| "per_class_f1": { |
| "lone_actor": 0.5438813349814586, |
| "organised_syndicate": 0.7479365079365079, |
| "raas_affiliate": 0.6453731343283582, |
| "nation_state_nexus": 0.6297488731487444 |
| }, |
| "confusion_matrix": { |
| "labels": [ |
| "lone_actor", |
| "organised_syndicate", |
| "raas_affiliate", |
| "nation_state_nexus" |
| ], |
| "matrix": [ |
| [ |
| 440, |
| 66, |
| 240, |
| 4 |
| ], |
| [ |
| 154, |
| 1767, |
| 230, |
| 174 |
| ], |
| [ |
| 169, |
| 412, |
| 1081, |
| 63 |
| ], |
| [ |
| 105, |
| 155, |
| 74, |
| 489 |
| ] |
| ] |
| }, |
| "macro_roc_auc_ovr": 0.8672596014037719 |
| }, |
| "delta_accuracy": 0.01813978303396757, |
| "delta_macro_f1": 0.033409739229485313 |
| }, |
| "no_engineered": { |
| "n_features": 57, |
| "dropped_count": 6, |
| "metrics": { |
| "model": "xgboost_no_engineered", |
| "accuracy": 0.6882447092299484, |
| "macro_f1": 0.6562913668551777, |
| "weighted_f1": 0.6881813027750402, |
| "per_class_f1": { |
| "lone_actor": 0.5686274509803921, |
| "organised_syndicate": 0.7419631375910845, |
| "raas_affiliate": 0.7053364269141531, |
| "nation_state_nexus": 0.6092384519350812 |
| }, |
| "confusion_matrix": { |
| "labels": [ |
| "lone_actor", |
| "organised_syndicate", |
| "raas_affiliate", |
| "nation_state_nexus" |
| ], |
| "matrix": [ |
| [ |
| 435, |
| 71, |
| 219, |
| 25 |
| ], |
| [ |
| 127, |
| 1731, |
| 287, |
| 180 |
| ], |
| [ |
| 107, |
| 316, |
| 1216, |
| 86 |
| ], |
| [ |
| 111, |
| 223, |
| 1, |
| 488 |
| ] |
| ] |
| }, |
| "macro_roc_auc_ovr": 0.874702950892121 |
| }, |
| "delta_accuracy": 0.0016005690912324066, |
| "delta_macro_f1": 0.018853334973074842 |
| } |
| } |
| } |
