| { |
| "purpose": "Quantify how much each feature group contributes to the headline XGBoost score. Identical architecture, same group-aware split, with one feature group dropped at a time.", |
| "full_model_metrics": { |
| "model": "xgboost", |
| "accuracy": 0.6547008547008547, |
| "macro_f1": 0.6401276666852063, |
| "weighted_f1": 0.657179533714298, |
| "per_class_f1": { |
| "target_reconnaissance": 0.8875739644970414, |
| "infrastructure_setup": 0.7115384615384616, |
| "lure_crafting": 0.6762589928057554, |
| "email_delivery": 0.7913669064748201, |
| "victim_engagement": 0.46938775510204084, |
| "credential_harvesting": 0.34074074074074073, |
| "post_compromise_escalation": 0.6040268456375839 |
| }, |
| "confusion_matrix": { |
| "labels": [ |
| "target_reconnaissance", |
| "infrastructure_setup", |
| "lure_crafting", |
| "email_delivery", |
| "victim_engagement", |
| "credential_harvesting", |
| "post_compromise_escalation" |
| ], |
| "matrix": [ |
| [ |
| 75, |
| 0, |
| 9, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 37, |
| 16, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 10, |
| 10, |
| 47, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 4, |
| 0, |
| 110, |
| 28, |
| 1, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 21, |
| 46, |
| 24, |
| 9 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 4, |
| 16, |
| 23, |
| 20 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 6, |
| 24, |
| 45 |
| ] |
| ] |
| }, |
| "macro_roc_auc_ovr": 0.935584434710217 |
| }, |
| "ablations": { |
| "no_topology": { |
| "n_features": 23, |
| "dropped_count": 30, |
| "metrics": { |
| "model": "xgboost_no_topology", |
| "accuracy": 0.6410256410256411, |
| "macro_f1": 0.626013906528604, |
| "weighted_f1": 0.6377089952999916, |
| "per_class_f1": { |
| "target_reconnaissance": 0.891566265060241, |
| "infrastructure_setup": 0.7586206896551724, |
| "lure_crafting": 0.676923076923077, |
| "email_delivery": 0.7598566308243727, |
| "victim_engagement": 0.40609137055837563, |
| "credential_harvesting": 0.2782608695652174, |
| "post_compromise_escalation": 0.6107784431137725 |
| }, |
| "confusion_matrix": { |
| "labels": [ |
| "target_reconnaissance", |
| "infrastructure_setup", |
| "lure_crafting", |
| "email_delivery", |
| "victim_engagement", |
| "credential_harvesting", |
| "post_compromise_escalation" |
| ], |
| "matrix": [ |
| [ |
| 74, |
| 0, |
| 10, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 44, |
| 9, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 8, |
| 15, |
| 44, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 4, |
| 0, |
| 106, |
| 30, |
| 3, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 26, |
| 40, |
| 16, |
| 18 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 4, |
| 20, |
| 16, |
| 23 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 7, |
| 17, |
| 51 |
| ] |
| ] |
| }, |
| "macro_roc_auc_ovr": 0.9341744835062434 |
| }, |
| "delta_accuracy": 0.013675213675213627, |
| "delta_macro_f1": 0.014113760156602262 |
| }, |
| "no_behavioural": { |
| "n_features": 36, |
| "dropped_count": 17, |
| "metrics": { |
| "model": "xgboost_no_behavioural", |
| "accuracy": 0.5794871794871795, |
| "macro_f1": 0.5734830391013238, |
| "weighted_f1": 0.5833619015067782, |
| "per_class_f1": { |
| "target_reconnaissance": 0.9024390243902439, |
| "infrastructure_setup": 0.4745762711864407, |
| "lure_crafting": 0.6619718309859155, |
| "email_delivery": 0.6390977443609023, |
| "victim_engagement": 0.3404255319148936, |
| "credential_harvesting": 0.3472222222222222, |
| "post_compromise_escalation": 0.6486486486486487 |
| }, |
| "confusion_matrix": { |
| "labels": [ |
| "target_reconnaissance", |
| "infrastructure_setup", |
| "lure_crafting", |
| "email_delivery", |
| "victim_engagement", |
| "credential_harvesting", |
| "post_compromise_escalation" |
| ], |
| "matrix": [ |
| [ |
| 74, |
| 0, |
| 10, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 28, |
| 16, |
| 9, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 6, |
| 13, |
| 47, |
| 1, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 23, |
| 2, |
| 85, |
| 30, |
| 3, |
| 0 |
| ], |
| [ |
| 0, |
| 1, |
| 0, |
| 26, |
| 32, |
| 34, |
| 7 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 2, |
| 18, |
| 25, |
| 18 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 8, |
| 19, |
| 48 |
| ] |
| ] |
| }, |
| "macro_roc_auc_ovr": 0.9187512184393106 |
| }, |
| "delta_accuracy": 0.07521367521367517, |
| "delta_macro_f1": 0.06664462758388245 |
| }, |
| "no_timestep": { |
| "n_features": 52, |
| "dropped_count": 1, |
| "metrics": { |
| "model": "xgboost_no_timestep", |
| "accuracy": 0.3623931623931624, |
| "macro_f1": 0.3138802646284953, |
| "weighted_f1": 0.3500013055228507, |
| "per_class_f1": { |
| "target_reconnaissance": 0.4419889502762431, |
| "infrastructure_setup": 0.24, |
| "lure_crafting": 0.2748091603053435, |
| "email_delivery": 0.5617283950617284, |
| "victim_engagement": 0.26666666666666666, |
| "credential_harvesting": 0.11666666666666667, |
| "post_compromise_escalation": 0.2953020134228188 |
| }, |
| "confusion_matrix": { |
| "labels": [ |
| "target_reconnaissance", |
| "infrastructure_setup", |
| "lure_crafting", |
| "email_delivery", |
| "victim_engagement", |
| "credential_harvesting", |
| "post_compromise_escalation" |
| ], |
| "matrix": [ |
| [ |
| 40, |
| 18, |
| 26, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 23, |
| 12, |
| 18, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 32, |
| 17, |
| 18, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 2, |
| 0, |
| 2, |
| 91, |
| 16, |
| 17, |
| 15 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 36, |
| 22, |
| 20, |
| 22 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 25, |
| 16, |
| 7, |
| 15 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 29, |
| 11, |
| 13, |
| 22 |
| ] |
| ] |
| }, |
| "macro_roc_auc_ovr": 0.8128267634071407 |
| }, |
| "delta_accuracy": 0.2923076923076923, |
| "delta_macro_f1": 0.326247402056711 |
| }, |
| "no_engineered": { |
| "n_features": 47, |
| "dropped_count": 6, |
| "metrics": { |
| "model": "xgboost_no_engineered", |
| "accuracy": 0.6581196581196581, |
| "macro_f1": 0.6401951204875947, |
| "weighted_f1": 0.6592473136316277, |
| "per_class_f1": { |
| "target_reconnaissance": 0.8809523809523809, |
| "infrastructure_setup": 0.7155963302752294, |
| "lure_crafting": 0.6518518518518519, |
| "email_delivery": 0.8, |
| "victim_engagement": 0.49473684210526314, |
| "credential_harvesting": 0.3484848484848485, |
| "post_compromise_escalation": 0.5897435897435898 |
| }, |
| "confusion_matrix": { |
| "labels": [ |
| "target_reconnaissance", |
| "infrastructure_setup", |
| "lure_crafting", |
| "email_delivery", |
| "victim_engagement", |
| "credential_harvesting", |
| "post_compromise_escalation" |
| ], |
| "matrix": [ |
| [ |
| 74, |
| 0, |
| 10, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 39, |
| 14, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 10, |
| 13, |
| 44, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 4, |
| 0, |
| 112, |
| 26, |
| 1, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 20, |
| 47, |
| 22, |
| 11 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 5, |
| 11, |
| 23, |
| 24 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 6, |
| 23, |
| 46 |
| ] |
| ] |
| }, |
| "macro_roc_auc_ovr": 0.9369503919262667 |
| }, |
| "delta_accuracy": -0.0034188034188034067, |
| "delta_macro_f1": -6.745380238848409e-05 |
| } |
| } |
| } |
