| { |
| "purpose": "Quantify how much each feature group contributes to the headline XGBoost score. Identical architecture, same group-aware split, with one feature group dropped at a time.", |
| "full_model_metrics": { |
| "model": "xgboost", |
| "accuracy": 0.9177777777777778, |
| "macro_f1": 0.7780699645112974, |
| "weighted_f1": 0.9064879129227142, |
| "per_class_f1": { |
| "c2_communication": 1.0, |
| "data_exfiltration": 0.9699570815450643, |
| "dormancy_dwell": 0.5301204819277109, |
| "initial_drop": 0.9453125, |
| "lateral_movement": 0.9917355371900827, |
| "payload_execution": 0.963302752293578, |
| "persistence_establishment": 0.9918032786885246, |
| "privilege_escalation": 0.9907407407407407, |
| "sandbox_evasion_stall": 0.125, |
| "self_destruct_cleanup": 0.2727272727272727 |
| }, |
| "confusion_matrix": { |
| "labels": [ |
| "c2_communication", |
| "data_exfiltration", |
| "dormancy_dwell", |
| "initial_drop", |
| "lateral_movement", |
| "payload_execution", |
| "persistence_establishment", |
| "privilege_escalation", |
| "sandbox_evasion_stall", |
| "self_destruct_cleanup" |
| ], |
| "matrix": [ |
| [ |
| 108, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 113, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 4, |
| 22, |
| 7, |
| 0, |
| 1, |
| 0, |
| 0, |
| 2, |
| 4 |
| ], |
| [ |
| 0, |
| 0, |
| 2, |
| 121, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 120, |
| 0, |
| 0, |
| 0, |
| 0, |
| 1 |
| ], |
| [ |
| 0, |
| 0, |
| 1, |
| 0, |
| 0, |
| 105, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 1, |
| 0, |
| 0, |
| 0, |
| 121, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 107, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 17, |
| 3, |
| 0, |
| 1, |
| 1, |
| 2, |
| 3, |
| 5 |
| ], |
| [ |
| 0, |
| 3, |
| 0, |
| 2, |
| 1, |
| 5, |
| 0, |
| 0, |
| 11, |
| 6 |
| ] |
| ] |
| }, |
| "macro_roc_auc_ovr": 0.979171667321058 |
| }, |
| "ablations": { |
| "no_pe_static": { |
| "n_features": 58, |
| "dropped_count": 11, |
| "metrics": { |
| "model": "xgboost_no_pe_static", |
| "accuracy": 0.9166666666666666, |
| "macro_f1": 0.7808429949060417, |
| "weighted_f1": 0.9063054516980296, |
| "per_class_f1": { |
| "c2_communication": 1.0, |
| "data_exfiltration": 0.9783549783549783, |
| "dormancy_dwell": 0.4675324675324675, |
| "initial_drop": 0.9494163424124513, |
| "lateral_movement": 0.995850622406639, |
| "payload_execution": 0.963302752293578, |
| "persistence_establishment": 0.9836065573770492, |
| "privilege_escalation": 0.9771689497716894, |
| "sandbox_evasion_stall": 0.16666666666666666, |
| "self_destruct_cleanup": 0.32653061224489793 |
| }, |
| "confusion_matrix": { |
| "labels": [ |
| "c2_communication", |
| "data_exfiltration", |
| "dormancy_dwell", |
| "initial_drop", |
| "lateral_movement", |
| "payload_execution", |
| "persistence_establishment", |
| "privilege_escalation", |
| "sandbox_evasion_stall", |
| "self_destruct_cleanup" |
| ], |
| "matrix": [ |
| [ |
| 108, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 113, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 3, |
| 18, |
| 7, |
| 0, |
| 1, |
| 0, |
| 0, |
| 6, |
| 5 |
| ], |
| [ |
| 0, |
| 0, |
| 1, |
| 122, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 120, |
| 0, |
| 0, |
| 0, |
| 0, |
| 1 |
| ], |
| [ |
| 0, |
| 0, |
| 1, |
| 0, |
| 0, |
| 105, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 1, |
| 0, |
| 0, |
| 0, |
| 120, |
| 0, |
| 0, |
| 1 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 107, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 15, |
| 3, |
| 0, |
| 1, |
| 1, |
| 2, |
| 4, |
| 6 |
| ], |
| [ |
| 0, |
| 2, |
| 1, |
| 2, |
| 0, |
| 5, |
| 1, |
| 3, |
| 6, |
| 8 |
| ] |
| ] |
| }, |
| "macro_roc_auc_ovr": 0.9785892106991877 |
| }, |
| "delta_accuracy": 0.0011111111111111738, |
| "delta_macro_f1": -0.0027730303947443025 |
| }, |
| "no_behavioural": { |
| "n_features": 60, |
| "dropped_count": 9, |
| "metrics": { |
| "model": "xgboost_no_behavioural", |
| "accuracy": 0.9088888888888889, |
| "macro_f1": 0.7578825763491894, |
| "weighted_f1": 0.8916039125438652, |
| "per_class_f1": { |
| "c2_communication": 1.0, |
| "data_exfiltration": 0.9372384937238494, |
| "dormancy_dwell": 0.463768115942029, |
| "initial_drop": 0.9494163424124513, |
| "lateral_movement": 0.9596774193548387, |
| "payload_execution": 0.9422222222222222, |
| "persistence_establishment": 0.9876543209876543, |
| "privilege_escalation": 0.9907407407407407, |
| "sandbox_evasion_stall": 0.24, |
| "self_destruct_cleanup": 0.10810810810810811 |
| }, |
| "confusion_matrix": { |
| "labels": [ |
| "c2_communication", |
| "data_exfiltration", |
| "dormancy_dwell", |
| "initial_drop", |
| "lateral_movement", |
| "payload_execution", |
| "persistence_establishment", |
| "privilege_escalation", |
| "sandbox_evasion_stall", |
| "self_destruct_cleanup" |
| ], |
| "matrix": [ |
| [ |
| 108, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 112, |
| 1, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 6, |
| 16, |
| 7, |
| 2, |
| 5, |
| 0, |
| 0, |
| 3, |
| 1 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 122, |
| 0, |
| 0, |
| 0, |
| 0, |
| 1, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 119, |
| 0, |
| 0, |
| 0, |
| 1, |
| 1 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 106, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 2, |
| 0, |
| 0, |
| 0, |
| 120, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 107, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 2, |
| 8, |
| 3, |
| 2, |
| 3, |
| 1, |
| 2, |
| 6, |
| 5 |
| ], |
| [ |
| 0, |
| 6, |
| 2, |
| 2, |
| 4, |
| 5, |
| 0, |
| 0, |
| 7, |
| 2 |
| ] |
| ] |
| }, |
| "macro_roc_auc_ovr": 0.9704768382021074 |
| }, |
| "delta_accuracy": 0.008888888888888946, |
| "delta_macro_f1": 0.020187388162107966 |
| }, |
| "no_timestep": { |
| "n_features": 68, |
| "dropped_count": 1, |
| "metrics": { |
| "model": "xgboost_no_timestep", |
| "accuracy": 0.6933333333333334, |
| "macro_f1": 0.5963303534115096, |
| "weighted_f1": 0.6919482762076271, |
| "per_class_f1": { |
| "c2_communication": 1.0, |
| "data_exfiltration": 0.7619047619047619, |
| "dormancy_dwell": 0.5882352941176471, |
| "initial_drop": 0.5072463768115942, |
| "lateral_movement": 0.6985645933014354, |
| "payload_execution": 0.5106382978723404, |
| "persistence_establishment": 0.8433734939759037, |
| "privilege_escalation": 0.9047619047619048, |
| "sandbox_evasion_stall": 0.05555555555555555, |
| "self_destruct_cleanup": 0.09302325581395349 |
| }, |
| "confusion_matrix": { |
| "labels": [ |
| "c2_communication", |
| "data_exfiltration", |
| "dormancy_dwell", |
| "initial_drop", |
| "lateral_movement", |
| "payload_execution", |
| "persistence_establishment", |
| "privilege_escalation", |
| "sandbox_evasion_stall", |
| "self_destruct_cleanup" |
| ], |
| "matrix": [ |
| [ |
| 108, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 96, |
| 0, |
| 4, |
| 9, |
| 2, |
| 1, |
| 0, |
| 0, |
| 1 |
| ], |
| [ |
| 0, |
| 0, |
| 25, |
| 10, |
| 0, |
| 1, |
| 0, |
| 0, |
| 4, |
| 0 |
| ], |
| [ |
| 0, |
| 2, |
| 6, |
| 70, |
| 1, |
| 12, |
| 7, |
| 0, |
| 22, |
| 3 |
| ], |
| [ |
| 0, |
| 39, |
| 0, |
| 1, |
| 73, |
| 7, |
| 0, |
| 1, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 1, |
| 0, |
| 37, |
| 5, |
| 48, |
| 2, |
| 1, |
| 5, |
| 7 |
| ], |
| [ |
| 0, |
| 0, |
| 1, |
| 7, |
| 0, |
| 2, |
| 105, |
| 6, |
| 1, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 2, |
| 9, |
| 95, |
| 1, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 13, |
| 12, |
| 0, |
| 2, |
| 1, |
| 0, |
| 2, |
| 2 |
| ], |
| [ |
| 0, |
| 1, |
| 0, |
| 12, |
| 0, |
| 6, |
| 2, |
| 0, |
| 5, |
| 2 |
| ] |
| ] |
| }, |
| "macro_roc_auc_ovr": 0.9263760295591874 |
| }, |
| "delta_accuracy": 0.22444444444444445, |
| "delta_macro_f1": 0.18173961109978776 |
| }, |
| "no_engineered": { |
| "n_features": 63, |
| "dropped_count": 6, |
| "metrics": { |
| "model": "xgboost_no_engineered", |
| "accuracy": 0.92, |
| "macro_f1": 0.7931081498668057, |
| "weighted_f1": 0.9099535506095557, |
| "per_class_f1": { |
| "c2_communication": 0.9906542056074766, |
| "data_exfiltration": 0.9617021276595744, |
| "dormancy_dwell": 0.5205479452054794, |
| "initial_drop": 0.9534883720930233, |
| "lateral_movement": 0.9958847736625515, |
| "payload_execution": 0.963302752293578, |
| "persistence_establishment": 0.9836065573770492, |
| "privilege_escalation": 0.9861751152073732, |
| "sandbox_evasion_stall": 0.23529411764705882, |
| "self_destruct_cleanup": 0.3404255319148936 |
| }, |
| "confusion_matrix": { |
| "labels": [ |
| "c2_communication", |
| "data_exfiltration", |
| "dormancy_dwell", |
| "initial_drop", |
| "lateral_movement", |
| "payload_execution", |
| "persistence_establishment", |
| "privilege_escalation", |
| "sandbox_evasion_stall", |
| "self_destruct_cleanup" |
| ], |
| "matrix": [ |
| [ |
| 106, |
| 2, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 113, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 4, |
| 19, |
| 7, |
| 0, |
| 1, |
| 0, |
| 0, |
| 4, |
| 5 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 123, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 121, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 1, |
| 0, |
| 0, |
| 105, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 120, |
| 0, |
| 1, |
| 1 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 107, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 13, |
| 3, |
| 0, |
| 1, |
| 1, |
| 3, |
| 6, |
| 5 |
| ], |
| [ |
| 0, |
| 3, |
| 0, |
| 2, |
| 1, |
| 5, |
| 1, |
| 0, |
| 8, |
| 8 |
| ] |
| ] |
| }, |
| "macro_roc_auc_ovr": 0.9796965243561164 |
| }, |
| "delta_accuracy": -0.0022222222222222365, |
| "delta_macro_f1": -0.015038185355508271 |
| } |
| } |
| } |
