| { |
| "purpose": "Quantify how much each feature group contributes to the headline XGBoost score. Identical architecture, same group-aware split, with one feature group dropped at a time.", |
| "full_model_metrics": { |
| "model": "xgboost", |
| "accuracy": 0.46831955922865015, |
| "macro_f1": 0.42549880749552066, |
| "weighted_f1": 0.440668872633435, |
| "per_class_f1": { |
| "dwell_idle": 0.040268456375838924, |
| "reconnaissance": 0.7532467532467533, |
| "initial_access": 0.6467661691542289, |
| "execution": 0.4406779661016949, |
| "persistence": 0.41304347826086957, |
| "privilege_escalation": 0.5, |
| "lateral_movement": 0.7422680412371134, |
| "collection": 0.22018348623853212, |
| "exfiltration": 0.2727272727272727, |
| "impact": 0.22580645161290322 |
| }, |
| "confusion_matrix": { |
| "labels": [ |
| "dwell_idle", |
| "reconnaissance", |
| "initial_access", |
| "execution", |
| "persistence", |
| "privilege_escalation", |
| "lateral_movement", |
| "collection", |
| "exfiltration", |
| "impact" |
| ], |
| "matrix": [ |
| [ |
| 3, |
| 23, |
| 23, |
| 18, |
| 21, |
| 18, |
| 2, |
| 17, |
| 9, |
| 7 |
| ], |
| [ |
| 2, |
| 87, |
| 2, |
| 21, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 1, |
| 5, |
| 65, |
| 5, |
| 3, |
| 26, |
| 1, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 2, |
| 4, |
| 1, |
| 39, |
| 24, |
| 3, |
| 1, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 1, |
| 12, |
| 38, |
| 9, |
| 0, |
| 18, |
| 1, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 3, |
| 8, |
| 4, |
| 44, |
| 3, |
| 5, |
| 1, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 6, |
| 6, |
| 36, |
| 2, |
| 0, |
| 4 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 2, |
| 1, |
| 0, |
| 12, |
| 15, |
| 10 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 5, |
| 0, |
| 0, |
| 4, |
| 9, |
| 13 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 2, |
| 1, |
| 0, |
| 11, |
| 0, |
| 7 |
| ] |
| ] |
| }, |
| "macro_roc_auc_ovr": 0.8598653258869782 |
| }, |
| "ablations": { |
| "no_topology": { |
| "n_features": 67, |
| "dropped_count": 23, |
| "metrics": { |
| "model": "xgboost_no_topology", |
| "accuracy": 0.46005509641873277, |
| "macro_f1": 0.4093395066167947, |
| "weighted_f1": 0.4281869072634682, |
| "per_class_f1": { |
| "dwell_idle": 0.013513513513513514, |
| "reconnaissance": 0.7574468085106383, |
| "initial_access": 0.6435643564356436, |
| "execution": 0.45348837209302323, |
| "persistence": 0.3829787234042553, |
| "privilege_escalation": 0.4943820224719101, |
| "lateral_movement": 0.72, |
| "collection": 0.205607476635514, |
| "exfiltration": 0.25, |
| "impact": 0.1724137931034483 |
| }, |
| "confusion_matrix": { |
| "labels": [ |
| "dwell_idle", |
| "reconnaissance", |
| "initial_access", |
| "execution", |
| "persistence", |
| "privilege_escalation", |
| "lateral_movement", |
| "collection", |
| "exfiltration", |
| "impact" |
| ], |
| "matrix": [ |
| [ |
| 1, |
| 24, |
| 24, |
| 16, |
| 24, |
| 16, |
| 4, |
| 15, |
| 10, |
| 7 |
| ], |
| [ |
| 2, |
| 89, |
| 2, |
| 16, |
| 3, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 1, |
| 6, |
| 65, |
| 4, |
| 3, |
| 26, |
| 1, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 1, |
| 4, |
| 1, |
| 39, |
| 25, |
| 3, |
| 1, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 1, |
| 0, |
| 0, |
| 16, |
| 36, |
| 9, |
| 0, |
| 16, |
| 1, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 3, |
| 7, |
| 4, |
| 44, |
| 3, |
| 5, |
| 2, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 1, |
| 0, |
| 5, |
| 9, |
| 36, |
| 2, |
| 0, |
| 1 |
| ], |
| [ |
| 1, |
| 0, |
| 0, |
| 0, |
| 2, |
| 2, |
| 1, |
| 11, |
| 11, |
| 12 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 5, |
| 0, |
| 0, |
| 6, |
| 8, |
| 12 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 2, |
| 1, |
| 0, |
| 12, |
| 1, |
| 5 |
| ] |
| ] |
| }, |
| "macro_roc_auc_ovr": 0.8625474585447981 |
| }, |
| "delta_accuracy": 0.008264462809917383, |
| "delta_macro_f1": 0.01615930087872597 |
| }, |
| "no_engineered": { |
| "n_features": 84, |
| "dropped_count": 6, |
| "metrics": { |
| "model": "xgboost_no_engineered", |
| "accuracy": 0.4641873278236915, |
| "macro_f1": 0.4239556593623024, |
| "weighted_f1": 0.4373277421758876, |
| "per_class_f1": { |
| "dwell_idle": 0.02631578947368421, |
| "reconnaissance": 0.7368421052631579, |
| "initial_access": 0.6305418719211823, |
| "execution": 0.46060606060606063, |
| "persistence": 0.4419889502762431, |
| "privilege_escalation": 0.49142857142857144, |
| "lateral_movement": 0.7346938775510204, |
| "collection": 0.24347826086956523, |
| "exfiltration": 0.2647058823529412, |
| "impact": 0.208955223880597 |
| }, |
| "confusion_matrix": { |
| "labels": [ |
| "dwell_idle", |
| "reconnaissance", |
| "initial_access", |
| "execution", |
| "persistence", |
| "privilege_escalation", |
| "lateral_movement", |
| "collection", |
| "exfiltration", |
| "impact" |
| ], |
| "matrix": [ |
| [ |
| 2, |
| 23, |
| 24, |
| 14, |
| 23, |
| 20, |
| 2, |
| 17, |
| 9, |
| 7 |
| ], |
| [ |
| 4, |
| 84, |
| 3, |
| 21, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 2, |
| 5, |
| 64, |
| 4, |
| 1, |
| 29, |
| 1, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 3, |
| 4, |
| 1, |
| 38, |
| 25, |
| 2, |
| 1, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 2, |
| 7, |
| 40, |
| 9, |
| 0, |
| 20, |
| 1, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 3, |
| 7, |
| 5, |
| 43, |
| 4, |
| 5, |
| 1, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 3, |
| 36, |
| 4, |
| 4, |
| 7 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 1, |
| 0, |
| 0, |
| 14, |
| 13, |
| 12 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 5, |
| 0, |
| 0, |
| 4, |
| 9, |
| 13 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 2, |
| 1, |
| 0, |
| 11, |
| 0, |
| 7 |
| ] |
| ] |
| }, |
| "macro_roc_auc_ovr": 0.8559080760692732 |
| }, |
| "delta_accuracy": 0.004132231404958664, |
| "delta_macro_f1": 0.001543148133218264 |
| }, |
| "no_timestep": { |
| "n_features": 89, |
| "dropped_count": 1, |
| "metrics": { |
| "model": "xgboost_no_timestep", |
| "accuracy": 0.32644628099173556, |
| "macro_f1": 0.31019209599143654, |
| "weighted_f1": 0.3273550154519158, |
| "per_class_f1": { |
| "dwell_idle": 0.06060606060606061, |
| "reconnaissance": 0.3728813559322034, |
| "initial_access": 0.5666666666666667, |
| "execution": 0.4090909090909091, |
| "persistence": 0.22818791946308725, |
| "privilege_escalation": 0.4520547945205479, |
| "lateral_movement": 0.7058823529411765, |
| "collection": 0.0975609756097561, |
| "exfiltration": 0.1836734693877551, |
| "impact": 0.02531645569620253 |
| }, |
| "confusion_matrix": { |
| "labels": [ |
| "dwell_idle", |
| "reconnaissance", |
| "initial_access", |
| "execution", |
| "persistence", |
| "privilege_escalation", |
| "lateral_movement", |
| "collection", |
| "exfiltration", |
| "impact" |
| ], |
| "matrix": [ |
| [ |
| 5, |
| 11, |
| 35, |
| 11, |
| 17, |
| 13, |
| 1, |
| 25, |
| 15, |
| 8 |
| ], |
| [ |
| 7, |
| 33, |
| 1, |
| 11, |
| 11, |
| 0, |
| 0, |
| 19, |
| 17, |
| 13 |
| ], |
| [ |
| 5, |
| 0, |
| 68, |
| 1, |
| 2, |
| 16, |
| 5, |
| 6, |
| 3, |
| 0 |
| ], |
| [ |
| 3, |
| 6, |
| 1, |
| 27, |
| 4, |
| 4, |
| 2, |
| 20, |
| 2, |
| 5 |
| ], |
| [ |
| 2, |
| 12, |
| 4, |
| 1, |
| 17, |
| 5, |
| 0, |
| 19, |
| 6, |
| 13 |
| ], |
| [ |
| 0, |
| 0, |
| 17, |
| 7, |
| 2, |
| 33, |
| 3, |
| 3, |
| 2, |
| 1 |
| ], |
| [ |
| 0, |
| 1, |
| 7, |
| 0, |
| 2, |
| 2, |
| 36, |
| 1, |
| 0, |
| 5 |
| ], |
| [ |
| 0, |
| 2, |
| 0, |
| 0, |
| 6, |
| 4, |
| 1, |
| 8, |
| 12, |
| 7 |
| ], |
| [ |
| 1, |
| 0, |
| 1, |
| 0, |
| 7, |
| 0, |
| 0, |
| 8, |
| 9, |
| 5 |
| ], |
| [ |
| 1, |
| 0, |
| 0, |
| 0, |
| 2, |
| 1, |
| 0, |
| 15, |
| 1, |
| 1 |
| ] |
| ] |
| }, |
| "macro_roc_auc_ovr": 0.7557281412642529 |
| }, |
| "delta_accuracy": 0.1418732782369146, |
| "delta_macro_f1": 0.11530671150408411 |
| }, |
| "no_detection_signals": { |
| "n_features": 76, |
| "dropped_count": 14, |
| "metrics": { |
| "model": "xgboost_no_detection_signals", |
| "accuracy": 0.4724517906336088, |
| "macro_f1": 0.4284152317167137, |
| "weighted_f1": 0.4449655177644492, |
| "per_class_f1": { |
| "dwell_idle": 0.039735099337748346, |
| "reconnaissance": 0.7456140350877193, |
| "initial_access": 0.6600985221674877, |
| "execution": 0.47126436781609193, |
| "persistence": 0.43333333333333335, |
| "privilege_escalation": 0.4971751412429379, |
| "lateral_movement": 0.7272727272727273, |
| "collection": 0.21818181818181817, |
| "exfiltration": 0.2727272727272727, |
| "impact": 0.21875 |
| }, |
| "confusion_matrix": { |
| "labels": [ |
| "dwell_idle", |
| "reconnaissance", |
| "initial_access", |
| "execution", |
| "persistence", |
| "privilege_escalation", |
| "lateral_movement", |
| "collection", |
| "exfiltration", |
| "impact" |
| ], |
| "matrix": [ |
| [ |
| 3, |
| 23, |
| 23, |
| 18, |
| 22, |
| 17, |
| 3, |
| 16, |
| 9, |
| 7 |
| ], |
| [ |
| 2, |
| 85, |
| 3, |
| 22, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 1, |
| 5, |
| 67, |
| 2, |
| 2, |
| 28, |
| 1, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 2, |
| 3, |
| 1, |
| 41, |
| 23, |
| 3, |
| 1, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 0, |
| 1, |
| 9, |
| 39, |
| 9, |
| 0, |
| 19, |
| 1, |
| 1 |
| ], |
| [ |
| 0, |
| 0, |
| 2, |
| 8, |
| 3, |
| 44, |
| 4, |
| 6, |
| 1, |
| 0 |
| ], |
| [ |
| 1, |
| 0, |
| 0, |
| 0, |
| 3, |
| 6, |
| 36, |
| 2, |
| 0, |
| 6 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 2, |
| 1, |
| 0, |
| 12, |
| 15, |
| 10 |
| ], |
| [ |
| 1, |
| 0, |
| 0, |
| 0, |
| 5, |
| 0, |
| 0, |
| 4, |
| 9, |
| 12 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 2, |
| 1, |
| 0, |
| 11, |
| 0, |
| 7 |
| ] |
| ] |
| }, |
| "macro_roc_auc_ovr": 0.8544378745036634 |
| }, |
| "delta_accuracy": -0.004132231404958664, |
| "delta_macro_f1": -0.002916424221193037 |
| } |
| } |
| } |
