Hugging Face's logo

xpertsystems
/
cyb010-baseline-classifier

Tabular Classification
PyTorch
cybersecurity
siem
security-logs
mitre-attack
apt
synthetic-data
xgboost
baseline
leakage-diagnostic
Eval Results (legacy)
Model card Files
xet
 Community
cyb010-baseline-classifier / ablation_results.json
pradeep-xpert's picture
pradeep-xpert
Initial release: attack_lifecycle_phase 5-class baseline + 11-oracle-path leakage diagnostic
e2c4702 verified
raw
history blame contribute delete
15.5 kB
{
 "purpose": "Quantify how much each feature group contributes to the headline XGBoost score. Identical architecture, same group-aware split, with one feature group dropped at a time.",
 "full_model_metrics": {
 "model": "xgboost",
 "accuracy": 0.9492753623188406,
 "macro_f1": 0.7780594102481514,
 "weighted_f1": 0.9522470071864876,
 "per_class_f1": {
 "benign_background": 0.9975996159385502,
 "initial_access": 0.7196652719665272,
 "lateral_movement": 0.48322147651006714,
 "persistence_establishment": 0.703030303030303,
 "exfiltration_or_impact": 0.9867803837953092
 },
 "confusion_matrix": {
 "labels": [
 "benign_background",
 "initial_access",
 "lateral_movement",
 "persistence_establishment",
 "exfiltration_or_impact"
 ],
 "matrix": [
 [
 2078,
 6,
 0,
 0,
 0
 ],
 [
 4,
 172,
 65,
 6,
 0
 ],
 [
 0,
 38,
 72,
 6,
 2
 ],
 [
 0,
 11,
 22,
 58,
 0
 ],
 [
 0,
 4,
 21,
 4,
 1157
 ]
 ]
 },
 "macro_roc_auc_ovr": 0.9904125505537232
 },
 "ablations": {
 "no_event_class": {
 "n_features": 75,
 "dropped_count": 12,
 "metrics": {
 "model": "xgboost_no_event_class",
 "accuracy": 0.9205582393988191,
 "macro_f1": 0.5968926085832369,
 "weighted_f1": 0.9214122465392139,
 "per_class_f1": {
 "benign_background": 0.9978412089230031,
 "initial_access": 0.5674044265593562,
 "lateral_movement": 0.3170731707317073,
 "persistence_establishment": 0.11965811965811966,
 "exfiltration_or_impact": 0.9824861170439982
 },
 "confusion_matrix": {
 "labels": [
 "benign_background",
 "initial_access",
 "lateral_movement",
 "persistence_establishment",
 "exfiltration_or_impact"
 ],
 "matrix": [
 [
 2080,
 4,
 0,
 0,
 0
 ],
 [
 4,
 141,
 94,
 6,
 2
 ],
 [
 0,
 54,
 52,
 9,
 3
 ],
 [
 1,
 40,
 43,
 7,
 0
 ],
 [
 0,
 11,
 21,
 4,
 1150
 ]
 ]
 },
 "macro_roc_auc_ovr": 0.9722802673741894
 },
 "delta_accuracy": 0.028717122920021487,
 "delta_macro_f1": 0.1811668016649145
 },
 "no_log_source": {
 "n_features": 79,
 "dropped_count": 8,
 "metrics": {
 "model": "xgboost_no_log_source",
 "accuracy": 0.9468599033816425,
 "macro_f1": 0.7655457635864822,
 "weighted_f1": 0.9496485129647918,
 "per_class_f1": {
 "benign_background": 0.9975996159385502,
 "initial_access": 0.7080745341614907,
 "lateral_movement": 0.4536082474226804,
 "persistence_establishment": 0.6829268292682927,
 "exfiltration_or_impact": 0.985519591141397
 },
 "confusion_matrix": {
 "labels": [
 "benign_background",
 "initial_access",
 "lateral_movement",
 "persistence_establishment",
 "exfiltration_or_impact"
 ],
 "matrix": [
 [
 2078,
 6,
 0,
 0,
 0
 ],
 [
 4,
 171,
 65,
 6,
 1
 ],
 [
 0,
 43,
 66,
 7,
 2
 ],
 [
 0,
 12,
 21,
 56,
 2
 ],
 [
 0,
 4,
 21,
 4,
 1157
 ]
 ]
 },
 "macro_roc_auc_ovr": 0.9902223408149018
 },
 "delta_accuracy": 0.0024154589371980784,
 "delta_macro_f1": 0.012513646661669209
 },
 "no_severity": {
 "n_features": 82,
 "dropped_count": 5,
 "metrics": {
 "model": "xgboost_no_severity",
 "accuracy": 0.9479334406870639,
 "macro_f1": 0.7688286964848263,
 "weighted_f1": 0.9505815101921871,
 "per_class_f1": {
 "benign_background": 0.9971195391262602,
 "initial_access": 0.7213114754098361,
 "lateral_movement": 0.4689655172413793,
 "persistence_establishment": 0.6708074534161491,
 "exfiltration_or_impact": 0.985939497230507
 },
 "confusion_matrix": {
 "labels": [
 "benign_background",
 "initial_access",
 "lateral_movement",
 "persistence_establishment",
 "exfiltration_or_impact"
 ],
 "matrix": [
 [
 2077,
 7,
 0,
 0,
 0
 ],
 [
 4,
 176,
 60,
 7,
 0
 ],
 [
 0,
 42,
 68,
 5,
 3
 ],
 [
 1,
 12,
 23,
 54,
 1
 ],
 [
 0,
 4,
 21,
 4,
 1157
 ]
 ]
 },
 "macro_roc_auc_ovr": 0.9901923411691304
 },
 "delta_accuracy": 0.0013419216317767102,
 "delta_macro_f1": 0.009230713763325071
 },
 "no_cvss": {
 "n_features": 84,
 "dropped_count": 3,
 "metrics": {
 "model": "xgboost_no_cvss",
 "accuracy": 0.9382716049382716,
 "macro_f1": 0.7475120671323378,
 "weighted_f1": 0.940926432572893,
 "per_class_f1": {
 "benign_background": 0.9930737998566993,
 "initial_access": 0.6948775055679287,
 "lateral_movement": 0.43278688524590164,
 "persistence_establishment": 0.6428571428571429,
 "exfiltration_or_impact": 0.9739650021340163
 },
 "confusion_matrix": {
 "labels": [
 "benign_background",
 "initial_access",
 "lateral_movement",
 "persistence_establishment",
 "exfiltration_or_impact"
 ],
 "matrix": [
 [
 2079,
 4,
 0,
 0,
 1
 ],
 [
 12,
 156,
 60,
 14,
 5
 ],
 [
 6,
 31,
 66,
 5,
 10
 ],
 [
 6,
 8,
 23,
 54,
 0
 ],
 [
 0,
 3,
 38,
 4,
 1141
 ]
 ]
 },
 "macro_roc_auc_ovr": 0.9812083795500166
 },
 "delta_accuracy": 0.011003757380569024,
 "delta_macro_f1": 0.03054734311581364
 },
 "no_host": {
 "n_features": 39,
 "dropped_count": 48,
 "metrics": {
 "model": "xgboost_no_host",
 "accuracy": 0.9522275899087493,
 "macro_f1": 0.7828011365615016,
 "weighted_f1": 0.9541737562003638,
 "per_class_f1": {
 "benign_background": 0.9983217453847998,
 "initial_access": 0.746268656716418,
 "lateral_movement": 0.4962962962962963,
 "persistence_establishment": 0.6871794871794872,
 "exfiltration_or_impact": 0.985939497230507
 },
 "confusion_matrix": {
 "labels": [
 "benign_background",
 "initial_access",
 "lateral_movement",
 "persistence_establishment",
 "exfiltration_or_impact"
 ],
 "matrix": [
 [
 2082,
 1,
 0,
 1,
 0
 ],
 [
 4,
 175,
 49,
 18,
 1
 ],
 [
 0,
 36,
 67,
 13,
 2
 ],
 [
 1,
 6,
 16,
 67,
 1
 ],
 [
 0,
 4,
 20,
 5,
 1157
 ]
 ]
 },
 "macro_roc_auc_ovr": 0.9917448228530954
 },
 "delta_accuracy": -0.0029522275899087624,
 "delta_macro_f1": -0.004741726313350236
 },
 "no_timing": {
 "n_features": 84,
 "dropped_count": 3,
 "metrics": {
 "model": "xgboost_no_timing",
 "accuracy": 0.9500805152979066,
 "macro_f1": 0.7730074031058032,
 "weighted_f1": 0.9527084816660557,
 "per_class_f1": {
 "benign_background": 0.9990407673860912,
 "initial_access": 0.7326315789473684,
 "lateral_movement": 0.48484848484848486,
 "persistence_establishment": 0.6625766871165644,
 "exfiltration_or_impact": 0.985939497230507
 },
 "confusion_matrix": {
 "labels": [
 "benign_background",
 "initial_access",
 "lateral_movement",
 "persistence_establishment",
 "exfiltration_or_impact"
 ],
 "matrix": [
 [
 2083,
 1,
 0,
 0,
 0
 ],
 [
 3,
 174,
 60,
 8,
 2
 ],
 [
 0,
 39,
 72,
 5,
 2
 ],
 [
 0,
 9,
 28,
 54,
 0
 ],
 [
 0,
 5,
 19,
 5,
 1157
 ]
 ]
 },
 "macro_roc_auc_ovr": 0.9906863118522171
 },
 "delta_accuracy": -0.0008051529790660261,
 "delta_macro_f1": 0.005052007142348214
 },
 "no_ports": {
 "n_features": 82,
 "dropped_count": 5,
 "metrics": {
 "model": "xgboost_no_ports",
 "accuracy": 0.9463231347289318,
 "macro_f1": 0.7620715002556177,
 "weighted_f1": 0.949550457691939,
 "per_class_f1": {
 "benign_background": 0.9978401727861771,
 "initial_access": 0.7036247334754797,
 "lateral_movement": 0.45544554455445546,
 "persistence_establishment": 0.6666666666666666,
 "exfiltration_or_impact": 0.9867803837953092
 },
 "confusion_matrix": {
 "labels": [
 "benign_background",
 "initial_access",
 "lateral_movement",
 "persistence_establishment",
 "exfiltration_or_impact"
 ],
 "matrix": [
 [
 2079,
 5,
 0,
 0,
 0
 ],
 [
 4,
 165,
 72,
 6,
 0
 ],
 [
 0,
 38,
 69,
 9,
 2
 ],
 [
 0,
 11,
 24,
 56,
 0
 ],
 [
 0,
 3,
 20,
 6,
 1157
 ]
 ]
 },
 "macro_roc_auc_ovr": 0.9902855327593585
 },
 "delta_accuracy": 0.0029522275899087624,
 "delta_macro_f1": 0.015987909992533744
 },
 "no_engineered": {
 "n_features": 79,
 "dropped_count": 8,
 "metrics": {
 "model": "xgboost_no_engineered",
 "accuracy": 0.9471282877079978,
 "macro_f1": 0.7655097846280253,
 "weighted_f1": 0.9499972622574527,
 "per_class_f1": {
 "benign_background": 0.9975984630163305,
 "initial_access": 0.7166666666666667,
 "lateral_movement": 0.4697986577181208,
 "persistence_establishment": 0.6583850931677019,
 "exfiltration_or_impact": 0.9851000425713069
 },
 "confusion_matrix": {
 "labels": [
 "benign_background",
 "initial_access",
 "lateral_movement",
 "persistence_establishment",
 "exfiltration_or_impact"
 ],
 "matrix": [
 [
 2077,
 7,
 0,
 0,
 0
 ],
 [
 3,
 172,
 63,
 8,
 1
 ],
 [
 0,
 40,
 70,
 5,
 3
 ],
 [
 0,
 10,
 26,
 53,
 2
 ],
 [
 0,
 4,
 21,
 4,
 1157
 ]
 ]
 },
 "macro_roc_auc_ovr": 0.9903013631552575
 },
 "delta_accuracy": 0.0021470746108427363,
 "delta_macro_f1": 0.01254962562012607
 },
 "no_tamper": {
 "n_features": 85,
 "dropped_count": 2,
 "metrics": {
 "model": "xgboost_no_tamper",
 "accuracy": 0.9468599033816425,
 "macro_f1": 0.7656884000157337,
 "weighted_f1": 0.9499631319237402,
 "per_class_f1": {
 "benign_background": 0.9980806142034548,
 "initial_access": 0.7048832271762208,
 "lateral_movement": 0.4605263157894737,
 "persistence_establishment": 0.6790123456790124,
 "exfiltration_or_impact": 0.985939497230507
 },
 "confusion_matrix": {
 "labels": [
 "benign_background",
 "initial_access",
 "lateral_movement",
 "persistence_establishment",
 "exfiltration_or_impact"
 ],
 "matrix": [
 [
 2080,
 4,
 0,
 0,
 0
 ],
 [
 4,
 166,
 70,
 6,
 1
 ],
 [
 0,
 39,
 70,
 7,
 2
 ],
 [
 0,
 11,
 24,
 55,
 1
 ],
 [
 0,
 4,
 22,
 3,
 1157
 ]
 ]
 },
 "macro_roc_auc_ovr": 0.9904534455006762
 },
 "delta_accuracy": 0.0024154589371980784,
 "delta_macro_f1": 0.012371010232417712
 }
 }
}