Hugging Face's logo

xpertsystems
/
cyb008-baseline-classifier

Tabular Classification
PyTorch
cybersecurity
soc-operations
alert-triage
mitre-attack
soar
siem
synthetic-data
xgboost
baseline
leakage-diagnostic
Eval Results (legacy)
Model card Files
xet
 Community
cyb008-baseline-classifier / ablation_results.json
pradeep-xpert's picture
pradeep-xpert
Initial release: XGBoost + MLP for SOC alert triage outcome classification, with structural-leakage and unlearnable-target diagnostic
001717c verified
raw
history blame contribute delete
15.6 kB
{
 "purpose": "Quantify how much each feature group contributes to the headline XGBoost score. Identical architecture, same stratified split, with one feature group dropped at a time.",
 "full_model_metrics": {
 "model": "xgboost",
 "accuracy": 0.7659420289855072,
 "macro_f1": 0.7429876131468711,
 "weighted_f1": 0.7669168766123218,
 "per_class_f1": {
 "auto_resolved_soar": 0.7572383073496659,
 "duplicate_merged": 0.7936507936507936,
 "false_positive_closed": 0.9038461538461539,
 "true_positive_remediated": 0.7012987012987013,
 "true_positive_escalated": 0.5589041095890411
 },
 "confusion_matrix": {
 "labels": [
 "auto_resolved_soar",
 "duplicate_merged",
 "false_positive_closed",
 "true_positive_remediated",
 "true_positive_escalated"
 ],
 "matrix": [
 [
 340,
 17,
 6,
 16,
 17
 ],
 [
 9,
 50,
 0,
 0,
 0
 ],
 [
 74,
 0,
 376,
 0,
 0
 ],
 [
 40,
 0,
 0,
 189,
 48
 ],
 [
 39,
 0,
 0,
 57,
 102
 ]
 ]
 },
 "macro_roc_auc_ovr": 0.9522005654044479
 },
 "ablations": {
 "no_severity": {
 "n_features": 46,
 "dropped_count": 7,
 "metrics": {
 "model": "xgboost_no_severity",
 "accuracy": 0.513768115942029,
 "macro_f1": 0.39328452803110936,
 "weighted_f1": 0.48887003837655496,
 "per_class_f1": {
 "auto_resolved_soar": 0.8058455114822547,
 "duplicate_merged": 0.0,
 "false_positive_closed": 0.4,
 "true_positive_remediated": 0.3155893536121673,
 "true_positive_escalated": 0.4449877750611247
 },
 "confusion_matrix": {
 "labels": [
 "auto_resolved_soar",
 "duplicate_merged",
 "false_positive_closed",
 "true_positive_remediated",
 "true_positive_escalated"
 ],
 "matrix": [
 [
 386,
 1,
 1,
 3,
 5
 ],
 [
 15,
 0,
 17,
 21,
 6
 ],
 [
 75,
 30,
 149,
 122,
 74
 ],
 [
 42,
 26,
 91,
 83,
 35
 ],
 [
 44,
 6,
 37,
 20,
 91
 ]
 ]
 },
 "macro_roc_auc_ovr": 0.7303857388456401
 },
 "delta_accuracy": 0.25217391304347825,
 "delta_macro_f1": 0.34970308511576176
 },
 "no_alert_source": {
 "n_features": 45,
 "dropped_count": 8,
 "metrics": {
 "model": "xgboost_no_alert_source",
 "accuracy": 0.763768115942029,
 "macro_f1": 0.7406277489805807,
 "weighted_f1": 0.764708131838635,
 "per_class_f1": {
 "auto_resolved_soar": 0.755011135857461,
 "duplicate_merged": 0.784,
 "false_positive_closed": 0.8984468339307049,
 "true_positive_remediated": 0.6981132075471698,
 "true_positive_escalated": 0.5675675675675675
 },
 "confusion_matrix": {
 "labels": [
 "auto_resolved_soar",
 "duplicate_merged",
 "false_positive_closed",
 "true_positive_remediated",
 "true_positive_escalated"
 ],
 "matrix": [
 [
 339,
 17,
 11,
 13,
 16
 ],
 [
 10,
 49,
 0,
 0,
 0
 ],
 [
 74,
 0,
 376,
 0,
 0
 ],
 [
 41,
 0,
 0,
 185,
 51
 ],
 [
 38,
 0,
 0,
 55,
 105
 ]
 ]
 },
 "macro_roc_auc_ovr": 0.9511218248098263
 },
 "delta_accuracy": 0.0021739130434782483,
 "delta_macro_f1": 0.002359864166290415
 },
 "no_tactic": {
 "n_features": 41,
 "dropped_count": 12,
 "metrics": {
 "model": "xgboost_no_tactic",
 "accuracy": 0.7811594202898551,
 "macro_f1": 0.7655889644647986,
 "weighted_f1": 0.7823552630061641,
 "per_class_f1": {
 "auto_resolved_soar": 0.7717750826901875,
 "duplicate_merged": 0.8346456692913385,
 "false_positive_closed": 0.908433734939759,
 "true_positive_remediated": 0.7088122605363985,
 "true_positive_escalated": 0.6042780748663101
 },
 "confusion_matrix": {
 "labels": [
 "auto_resolved_soar",
 "duplicate_merged",
 "false_positive_closed",
 "true_positive_remediated",
 "true_positive_escalated"
 ],
 "matrix": [
 [
 350,
 15,
 3,
 15,
 13
 ],
 [
 6,
 53,
 0,
 0,
 0
 ],
 [
 73,
 0,
 377,
 0,
 0
 ],
 [
 42,
 0,
 0,
 185,
 50
 ],
 [
 40,
 0,
 0,
 45,
 113
 ]
 ]
 },
 "macro_roc_auc_ovr": 0.9529923809161402
 },
 "delta_accuracy": -0.01521739130434785,
 "delta_macro_f1": -0.02260135131792751
 },
 "no_siem": {
 "n_features": 45,
 "dropped_count": 8,
 "metrics": {
 "model": "xgboost_no_siem",
 "accuracy": 0.7681159420289855,
 "macro_f1": 0.747392848800313,
 "weighted_f1": 0.7695871955675133,
 "per_class_f1": {
 "auto_resolved_soar": 0.7577777777777778,
 "duplicate_merged": 0.8,
 "false_positive_closed": 0.9025270758122743,
 "true_positive_remediated": 0.706766917293233,
 "true_positive_escalated": 0.5698924731182796
 },
 "confusion_matrix": {
 "labels": [
 "auto_resolved_soar",
 "duplicate_merged",
 "false_positive_closed",
 "true_positive_remediated",
 "true_positive_escalated"
 ],
 "matrix": [
 [
 341,
 16,
 6,
 15,
 18
 ],
 [
 9,
 50,
 0,
 0,
 0
 ],
 [
 75,
 0,
 375,
 0,
 0
 ],
 [
 39,
 0,
 0,
 188,
 50
 ],
 [
 40,
 0,
 0,
 52,
 106
 ]
 ]
 },
 "macro_roc_auc_ovr": 0.9521514669693077
 },
 "delta_accuracy": -0.0021739130434782483,
 "delta_macro_f1": -0.0044052356534418635
 },
 "no_analyst_tier": {
 "n_features": 50,
 "dropped_count": 3,
 "metrics": {
 "model": "xgboost_no_analyst_tier",
 "accuracy": 0.7717391304347826,
 "macro_f1": 0.7470947169858246,
 "weighted_f1": 0.7727339237745289,
 "per_class_f1": {
 "auto_resolved_soar": 0.768893756845564,
 "duplicate_merged": 0.784,
 "false_positive_closed": 0.9071170084439083,
 "true_positive_remediated": 0.6948176583493282,
 "true_positive_escalated": 0.5806451612903226
 },
 "confusion_matrix": {
 "labels": [
 "auto_resolved_soar",
 "duplicate_merged",
 "false_positive_closed",
 "true_positive_remediated",
 "true_positive_escalated"
 ],
 "matrix": [
 [
 351,
 17,
 3,
 14,
 11
 ],
 [
 10,
 49,
 0,
 0,
 0
 ],
 [
 74,
 0,
 376,
 0,
 0
 ],
 [
 41,
 0,
 0,
 181,
 55
 ],
 [
 41,
 0,
 0,
 49,
 108
 ]
 ]
 },
 "macro_roc_auc_ovr": 0.9524262361561989
 },
 "delta_accuracy": -0.005797101449275366,
 "delta_macro_f1": -0.004107103838953519
 },
 "no_timing": {
 "n_features": 48,
 "dropped_count": 5,
 "metrics": {
 "model": "xgboost_no_timing",
 "accuracy": 0.777536231884058,
 "macro_f1": 0.7572452763946715,
 "weighted_f1": 0.7795520836463574,
 "per_class_f1": {
 "auto_resolved_soar": 0.7676991150442478,
 "duplicate_merged": 0.8031496062992126,
 "false_positive_closed": 0.9071170084439083,
 "true_positive_remediated": 0.723404255319149,
 "true_positive_escalated": 0.5848563968668408
 },
 "confusion_matrix": {
 "labels": [
 "auto_resolved_soar",
 "duplicate_merged",
 "false_positive_closed",
 "true_positive_remediated",
 "true_positive_escalated"
 ],
 "matrix": [
 [
 347,
 17,
 3,
 9,
 20
 ],
 [
 8,
 51,
 0,
 0,
 0
 ],
 [
 74,
 0,
 376,
 0,
 0
 ],
 [
 37,
 0,
 0,
 187,
 53
 ],
 [
 42,
 0,
 0,
 44,
 112
 ]
 ]
 },
 "macro_roc_auc_ovr": 0.9546713378957848
 },
 "delta_accuracy": -0.011594202898550732,
 "delta_macro_f1": -0.014257663247800423
 },
 "no_scores": {
 "n_features": 48,
 "dropped_count": 5,
 "metrics": {
 "model": "xgboost_no_scores",
 "accuracy": 0.7710144927536232,
 "macro_f1": 0.7569411600325896,
 "weighted_f1": 0.7729871790343515,
 "per_class_f1": {
 "auto_resolved_soar": 0.7531285551763367,
 "duplicate_merged": 0.8253968253968254,
 "false_positive_closed": 0.9019138755980861,
 "true_positive_remediated": 0.7047970479704797,
 "true_positive_escalated": 0.5994694960212201
 },
 "confusion_matrix": {
 "labels": [
 "auto_resolved_soar",
 "duplicate_merged",
 "false_positive_closed",
 "true_positive_remediated",
 "true_positive_escalated"
 ],
 "matrix": [
 [
 331,
 15,
 9,
 23,
 18
 ],
 [
 7,
 52,
 0,
 0,
 0
 ],
 [
 73,
 0,
 377,
 0,
 0
 ],
 [
 38,
 0,
 0,
 191,
 48
 ],
 [
 34,
 0,
 0,
 51,
 113
 ]
 ]
 },
 "macro_roc_auc_ovr": 0.9541430544791097
 },
 "delta_accuracy": -0.005072463768115987,
 "delta_macro_f1": -0.013953546885718482
 },
 "no_soar": {
 "n_features": 52,
 "dropped_count": 1,
 "metrics": {
 "model": "xgboost_no_soar",
 "accuracy": 0.618840579710145,
 "macro_f1": 0.5773360587813117,
 "weighted_f1": 0.5258347983183296,
 "per_class_f1": {
 "auto_resolved_soar": 0.028846153846153848,
 "duplicate_merged": 0.8194444444444444,
 "false_positive_closed": 0.8424068767908309,
 "true_positive_remediated": 0.6328358208955224,
 "true_positive_escalated": 0.5631469979296067
 },
 "confusion_matrix": {
 "labels": [
 "auto_resolved_soar",
 "duplicate_merged",
 "false_positive_closed",
 "true_positive_remediated",
 "true_positive_escalated"
 ],
 "matrix": [
 [
 6,
 26,
 156,
 122,
 86
 ],
 [
 0,
 59,
 0,
 0,
 0
 ],
 [
 9,
 0,
 441,
 0,
 0
 ],
 [
 2,
 0,
 0,
 212,
 63
 ],
 [
 3,
 0,
 0,
 59,
 136
 ]
 ]
 },
 "macro_roc_auc_ovr": 0.8369099942380366
 },
 "delta_accuracy": 0.14710144927536228,
 "delta_macro_f1": 0.16565155436555945
 },
 "no_engineered": {
 "n_features": 47,
 "dropped_count": 6,
 "metrics": {
 "model": "xgboost_no_engineered",
 "accuracy": 0.7681159420289855,
 "macro_f1": 0.7479996795268518,
 "weighted_f1": 0.7700206321761683,
 "per_class_f1": {
 "auto_resolved_soar": 0.7542087542087542,
 "duplicate_merged": 0.796875,
 "false_positive_closed": 0.9027611044417767,
 "true_positive_remediated": 0.7094339622641509,
 "true_positive_escalated": 0.5767195767195767
 },
 "confusion_matrix": {
 "labels": [
 "auto_resolved_soar",
 "duplicate_merged",
 "false_positive_closed",
 "true_positive_remediated",
 "true_positive_escalated"
 ],
 "matrix": [
 [
 336,
 18,
 7,
 13,
 22
 ],
 [
 8,
 51,
 0,
 0,
 0
 ],
 [
 74,
 0,
 376,
 0,
 0
 ],
 [
 40,
 0,
 0,
 188,
 49
 ],
 [
 37,
 0,
 0,
 52,
 109
 ]
 ]
 },
 "macro_roc_auc_ovr": 0.9533153727185603
 },
 "delta_accuracy": -0.0021739130434782483,
 "delta_macro_f1": -0.00501206637998064
 }
 }
}