{
  "purpose": "Quantify how much each feature group contributes to the headline XGBoost score. Identical architecture, same stratified split, with one feature group dropped at a time.",
  "full_model_metrics": {
    "model": "xgboost",
    "accuracy": 0.7659420289855072,
    "macro_f1": 0.7429876131468711,
    "weighted_f1": 0.7669168766123218,
    "per_class_f1": {
      "auto_resolved_soar": 0.7572383073496659,
      "duplicate_merged": 0.7936507936507936,
      "false_positive_closed": 0.9038461538461539,
      "true_positive_remediated": 0.7012987012987013,
      "true_positive_escalated": 0.5589041095890411
    },
    "confusion_matrix": {
      "labels": [
        "auto_resolved_soar",
        "duplicate_merged",
        "false_positive_closed",
        "true_positive_remediated",
        "true_positive_escalated"
      ],
      "matrix": [
        [
          340,
          17,
          6,
          16,
          17
        ],
        [
          9,
          50,
          0,
          0,
          0
        ],
        [
          74,
          0,
          376,
          0,
          0
        ],
        [
          40,
          0,
          0,
          189,
          48
        ],
        [
          39,
          0,
          0,
          57,
          102
        ]
      ]
    },
    "macro_roc_auc_ovr": 0.9522005654044479
  },
  "ablations": {
    "no_severity": {
      "n_features": 46,
      "dropped_count": 7,
      "metrics": {
        "model": "xgboost_no_severity",
        "accuracy": 0.513768115942029,
        "macro_f1": 0.39328452803110936,
        "weighted_f1": 0.48887003837655496,
        "per_class_f1": {
          "auto_resolved_soar": 0.8058455114822547,
          "duplicate_merged": 0.0,
          "false_positive_closed": 0.4,
          "true_positive_remediated": 0.3155893536121673,
          "true_positive_escalated": 0.4449877750611247
        },
        "confusion_matrix": {
          "labels": [
            "auto_resolved_soar",
            "duplicate_merged",
            "false_positive_closed",
            "true_positive_remediated",
            "true_positive_escalated"
          ],
          "matrix": [
            [
              386,
              1,
              1,
              3,
              5
            ],
            [
              15,
              0,
              17,
              21,
              6
            ],
            [
              75,
              30,
              149,
              122,
              74
            ],
            [
              42,
              26,
              91,
              83,
              35
            ],
            [
              44,
              6,
              37,
              20,
              91
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.7303857388456401
      },
      "delta_accuracy": 0.25217391304347825,
      "delta_macro_f1": 0.34970308511576176
    },
    "no_alert_source": {
      "n_features": 45,
      "dropped_count": 8,
      "metrics": {
        "model": "xgboost_no_alert_source",
        "accuracy": 0.763768115942029,
        "macro_f1": 0.7406277489805807,
        "weighted_f1": 0.764708131838635,
        "per_class_f1": {
          "auto_resolved_soar": 0.755011135857461,
          "duplicate_merged": 0.784,
          "false_positive_closed": 0.8984468339307049,
          "true_positive_remediated": 0.6981132075471698,
          "true_positive_escalated": 0.5675675675675675
        },
        "confusion_matrix": {
          "labels": [
            "auto_resolved_soar",
            "duplicate_merged",
            "false_positive_closed",
            "true_positive_remediated",
            "true_positive_escalated"
          ],
          "matrix": [
            [
              339,
              17,
              11,
              13,
              16
            ],
            [
              10,
              49,
              0,
              0,
              0
            ],
            [
              74,
              0,
              376,
              0,
              0
            ],
            [
              41,
              0,
              0,
              185,
              51
            ],
            [
              38,
              0,
              0,
              55,
              105
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.9511218248098263
      },
      "delta_accuracy": 0.0021739130434782483,
      "delta_macro_f1": 0.002359864166290415
    },
    "no_tactic": {
      "n_features": 41,
      "dropped_count": 12,
      "metrics": {
        "model": "xgboost_no_tactic",
        "accuracy": 0.7811594202898551,
        "macro_f1": 0.7655889644647986,
        "weighted_f1": 0.7823552630061641,
        "per_class_f1": {
          "auto_resolved_soar": 0.7717750826901875,
          "duplicate_merged": 0.8346456692913385,
          "false_positive_closed": 0.908433734939759,
          "true_positive_remediated": 0.7088122605363985,
          "true_positive_escalated": 0.6042780748663101
        },
        "confusion_matrix": {
          "labels": [
            "auto_resolved_soar",
            "duplicate_merged",
            "false_positive_closed",
            "true_positive_remediated",
            "true_positive_escalated"
          ],
          "matrix": [
            [
              350,
              15,
              3,
              15,
              13
            ],
            [
              6,
              53,
              0,
              0,
              0
            ],
            [
              73,
              0,
              377,
              0,
              0
            ],
            [
              42,
              0,
              0,
              185,
              50
            ],
            [
              40,
              0,
              0,
              45,
              113
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.9529923809161402
      },
      "delta_accuracy": -0.01521739130434785,
      "delta_macro_f1": -0.02260135131792751
    },
    "no_siem": {
      "n_features": 45,
      "dropped_count": 8,
      "metrics": {
        "model": "xgboost_no_siem",
        "accuracy": 0.7681159420289855,
        "macro_f1": 0.747392848800313,
        "weighted_f1": 0.7695871955675133,
        "per_class_f1": {
          "auto_resolved_soar": 0.7577777777777778,
          "duplicate_merged": 0.8,
          "false_positive_closed": 0.9025270758122743,
          "true_positive_remediated": 0.706766917293233,
          "true_positive_escalated": 0.5698924731182796
        },
        "confusion_matrix": {
          "labels": [
            "auto_resolved_soar",
            "duplicate_merged",
            "false_positive_closed",
            "true_positive_remediated",
            "true_positive_escalated"
          ],
          "matrix": [
            [
              341,
              16,
              6,
              15,
              18
            ],
            [
              9,
              50,
              0,
              0,
              0
            ],
            [
              75,
              0,
              375,
              0,
              0
            ],
            [
              39,
              0,
              0,
              188,
              50
            ],
            [
              40,
              0,
              0,
              52,
              106
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.9521514669693077
      },
      "delta_accuracy": -0.0021739130434782483,
      "delta_macro_f1": -0.0044052356534418635
    },
    "no_analyst_tier": {
      "n_features": 50,
      "dropped_count": 3,
      "metrics": {
        "model": "xgboost_no_analyst_tier",
        "accuracy": 0.7717391304347826,
        "macro_f1": 0.7470947169858246,
        "weighted_f1": 0.7727339237745289,
        "per_class_f1": {
          "auto_resolved_soar": 0.768893756845564,
          "duplicate_merged": 0.784,
          "false_positive_closed": 0.9071170084439083,
          "true_positive_remediated": 0.6948176583493282,
          "true_positive_escalated": 0.5806451612903226
        },
        "confusion_matrix": {
          "labels": [
            "auto_resolved_soar",
            "duplicate_merged",
            "false_positive_closed",
            "true_positive_remediated",
            "true_positive_escalated"
          ],
          "matrix": [
            [
              351,
              17,
              3,
              14,
              11
            ],
            [
              10,
              49,
              0,
              0,
              0
            ],
            [
              74,
              0,
              376,
              0,
              0
            ],
            [
              41,
              0,
              0,
              181,
              55
            ],
            [
              41,
              0,
              0,
              49,
              108
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.9524262361561989
      },
      "delta_accuracy": -0.005797101449275366,
      "delta_macro_f1": -0.004107103838953519
    },
    "no_timing": {
      "n_features": 48,
      "dropped_count": 5,
      "metrics": {
        "model": "xgboost_no_timing",
        "accuracy": 0.777536231884058,
        "macro_f1": 0.7572452763946715,
        "weighted_f1": 0.7795520836463574,
        "per_class_f1": {
          "auto_resolved_soar": 0.7676991150442478,
          "duplicate_merged": 0.8031496062992126,
          "false_positive_closed": 0.9071170084439083,
          "true_positive_remediated": 0.723404255319149,
          "true_positive_escalated": 0.5848563968668408
        },
        "confusion_matrix": {
          "labels": [
            "auto_resolved_soar",
            "duplicate_merged",
            "false_positive_closed",
            "true_positive_remediated",
            "true_positive_escalated"
          ],
          "matrix": [
            [
              347,
              17,
              3,
              9,
              20
            ],
            [
              8,
              51,
              0,
              0,
              0
            ],
            [
              74,
              0,
              376,
              0,
              0
            ],
            [
              37,
              0,
              0,
              187,
              53
            ],
            [
              42,
              0,
              0,
              44,
              112
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.9546713378957848
      },
      "delta_accuracy": -0.011594202898550732,
      "delta_macro_f1": -0.014257663247800423
    },
    "no_scores": {
      "n_features": 48,
      "dropped_count": 5,
      "metrics": {
        "model": "xgboost_no_scores",
        "accuracy": 0.7710144927536232,
        "macro_f1": 0.7569411600325896,
        "weighted_f1": 0.7729871790343515,
        "per_class_f1": {
          "auto_resolved_soar": 0.7531285551763367,
          "duplicate_merged": 0.8253968253968254,
          "false_positive_closed": 0.9019138755980861,
          "true_positive_remediated": 0.7047970479704797,
          "true_positive_escalated": 0.5994694960212201
        },
        "confusion_matrix": {
          "labels": [
            "auto_resolved_soar",
            "duplicate_merged",
            "false_positive_closed",
            "true_positive_remediated",
            "true_positive_escalated"
          ],
          "matrix": [
            [
              331,
              15,
              9,
              23,
              18
            ],
            [
              7,
              52,
              0,
              0,
              0
            ],
            [
              73,
              0,
              377,
              0,
              0
            ],
            [
              38,
              0,
              0,
              191,
              48
            ],
            [
              34,
              0,
              0,
              51,
              113
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.9541430544791097
      },
      "delta_accuracy": -0.005072463768115987,
      "delta_macro_f1": -0.013953546885718482
    },
    "no_soar": {
      "n_features": 52,
      "dropped_count": 1,
      "metrics": {
        "model": "xgboost_no_soar",
        "accuracy": 0.618840579710145,
        "macro_f1": 0.5773360587813117,
        "weighted_f1": 0.5258347983183296,
        "per_class_f1": {
          "auto_resolved_soar": 0.028846153846153848,
          "duplicate_merged": 0.8194444444444444,
          "false_positive_closed": 0.8424068767908309,
          "true_positive_remediated": 0.6328358208955224,
          "true_positive_escalated": 0.5631469979296067
        },
        "confusion_matrix": {
          "labels": [
            "auto_resolved_soar",
            "duplicate_merged",
            "false_positive_closed",
            "true_positive_remediated",
            "true_positive_escalated"
          ],
          "matrix": [
            [
              6,
              26,
              156,
              122,
              86
            ],
            [
              0,
              59,
              0,
              0,
              0
            ],
            [
              9,
              0,
              441,
              0,
              0
            ],
            [
              2,
              0,
              0,
              212,
              63
            ],
            [
              3,
              0,
              0,
              59,
              136
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.8369099942380366
      },
      "delta_accuracy": 0.14710144927536228,
      "delta_macro_f1": 0.16565155436555945
    },
    "no_engineered": {
      "n_features": 47,
      "dropped_count": 6,
      "metrics": {
        "model": "xgboost_no_engineered",
        "accuracy": 0.7681159420289855,
        "macro_f1": 0.7479996795268518,
        "weighted_f1": 0.7700206321761683,
        "per_class_f1": {
          "auto_resolved_soar": 0.7542087542087542,
          "duplicate_merged": 0.796875,
          "false_positive_closed": 0.9027611044417767,
          "true_positive_remediated": 0.7094339622641509,
          "true_positive_escalated": 0.5767195767195767
        },
        "confusion_matrix": {
          "labels": [
            "auto_resolved_soar",
            "duplicate_merged",
            "false_positive_closed",
            "true_positive_remediated",
            "true_positive_escalated"
          ],
          "matrix": [
            [
              336,
              18,
              7,
              13,
              22
            ],
            [
              8,
              51,
              0,
              0,
              0
            ],
            [
              74,
              0,
              376,
              0,
              0
            ],
            [
              40,
              0,
              0,
              188,
              49
            ],
            [
              37,
              0,
              0,
              52,
              109
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.9533153727185603
      },
      "delta_accuracy": -0.0021739130434782483,
      "delta_macro_f1": -0.00501206637998064
    }
  }
}