{
  "purpose": "Quantify how much each feature group contributes to the headline XGBoost score. Identical architecture, same group-aware split, with one feature group dropped at a time.",
  "full_model_metrics": {
    "model": "xgboost",
    "accuracy": 0.6547008547008547,
    "macro_f1": 0.6401276666852063,
    "weighted_f1": 0.657179533714298,
    "per_class_f1": {
      "target_reconnaissance": 0.8875739644970414,
      "infrastructure_setup": 0.7115384615384616,
      "lure_crafting": 0.6762589928057554,
      "email_delivery": 0.7913669064748201,
      "victim_engagement": 0.46938775510204084,
      "credential_harvesting": 0.34074074074074073,
      "post_compromise_escalation": 0.6040268456375839
    },
    "confusion_matrix": {
      "labels": [
        "target_reconnaissance",
        "infrastructure_setup",
        "lure_crafting",
        "email_delivery",
        "victim_engagement",
        "credential_harvesting",
        "post_compromise_escalation"
      ],
      "matrix": [
        [
          75,
          0,
          9,
          0,
          0,
          0,
          0
        ],
        [
          0,
          37,
          16,
          0,
          0,
          0,
          0
        ],
        [
          10,
          10,
          47,
          0,
          0,
          0,
          0
        ],
        [
          0,
          4,
          0,
          110,
          28,
          1,
          0
        ],
        [
          0,
          0,
          0,
          21,
          46,
          24,
          9
        ],
        [
          0,
          0,
          0,
          4,
          16,
          23,
          20
        ],
        [
          0,
          0,
          0,
          0,
          6,
          24,
          45
        ]
      ]
    },
    "macro_roc_auc_ovr": 0.935584434710217
  },
  "ablations": {
    "no_topology": {
      "n_features": 23,
      "dropped_count": 30,
      "metrics": {
        "model": "xgboost_no_topology",
        "accuracy": 0.6410256410256411,
        "macro_f1": 0.626013906528604,
        "weighted_f1": 0.6377089952999916,
        "per_class_f1": {
          "target_reconnaissance": 0.891566265060241,
          "infrastructure_setup": 0.7586206896551724,
          "lure_crafting": 0.676923076923077,
          "email_delivery": 0.7598566308243727,
          "victim_engagement": 0.40609137055837563,
          "credential_harvesting": 0.2782608695652174,
          "post_compromise_escalation": 0.6107784431137725
        },
        "confusion_matrix": {
          "labels": [
            "target_reconnaissance",
            "infrastructure_setup",
            "lure_crafting",
            "email_delivery",
            "victim_engagement",
            "credential_harvesting",
            "post_compromise_escalation"
          ],
          "matrix": [
            [
              74,
              0,
              10,
              0,
              0,
              0,
              0
            ],
            [
              0,
              44,
              9,
              0,
              0,
              0,
              0
            ],
            [
              8,
              15,
              44,
              0,
              0,
              0,
              0
            ],
            [
              0,
              4,
              0,
              106,
              30,
              3,
              0
            ],
            [
              0,
              0,
              0,
              26,
              40,
              16,
              18
            ],
            [
              0,
              0,
              0,
              4,
              20,
              16,
              23
            ],
            [
              0,
              0,
              0,
              0,
              7,
              17,
              51
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.9341744835062434
      },
      "delta_accuracy": 0.013675213675213627,
      "delta_macro_f1": 0.014113760156602262
    },
    "no_behavioural": {
      "n_features": 36,
      "dropped_count": 17,
      "metrics": {
        "model": "xgboost_no_behavioural",
        "accuracy": 0.5794871794871795,
        "macro_f1": 0.5734830391013238,
        "weighted_f1": 0.5833619015067782,
        "per_class_f1": {
          "target_reconnaissance": 0.9024390243902439,
          "infrastructure_setup": 0.4745762711864407,
          "lure_crafting": 0.6619718309859155,
          "email_delivery": 0.6390977443609023,
          "victim_engagement": 0.3404255319148936,
          "credential_harvesting": 0.3472222222222222,
          "post_compromise_escalation": 0.6486486486486487
        },
        "confusion_matrix": {
          "labels": [
            "target_reconnaissance",
            "infrastructure_setup",
            "lure_crafting",
            "email_delivery",
            "victim_engagement",
            "credential_harvesting",
            "post_compromise_escalation"
          ],
          "matrix": [
            [
              74,
              0,
              10,
              0,
              0,
              0,
              0
            ],
            [
              0,
              28,
              16,
              9,
              0,
              0,
              0
            ],
            [
              6,
              13,
              47,
              1,
              0,
              0,
              0
            ],
            [
              0,
              23,
              2,
              85,
              30,
              3,
              0
            ],
            [
              0,
              1,
              0,
              26,
              32,
              34,
              7
            ],
            [
              0,
              0,
              0,
              2,
              18,
              25,
              18
            ],
            [
              0,
              0,
              0,
              0,
              8,
              19,
              48
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.9187512184393106
      },
      "delta_accuracy": 0.07521367521367517,
      "delta_macro_f1": 0.06664462758388245
    },
    "no_timestep": {
      "n_features": 52,
      "dropped_count": 1,
      "metrics": {
        "model": "xgboost_no_timestep",
        "accuracy": 0.3623931623931624,
        "macro_f1": 0.3138802646284953,
        "weighted_f1": 0.3500013055228507,
        "per_class_f1": {
          "target_reconnaissance": 0.4419889502762431,
          "infrastructure_setup": 0.24,
          "lure_crafting": 0.2748091603053435,
          "email_delivery": 0.5617283950617284,
          "victim_engagement": 0.26666666666666666,
          "credential_harvesting": 0.11666666666666667,
          "post_compromise_escalation": 0.2953020134228188
        },
        "confusion_matrix": {
          "labels": [
            "target_reconnaissance",
            "infrastructure_setup",
            "lure_crafting",
            "email_delivery",
            "victim_engagement",
            "credential_harvesting",
            "post_compromise_escalation"
          ],
          "matrix": [
            [
              40,
              18,
              26,
              0,
              0,
              0,
              0
            ],
            [
              23,
              12,
              18,
              0,
              0,
              0,
              0
            ],
            [
              32,
              17,
              18,
              0,
              0,
              0,
              0
            ],
            [
              2,
              0,
              2,
              91,
              16,
              17,
              15
            ],
            [
              0,
              0,
              0,
              36,
              22,
              20,
              22
            ],
            [
              0,
              0,
              0,
              25,
              16,
              7,
              15
            ],
            [
              0,
              0,
              0,
              29,
              11,
              13,
              22
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.8128267634071407
      },
      "delta_accuracy": 0.2923076923076923,
      "delta_macro_f1": 0.326247402056711
    },
    "no_engineered": {
      "n_features": 47,
      "dropped_count": 6,
      "metrics": {
        "model": "xgboost_no_engineered",
        "accuracy": 0.6581196581196581,
        "macro_f1": 0.6401951204875947,
        "weighted_f1": 0.6592473136316277,
        "per_class_f1": {
          "target_reconnaissance": 0.8809523809523809,
          "infrastructure_setup": 0.7155963302752294,
          "lure_crafting": 0.6518518518518519,
          "email_delivery": 0.8,
          "victim_engagement": 0.49473684210526314,
          "credential_harvesting": 0.3484848484848485,
          "post_compromise_escalation": 0.5897435897435898
        },
        "confusion_matrix": {
          "labels": [
            "target_reconnaissance",
            "infrastructure_setup",
            "lure_crafting",
            "email_delivery",
            "victim_engagement",
            "credential_harvesting",
            "post_compromise_escalation"
          ],
          "matrix": [
            [
              74,
              0,
              10,
              0,
              0,
              0,
              0
            ],
            [
              0,
              39,
              14,
              0,
              0,
              0,
              0
            ],
            [
              10,
              13,
              44,
              0,
              0,
              0,
              0
            ],
            [
              0,
              4,
              0,
              112,
              26,
              1,
              0
            ],
            [
              0,
              0,
              0,
              20,
              47,
              22,
              11
            ],
            [
              0,
              0,
              0,
              5,
              11,
              23,
              24
            ],
            [
              0,
              0,
              0,
              0,
              6,
              23,
              46
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.9369503919262667
      },
      "delta_accuracy": -0.0034188034188034067,
      "delta_macro_f1": -6.745380238848409e-05
    }
  }
}