ablation_results.json

{
  "purpose": "Quantify how much each feature group contributes to the headline XGBoost score. Identical architecture, same group-aware split, with one feature group dropped at a time.",
  "full_model_metrics": {
    "model": "xgboost",
    "accuracy": 0.8529230769230769,
    "macro_f1": 0.8495931102241494,
    "weighted_f1": 0.8518585237469937,
    "per_class_f1": {
      "negligent_user": 0.8762557077625571,
      "malicious_employee": 0.8262571514604035,
      "privileged_insider": 0.8462664714494875
    },
    "confusion_matrix": {
      "labels": [
        "negligent_user",
        "malicious_employee",
        "privileged_insider"
      ],
      "matrix": [
        [
          1919,
          111,
          50
        ],
        [
          291,
          1372,
          92
        ],
        [
          90,
          83,
          867
        ]
      ]
    },
    "macro_roc_auc_ovr": 0.9627526877302969
  },
  "ablations": {
    "no_volume": {
      "n_features": 23,
      "dropped_count": 5,
      "metrics": {
        "model": "xgboost_no_volume",
        "accuracy": 0.489025641025641,
        "macro_f1": 0.47358930080150813,
        "weighted_f1": 0.48784413847470176,
        "per_class_f1": {
          "negligent_user": 0.5617715617715617,
          "malicious_employee": 0.44251626898047725,
          "privileged_insider": 0.41648007165248546
        },
        "confusion_matrix": {
          "labels": [
            "negligent_user",
            "malicious_employee",
            "privileged_insider"
          ],
          "matrix": [
            [
              1205,
              483,
              392
            ],
            [
              705,
              714,
              336
            ],
            [
              300,
              275,
              465
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.6827532681591143
      },
      "delta_accuracy": 0.3638974358974359,
      "delta_macro_f1": 0.3760038094226413
    },
    "no_behavioural": {
      "n_features": 18,
      "dropped_count": 10,
      "metrics": {
        "model": "xgboost_no_behavioural",
        "accuracy": 0.7126153846153847,
        "macro_f1": 0.7054601986097401,
        "weighted_f1": 0.7141318275968602,
        "per_class_f1": {
          "negligent_user": 0.7372585524784734,
          "malicious_employee": 0.7183327906219472,
          "privileged_insider": 0.6607892527287993
        },
        "confusion_matrix": {
          "labels": [
            "negligent_user",
            "malicious_employee",
            "privileged_insider"
          ],
          "matrix": [
            [
              1584,
              154,
              342
            ],
            [
              439,
              1103,
              213
            ],
            [
              194,
              59,
              787
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.896141715091384
      },
      "delta_accuracy": 0.14030769230769224,
      "delta_macro_f1": 0.14413291161440933
    },
    "no_timestep": {
      "n_features": 26,
      "dropped_count": 2,
      "metrics": {
        "model": "xgboost_no_timestep",
        "accuracy": 0.8393846153846154,
        "macro_f1": 0.8335587554093177,
        "weighted_f1": 0.838097363099834,
        "per_class_f1": {
          "negligent_user": 0.8618759794045221,
          "malicious_employee": 0.8233151183970856,
          "privileged_insider": 0.8154851684263449
        },
        "confusion_matrix": {
          "labels": [
            "negligent_user",
            "malicious_employee",
            "privileged_insider"
          ],
          "matrix": [
            [
              1925,
              97,
              58
            ],
            [
              319,
              1356,
              80
            ],
            [
              143,
              86,
              811
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.9568593124770418
      },
      "delta_accuracy": 0.0135384615384615,
      "delta_macro_f1": 0.01603435481483173
    },
    "no_context": {
      "n_features": 17,
      "dropped_count": 11,
      "metrics": {
        "model": "xgboost_no_context",
        "accuracy": 0.8543589743589743,
        "macro_f1": 0.8489739255889375,
        "weighted_f1": 0.8531648766003023,
        "per_class_f1": {
          "negligent_user": 0.8806546942486929,
          "malicious_employee": 0.8314674735249622,
          "privileged_insider": 0.8347996089931574
        },
        "confusion_matrix": {
          "labels": [
            "negligent_user",
            "malicious_employee",
            "privileged_insider"
          ],
          "matrix": [
            [
              1937,
              92,
              51
            ],
            [
              280,
              1374,
              101
            ],
            [
              102,
              84,
              854
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.9632029829754446
      },
      "delta_accuracy": -0.0014358974358974486,
      "delta_macro_f1": 0.0006191846352119335
    },
    "no_engineered": {
      "n_features": 22,
      "dropped_count": 6,
      "metrics": {
        "model": "xgboost_no_engineered",
        "accuracy": 0.8596923076923076,
        "macro_f1": 0.8559750404567971,
        "weighted_f1": 0.8586557301112084,
        "per_class_f1": {
          "negligent_user": 0.8818575005690872,
          "malicious_employee": 0.8366052552099064,
          "privileged_insider": 0.8494623655913979
        },
        "confusion_matrix": {
          "labels": [
            "negligent_user",
            "malicious_employee",
            "privileged_insider"
          ],
          "matrix": [
            [
              1937,
              91,
              52
            ],
            [
              285,
              1385,
              85
            ],
            [
              91,
              80,
              869
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.9629058321133872
      },
      "delta_accuracy": -0.00676923076923075,
      "delta_macro_f1": -0.006381930232647659
    }
  }
}