ablation_results.json

{
  "purpose": "Quantify how much each feature group contributes to the headline XGBoost score. Identical architecture, same stratified split, with one feature group dropped at a time.",
  "full_model_metrics": {
    "model": "xgboost",
    "accuracy": 0.6666666666666666,
    "macro_f1": 0.6453546453546454,
    "weighted_f1": 0.6634032634032633,
    "per_class_f1": {
      "low": 0.7272727272727273,
      "medium": 0.2857142857142857,
      "high": 0.9230769230769231
    },
    "confusion_matrix": {
      "labels": [
        "low",
        "medium",
        "high"
      ],
      "matrix": [
        [
          12,
          5,
          0
        ],
        [
          4,
          2,
          1
        ],
        [
          0,
          0,
          6
        ]
      ]
    },
    "macro_roc_auc_ovr": 0.8016919142238835
  },
  "ablations": {
    "no_session_aggregates": {
      "n_features": 26,
      "dropped_count": 8,
      "metrics": {
        "model": "xgboost_no_session_aggregates",
        "accuracy": 0.7,
        "macro_f1": 0.6129870129870131,
        "weighted_f1": 0.6671861471861472,
        "per_class_f1": {
          "low": 0.8,
          "medium": 0.18181818181818182,
          "high": 0.8571428571428571
        },
        "confusion_matrix": {
          "labels": [
            "low",
            "medium",
            "high"
          ],
          "matrix": [
            [
              14,
              3,
              0
            ],
            [
              4,
              1,
              2
            ],
            [
              0,
              0,
              6
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.7625392687732843
      },
      "delta_accuracy": -0.033333333333333326,
      "delta_macro_f1": 0.03236763236763229
    },
    "no_user_aggregates": {
      "n_features": 26,
      "dropped_count": 8,
      "metrics": {
        "model": "xgboost_no_user_aggregates",
        "accuracy": 0.5333333333333333,
        "macro_f1": 0.45864045864045866,
        "weighted_f1": 0.5130221130221131,
        "per_class_f1": {
          "low": 0.6486486486486487,
          "medium": 0.0,
          "high": 0.7272727272727273
        },
        "confusion_matrix": {
          "labels": [
            "low",
            "medium",
            "high"
          ],
          "matrix": [
            [
              12,
              4,
              1
            ],
            [
              7,
              0,
              0
            ],
            [
              1,
              1,
              4
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.7042183744549474
      },
      "delta_accuracy": 0.1333333333333333,
      "delta_macro_f1": 0.18671418671418671
    },
    "no_risk_scores": {
      "n_features": 28,
      "dropped_count": 6,
      "metrics": {
        "model": "xgboost_no_risk_scores",
        "accuracy": 0.5666666666666667,
        "macro_f1": 0.5300213675213675,
        "weighted_f1": 0.5745405982905983,
        "per_class_f1": {
          "low": 0.6875,
          "medium": 0.13333333333333333,
          "high": 0.7692307692307693
        },
        "confusion_matrix": {
          "labels": [
            "low",
            "medium",
            "high"
          ],
          "matrix": [
            [
              11,
              6,
              0
            ],
            [
              4,
              1,
              2
            ],
            [
              0,
              1,
              5
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.7397649416511309
      },
      "delta_accuracy": 0.09999999999999998,
      "delta_macro_f1": 0.11533327783327785
    },
    "no_engineered": {
      "n_features": 28,
      "dropped_count": 6,
      "metrics": {
        "model": "xgboost_no_engineered",
        "accuracy": 0.5666666666666667,
        "macro_f1": 0.5444444444444444,
        "weighted_f1": 0.5755555555555555,
        "per_class_f1": {
          "low": 0.6666666666666666,
          "medium": 0.13333333333333333,
          "high": 0.8333333333333334
        },
        "confusion_matrix": {
          "labels": [
            "low",
            "medium",
            "high"
          ],
          "matrix": [
            [
              11,
              6,
              0
            ],
            [
              5,
              1,
              1
            ],
            [
              0,
              1,
              5
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.7972402822147068
      },
      "delta_accuracy": 0.09999999999999998,
      "delta_macro_f1": 0.10091020091020098
    }
  }
}