{
  "purpose": "Quantify how much each feature group contributes to the honest XGBoost score. Identical architecture, same stratified split, with one feature group dropped at a time.",
  "full_model_metrics": {
    "model": "xgboost",
    "accuracy": 0.23737373737373738,
    "macro_f1": 0.22437482872901052,
    "weighted_f1": 0.23213786276177156,
    "per_class_f1": {
      "auth_access_control": 0.14583333333333334,
      "cryptographic_failure": 0.21686746987951808,
      "information_disclosure": 0.2909090909090909,
      "injection_family": 0.23728813559322035,
      "logic_flaw": 0.08955223880597014,
      "memory_corruption": 0.3333333333333333,
      "misconfiguration": 0.2589928057553957,
      "supply_chain_weakness": 0.2222222222222222
    },
    "confusion_matrix": {
      "labels": [
        "auth_access_control",
        "cryptographic_failure",
        "information_disclosure",
        "injection_family",
        "logic_flaw",
        "memory_corruption",
        "misconfiguration",
        "supply_chain_weakness"
      ],
      "matrix": [
        [
          7,
          7,
          0,
          11,
          6,
          10,
          7,
          5
        ],
        [
          4,
          9,
          3,
          5,
          3,
          5,
          16,
          0
        ],
        [
          3,
          0,
          8,
          1,
          4,
          0,
          7,
          0
        ],
        [
          3,
          6,
          1,
          14,
          8,
          20,
          6,
          7
        ],
        [
          4,
          4,
          5,
          3,
          3,
          2,
          13,
          0
        ],
        [
          11,
          3,
          0,
          13,
          3,
          27,
          5,
          8
        ],
        [
          6,
          9,
          15,
          2,
          5,
          7,
          18,
          3
        ],
        [
          5,
          0,
          0,
          4,
          1,
          21,
          2,
          8
        ]
      ]
    },
    "macro_roc_auc_ovr": 0.6837125710196055
  },
  "ablations": {
    "no_cvss": {
      "n_features": 55,
      "dropped_count": 2,
      "metrics": {
        "model": "xgboost_no_cvss",
        "accuracy": 0.21212121212121213,
        "macro_f1": 0.19261691542621184,
        "weighted_f1": 0.20621456669040633,
        "per_class_f1": {
          "auth_access_control": 0.14285714285714285,
          "cryptographic_failure": 0.09523809523809523,
          "information_disclosure": 0.14705882352941177,
          "injection_family": 0.23728813559322035,
          "logic_flaw": 0.16216216216216217,
          "memory_corruption": 0.33121019108280253,
          "misconfiguration": 0.2028985507246377,
          "supply_chain_weakness": 0.2222222222222222
        },
        "confusion_matrix": {
          "labels": [
            "auth_access_control",
            "cryptographic_failure",
            "information_disclosure",
            "injection_family",
            "logic_flaw",
            "memory_corruption",
            "misconfiguration",
            "supply_chain_weakness"
          ],
          "matrix": [
            [
              6,
              3,
              0,
              13,
              7,
              12,
              7,
              5
            ],
            [
              3,
              3,
              8,
              3,
              7,
              5,
              12,
              4
            ],
            [
              2,
              1,
              5,
              0,
              5,
              2,
              8,
              0
            ],
            [
              1,
              3,
              3,
              14,
              2,
              20,
              10,
              12
            ],
            [
              1,
              2,
              7,
              2,
              6,
              1,
              15,
              0
            ],
            [
              10,
              2,
              1,
              13,
              2,
              26,
              5,
              11
            ],
            [
              4,
              3,
              20,
              3,
              9,
              5,
              14,
              7
            ],
            [
              4,
              1,
              1,
              5,
              2,
              16,
              2,
              10
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.669002340507073
      },
      "delta_accuracy": 0.02525252525252525,
      "delta_macro_f1": 0.031757913302798674
    },
    "no_epss": {
      "n_features": 54,
      "dropped_count": 3,
      "metrics": {
        "model": "xgboost_no_epss",
        "accuracy": 0.2474747474747475,
        "macro_f1": 0.2237319833172186,
        "weighted_f1": 0.24186505327006125,
        "per_class_f1": {
          "auth_access_control": 0.17204301075268819,
          "cryptographic_failure": 0.08,
          "information_disclosure": 0.25,
          "injection_family": 0.3089430894308943,
          "logic_flaw": 0.11904761904761904,
          "memory_corruption": 0.4050632911392405,
          "misconfiguration": 0.25757575757575757,
          "supply_chain_weakness": 0.19718309859154928
        },
        "confusion_matrix": {
          "labels": [
            "auth_access_control",
            "cryptographic_failure",
            "information_disclosure",
            "injection_family",
            "logic_flaw",
            "memory_corruption",
            "misconfiguration",
            "supply_chain_weakness"
          ],
          "matrix": [
            [
              8,
              6,
              0,
              12,
              7,
              11,
              5,
              4
            ],
            [
              6,
              3,
              3,
              5,
              10,
              4,
              12,
              2
            ],
            [
              2,
              2,
              7,
              2,
              3,
              0,
              7,
              0
            ],
            [
              2,
              5,
              2,
              19,
              6,
              20,
              6,
              5
            ],
            [
              2,
              3,
              5,
              2,
              5,
              1,
              15,
              1
            ],
            [
              9,
              6,
              0,
              10,
              2,
              32,
              4,
              7
            ],
            [
              6,
              2,
              16,
              1,
              15,
              4,
              17,
              4
            ],
            [
              5,
              3,
              0,
              7,
              2,
              16,
              1,
              7
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.6925718594708064
      },
      "delta_accuracy": -0.01010101010101011,
      "delta_macro_f1": 0.0006428454117919091
    },
    "no_flags": {
      "n_features": 48,
      "dropped_count": 9,
      "metrics": {
        "model": "xgboost_no_flags",
        "accuracy": 0.22727272727272727,
        "macro_f1": 0.21140688534448485,
        "weighted_f1": 0.2214593080677342,
        "per_class_f1": {
          "auth_access_control": 0.13186813186813187,
          "cryptographic_failure": 0.1686746987951807,
          "information_disclosure": 0.3333333333333333,
          "injection_family": 0.2764227642276423,
          "logic_flaw": 0.08450704225352113,
          "memory_corruption": 0.34838709677419355,
          "misconfiguration": 0.24806201550387597,
          "supply_chain_weakness": 0.1
        },
        "confusion_matrix": {
          "labels": [
            "auth_access_control",
            "cryptographic_failure",
            "information_disclosure",
            "injection_family",
            "logic_flaw",
            "memory_corruption",
            "misconfiguration",
            "supply_chain_weakness"
          ],
          "matrix": [
            [
              6,
              6,
              1,
              9,
              5,
              10,
              6,
              10
            ],
            [
              5,
              7,
              3,
              5,
              5,
              4,
              14,
              2
            ],
            [
              3,
              0,
              10,
              1,
              4,
              0,
              5,
              0
            ],
            [
              3,
              7,
              1,
              17,
              7,
              18,
              4,
              8
            ],
            [
              3,
              5,
              6,
              2,
              3,
              2,
              13,
              0
            ],
            [
              8,
              3,
              0,
              14,
              3,
              27,
              4,
              11
            ],
            [
              4,
              10,
              16,
              2,
              7,
              6,
              16,
              4
            ],
            [
              6,
              0,
              0,
              8,
              3,
              18,
              2,
              4
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.6776398959263554
      },
      "delta_accuracy": 0.01010101010101011,
      "delta_macro_f1": 0.01296794338452567
    },
    "no_asset": {
      "n_features": 18,
      "dropped_count": 39,
      "metrics": {
        "model": "xgboost_no_asset",
        "accuracy": 0.21717171717171718,
        "macro_f1": 0.19672873773465777,
        "weighted_f1": 0.2140924517062793,
        "per_class_f1": {
          "auth_access_control": 0.10526315789473684,
          "cryptographic_failure": 0.13043478260869565,
          "information_disclosure": 0.13793103448275862,
          "injection_family": 0.17857142857142858,
          "logic_flaw": 0.08695652173913043,
          "memory_corruption": 0.37333333333333335,
          "misconfiguration": 0.26865671641791045,
          "supply_chain_weakness": 0.2926829268292683
        },
        "confusion_matrix": {
          "labels": [
            "auth_access_control",
            "cryptographic_failure",
            "information_disclosure",
            "injection_family",
            "logic_flaw",
            "memory_corruption",
            "misconfiguration",
            "supply_chain_weakness"
          ],
          "matrix": [
            [
              5,
              6,
              1,
              8,
              5,
              16,
              7,
              5
            ],
            [
              5,
              6,
              6,
              5,
              5,
              3,
              13,
              2
            ],
            [
              2,
              2,
              4,
              1,
              4,
              1,
              8,
              1
            ],
            [
              11,
              7,
              1,
              10,
              8,
              15,
              7,
              6
            ],
            [
              1,
              6,
              8,
              2,
              3,
              1,
              12,
              1
            ],
            [
              9,
              9,
              0,
              9,
              2,
              28,
              3,
              10
            ],
            [
              4,
              10,
              15,
              7,
              5,
              2,
              18,
              4
            ],
            [
              5,
              1,
              0,
              5,
              3,
              14,
              1,
              12
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.6869647093980484
      },
      "delta_accuracy": 0.020202020202020193,
      "delta_macro_f1": 0.02764609099435275
    },
    "no_severity": {
      "n_features": 53,
      "dropped_count": 4,
      "metrics": {
        "model": "xgboost_no_severity",
        "accuracy": 0.22727272727272727,
        "macro_f1": 0.21747488568762768,
        "weighted_f1": 0.2268764018926795,
        "per_class_f1": {
          "auth_access_control": 0.14893617021276595,
          "cryptographic_failure": 0.19047619047619047,
          "information_disclosure": 0.23333333333333334,
          "injection_family": 0.288135593220339,
          "logic_flaw": 0.12658227848101267,
          "memory_corruption": 0.28205128205128205,
          "misconfiguration": 0.24806201550387597,
          "supply_chain_weakness": 0.2222222222222222
        },
        "confusion_matrix": {
          "labels": [
            "auth_access_control",
            "cryptographic_failure",
            "information_disclosure",
            "injection_family",
            "logic_flaw",
            "memory_corruption",
            "misconfiguration",
            "supply_chain_weakness"
          ],
          "matrix": [
            [
              7,
              7,
              0,
              9,
              7,
              12,
              7,
              4
            ],
            [
              5,
              8,
              3,
              2,
              8,
              5,
              14,
              0
            ],
            [
              3,
              0,
              7,
              1,
              7,
              0,
              5,
              0
            ],
            [
              3,
              6,
              2,
              17,
              5,
              20,
              7,
              5
            ],
            [
              3,
              5,
              7,
              3,
              5,
              2,
              9,
              0
            ],
            [
              10,
              7,
              0,
              13,
              4,
              22,
              4,
              10
            ],
            [
              5,
              6,
              18,
              2,
              8,
              6,
              16,
              4
            ],
            [
              5,
              0,
              0,
              6,
              1,
              19,
              2,
              8
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.6857295225029008
      },
      "delta_accuracy": 0.01010101010101011,
      "delta_macro_f1": 0.006899943041382833
    },
    "no_engineered": {
      "n_features": 52,
      "dropped_count": 5,
      "metrics": {
        "model": "xgboost_no_engineered",
        "accuracy": 0.23232323232323232,
        "macro_f1": 0.22158389829583944,
        "weighted_f1": 0.22713804092389037,
        "per_class_f1": {
          "auth_access_control": 0.15053763440860216,
          "cryptographic_failure": 0.14285714285714285,
          "information_disclosure": 0.3157894736842105,
          "injection_family": 0.23931623931623933,
          "logic_flaw": 0.12987012987012986,
          "memory_corruption": 0.345679012345679,
          "misconfiguration": 0.23809523809523808,
          "supply_chain_weakness": 0.21052631578947367
        },
        "confusion_matrix": {
          "labels": [
            "auth_access_control",
            "cryptographic_failure",
            "information_disclosure",
            "injection_family",
            "logic_flaw",
            "memory_corruption",
            "misconfiguration",
            "supply_chain_weakness"
          ],
          "matrix": [
            [
              7,
              5,
              0,
              9,
              9,
              13,
              6,
              4
            ],
            [
              5,
              6,
              2,
              3,
              7,
              4,
              15,
              3
            ],
            [
              3,
              1,
              9,
              1,
              6,
              0,
              3,
              0
            ],
            [
              5,
              8,
              2,
              14,
              6,
              19,
              3,
              8
            ],
            [
              2,
              4,
              4,
              3,
              5,
              2,
              14,
              0
            ],
            [
              8,
              6,
              0,
              13,
              3,
              28,
              4,
              8
            ],
            [
              5,
              9,
              17,
              2,
              6,
              7,
              15,
              4
            ],
            [
              5,
              0,
              0,
              7,
              1,
              19,
              1,
              8
            ]
          ]
        },
        "macro_roc_auc_ovr": 0.6871096699405611
      },
      "delta_accuracy": 0.005050505050505055,
      "delta_macro_f1": 0.0027909304331710794
    }
  }
}