Hugging Face's logo

xpertsystems
/
cyb009-baseline-classifier

Tabular Classification
PyTorch
cybersecurity
vulnerability-management
cve
cvss
epss
cisa-kev
synthetic-data
xgboost
baseline
leakage-diagnostic
data-quality-audit
Eval Results (legacy)
Model card Files
xet
 Community
cyb009-baseline-classifier / ablation_results.json
pradeep-xpert's picture
pradeep-xpert
Initial release: vulnerability_class baseline + comprehensive 8-oracle-path leakage diagnostic on CYB009 sample
e520bf1 verified
raw
history blame contribute delete
17.5 kB
{
 "purpose": "Quantify how much each feature group contributes to the honest XGBoost score. Identical architecture, same stratified split, with one feature group dropped at a time.",
 "full_model_metrics": {
 "model": "xgboost",
 "accuracy": 0.23737373737373738,
 "macro_f1": 0.22437482872901052,
 "weighted_f1": 0.23213786276177156,
 "per_class_f1": {
 "auth_access_control": 0.14583333333333334,
 "cryptographic_failure": 0.21686746987951808,
 "information_disclosure": 0.2909090909090909,
 "injection_family": 0.23728813559322035,
 "logic_flaw": 0.08955223880597014,
 "memory_corruption": 0.3333333333333333,
 "misconfiguration": 0.2589928057553957,
 "supply_chain_weakness": 0.2222222222222222
 },
 "confusion_matrix": {
 "labels": [
 "auth_access_control",
 "cryptographic_failure",
 "information_disclosure",
 "injection_family",
 "logic_flaw",
 "memory_corruption",
 "misconfiguration",
 "supply_chain_weakness"
 ],
 "matrix": [
 [
 7,
 7,
 0,
 11,
 6,
 10,
 7,
 5
 ],
 [
 4,
 9,
 3,
 5,
 3,
 5,
 16,
 0
 ],
 [
 3,
 0,
 8,
 1,
 4,
 0,
 7,
 0
 ],
 [
 3,
 6,
 1,
 14,
 8,
 20,
 6,
 7
 ],
 [
 4,
 4,
 5,
 3,
 3,
 2,
 13,
 0
 ],
 [
 11,
 3,
 0,
 13,
 3,
 27,
 5,
 8
 ],
 [
 6,
 9,
 15,
 2,
 5,
 7,
 18,
 3
 ],
 [
 5,
 0,
 0,
 4,
 1,
 21,
 2,
 8
 ]
 ]
 },
 "macro_roc_auc_ovr": 0.6837125710196055
 },
 "ablations": {
 "no_cvss": {
 "n_features": 55,
 "dropped_count": 2,
 "metrics": {
 "model": "xgboost_no_cvss",
 "accuracy": 0.21212121212121213,
 "macro_f1": 0.19261691542621184,
 "weighted_f1": 0.20621456669040633,
 "per_class_f1": {
 "auth_access_control": 0.14285714285714285,
 "cryptographic_failure": 0.09523809523809523,
 "information_disclosure": 0.14705882352941177,
 "injection_family": 0.23728813559322035,
 "logic_flaw": 0.16216216216216217,
 "memory_corruption": 0.33121019108280253,
 "misconfiguration": 0.2028985507246377,
 "supply_chain_weakness": 0.2222222222222222
 },
 "confusion_matrix": {
 "labels": [
 "auth_access_control",
 "cryptographic_failure",
 "information_disclosure",
 "injection_family",
 "logic_flaw",
 "memory_corruption",
 "misconfiguration",
 "supply_chain_weakness"
 ],
 "matrix": [
 [
 6,
 3,
 0,
 13,
 7,
 12,
 7,
 5
 ],
 [
 3,
 3,
 8,
 3,
 7,
 5,
 12,
 4
 ],
 [
 2,
 1,
 5,
 0,
 5,
 2,
 8,
 0
 ],
 [
 1,
 3,
 3,
 14,
 2,
 20,
 10,
 12
 ],
 [
 1,
 2,
 7,
 2,
 6,
 1,
 15,
 0
 ],
 [
 10,
 2,
 1,
 13,
 2,
 26,
 5,
 11
 ],
 [
 4,
 3,
 20,
 3,
 9,
 5,
 14,
 7
 ],
 [
 4,
 1,
 1,
 5,
 2,
 16,
 2,
 10
 ]
 ]
 },
 "macro_roc_auc_ovr": 0.669002340507073
 },
 "delta_accuracy": 0.02525252525252525,
 "delta_macro_f1": 0.031757913302798674
 },
 "no_epss": {
 "n_features": 54,
 "dropped_count": 3,
 "metrics": {
 "model": "xgboost_no_epss",
 "accuracy": 0.2474747474747475,
 "macro_f1": 0.2237319833172186,
 "weighted_f1": 0.24186505327006125,
 "per_class_f1": {
 "auth_access_control": 0.17204301075268819,
 "cryptographic_failure": 0.08,
 "information_disclosure": 0.25,
 "injection_family": 0.3089430894308943,
 "logic_flaw": 0.11904761904761904,
 "memory_corruption": 0.4050632911392405,
 "misconfiguration": 0.25757575757575757,
 "supply_chain_weakness": 0.19718309859154928
 },
 "confusion_matrix": {
 "labels": [
 "auth_access_control",
 "cryptographic_failure",
 "information_disclosure",
 "injection_family",
 "logic_flaw",
 "memory_corruption",
 "misconfiguration",
 "supply_chain_weakness"
 ],
 "matrix": [
 [
 8,
 6,
 0,
 12,
 7,
 11,
 5,
 4
 ],
 [
 6,
 3,
 3,
 5,
 10,
 4,
 12,
 2
 ],
 [
 2,
 2,
 7,
 2,
 3,
 0,
 7,
 0
 ],
 [
 2,
 5,
 2,
 19,
 6,
 20,
 6,
 5
 ],
 [
 2,
 3,
 5,
 2,
 5,
 1,
 15,
 1
 ],
 [
 9,
 6,
 0,
 10,
 2,
 32,
 4,
 7
 ],
 [
 6,
 2,
 16,
 1,
 15,
 4,
 17,
 4
 ],
 [
 5,
 3,
 0,
 7,
 2,
 16,
 1,
 7
 ]
 ]
 },
 "macro_roc_auc_ovr": 0.6925718594708064
 },
 "delta_accuracy": -0.01010101010101011,
 "delta_macro_f1": 0.0006428454117919091
 },
 "no_flags": {
 "n_features": 48,
 "dropped_count": 9,
 "metrics": {
 "model": "xgboost_no_flags",
 "accuracy": 0.22727272727272727,
 "macro_f1": 0.21140688534448485,
 "weighted_f1": 0.2214593080677342,
 "per_class_f1": {
 "auth_access_control": 0.13186813186813187,
 "cryptographic_failure": 0.1686746987951807,
 "information_disclosure": 0.3333333333333333,
 "injection_family": 0.2764227642276423,
 "logic_flaw": 0.08450704225352113,
 "memory_corruption": 0.34838709677419355,
 "misconfiguration": 0.24806201550387597,
 "supply_chain_weakness": 0.1
 },
 "confusion_matrix": {
 "labels": [
 "auth_access_control",
 "cryptographic_failure",
 "information_disclosure",
 "injection_family",
 "logic_flaw",
 "memory_corruption",
 "misconfiguration",
 "supply_chain_weakness"
 ],
 "matrix": [
 [
 6,
 6,
 1,
 9,
 5,
 10,
 6,
 10
 ],
 [
 5,
 7,
 3,
 5,
 5,
 4,
 14,
 2
 ],
 [
 3,
 0,
 10,
 1,
 4,
 0,
 5,
 0
 ],
 [
 3,
 7,
 1,
 17,
 7,
 18,
 4,
 8
 ],
 [
 3,
 5,
 6,
 2,
 3,
 2,
 13,
 0
 ],
 [
 8,
 3,
 0,
 14,
 3,
 27,
 4,
 11
 ],
 [
 4,
 10,
 16,
 2,
 7,
 6,
 16,
 4
 ],
 [
 6,
 0,
 0,
 8,
 3,
 18,
 2,
 4
 ]
 ]
 },
 "macro_roc_auc_ovr": 0.6776398959263554
 },
 "delta_accuracy": 0.01010101010101011,
 "delta_macro_f1": 0.01296794338452567
 },
 "no_asset": {
 "n_features": 18,
 "dropped_count": 39,
 "metrics": {
 "model": "xgboost_no_asset",
 "accuracy": 0.21717171717171718,
 "macro_f1": 0.19672873773465777,
 "weighted_f1": 0.2140924517062793,
 "per_class_f1": {
 "auth_access_control": 0.10526315789473684,
 "cryptographic_failure": 0.13043478260869565,
 "information_disclosure": 0.13793103448275862,
 "injection_family": 0.17857142857142858,
 "logic_flaw": 0.08695652173913043,
 "memory_corruption": 0.37333333333333335,
 "misconfiguration": 0.26865671641791045,
 "supply_chain_weakness": 0.2926829268292683
 },
 "confusion_matrix": {
 "labels": [
 "auth_access_control",
 "cryptographic_failure",
 "information_disclosure",
 "injection_family",
 "logic_flaw",
 "memory_corruption",
 "misconfiguration",
 "supply_chain_weakness"
 ],
 "matrix": [
 [
 5,
 6,
 1,
 8,
 5,
 16,
 7,
 5
 ],
 [
 5,
 6,
 6,
 5,
 5,
 3,
 13,
 2
 ],
 [
 2,
 2,
 4,
 1,
 4,
 1,
 8,
 1
 ],
 [
 11,
 7,
 1,
 10,
 8,
 15,
 7,
 6
 ],
 [
 1,
 6,
 8,
 2,
 3,
 1,
 12,
 1
 ],
 [
 9,
 9,
 0,
 9,
 2,
 28,
 3,
 10
 ],
 [
 4,
 10,
 15,
 7,
 5,
 2,
 18,
 4
 ],
 [
 5,
 1,
 0,
 5,
 3,
 14,
 1,
 12
 ]
 ]
 },
 "macro_roc_auc_ovr": 0.6869647093980484
 },
 "delta_accuracy": 0.020202020202020193,
 "delta_macro_f1": 0.02764609099435275
 },
 "no_severity": {
 "n_features": 53,
 "dropped_count": 4,
 "metrics": {
 "model": "xgboost_no_severity",
 "accuracy": 0.22727272727272727,
 "macro_f1": 0.21747488568762768,
 "weighted_f1": 0.2268764018926795,
 "per_class_f1": {
 "auth_access_control": 0.14893617021276595,
 "cryptographic_failure": 0.19047619047619047,
 "information_disclosure": 0.23333333333333334,
 "injection_family": 0.288135593220339,
 "logic_flaw": 0.12658227848101267,
 "memory_corruption": 0.28205128205128205,
 "misconfiguration": 0.24806201550387597,
 "supply_chain_weakness": 0.2222222222222222
 },
 "confusion_matrix": {
 "labels": [
 "auth_access_control",
 "cryptographic_failure",
 "information_disclosure",
 "injection_family",
 "logic_flaw",
 "memory_corruption",
 "misconfiguration",
 "supply_chain_weakness"
 ],
 "matrix": [
 [
 7,
 7,
 0,
 9,
 7,
 12,
 7,
 4
 ],
 [
 5,
 8,
 3,
 2,
 8,
 5,
 14,
 0
 ],
 [
 3,
 0,
 7,
 1,
 7,
 0,
 5,
 0
 ],
 [
 3,
 6,
 2,
 17,
 5,
 20,
 7,
 5
 ],
 [
 3,
 5,
 7,
 3,
 5,
 2,
 9,
 0
 ],
 [
 10,
 7,
 0,
 13,
 4,
 22,
 4,
 10
 ],
 [
 5,
 6,
 18,
 2,
 8,
 6,
 16,
 4
 ],
 [
 5,
 0,
 0,
 6,
 1,
 19,
 2,
 8
 ]
 ]
 },
 "macro_roc_auc_ovr": 0.6857295225029008
 },
 "delta_accuracy": 0.01010101010101011,
 "delta_macro_f1": 0.006899943041382833
 },
 "no_engineered": {
 "n_features": 52,
 "dropped_count": 5,
 "metrics": {
 "model": "xgboost_no_engineered",
 "accuracy": 0.23232323232323232,
 "macro_f1": 0.22158389829583944,
 "weighted_f1": 0.22713804092389037,
 "per_class_f1": {
 "auth_access_control": 0.15053763440860216,
 "cryptographic_failure": 0.14285714285714285,
 "information_disclosure": 0.3157894736842105,
 "injection_family": 0.23931623931623933,
 "logic_flaw": 0.12987012987012986,
 "memory_corruption": 0.345679012345679,
 "misconfiguration": 0.23809523809523808,
 "supply_chain_weakness": 0.21052631578947367
 },
 "confusion_matrix": {
 "labels": [
 "auth_access_control",
 "cryptographic_failure",
 "information_disclosure",
 "injection_family",
 "logic_flaw",
 "memory_corruption",
 "misconfiguration",
 "supply_chain_weakness"
 ],
 "matrix": [
 [
 7,
 5,
 0,
 9,
 9,
 13,
 6,
 4
 ],
 [
 5,
 6,
 2,
 3,
 7,
 4,
 15,
 3
 ],
 [
 3,
 1,
 9,
 1,
 6,
 0,
 3,
 0
 ],
 [
 5,
 8,
 2,
 14,
 6,
 19,
 3,
 8
 ],
 [
 2,
 4,
 4,
 3,
 5,
 2,
 14,
 0
 ],
 [
 8,
 6,
 0,
 13,
 3,
 28,
 4,
 8
 ],
 [
 5,
 9,
 17,
 2,
 6,
 7,
 15,
 4
 ],
 [
 5,
 0,
 0,
 7,
 1,
 19,
 1,
 8
 ]
 ]
 },
 "macro_roc_auc_ovr": 0.6871096699405611
 },
 "delta_accuracy": 0.005050505050505055,
 "delta_macro_f1": 0.0027909304331710794
 }
 }
}