{ "analyzer": "ScriptedAnalyzer (rule-based baseline)", "n": 110, "detection_rate": 0.7666666666666667, "false_positive_rate": 0.3, "precision": 0.92, "recall": 0.7666666666666667, "f1": 0.8363636363636363, "accuracy": 0.7545454545454545, "detection_ci_95": [ 0.6666666666666666, 0.8444444444444444 ], "per_category": { "benign": { "n": 15, "detection_rate": 0.0, "false_positive_rate": 0.26666666666666666, "precision": 0.0, "recall": 0.0, "f1": 0.0, "accuracy": 0.7333333333333333 }, "borderline": { "n": 5, "detection_rate": 0.0, "false_positive_rate": 0.4, "precision": 0.0, "recall": 0.0, "f1": 0.0, "accuracy": 0.6 }, "impersonation": { "n": 19, "detection_rate": 0.8421052631578947, "false_positive_rate": 0.0, "precision": 1.0, "recall": 0.8421052631578947, "f1": 0.9142857142857143, "accuracy": 0.8421052631578947 }, "investment_fraud": { "n": 20, "detection_rate": 0.45, "false_positive_rate": 0.0, "precision": 1.0, "recall": 0.45, "f1": 0.6206896551724138, "accuracy": 0.45 }, "kyc_fraud": { "n": 17, "detection_rate": 0.9411764705882353, "false_positive_rate": 0.0, "precision": 1.0, "recall": 0.9411764705882353, "f1": 0.9696969696969697, "accuracy": 0.9411764705882353 }, "loan_app_fraud": { "n": 16, "detection_rate": 0.6875, "false_positive_rate": 0.0, "precision": 1.0, "recall": 0.6875, "f1": 0.8148148148148148, "accuracy": 0.6875 }, "otp_theft": { "n": 18, "detection_rate": 0.9444444444444444, "false_positive_rate": 0.0, "precision": 1.0, "recall": 0.9444444444444444, "f1": 0.9714285714285714, "accuracy": 0.9444444444444444 } }, "per_difficulty": { "easy": { "n": 29, "detection_rate": 0.88, "false_positive_rate": 0.0, "precision": 1.0, "recall": 0.88, "f1": 0.9361702127659575, "accuracy": 0.896551724137931 }, "hard": { "n": 17, "detection_rate": 0.42857142857142855, "false_positive_rate": 0.5, "precision": 0.375, "recall": 0.42857142857142855, "f1": 0.39999999999999997, "accuracy": 0.47058823529411764 }, "medium": { "n": 59, "detection_rate": 0.8113207547169812, "false_positive_rate": 0.16666666666666666, "precision": 0.9772727272727273, "recall": 0.8113207547169812, "f1": 0.88659793814433, "accuracy": 0.8135593220338984 }, "novel": { "n": 5, "detection_rate": 0.2, "false_positive_rate": 0.0, "precision": 1.0, "recall": 0.2, "f1": 0.33333333333333337, "accuracy": 0.2 } }, "scenarios": [ { "scenario_id": "modec_001", "is_scam_truth": true, "predicted_score": 1.0, "predicted_flag": true, "correct": true, "category": "otp_theft", "difficulty": "easy" }, { "scenario_id": "modec_002", "is_scam_truth": true, "predicted_score": 1.0, "predicted_flag": true, "correct": true, "category": "otp_theft", "difficulty": "easy" }, { "scenario_id": "modec_003", "is_scam_truth": true, "predicted_score": 0.71, "predicted_flag": true, "correct": true, "category": "otp_theft", "difficulty": "medium" }, { "scenario_id": "modec_004", "is_scam_truth": true, "predicted_score": 1.0, "predicted_flag": true, "correct": true, "category": "otp_theft", "difficulty": "easy" }, { "scenario_id": "modec_005", "is_scam_truth": true, "predicted_score": 1.0, "predicted_flag": true, "correct": true, "category": "otp_theft", "difficulty": "easy" }, { "scenario_id": "modec_006", "is_scam_truth": true, "predicted_score": 1.0, "predicted_flag": true, "correct": true, "category": "otp_theft", "difficulty": "easy" }, { "scenario_id": "modec_007", "is_scam_truth": true, "predicted_score": 0.9100000000000001, "predicted_flag": true, "correct": true, "category": "otp_theft", "difficulty": "medium" }, { "scenario_id": "modec_008", "is_scam_truth": true, "predicted_score": 0.71, "predicted_flag": true, "correct": true, "category": "otp_theft", "difficulty": "medium" }, { "scenario_id": "modec_009", "is_scam_truth": true, "predicted_score": 1.0, "predicted_flag": true, "correct": true, "category": "otp_theft", "difficulty": "easy" }, { "scenario_id": "modec_010", "is_scam_truth": true, "predicted_score": 1.0, "predicted_flag": true, "correct": true, "category": "otp_theft", "difficulty": "medium" }, { "scenario_id": "modec_011", "is_scam_truth": true, "predicted_score": 1.0, "predicted_flag": true, "correct": true, "category": "otp_theft", "difficulty": "medium" }, { "scenario_id": "modec_012", "is_scam_truth": true, "predicted_score": 0.76, "predicted_flag": true, "correct": true, "category": "otp_theft", "difficulty": "easy" }, { "scenario_id": "modec_013", "is_scam_truth": true, "predicted_score": 0.9100000000000001, "predicted_flag": true, "correct": true, "category": "otp_theft", "difficulty": "medium" }, { "scenario_id": "modec_014", "is_scam_truth": true, "predicted_score": 0.71, "predicted_flag": true, "correct": true, "category": "otp_theft", "difficulty": "medium" }, { "scenario_id": "modec_015", "is_scam_truth": true, "predicted_score": 1.0, "predicted_flag": true, "correct": true, "category": "otp_theft", "difficulty": "medium" }, { "scenario_id": "modec_016", "is_scam_truth": true, "predicted_score": 1.0, "predicted_flag": true, "correct": true, "category": "otp_theft", "difficulty": "easy" }, { "scenario_id": "modec_017", "is_scam_truth": true, "predicted_score": 1.0, "predicted_flag": true, "correct": true, "category": "otp_theft", "difficulty": "medium" }, { "scenario_id": "modec_018", "is_scam_truth": true, "predicted_score": 0.77, "predicted_flag": true, "correct": true, "category": "kyc_fraud", "difficulty": "easy" }, { "scenario_id": "modec_019", "is_scam_truth": true, "predicted_score": 0.77, "predicted_flag": true, "correct": true, "category": "kyc_fraud", "difficulty": "easy" }, { "scenario_id": "modec_020", "is_scam_truth": true, "predicted_score": 0.71, "predicted_flag": true, "correct": true, "category": "kyc_fraud", "difficulty": "easy" }, { "scenario_id": "modec_021", "is_scam_truth": true, "predicted_score": 0.77, "predicted_flag": true, "correct": true, "category": "kyc_fraud", "difficulty": "easy" }, { "scenario_id": "modec_022", "is_scam_truth": true, "predicted_score": 0.52, "predicted_flag": true, "correct": true, "category": "kyc_fraud", "difficulty": "easy" }, { "scenario_id": "modec_023", "is_scam_truth": true, "predicted_score": 1.0, "predicted_flag": true, "correct": true, "category": "kyc_fraud", "difficulty": "medium" }, { "scenario_id": "modec_024", "is_scam_truth": true, "predicted_score": 0.71, "predicted_flag": true, "correct": true, "category": "kyc_fraud", "difficulty": "easy" }, { "scenario_id": "modec_025", "is_scam_truth": true, "predicted_score": 0.52, "predicted_flag": true, "correct": true, "category": "kyc_fraud", "difficulty": "easy" }, { "scenario_id": "modec_026", "is_scam_truth": true, "predicted_score": 0.52, "predicted_flag": true, "correct": true, "category": "kyc_fraud", "difficulty": "medium" }, { "scenario_id": "modec_027", "is_scam_truth": true, "predicted_score": 0.52, "predicted_flag": true, "correct": true, "category": "kyc_fraud", "difficulty": "medium" }, { "scenario_id": "modec_028", "is_scam_truth": true, "predicted_score": 0.52, "predicted_flag": true, "correct": true, "category": "kyc_fraud", "difficulty": "medium" }, { "scenario_id": "modec_029", "is_scam_truth": true, "predicted_score": 0.52, "predicted_flag": true, "correct": true, "category": "kyc_fraud", "difficulty": "medium" }, { "scenario_id": "modec_030", "is_scam_truth": true, "predicted_score": 0.52, "predicted_flag": true, "correct": true, "category": "kyc_fraud", "difficulty": "medium" }, { "scenario_id": "modec_031", "is_scam_truth": true, "predicted_score": 0.96, "predicted_flag": true, "correct": true, "category": "kyc_fraud", "difficulty": "medium" }, { "scenario_id": "modec_032", "is_scam_truth": true, "predicted_score": 0.77, "predicted_flag": true, "correct": true, "category": "kyc_fraud", "difficulty": "medium" }, { "scenario_id": "modec_033", "is_scam_truth": true, "predicted_score": 0.72, "predicted_flag": true, "correct": true, "category": "kyc_fraud", "difficulty": "hard" }, { "scenario_id": "modec_034", "is_scam_truth": true, "predicted_score": 0.3, "predicted_flag": false, "correct": false, "category": "kyc_fraud", "difficulty": "medium" }, { "scenario_id": "modec_035", "is_scam_truth": true, "predicted_score": 0.47000000000000003, "predicted_flag": false, "correct": false, "category": "loan_app_fraud", "difficulty": "easy" }, { "scenario_id": "modec_036", "is_scam_truth": true, "predicted_score": 0.75, "predicted_flag": true, "correct": true, "category": "loan_app_fraud", "difficulty": "easy" }, { "scenario_id": "modec_037", "is_scam_truth": true, "predicted_score": 0.97, "predicted_flag": true, "correct": true, "category": "loan_app_fraud", "difficulty": "easy" }, { "scenario_id": "modec_038", "is_scam_truth": true, "predicted_score": 0.5, "predicted_flag": true, "correct": true, "category": "loan_app_fraud", "difficulty": "medium" }, { "scenario_id": "modec_039", "is_scam_truth": true, "predicted_score": 0.72, "predicted_flag": true, "correct": true, "category": "loan_app_fraud", "difficulty": "medium" }, { "scenario_id": "modec_040", "is_scam_truth": true, "predicted_score": 0.5, "predicted_flag": true, "correct": true, "category": "loan_app_fraud", "difficulty": "medium" }, { "scenario_id": "modec_041", "is_scam_truth": true, "predicted_score": 0.27, "predicted_flag": false, "correct": false, "category": "loan_app_fraud", "difficulty": "medium" }, { "scenario_id": "modec_042", "is_scam_truth": true, "predicted_score": 0.25, "predicted_flag": false, "correct": false, "category": "loan_app_fraud", "difficulty": "medium" }, { "scenario_id": "modec_043", "is_scam_truth": true, "predicted_score": 0.5, "predicted_flag": true, "correct": true, "category": "loan_app_fraud", "difficulty": "medium" }, { "scenario_id": "modec_044", "is_scam_truth": true, "predicted_score": 0.05, "predicted_flag": false, "correct": false, "category": "loan_app_fraud", "difficulty": "hard" }, { "scenario_id": "modec_045", "is_scam_truth": true, "predicted_score": 0.6300000000000001, "predicted_flag": true, "correct": true, "category": "loan_app_fraud", "difficulty": "medium" }, { "scenario_id": "modec_046", "is_scam_truth": true, "predicted_score": 0.5, "predicted_flag": true, "correct": true, "category": "loan_app_fraud", "difficulty": "medium" }, { "scenario_id": "modec_047", "is_scam_truth": true, "predicted_score": 0.72, "predicted_flag": true, "correct": true, "category": "loan_app_fraud", "difficulty": "medium" }, { "scenario_id": "modec_048", "is_scam_truth": true, "predicted_score": 0.72, "predicted_flag": true, "correct": true, "category": "loan_app_fraud", "difficulty": "medium" }, { "scenario_id": "modec_049", "is_scam_truth": true, "predicted_score": 0.25, "predicted_flag": false, "correct": false, "category": "loan_app_fraud", "difficulty": "medium" }, { "scenario_id": "modec_050", "is_scam_truth": true, "predicted_score": 0.72, "predicted_flag": true, "correct": true, "category": "loan_app_fraud", "difficulty": "hard" }, { "scenario_id": "modec_051", "is_scam_truth": true, "predicted_score": 0.25, "predicted_flag": false, "correct": false, "category": "investment_fraud", "difficulty": "easy" }, { "scenario_id": "modec_052", "is_scam_truth": true, "predicted_score": 0.5, "predicted_flag": true, "correct": true, "category": "investment_fraud", "difficulty": "medium" }, { "scenario_id": "modec_053", "is_scam_truth": true, "predicted_score": 0.97, "predicted_flag": true, "correct": true, "category": "investment_fraud", "difficulty": "easy" }, { "scenario_id": "modec_054", "is_scam_truth": true, "predicted_score": 0.5, "predicted_flag": true, "correct": true, "category": "investment_fraud", "difficulty": "medium" }, { "scenario_id": "modec_055", "is_scam_truth": true, "predicted_score": 0.25, "predicted_flag": false, "correct": false, "category": "investment_fraud", "difficulty": "medium" }, { "scenario_id": "modec_056", "is_scam_truth": true, "predicted_score": 0.5, "predicted_flag": true, "correct": true, "category": "investment_fraud", "difficulty": "easy" }, { "scenario_id": "modec_057", "is_scam_truth": true, "predicted_score": 0.72, "predicted_flag": true, "correct": true, "category": "investment_fraud", "difficulty": "medium" }, { "scenario_id": "modec_058", "is_scam_truth": true, "predicted_score": 0.96, "predicted_flag": true, "correct": true, "category": "investment_fraud", "difficulty": "medium" }, { "scenario_id": "modec_059", "is_scam_truth": true, "predicted_score": 0.25, "predicted_flag": false, "correct": false, "category": "investment_fraud", "difficulty": "medium" }, { "scenario_id": "modec_060", "is_scam_truth": true, "predicted_score": 0.05, "predicted_flag": false, "correct": false, "category": "investment_fraud", "difficulty": "medium" }, { "scenario_id": "modec_061", "is_scam_truth": true, "predicted_score": 0.25, "predicted_flag": false, "correct": false, "category": "investment_fraud", "difficulty": "hard" }, { "scenario_id": "modec_062", "is_scam_truth": true, "predicted_score": 0.55, "predicted_flag": true, "correct": true, "category": "investment_fraud", "difficulty": "medium" }, { "scenario_id": "modec_063", "is_scam_truth": true, "predicted_score": 0.25, "predicted_flag": false, "correct": false, "category": "investment_fraud", "difficulty": "easy" }, { "scenario_id": "modec_064", "is_scam_truth": true, "predicted_score": 0.27, "predicted_flag": false, "correct": false, "category": "investment_fraud", "difficulty": "medium" }, { "scenario_id": "modec_065", "is_scam_truth": true, "predicted_score": 0.05, "predicted_flag": false, "correct": false, "category": "investment_fraud", "difficulty": "hard" }, { "scenario_id": "modec_066", "is_scam_truth": true, "predicted_score": 0.5, "predicted_flag": true, "correct": true, "category": "investment_fraud", "difficulty": "medium" }, { "scenario_id": "modec_067", "is_scam_truth": true, "predicted_score": 0.72, "predicted_flag": true, "correct": true, "category": "impersonation", "difficulty": "medium" }, { "scenario_id": "modec_068", "is_scam_truth": true, "predicted_score": 0.97, "predicted_flag": true, "correct": true, "category": "impersonation", "difficulty": "medium" }, { "scenario_id": "modec_069", "is_scam_truth": true, "predicted_score": 1.0, "predicted_flag": true, "correct": true, "category": "impersonation", "difficulty": "easy" }, { "scenario_id": "modec_070", "is_scam_truth": true, "predicted_score": 0.72, "predicted_flag": true, "correct": true, "category": "impersonation", "difficulty": "medium" }, { "scenario_id": "modec_071", "is_scam_truth": true, "predicted_score": 1.0, "predicted_flag": true, "correct": true, "category": "impersonation", "difficulty": "medium" }, { "scenario_id": "modec_072", "is_scam_truth": true, "predicted_score": 1.0, "predicted_flag": true, "correct": true, "category": "impersonation", "difficulty": "medium" }, { "scenario_id": "modec_073", "is_scam_truth": true, "predicted_score": 0.9100000000000001, "predicted_flag": true, "correct": true, "category": "impersonation", "difficulty": "easy" }, { "scenario_id": "modec_074", "is_scam_truth": true, "predicted_score": 0.47000000000000003, "predicted_flag": false, "correct": false, "category": "impersonation", "difficulty": "medium" }, { "scenario_id": "modec_075", "is_scam_truth": true, "predicted_score": 0.52, "predicted_flag": true, "correct": true, "category": "impersonation", "difficulty": "medium" }, { "scenario_id": "modec_076", "is_scam_truth": true, "predicted_score": 0.75, "predicted_flag": true, "correct": true, "category": "impersonation", "difficulty": "medium" }, { "scenario_id": "modec_077", "is_scam_truth": true, "predicted_score": 0.71, "predicted_flag": true, "correct": true, "category": "impersonation", "difficulty": "medium" }, { "scenario_id": "modec_078", "is_scam_truth": true, "predicted_score": 0.52, "predicted_flag": true, "correct": true, "category": "impersonation", "difficulty": "medium" }, { "scenario_id": "modec_079", "is_scam_truth": true, "predicted_score": 0.27, "predicted_flag": false, "correct": false, "category": "impersonation", "difficulty": "hard" }, { "scenario_id": "modec_080", "is_scam_truth": true, "predicted_score": 0.52, "predicted_flag": true, "correct": true, "category": "impersonation", "difficulty": "hard" }, { "scenario_id": "modec_081", "is_scam_truth": true, "predicted_score": 1.0, "predicted_flag": true, "correct": true, "category": "impersonation", "difficulty": "easy" }, { "scenario_id": "modec_082", "is_scam_truth": true, "predicted_score": 0.97, "predicted_flag": true, "correct": true, "category": "impersonation", "difficulty": "medium" }, { "scenario_id": "modec_083", "is_scam_truth": true, "predicted_score": 0.71, "predicted_flag": true, "correct": true, "category": "impersonation", "difficulty": "medium" }, { "scenario_id": "modec_084", "is_scam_truth": true, "predicted_score": 0.27, "predicted_flag": false, "correct": false, "category": "impersonation", "difficulty": "medium" }, { "scenario_id": "modec_085", "is_scam_truth": true, "predicted_score": 0.5, "predicted_flag": true, "correct": true, "category": "impersonation", "difficulty": "medium" }, { "scenario_id": "modec_086", "is_scam_truth": false, "predicted_score": 0.27, "predicted_flag": false, "correct": true, "category": "benign", "difficulty": "medium" }, { "scenario_id": "modec_087", "is_scam_truth": false, "predicted_score": 0.27, "predicted_flag": false, "correct": true, "category": "benign", "difficulty": "medium" }, { "scenario_id": "modec_088", "is_scam_truth": false, "predicted_score": 0.27, "predicted_flag": false, "correct": true, "category": "benign", "difficulty": "easy" }, { "scenario_id": "modec_089", "is_scam_truth": false, "predicted_score": 1.0, "predicted_flag": true, "correct": false, "category": "benign", "difficulty": "hard" }, { "scenario_id": "modec_090", "is_scam_truth": false, "predicted_score": 0.27, "predicted_flag": false, "correct": true, "category": "benign", "difficulty": "easy" }, { "scenario_id": "modec_091", "is_scam_truth": false, "predicted_score": 0.5, "predicted_flag": true, "correct": false, "category": "benign", "difficulty": "medium" }, { "scenario_id": "modec_092", "is_scam_truth": false, "predicted_score": 0.5, "predicted_flag": true, "correct": false, "category": "benign", "difficulty": "hard" }, { "scenario_id": "modec_093", "is_scam_truth": false, "predicted_score": 0.25, "predicted_flag": false, "correct": true, "category": "benign", "difficulty": "medium" }, { "scenario_id": "modec_094", "is_scam_truth": false, "predicted_score": 0.05, "predicted_flag": false, "correct": true, "category": "benign", "difficulty": "hard" }, { "scenario_id": "modec_095", "is_scam_truth": false, "predicted_score": 0.47000000000000003, "predicted_flag": false, "correct": true, "category": "benign", "difficulty": "easy" }, { "scenario_id": "modec_096", "is_scam_truth": false, "predicted_score": 0.27, "predicted_flag": false, "correct": true, "category": "benign", "difficulty": "medium" }, { "scenario_id": "modec_097", "is_scam_truth": false, "predicted_score": 0.05, "predicted_flag": false, "correct": true, "category": "benign", "difficulty": "medium" }, { "scenario_id": "modec_098", "is_scam_truth": false, "predicted_score": 1.0, "predicted_flag": true, "correct": false, "category": "benign", "difficulty": "hard" }, { "scenario_id": "modec_099", "is_scam_truth": false, "predicted_score": 0.05, "predicted_flag": false, "correct": true, "category": "benign", "difficulty": "easy" }, { "scenario_id": "modec_100", "is_scam_truth": false, "predicted_score": 0.27, "predicted_flag": false, "correct": true, "category": "benign", "difficulty": "hard" }, { "scenario_id": "modec_101", "is_scam_truth": false, "predicted_score": 0.3, "predicted_flag": false, "correct": true, "category": "borderline", "difficulty": "hard" }, { "scenario_id": "modec_102", "is_scam_truth": false, "predicted_score": 1.0, "predicted_flag": true, "correct": false, "category": "borderline", "difficulty": "hard" }, { "scenario_id": "modec_103", "is_scam_truth": false, "predicted_score": 0.47000000000000003, "predicted_flag": false, "correct": true, "category": "borderline", "difficulty": "hard" }, { "scenario_id": "modec_104", "is_scam_truth": false, "predicted_score": 0.72, "predicted_flag": true, "correct": false, "category": "borderline", "difficulty": "hard" }, { "scenario_id": "modec_105", "is_scam_truth": false, "predicted_score": 0.27, "predicted_flag": false, "correct": true, "category": "borderline", "difficulty": "hard" }, { "scenario_id": "modec_106", "is_scam_truth": true, "predicted_score": 0.5, "predicted_flag": true, "correct": true, "category": "investment_fraud", "difficulty": "novel" }, { "scenario_id": "modec_107", "is_scam_truth": true, "predicted_score": 0.05, "predicted_flag": false, "correct": false, "category": "investment_fraud", "difficulty": "novel" }, { "scenario_id": "modec_108", "is_scam_truth": true, "predicted_score": 0.43, "predicted_flag": false, "correct": false, "category": "otp_theft", "difficulty": "novel" }, { "scenario_id": "modec_109", "is_scam_truth": true, "predicted_score": 0.05, "predicted_flag": false, "correct": false, "category": "investment_fraud", "difficulty": "novel" }, { "scenario_id": "modec_110", "is_scam_truth": true, "predicted_score": 0.3, "predicted_flag": false, "correct": false, "category": "investment_fraud", "difficulty": "novel" } ] }