diff --git "a/analysis/stage1_examples/failure_examples.json" "b/analysis/stage1_examples/failure_examples.json" new file mode 100644--- /dev/null +++ "b/analysis/stage1_examples/failure_examples.json" @@ -0,0 +1,2914 @@ +[ + { + "split": "test_in_distribution", + "kind": "failure", + "id": "d18c73f3a6cc", + "target_layer": "o1_nrm", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.2608695652173913, + "norm_field_f1": 0.3, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-044\",\n \"GNBDUFunction\": {\n \"id\": \"DU-045\",\n \"gNBId\": 44,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-045-002\",\n \"cellLocalId\": 2,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n258\",\n \"arfcnDL\": 2017504,\n \"bSChannelBwDL\": 200,\n \"nrPCI\": 145,\n \"nrTAC\": 3193,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 1,\n \"sd\": \"000002\"\n },\n \"rrmPolicyMinRatio\": 0.07,\n \"rrmPolicyMaxRatio\": 0.33,\n \"rrmPolicyDedicatedRatio\": 0.2,\n \"rrmPolicyMaxRatioPerSlice\": 0.33\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"234\",\n \"mnc\": \"260\",\n \"sNSSAIList\": [\n {\n \"sst\": 1,\n \"sd\": \"000003\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-016\",\n \"GNBDUFunction\": {\n \"id\": \"DU-001\",\n \"gNBId\": 16,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-001-001\",\n \"cellLocalId\": 1,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n78\",\n \"arfcnDL\": 629820,\n \"bSChannelBwDL\": 40,\n \"nrPCI\": 812,\n \"nrTAC\": 18111,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.07,\n \"rrmPolicyMaxRatio\": 0.31,\n \"rrmPolicyDedicatedRatio\": 0.19,\n \"rrmPolicyMaxRatioPerSlice\": 0.31\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "9c96f9cfe92f", + "target_layer": "o1_nrm", + "slice_type": "URLLC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.2608695652173913, + "norm_field_f1": 0.3, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-035\",\n \"GNBDUFunction\": {\n \"id\": \"DU-015\",\n \"gNBId\": 35,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-015-006\",\n \"cellLocalId\": 6,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n77\",\n \"arfcnDL\": 635915,\n \"bSChannelBwDL\": 20,\n \"nrPCI\": 716,\n \"nrTAC\": 63923,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 2,\n \"sd\": \"000002\"\n },\n \"rrmPolicyMinRatio\": 0.17,\n \"rrmPolicyMaxRatio\": 0.46,\n \"rrmPolicyDedicatedRatio\": 0.32,\n \"rrmPolicyMaxRatioPerSlice\": 0.46\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"001\",\n \"mnc\": \"15\",\n \"sNSSAIList\": [\n {\n \"sst\": 2,\n \"sd\": \"000001\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-006\",\n \"GNBDUFunction\": {\n \"id\": \"DU-008\",\n \"gNBId\": 6,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-008-002\",\n \"cellLocalId\": 2,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n78\",\n \"arfcnDL\": 629819,\n \"bSChannelBwDL\": 40,\n \"nrPCI\": 812,\n \"nrTAC\": 58981,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 2,\n \"sd\": \"000001\"\n },\n \"rrmPolicyMinRatio\": 0.17,\n \"rrmPolicyMaxRatio\": 0.41,\n \"rrmPolicyDedicatedRatio\": 0.29,\n \"rrmPolicyMaxRatioPerSlice\": 0.41\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 2,\n \"sd\": \"000003\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "0b3a40c9ef29", + "target_layer": "o1_nrm", + "slice_type": "V2X", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.2608695652173913, + "norm_field_f1": 0.3, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-034\",\n \"GNBDUFunction\": {\n \"id\": \"DU-020\",\n \"gNBId\": 34,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-020-001\",\n \"cellLocalId\": 1,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n258\",\n \"arfcnDL\": 2066044,\n \"bSChannelBwDL\": 100,\n \"nrPCI\": 399,\n \"nrTAC\": 13260,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 4,\n \"sd\": \"000002\"\n },\n \"rrmPolicyMinRatio\": 0.11,\n \"rrmPolicyMaxRatio\": 0.51,\n \"rrmPolicyDedicatedRatio\": 0.31,\n \"rrmPolicyMaxRatioPerSlice\": 0.51\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"234\",\n \"mnc\": \"02\",\n \"sNSSAIList\": [\n {\n \"sst\": 4,\n \"sd\": \"000002\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-018\",\n \"GNBDUFunction\": {\n \"id\": \"DU-008\",\n \"gNBId\": 18,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-008-002\",\n \"cellLocalId\": 2,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n78\",\n \"arfcnDL\": 620812,\n \"bSChannelBwDL\": 40,\n \"nrPCI\": 812,\n \"nrTAC\": 38501,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 4,\n \"sd\": \"000001\"\n },\n \"rrmPolicyMinRatio\": 0.11,\n \"rrmPolicyMaxRatio\": 0.36,\n \"rrmPolicyDedicatedRatio\": 0.23,\n \"rrmPolicyMaxRatioPerSlice\": 0.36\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 4,\n \"sd\": \"000001\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "success", + "id": "9bb20702c4ed", + "target_layer": "o1_nrm", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5217391304347826, + "norm_field_f1": 0.6, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-041\",\n \"GNBDUFunction\": {\n \"id\": \"DU-041\",\n \"gNBId\": 41,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-041-010\",\n \"cellLocalId\": 10,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n77\",\n \"arfcnDL\": 650829,\n \"bSChannelBwDL\": 20,\n \"nrPCI\": 592,\n \"nrTAC\": 17343,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 3,\n \"sd\": \"000001\"\n },\n \"rrmPolicyMinRatio\": 0.05,\n \"rrmPolicyMaxRatio\": 0.23,\n \"rrmPolicyDedicatedRatio\": 0.14,\n \"rrmPolicyMaxRatioPerSlice\": 0.23\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 3,\n \"sd\": \"000002\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-016\",\n \"GNBDUFunction\": {\n \"id\": \"DU-008\",\n \"gNBId\": 16,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-008-001\",\n \"cellLocalId\": 1,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n77\",\n \"arfcnDL\": 629813,\n \"bSChannelBwDL\": 40,\n \"nrPCI\": 100,\n \"nrTAC\": 58461,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.05,\n \"rrmPolicyMaxRatio\": 0.23,\n \"rrmPolicyDedicatedRatio\": 0.14,\n \"rrmPolicyMaxRatioPerSlice\": 0.23\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 3,\n \"sd\": \"000001\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "694d0090fddd", + "target_layer": "a1_policy", + "slice_type": "URLLC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.42105263157894735, + "norm_field_f1": 0.5, + "norm_key_f1": 1.0, + "error_label": "a1_policy_value_error", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-urllc-032b45\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-urllc-6ddfa6\",\n \"cellId\": \"cell-006\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000002\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 150,\n \"mfbr\": 180.0,\n \"priorityLevel\": 1,\n \"5qi\": 2,\n \"pdb\": 150,\n \"per\": 1e-05\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 15,\n \"max\": 44\n },\n \"schedulerWeight\": 0.3,\n \"maxUEs\": 50\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-urllc-0d0d61\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-urllc-10000b\",\n \"cellId\": \"cell-010\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000001\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 150,\n \"mfbr\": 180.0,\n \"priorityLevel\": 3,\n \"5qi\": 85,\n \"pdb\": 5,\n \"per\": 1e-06\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 12,\n \"max\": 32\n },\n \"schedulerWeight\": 0.46,\n \"maxUEs\": 50\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "203521595cbf", + "target_layer": "a1_policy", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.47368421052631576, + "norm_field_f1": 0.5555555555555556, + "norm_key_f1": 1.0, + "error_label": "a1_policy_value_error", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-embb-df7deb\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-embb-1e007e\",\n \"cellId\": \"cell-092\",\n \"sNSSAI\": {\n \"sst\": 1,\n \"sd\": \"000002\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 200,\n \"mfbr\": 240.0,\n \"priorityLevel\": 6,\n \"5qi\": 7,\n \"pdb\": 100,\n \"per\": 0.0001\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 11,\n \"max\": 24\n },\n \"schedulerWeight\": 0.33,\n \"maxUEs\": 5000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-embb-11440c\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-embb-11440c\",\n \"cellId\": \"cell-064\",\n \"sNSSAI\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 200,\n \"mfbr\": 240.0,\n \"priorityLevel\": 5,\n \"5qi\": 6,\n \"pdb\": 300,\n \"per\": 0.0005\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 11,\n \"max\": 31\n },\n \"schedulerWeight\": 0.46,\n \"maxUEs\": 5000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "2414e525b358", + "target_layer": "a1_policy", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.47368421052631576, + "norm_field_f1": 0.5555555555555556, + "norm_key_f1": 1.0, + "error_label": "a1_policy_value_error", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-embb-f79e96\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-embb-59c165\",\n \"cellId\": \"cell-094\",\n \"sNSSAI\": {\n \"sst\": 1,\n \"sd\": \"000002\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 300,\n \"mfbr\": 360.0,\n \"priorityLevel\": 6,\n \"5qi\": 7,\n \"pdb\": 100,\n \"per\": 0.005\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 20,\n \"max\": 47\n },\n \"schedulerWeight\": 0.48,\n \"maxUEs\": 100\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-embb-11000d\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-embb-00000d\",\n \"cellId\": \"cell-086\",\n \"sNSSAI\": {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 300,\n \"mfbr\": 360.0,\n \"priorityLevel\": 5,\n \"5qi\": 6,\n \"pdb\": 300,\n \"per\": 0.0005\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 20,\n \"max\": 38\n },\n \"schedulerWeight\": 0.36,\n \"maxUEs\": 100\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "success", + "id": "abc1c25149ad", + "target_layer": "a1_policy", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7368421052631579, + "norm_field_f1": 0.8333333333333334, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-embb-083c68\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-embb-edadec\",\n \"cellId\": \"cell-024\",\n \"sNSSAI\": {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 100,\n \"mfbr\": 120.0,\n \"priorityLevel\": 5,\n \"5qi\": 6,\n \"pdb\": 300,\n \"per\": 0.0001\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 15,\n \"max\": 41\n },\n \"schedulerWeight\": 0.2,\n \"maxUEs\": 5000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-embb-110111\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-embb-110111\",\n \"cellId\": \"cell-061\",\n \"sNSSAI\": {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 100,\n \"mfbr\": 120.0,\n \"priorityLevel\": 5,\n \"5qi\": 6,\n \"pdb\": 300,\n \"per\": 0.0001\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 15,\n \"max\": 32\n },\n \"schedulerWeight\": 0.36,\n \"maxUEs\": 5000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "c6c087a3ecb5", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "V2X", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.20634920634920634, + "norm_field_f1": 0.12698412698412698, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-c9ffd5f8f042\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 1,\n \"measurements\": [\n 0.72,\n 0.85,\n 0.91,\n 0.9,\n 0.67\n ],\n \"average\": 0.81,\n \"min\": 0.67,\n \"max\": 0.91,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 50,\n \"measurements\": [\n 52.1,\n 57.1,\n 49.6,\n 48.6,\n 46.9\n ],\n \"average\": 50.86,\n \"min\": 46.9,\n \"max\": 57.1,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 50,\n \"measurements\": [\n 59.4,\n 47.4,\n 52.8,\n 54.5,\n 56.6\n ],\n \"average\": 54.14,\n \"min\": 47.4,\n \"max\": 59.4,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99,\n \"measurements\": [\n 99.983253,\n 99.997202,\n 100,\n 99.967519,\n 100\n ],\n \"average\": 99.99,\n \"min\": 99.967519,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 1000,\n \"measurements\": [\n 475,\n 202,\n 482,\n 989,\n 603\n ],\n \"average\": 550.2,\n \"min\": 202,\n \"max\": 989,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": false,\n \"resourceUtilization\": {\n \"cpuUsage\": 25.5,\n \"memoryUsage\": 41.4,\n \"prbUtilization\": 78.2\n },\n \"@type\": \"IntentAssuranceReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-111111111111\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 10,\n \"measurements\": [\n 10.26,\n 6.43,\n 8.56,\n 10.05,\n 9.06\n ],\n \"average\": 8.67,\n \"min\": 6.43,\n \"max\": 10.26,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 150,\n \"measurements\": [\n 158.4,\n 139.1,\n 168.4,\n 149.4,\n 168.0\n ],\n \"average\": 152.66,\n \"min\": 139.1,\n \"max\": 168.4,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measurements\": [\n 115.0,\n 112.1,\n 99.4,\n 118.2,\n 118.4\n ],\n \"average\": 110.62,\n \"min\": 99.4,\n \"max\": 118.4,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.999,\n \"measurements\": [\n 99.903671,\n 99.998011,\n 99.999622,\n 99.998972,\n 99.999811\n ],\n \"average\": 99.98,\n \"min\": 99.903671,\n \"max\": 99.999811,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 5000,\n \"measurements\": [\n 1643,\n 2822,\n 4563,\n 4865,\n 4865\n ],\n \"average\": 3571.6,\n \"min\": 1643,\n \"max\": 4865,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 77.7,\n \"memoryUsage\": 28.6,\n \"prbUtilization\": 82\n......" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "a0dde7740590", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "eMBB", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.23809523809523808, + "norm_field_f1": 0.12698412698412698, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-bb2d09f58a92\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 50,\n \"measurements\": [\n 53.36,\n 38.17,\n 53.78,\n 45.41,\n 47.52\n ],\n \"average\": 47.648,\n \"min\": 38.17,\n \"max\": 53.78,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 1000,\n \"measurements\": [\n 1098.1,\n 1095.5,\n 905.5,\n 1171.4,\n 935.4\n ],\n \"average\": 1041.18,\n \"min\": 905.5,\n \"max\": 1171.4,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 75,\n \"measurements\": [\n 78.5,\n 75.9,\n 79.6,\n 77.9,\n 80.3\n ],\n \"average\": 78.44,\n \"min\": 75.9,\n \"max\": 80.3,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.95,\n \"measurements\": [\n 99.993633,\n 99.904768,\n 100,\n 99.940096,\n 99.980962\n ],\n \"average\": 99.964,\n \"min\": 99.904768,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 10000,\n \"measurements\": [\n 2978,\n 4838,\n 3232,\n 5089,\n 4608\n ],\n \"average\": 4149.0,\n \"min\": 2978,\n \"max\": 5089,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 65.1,\n \"memoryUsage\": 61.2,\n \"prbUtilization\": 48.9\n },\n \"@type\": \"I\n......", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-bb2d09f58a92\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 10,\n \"measurements\": [\n 6.23,\n 9.5,\n 9.84,\n 10.0,\n 8.54\n ],\n \"average\": 8.822,\n \"min\": 6.23,\n \"max\": 10.0,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 100,\n \"measurements\": [\n 108.4,\n 112.1,\n 108.4,\n 118.4,\n 112.1\n ],\n \"average\": 111.88,\n \"min\": 108.4,\n \"max\": 118.4,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measurements\": [\n 99.4,\n 100.6,\n 95.2,\n 99.8,\n 115.6\n ],\n \"average\": 102.12,\n \"min\": 95.2,\n \"max\": 115.6,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.999,\n \"measurements\": [\n 99.903575,\n 99.998042,\n 99.998185,\n 99.998011,\n 99.999434\n ],\n \"average\": 99.979,\n \"min\": 99.903575,\n \"max\": 99.999434,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 5000,\n \"measurements\": [\n 1802,\n 1802,\n 2834,\n 4563,\n 4644\n ],\n \"average\": 3129.0,\n \"min\": 1802,\n \"max\": 4644,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 66.6,\n \"memoryUsage\": 28.6,\n \"prbUtilization\": 82.3\n \n......" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "1cf218c12177", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "MPS", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.25396825396825395, + "norm_field_f1": 0.14285714285714285, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-63282990ad3a\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 1,\n \"measurements\": [\n 0.78,\n 0.68,\n 0.61,\n 1.09,\n 0.98\n ],\n \"average\": 0.828,\n \"min\": 0.61,\n \"max\": 1.09,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 25,\n \"measurements\": [\n 25.9,\n 26.3,\n 24.3,\n 28.7,\n 22.8\n ],\n \"average\": 25.6,\n \"min\": 22.8,\n \"max\": 28.7,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 25,\n \"measurements\": [\n 27.6,\n 25.2,\n 24.7,\n 25.5,\n 22.6\n ],\n \"average\": 25.12,\n \"min\": 22.6,\n \"max\": 27.6,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9999,\n \"measurements\": [\n 99.900581,\n 100,\n 100,\n 99.960229,\n 99.947729\n ],\n \"average\": 99.962,\n \"min\": 99.900581,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 50,\n \"measurements\": [\n 21,\n 12,\n 41,\n 38,\n 41\n ],\n \"average\": 30.6,\n \"min\": 12,\n \"max\": 41,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 56.4,\n \"memoryUsage\": 56.9,\n \"prbUtilization\": 54.7\n },\n \"@type\": \"IntentAssuranceReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-63282990ad3a\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 100,\n \"measurements\": [\n 66.06,\n 100.89,\n 100.0,\n 61.1,\n 89.0\n ],\n \"average\": 81.41,\n \"min\": 61.1,\n \"max\": 100.89,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 50,\n \"measurements\": [\n 58.1,\n 58.8,\n 58.4,\n 52.1,\n 52.2\n ],\n \"average\": 55.72,\n \"min\": 52.1,\n \"max\": 58.8,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 25,\n \"measurements\": [\n 29.1,\n 23.1,\n 28.4,\n 23.6,\n 29.6\n ],\n \"average\": 26.96,\n \"min\": 23.1,\n \"max\": 29.6,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9,\n \"measurements\": [\n 99.966136,\n 99.998141,\n 99.998042,\n 99.998176,\n 99.998042\n ],\n \"average\": 99.973,\n \"min\": 99.966136,\n \"max\": 99.998176,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 5000,\n \"measurements\": [\n 1666,\n 1808,\n 4166,\n 4646,\n 1865\n ],\n \"average\": 2810.2,\n \"min\": 1666,\n \"max\": 4646,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 62.1,\n \"memoryUsage\": 23.3,\n \"prbUtilization\": 82.3\n },\n \n......" + }, + { + "split": "test_in_distribution", + "kind": "success", + "id": "951a51750d41", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "mMTC", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.2222222222222222, + "norm_field_f1": 0.2222222222222222, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-6a4c9dffc32d\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 100,\n \"measurements\": [\n 87.74,\n 69.75,\n 92.13,\n 97.59,\n 71.95\n ],\n \"average\": 83.832,\n \"min\": 69.75,\n \"max\": 97.59,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 50,\n \"measurements\": [\n 50.4,\n 45.8,\n 52.7,\n 46.6,\n 58.7\n ],\n \"average\": 50.84,\n \"min\": 45.8,\n \"max\": 58.7,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 0.5,\n \"measurements\": [\n 0.5,\n 0.5,\n 0.5,\n 0.5,\n 0.6\n ],\n \"average\": 0.52,\n \"min\": 0.5,\n \"max\": 0.6,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.5,\n \"measurements\": [\n 99.5203,\n 99.520712,\n 99.496279,\n 99.571727,\n 99.415768\n ],\n \"average\": 99.505,\n \"min\": 99.415768,\n \"max\": 99.571727,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 1000000,\n \"measurements\": [\n 984187,\n 335768,\n 486668,\n 566126,\n 800523\n ],\n \"average\": 634654.4,\n \"min\": 335768,\n \"max\": 984187,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": false,\n \"resourceUtilization\": {\n \"cpuUsage\": 54.3,\n \"memoryUsage\": 29.6,\n \"prbUtilization\": 84.5\n },\n \"@ty\n......", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-6a4c9dffc32d\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 10,\n \"measurements\": [\n 6.46,\n 6.86,\n 10.3,\n 10.0,\n 8.56\n ],\n \"average\": 8.236,\n \"min\": 6.46,\n \"max\": 10.3,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 100,\n \"measurements\": [\n 108.2,\n 112.1,\n 100.6,\n 99.6,\n 118.6\n ],\n \"average\": 107.82,\n \"min\": 99.6,\n \"max\": 118.6,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measurements\": [\n 109.0,\n 108.4,\n 99.1,\n 112.1,\n 99.8\n ],\n \"average\": 105.88,\n \"min\": 99.1,\n \"max\": 112.1,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9999,\n \"measurements\": [\n 99.903667,\n 99.999821,\n 99.999971,\n 100,\n 99.999975\n ],\n \"average\": 99.98,\n \"min\": 99.903667,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 5000,\n \"measurements\": [\n 1866,\n 1808,\n 2820,\n 4565,\n 4444\n ],\n \"average\": 3080.6,\n \"min\": 1808,\n \"max\": 4565,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 77.7,\n \"memoryUsage\": 28.6,\n \"prbUtilization\": 82.3\n },\n \"@\n......" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "a790d5b62fbc", + "target_layer": "tmf921_lifecycle_monitor", + "slice_type": "eMBB", + "lifecycle_operation": "monitor", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5666666666666667, + "norm_field_f1": 0.2413793103448276, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentFulfillmentReport" + ], + "prediction_json_keys": [ + "intentFulfillmentReport" + ], + "gold": "{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-85ac305b5e9c\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Fulfilled\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 30,\n \"measured\": 22.47,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 500,\n \"measured\": 507.9,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 200,\n \"measured\": 200.2,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99,\n \"measured\": 100,\n \"unit\": \"percent\",\n \"compliant\": true\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 100,\n \"measured\": 95,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": true,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-85ac305b5e9c\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 100,\n \"measured\": 75.01,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 1000,\n \"measured\": 994.1,\n \"unit\": \"Mbps\",\n \"compliant\": false\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measured\": 109.0,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99,\n \"measured\": 99.984066,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 10000,\n \"measured\": 8942,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "5efed41ac432", + "target_layer": "tmf921_lifecycle_monitor", + "slice_type": "mMTC", + "lifecycle_operation": "monitor", + "parse_json": true, + "exact_match": false, + "field_f1": 0.6333333333333333, + "norm_field_f1": 0.41379310344827586, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentFulfillmentReport" + ], + "prediction_json_keys": [ + "intentFulfillmentReport" + ], + "gold": "{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-54f14eac901c\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 200,\n \"measured\": 197.74,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 25,\n \"measured\": 24.7,\n \"unit\": \"Mbps\",\n \"compliant\": false\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 25,\n \"measured\": 27.4,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.5,\n \"measured\": 99.455458,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 1000000,\n \"measured\": 13690,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-54f14eac901c\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 500,\n \"measured\": 361.8,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 1,\n \"measured\": 1.0,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 1,\n \"measured\": 1.1,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99,\n \"measured\": 98.981072,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 10000,\n \"measured\": 8142,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "50a4e9fa9db3", + "target_layer": "tmf921_lifecycle_monitor", + "slice_type": "mMTC", + "lifecycle_operation": "monitor", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5666666666666667, + "norm_field_f1": 0.4482758620689655, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentFulfillmentReport" + ], + "prediction_json_keys": [ + "intentFulfillmentReport" + ], + "gold": "{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-ea956a3226f7\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 1000,\n \"measured\": 751.73,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 25,\n \"measured\": 24.7,\n \"unit\": \"Mbps\",\n \"compliant\": false\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 0.1,\n \"measured\": 0.1,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 90,\n \"measured\": 90.008003,\n \"unit\": \"percent\",\n \"compliant\": true\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 1000000,\n \"measured\": 801042,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-000000000001\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 500,\n \"measured\": 361.1,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 1,\n \"measured\": 1.1,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 1,\n \"measured\": 1.1,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99,\n \"measured\": 98.981072,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 10000,\n \"measured\": 8142,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "success", + "id": "1a6b5d210b5d", + "target_layer": "tmf921_lifecycle_monitor", + "slice_type": "URLLC", + "lifecycle_operation": "monitor", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7666666666666667, + "norm_field_f1": 0.7586206896551724, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "intentFulfillmentReport" + ], + "prediction_json_keys": [ + "intentFulfillmentReport" + ], + "gold": "{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-272abd35749b\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 1,\n \"measured\": 0.74,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 150,\n \"measured\": 160.6,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measured\": 111.9,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.999,\n \"measured\": 99.927269,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 1000,\n \"measured\": 373,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-272abd35749b\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 10,\n \"measured\": 8.06,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 150,\n \"measured\": 159.8,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measured\": 109.0,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.999,\n \"measured\": 99.90356,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 50,\n \"measured\": 40,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "2e170e5d2f68", + "target_layer": "tmf921", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7692307692307693, + "norm_field_f1": 0.8656716417910447, + "norm_key_f1": 0.9402985074626866, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-6830bb1fb139\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-6830bb1fb139\",\n \"name\": \"eMBB Slice - Electronic Warfare Spectrum Monitoring\",\n \"description\": \"Intent for electronic warfare spectrum monitoring at Forward Operating Area with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 5,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-6b2137891dca\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-b74b1e\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000002\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 25,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-111111111111\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-111111111111\",\n \"name\": \"eMBB Slice - Electronic Warfare Spectrum Monitoring\",\n \"description\": \"Intent for electronic warfare spectrum monitoring at Forward Operating Area with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-000000000001\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-000001\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 25,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NR\n......" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "6a37983cc8b1", + "target_layer": "tmf921", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7692307692307693, + "norm_field_f1": 0.8656716417910447, + "norm_key_f1": 0.9402985074626866, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-0f94edf4e5fd\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-0f94edf4e5fd\",\n \"name\": \"eMBB Slice - Fleet Route Optimization\",\n \"description\": \"Intent for fleet route optimization at Factory Complex A with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 5,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-5ebd3bd193ab\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-08017d\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000002\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1000,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 150,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n \n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-10b14040b116\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-10b14040b116\",\n \"name\": \"eMBB Slice - Fleet Route Optimization\",\n \"description\": \"Intent for fleet route optimization at Factory Complex A with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-1b10b140b116\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-10b140\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1000,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 150,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"M\n......" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "32c3a5d4c830", + "target_layer": "tmf921", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7692307692307693, + "norm_field_f1": 0.8656716417910447, + "norm_key_f1": 0.9402985074626866, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-e739759aea88\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-e739759aea88\",\n \"name\": \"eMBB Slice - Urban Noise Monitoring\",\n \"description\": \"Intent for urban noise monitoring at Hospital Campus with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 5,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-8032d4ff1385\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-4f8120\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1000,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n \n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-11b1b1b1b1b1\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-11b1b1b1b1b1\",\n \"name\": \"eMBB Slice - Urban Noise Monitoring\",\n \"description\": \"Intent for urban noise monitoring at Hospital Campus with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-11b1b1b1b1b1\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-11b1b1\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1000,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n \n......" + }, + { + "split": "test_in_distribution", + "kind": "success", + "id": "95c2a0375b70", + "target_layer": "tmf921", + "slice_type": "URLLC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.8860759493670886, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-1672dbf0d214\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-1672dbf0d214\",\n \"name\": \"URLLC Slice - Terminal Crowd Management\",\n \"description\": \"Intent for terminal crowd management at Runway Complex with URLLC service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 1,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-30f8c5e05aee\",\n \"name\": \"URLLC QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-urllc-5abeba\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"URLLC\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 2,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 300,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 75,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-1b1111111111\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-1b1111111111\",\n \"name\": \"URLLC Slice - Terminal Crowd Management\",\n \"description\": \"Intent for terminal crowd management at Runway Complex with URLLC service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 1,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-1b1111111111\",\n \"name\": \"URLLC QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-urllc-1b1111\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"URLLC\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 2,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 300,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 75,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \n......" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "86beccd364f5", + "target_layer": "camara", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5, + "norm_field_f1": 0.7142857142857143, + "norm_key_f1": 0.8571428571428571, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"eMBB\",\n \"sliceProfile\": {\n \"latencyTarget\": 80,\n \"reliability\": 99.95,\n \"throughput\": {\n \"downlink\": 1000,\n \"uplink\": 200\n },\n \"maxNumberOfUEs\": 50000\n },\n \"areaOfService\": {\n \"region\": \"Smart Farm Sector\",\n \"center\": {\n \"latitude\": 41.8781,\n \"longitude\": -93.0977\n },\n \"radius\": 2000,\n \"cells\": [\n \"cell-082\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-07-26T12:00:00Z\",\n \"endTime\": \"2025-07-27T12:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/56bbfa0f18e8\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"eMBB\",\n \"sliceProfile\": {\n \"latencyTarget\": 80,\n \"reliability\": 99.9,\n \"throughput\": {\n \"downlink\": 1000,\n \"uplink\": 200\n },\n \"maxNumberOfUEs\": 50000\n },\n \"areaOfService\": {\n \"region\": \"Smart Farm Sector\",\n \"center\": {\n \"latitude\": 41.8781,\n \"longitude\": -93.0977\n },\n \"radius\": 500,\n \"cells\": [\n \"cell-086\",\n \"cell-080\",\n \"cell-080\",\n \"cell-080\",\n \"cell-080\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-18T11:00:00Z\",\n \"endTime\": \"2025-01-18T12:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/71b104411101\"\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "a433985ab22e", + "target_layer": "camara", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5, + "norm_field_f1": 0.7142857142857143, + "norm_key_f1": 0.8571428571428571, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"eMBB\",\n \"sliceProfile\": {\n \"latencyTarget\": 20,\n \"reliability\": 99.5,\n \"throughput\": {\n \"downlink\": 800,\n \"uplink\": 100\n },\n \"maxNumberOfUEs\": 5000\n },\n \"areaOfService\": {\n \"region\": \"Cell Site Cluster\",\n \"center\": {\n \"latitude\": 47.6062,\n \"longitude\": -122.3321\n },\n \"radius\": 2000,\n \"cells\": [\n \"cell-064\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-05-14T13:00:00Z\",\n \"endTime\": \"2025-05-15T01:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/6dde561161ba\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"eMBB\",\n \"sliceProfile\": {\n \"latencyTarget\": 20,\n \"reliability\": 99.9,\n \"throughput\": {\n \"downlink\": 800,\n \"uplink\": 100\n },\n \"maxNumberOfUEs\": 5000\n },\n \"areaOfService\": {\n \"region\": \"Cell Site Cluster\",\n \"center\": {\n \"latitude\": 47.6062,\n \"longitude\": -122.3321\n },\n \"radius\": 500,\n \"cells\": [\n \"cell-086\",\n \"cell-080\",\n \"cell-088\",\n \"cell-085\",\n \"cell-089\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-18T01:00:00Z\",\n \"endTime\": \"2025-01-18T03:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/100000000000\"\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "c7b67713cea0", + "target_layer": "camara", + "slice_type": "V2X", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5, + "norm_field_f1": 0.7142857142857143, + "norm_key_f1": 0.8571428571428571, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"V2X\",\n \"sliceProfile\": {\n \"latencyTarget\": 3,\n \"reliability\": 99.99,\n \"throughput\": {\n \"downlink\": 25,\n \"uplink\": 75\n },\n \"maxNumberOfUEs\": 10000\n },\n \"areaOfService\": {\n \"region\": \"Industrial Zone North\",\n \"center\": {\n \"latitude\": 52.52,\n \"longitude\": 13.405\n },\n \"radius\": 2000,\n \"cells\": [\n \"cell-089\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-04-11T00:00:00Z\",\n \"endTime\": \"2025-04-11T06:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/1ef93372cd51\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"V2X\",\n \"sliceProfile\": {\n \"latencyTarget\": 3,\n \"reliability\": 99.999,\n \"throughput\": {\n \"downlink\": 25,\n \"uplink\": 75\n },\n \"maxNumberOfUEs\": 10000\n },\n \"areaOfService\": {\n \"region\": \"Industrial Zone North\",\n \"center\": {\n \"latitude\": 52.52,\n \"longitude\": 13.405\n },\n \"radius\": 500,\n \"cells\": [\n \"cell-086\",\n \"cell-080\",\n \"cell-080\",\n \"cell-080\",\n \"cell-080\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-04T08:00:00Z\",\n \"endTime\": \"2025-01-04T10:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/114001111111\"\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "success", + "id": "734b0b0d58e4", + "target_layer": "camara", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5882352941176471, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"mMTC\",\n \"sliceProfile\": {\n \"latencyTarget\": 200,\n \"reliability\": 99.9,\n \"throughput\": {\n \"downlink\": 50,\n \"uplink\": 0.1\n },\n \"maxNumberOfUEs\": 5000\n },\n \"areaOfService\": {\n \"region\": \"Building Development Zone\",\n \"center\": {\n \"latitude\": 55.7558,\n \"longitude\": 37.6173\n },\n \"radius\": 500,\n \"cells\": [\n \"cell-054\",\n \"cell-039\",\n \"cell-057\",\n \"cell-095\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-08T21:00:00Z\",\n \"endTime\": \"2025-01-09T03:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/884603d08f0b\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"mMTC\",\n \"sliceProfile\": {\n \"latencyTarget\": 200,\n \"reliability\": 99.9,\n \"throughput\": {\n \"downlink\": 50,\n \"uplink\": 0.1\n },\n \"maxNumberOfUEs\": 5000\n },\n \"areaOfService\": {\n \"region\": \"Building Development Zone\",\n \"center\": {\n \"latitude\": 55.7558,\n \"longitude\": 37.6173\n },\n \"radius\": 500,\n \"cells\": [\n \"cell-081\",\n \"cell-080\",\n \"cell-082\",\n \"cell-083\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-01T08:00:00Z\",\n \"endTime\": \"2025-01-01T10:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/194310990111\"\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "9777a5690c22", + "target_layer": "intent_3gpp", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.775, + "norm_field_f1": 0.6578947368421053, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-mMTC-75f0f278206a\",\n \"name\": \"Industrial Robot Coordination mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-106246427132\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000002\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 0.5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 25,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 200,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 90,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Hospital Campus\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"indu\n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-mMTC-0d111010000d\",\n \"name\": \"Industrial Robot Coordination mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-000000000000\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 0.5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 25,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 200,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 90,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Hospital Campus\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contex\n......" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "792a2e7768b4", + "target_layer": "intent_3gpp", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.775, + "norm_field_f1": 0.6578947368421053, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-mMTC-7563ca2e916a\",\n \"name\": \"Hazmat Incident Command mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-11b912607ff5\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000002\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 50,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 0.5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 30,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99.5,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 500000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Disaster Response Area\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"\n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-mMTC-11b1b1b1b1b1\",\n \"name\": \"Hazmat Incident Command mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-11b1b1b1b1b1\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 0.5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 30,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 99.5,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 500000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Disaster Response Area\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"co\n......" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "1365f89d3e33", + "target_layer": "intent_3gpp", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.775, + "norm_field_f1": 0.6578947368421053, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-mMTC-3e24f2da03ac\",\n \"name\": \"Soil Moisture Sensor Mesh mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-ae8ba2fb4584\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000002\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 0.1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 30,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Rural Coverage Area\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"soil \n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-mMTC-0d0010000000\",\n \"name\": \"Soil Moisture Sensor Mesh mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-000000000000\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 0.1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 30,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 99,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Rural Coverage Area\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"conte\n......" + }, + { + "split": "test_in_distribution", + "kind": "success", + "id": "32ddbb98e457", + "target_layer": "intent_3gpp", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.925, + "norm_field_f1": 0.9736842105263158, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-eMBB-a5540fd9c553\",\n \"name\": \"Concrete Pour Monitoring eMBB Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-6b4bf57e7434\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"eMBB\",\n \"sNSSAI\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 1000,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 200,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 80,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99.95,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"maxNumberOfUEs\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Construction Site Gamma\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-eMBB-0d0b10b11006\",\n \"name\": \"Concrete Pour Monitoring eMBB Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-1b1b1b1b1b1b\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"eMBB\",\n \"sNSSAI\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 1000,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 200,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 80,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99.95,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"maxNumberOfUEs\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Construction Site Gamma\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \n......" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "e76a129ef0e9", + "target_layer": "o1_nrm", + "slice_type": "URLLC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.17391304347826086, + "norm_field_f1": 0.20000000000000004, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-009\",\n \"GNBDUFunction\": {\n \"id\": \"DU-004\",\n \"gNBId\": 9,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-004-003\",\n \"cellLocalId\": 3,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n79\",\n \"arfcnDL\": 708059,\n \"bSChannelBwDL\": 50,\n \"nrPCI\": 176,\n \"nrTAC\": 14794,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 2,\n \"sd\": \"000002\"\n },\n \"rrmPolicyMinRatio\": 0.05,\n \"rrmPolicyMaxRatio\": 0.25,\n \"rrmPolicyDedicatedRatio\": 0.15,\n \"rrmPolicyMaxRatioPerSlice\": 0.25\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"001\",\n \"mnc\": \"260\",\n \"sNSSAIList\": [\n {\n \"sst\": 2,\n \"sd\": \"000002\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-001\",\n \"GNBDUFunction\": {\n \"id\": \"DU-001\",\n \"gNBId\": 1,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-001-001\",\n \"cellLocalId\": 1,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n78\",\n \"arfcnDL\": 629820,\n \"bSChannelBwDL\": 40,\n \"nrPCI\": 816,\n \"nrTAC\": 3895,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 5,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.05,\n \"rrmPolicyMaxRatio\": 0.3,\n \"rrmPolicyDedicatedRatio\": 0.17,\n \"rrmPolicyMaxRatioPerSlice\": 0.3\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 5,\n \"sd\": \"000003\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "60c525414a65", + "target_layer": "o1_nrm", + "slice_type": "V2X", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.21739130434782608, + "norm_field_f1": 0.25, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-015\",\n \"GNBDUFunction\": {\n \"id\": \"DU-007\",\n \"gNBId\": 15,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-007-005\",\n \"cellLocalId\": 5,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n77\",\n \"arfcnDL\": 669944,\n \"bSChannelBwDL\": 20,\n \"nrPCI\": 815,\n \"nrTAC\": 61118,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 4,\n \"sd\": \"000001\"\n },\n \"rrmPolicyMinRatio\": 0.06,\n \"rrmPolicyMaxRatio\": 0.4,\n \"rrmPolicyDedicatedRatio\": 0.23,\n \"rrmPolicyMaxRatioPerSlice\": 0.4\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"15\",\n \"sNSSAIList\": [\n {\n \"sst\": 4,\n \"sd\": \"000002\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-018\",\n \"GNBDUFunction\": {\n \"id\": \"DU-008\",\n \"gNBId\": 18,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-008-001\",\n \"cellLocalId\": 1,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n78\",\n \"arfcnDL\": 629819,\n \"bSChannelBwDL\": 40,\n \"nrPCI\": 812,\n \"nrTAC\": 56681,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 5,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.06,\n \"rrmPolicyMaxRatio\": 0.31,\n \"rrmPolicyDedicatedRatio\": 0.18,\n \"rrmPolicyMaxRatioPerSlice\": 0.31\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 5,\n \"sd\": \"000003\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "3ab70dc309fc", + "target_layer": "o1_nrm", + "slice_type": "V2X", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.21739130434782608, + "norm_field_f1": 0.25, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-027\",\n \"GNBDUFunction\": {\n \"id\": \"DU-013\",\n \"gNBId\": 27,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-013-007\",\n \"cellLocalId\": 7,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n41\",\n \"arfcnDL\": 533808,\n \"bSChannelBwDL\": 40,\n \"nrPCI\": 347,\n \"nrTAC\": 28252,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 4,\n \"sd\": \"000001\"\n },\n \"rrmPolicyMinRatio\": 0.15,\n \"rrmPolicyMaxRatio\": 0.31,\n \"rrmPolicyDedicatedRatio\": 0.23,\n \"rrmPolicyMaxRatioPerSlice\": 0.31\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"001\",\n \"mnc\": \"01\",\n \"sNSSAIList\": [\n {\n \"sst\": 4,\n \"sd\": \"000001\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-016\",\n \"GNBDUFunction\": {\n \"id\": \"DU-008\",\n \"gNBId\": 16,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-008-002\",\n \"cellLocalId\": 2,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n77\",\n \"arfcnDL\": 629819,\n \"bSChannelBwDL\": 20,\n \"nrPCI\": 100,\n \"nrTAC\": 5846,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 2,\n \"sd\": \"000001\"\n },\n \"rrmPolicyMinRatio\": 0.15,\n \"rrmPolicyMaxRatio\": 0.4,\n \"rrmPolicyDedicatedRatio\": 0.28,\n \"rrmPolicyMaxRatioPerSlice\": 0.4\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 2,\n \"sd\": \"000003\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_template_ood", + "kind": "success", + "id": "f99a8ab43ed3", + "target_layer": "o1_nrm", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5652173913043478, + "norm_field_f1": 0.6, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-016\",\n \"GNBDUFunction\": {\n \"id\": \"DU-032\",\n \"gNBId\": 16,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-032-009\",\n \"cellLocalId\": 9,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n78\",\n \"arfcnDL\": 633919,\n \"bSChannelBwDL\": 50,\n \"nrPCI\": 688,\n \"nrTAC\": 4707,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.05,\n \"rrmPolicyMaxRatio\": 0.23,\n \"rrmPolicyDedicatedRatio\": 0.14,\n \"rrmPolicyMaxRatioPerSlice\": 0.23\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"001\",\n \"mnc\": \"15\",\n \"sNSSAIList\": [\n {\n \"sst\": 3,\n \"sd\": \"000002\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-016\",\n \"GNBDUFunction\": {\n \"id\": \"DU-008\",\n \"gNBId\": 16,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-008-002\",\n \"cellLocalId\": 2,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n78\",\n \"arfcnDL\": 620896,\n \"bSChannelBwDL\": 40,\n \"nrPCI\": 812,\n \"nrTAC\": 18621,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.05,\n \"rrmPolicyMaxRatio\": 0.23,\n \"rrmPolicyDedicatedRatio\": 0.14,\n \"rrmPolicyMaxRatioPerSlice\": 0.23\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 3,\n \"sd\": \"000001\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "14348f59be66", + "target_layer": "a1_policy", + "slice_type": "MPS", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.42105263157894735, + "norm_field_f1": 0.5, + "norm_key_f1": 1.0, + "error_label": "a1_policy_value_error", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-mps-eff6d6\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-mps-59d80a\",\n \"cellId\": \"cell-079\",\n \"sNSSAI\": {\n \"sst\": 5,\n \"sd\": \"000003\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 200,\n \"mfbr\": 240.0,\n \"priorityLevel\": 1,\n \"5qi\": 69,\n \"pdb\": 60,\n \"per\": 1e-05\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 15,\n \"max\": 41\n },\n \"schedulerWeight\": 0.71,\n \"maxUEs\": 5000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-urllc-111111\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-urllc-111111\",\n \"cellId\": \"cell-081\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000001\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 200,\n \"mfbr\": 240.0,\n \"priorityLevel\": 3,\n \"5qi\": 85,\n \"pdb\": 5,\n \"per\": 1e-06\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 15,\n \"max\": 33\n },\n \"schedulerWeight\": 0.48,\n \"maxUEs\": 5000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "2a4c961b861b", + "target_layer": "a1_policy", + "slice_type": "V2X", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.42105263157894735, + "norm_field_f1": 0.5, + "norm_key_f1": 1.0, + "error_label": "a1_policy_value_error", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-v2x-afe37b\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-v2x-f87cb8\",\n \"cellId\": \"cell-047\",\n \"sNSSAI\": {\n \"sst\": 4,\n \"sd\": \"000002\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 150,\n \"mfbr\": 180.0,\n \"priorityLevel\": 2,\n \"5qi\": 84,\n \"pdb\": 30,\n \"per\": 0.0001\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 11,\n \"max\": 39\n },\n \"schedulerWeight\": 0.48,\n \"maxUEs\": 10000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-mps-111111\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-mps-000001\",\n \"cellId\": \"cell-010\",\n \"sNSSAI\": {\n \"sst\": 5,\n \"sd\": \"000003\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 150,\n \"mfbr\": 180.0,\n \"priorityLevel\": 1,\n \"5qi\": 66,\n \"pdb\": 100,\n \"per\": 1e-05\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 11,\n \"max\": 31\n },\n \"schedulerWeight\": 0.59,\n \"maxUEs\": 10000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "d5736ff86f3e", + "target_layer": "a1_policy", + "slice_type": "URLLC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.42105263157894735, + "norm_field_f1": 0.5, + "norm_key_f1": 1.0, + "error_label": "a1_policy_value_error", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-urllc-032b45\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-urllc-6ddfa6\",\n \"cellId\": \"cell-006\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000002\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 150,\n \"mfbr\": 180.0,\n \"priorityLevel\": 1,\n \"5qi\": 2,\n \"pdb\": 150,\n \"per\": 1e-05\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 15,\n \"max\": 44\n },\n \"schedulerWeight\": 0.3,\n \"maxUEs\": 50\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-urllc-111111\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-urllc-111111\",\n \"cellId\": \"cell-081\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000001\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 150,\n \"mfbr\": 180.0,\n \"priorityLevel\": 3,\n \"5qi\": 85,\n \"pdb\": 5,\n \"per\": 1e-06\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 12,\n \"max\": 32\n },\n \"schedulerWeight\": 0.59,\n \"maxUEs\": 50\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_template_ood", + "kind": "success", + "id": "bfc4e60b96a9", + "target_layer": "a1_policy", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7368421052631579, + "norm_field_f1": 0.8333333333333334, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-mmtc-2ff4af\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-mmtc-1e2628\",\n \"cellId\": \"cell-052\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000001\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 1,\n \"mfbr\": 1.2,\n \"priorityLevel\": 9,\n \"5qi\": 8,\n \"pdb\": 300,\n \"per\": 0.005\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 5,\n \"max\": 23\n },\n \"schedulerWeight\": 0.45,\n \"maxUEs\": 1000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-mmtc-110006\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-mmtc-110006\",\n \"cellId\": \"cell-086\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 1,\n \"mfbr\": 1.2,\n \"priorityLevel\": 9,\n \"5qi\": 8,\n \"pdb\": 300,\n \"per\": 0.005\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 5,\n \"max\": 23\n },\n \"schedulerWeight\": 0.46,\n \"maxUEs\": 1000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "a43297d5ddee", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "HMTC", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.2222222222222222, + "norm_field_f1": 0.1746031746031746, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-ed3943082134\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 5,\n \"measurements\": [\n 3.79,\n 4.98,\n 3.83,\n 4.78,\n 5.43\n ],\n \"average\": 4.562,\n \"min\": 3.79,\n \"max\": 5.43,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 100,\n \"measurements\": [\n 110.9,\n 99.1,\n 119.2,\n 119.3,\n 110.7\n ],\n \"average\": 111.84,\n \"min\": 99.1,\n \"max\": 119.3,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 500,\n \"measurements\": [\n 508.6,\n 506.8,\n 503.3,\n 541.3,\n 559.8\n ],\n \"average\": 523.96,\n \"min\": 503.3,\n \"max\": 559.8,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99999,\n \"measurements\": [\n 99.923372,\n 100,\n 99.961583,\n 99.913077,\n 99.998417\n ],\n \"average\": 99.959,\n \"min\": 99.913077,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 500,\n \"measurements\": [\n 73,\n 119,\n 268,\n 421,\n 487\n ],\n \"average\": 273.6,\n \"min\": 73,\n \"max\": 487,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 72.9,\n \"memoryUsage\": 47.5,\n \"prbUtilization\": 32.4\n },\n \"@type\": \"IntentAssuranceRep\n......", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-111111111111\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 3,\n \"measurements\": [\n 2.26,\n 2.9,\n 2.84,\n 2.89,\n 2.89\n ],\n \"average\": 2.776,\n \"min\": 2.26,\n \"max\": 2.9,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 500,\n \"measurements\": [\n 585.1,\n 519.4,\n 587.6,\n 539.8,\n 580.5\n ],\n \"average\": 548.48,\n \"min\": 519.4,\n \"max\": 587.6,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measurements\": [\n 112.1,\n 112.1,\n 99.9,\n 119.5,\n 118.5\n ],\n \"average\": 110.42,\n \"min\": 99.9,\n \"max\": 119.5,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9999,\n \"measurements\": [\n 99.903676,\n 99.999611,\n 99.999811,\n 99.999962,\n 99.999962\n ],\n \"average\": 99.98,\n \"min\": 99.903676,\n \"max\": 99.999962,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 50000,\n \"measurements\": [\n 14462,\n 28641,\n 28446,\n 14462,\n 28446\n ],\n \"average\": 22991.4,\n \"min\": 14462,\n \"max\": 28641,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 77.7,\n \"memoryUsage\": 28.6,\n \"prbUtilization\n......" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "b4ae6c9d9034", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "HMTC", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.2222222222222222, + "norm_field_f1": 0.1746031746031746, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-a1c5e0cd2107\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 1,\n \"measurements\": [\n 0.77,\n 1.07,\n 0.92,\n 0.82,\n 0.88\n ],\n \"average\": 0.892,\n \"min\": 0.77,\n \"max\": 1.07,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 100,\n \"measurements\": [\n 102.5,\n 114.4,\n 112.1,\n 106.4,\n 109.8\n ],\n \"average\": 109.04,\n \"min\": 102.5,\n \"max\": 114.4,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 500,\n \"measurements\": [\n 515.7,\n 460.6,\n 538.9,\n 552.5,\n 593.1\n ],\n \"average\": 532.16,\n \"min\": 460.6,\n \"max\": 593.1,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99999,\n \"measurements\": [\n 99.949115,\n 99.978169,\n 99.934188,\n 99.978159,\n 100\n ],\n \"average\": 99.968,\n \"min\": 99.934188,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 500,\n \"measurements\": [\n 393,\n 192,\n 246,\n 45,\n 280\n ],\n \"average\": 231.2,\n \"min\": 45,\n \"max\": 393,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 56.7,\n \"memoryUsage\": 39.4,\n \"prbUtilization\": 69.3\n },\n \"@type\": \"IntentAssuranceR\n......", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-000000000001\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 3,\n \"measurements\": [\n 2.26,\n 2.9,\n 2.84,\n 2.89,\n 2.89\n ],\n \"average\": 2.776,\n \"min\": 2.26,\n \"max\": 2.9,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 500,\n \"measurements\": [\n 585.1,\n 519.8,\n 587.6,\n 538.6,\n 521.1\n ],\n \"average\": 548.42,\n \"min\": 519.8,\n \"max\": 587.6,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measurements\": [\n 115.6,\n 112.1,\n 99.9,\n 112.6,\n 112.6\n ],\n \"average\": 110.56,\n \"min\": 99.9,\n \"max\": 115.6,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9999,\n \"measurements\": [\n 99.903676,\n 99.999621,\n 99.99961,\n 99.999812,\n 99.999989\n ],\n \"average\": 99.98,\n \"min\": 99.903676,\n \"max\": 99.999989,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 500000,\n \"measurements\": [\n 146446,\n 289212,\n 289212,\n 146446,\n 289212\n ],\n \"average\": 234265.2,\n \"min\": 146446,\n \"max\": 289212,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 77.7,\n \"memoryUsage\": 28.6,\n \"prbUti\n......" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "7f4c7e3c8d4e", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "HMTC", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.23809523809523808, + "norm_field_f1": 0.19047619047619047, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-676527391acc\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 3,\n \"measurements\": [\n 1.8,\n 2.04,\n 2.99,\n 2.57,\n 2.84\n ],\n \"average\": 2.448,\n \"min\": 1.8,\n \"max\": 2.99,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 100,\n \"measurements\": [\n 95.7,\n 108.9,\n 98.5,\n 104.4,\n 118.3\n ],\n \"average\": 105.16,\n \"min\": 95.7,\n \"max\": 118.3,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 1000,\n \"measurements\": [\n 931.2,\n 912.2,\n 1194.4,\n 1192.6,\n 946.9\n ],\n \"average\": 1035.46,\n \"min\": 912.2,\n \"max\": 1194.4,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99999,\n \"measurements\": [\n 100,\n 100,\n 99.95766,\n 99.988027,\n 99.93028\n ],\n \"average\": 99.975,\n \"min\": 99.93028,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 100000,\n \"measurements\": [\n 60415,\n 88288,\n 41560,\n 77445,\n 94786\n ],\n \"average\": 72498.8,\n \"min\": 41560,\n \"max\": 94786,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 56.7,\n \"memoryUsage\": 69.1,\n \"prbUtilization\": 59.8\n },\n \"@type\": \"Inte\n......", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-110011110000\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 3,\n \"measurements\": [\n 2.89,\n 2.86,\n 2.86,\n 2.86,\n 2.86\n ],\n \"average\": 2.866,\n \"min\": 2.86,\n \"max\": 2.89,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 1000,\n \"measurements\": [\n 1016.1,\n 1169.4,\n 1128.4,\n 1098.1,\n 1128.4\n ],\n \"average\": 1108.08,\n \"min\": 1016.1,\n \"max\": 1169.4,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 50,\n \"measurements\": [\n 58.6,\n 58.3,\n 58.8,\n 52.1,\n 52.6\n ],\n \"average\": 54.48,\n \"min\": 52.1,\n \"max\": 58.8,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9999,\n \"measurements\": [\n 99.903667,\n 99.999611,\n 99.99996,\n 99.999962,\n 99.999966\n ],\n \"average\": 99.98,\n \"min\": 99.903667,\n \"max\": 99.999966,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 500000,\n \"measurements\": [\n 136446,\n 19801,\n 289202,\n 289262,\n 289262\n ],\n \"average\": 188838.6,\n \"min\": 19801,\n \"max\": 289262,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 77.7,\n \"memoryUsage\": 28.6,\n \"prbU\n......" + }, + { + "split": "test_template_ood", + "kind": "success", + "id": "7f4c7e3c8d4e", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "HMTC", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.23809523809523808, + "norm_field_f1": 0.19047619047619047, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-676527391acc\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 3,\n \"measurements\": [\n 1.8,\n 2.04,\n 2.99,\n 2.57,\n 2.84\n ],\n \"average\": 2.448,\n \"min\": 1.8,\n \"max\": 2.99,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 100,\n \"measurements\": [\n 95.7,\n 108.9,\n 98.5,\n 104.4,\n 118.3\n ],\n \"average\": 105.16,\n \"min\": 95.7,\n \"max\": 118.3,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 1000,\n \"measurements\": [\n 931.2,\n 912.2,\n 1194.4,\n 1192.6,\n 946.9\n ],\n \"average\": 1035.46,\n \"min\": 912.2,\n \"max\": 1194.4,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99999,\n \"measurements\": [\n 100,\n 100,\n 99.95766,\n 99.988027,\n 99.93028\n ],\n \"average\": 99.975,\n \"min\": 99.93028,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 100000,\n \"measurements\": [\n 60415,\n 88288,\n 41560,\n 77445,\n 94786\n ],\n \"average\": 72498.8,\n \"min\": 41560,\n \"max\": 94786,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 56.7,\n \"memoryUsage\": 69.1,\n \"prbUtilization\": 59.8\n },\n \"@type\": \"Inte\n......", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-110011110000\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 3,\n \"measurements\": [\n 2.89,\n 2.86,\n 2.86,\n 2.86,\n 2.86\n ],\n \"average\": 2.866,\n \"min\": 2.86,\n \"max\": 2.89,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 1000,\n \"measurements\": [\n 1016.1,\n 1169.4,\n 1128.4,\n 1098.1,\n 1128.4\n ],\n \"average\": 1108.08,\n \"min\": 1016.1,\n \"max\": 1169.4,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 50,\n \"measurements\": [\n 58.6,\n 58.3,\n 58.8,\n 52.1,\n 52.6\n ],\n \"average\": 54.48,\n \"min\": 52.1,\n \"max\": 58.8,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9999,\n \"measurements\": [\n 99.903667,\n 99.999611,\n 99.99996,\n 99.999962,\n 99.999966\n ],\n \"average\": 99.98,\n \"min\": 99.903667,\n \"max\": 99.999966,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 500000,\n \"measurements\": [\n 136446,\n 19801,\n 289202,\n 289262,\n 289262\n ],\n \"average\": 188838.6,\n \"min\": 19801,\n \"max\": 289262,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 77.7,\n \"memoryUsage\": 28.6,\n \"prbU\n......" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "75ff5712aae2", + "target_layer": "tmf921", + "slice_type": "MPS", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.717948717948718, + "norm_field_f1": 0.8507462686567164, + "norm_key_f1": 0.9402985074626866, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-57fe3dc7da91\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-57fe3dc7da91\",\n \"name\": \"MPS Slice - Public Wifi Mesh Backhaul\",\n \"description\": \"Intent for public WiFi mesh backhaul at Urban Innovation District with MPS service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-c2805cb10e14\",\n \"name\": \"MPS QoS Delivery Expectation\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-mps-a4d53b\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"MPS\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 5,\n \"sd\": \"000004\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-11b144b1b106\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-11b144b1b106\",\n \"name\": \"URLLC Slice - Public Wifi Mesh Backhaul\",\n \"description\": \"Intent for public WiFi mesh backhaul at Urban Innovation District with URLLC service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 1,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-10b1b1b1b106\",\n \"name\": \"URLLC QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-urllc-10b1b1\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"URLLC\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 2,\n \"sd\": \"000001\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \n......" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "27ff1aefa833", + "target_layer": "tmf921", + "slice_type": "URLLC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7692307692307693, + "norm_field_f1": 0.8656716417910447, + "norm_key_f1": 0.9402985074626866, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-6db2cf6a195a\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-6db2cf6a195a\",\n \"name\": \"URLLC Slice - V2X Intersection Management\",\n \"description\": \"Intent for V2X intersection management at Hospital Campus with URLLC service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 2,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-8e8fc9a670b9\",\n \"name\": \"URLLC QoS Delivery Expectation\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-urllc-20c617\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"URLLC\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 2,\n \"sd\": \"000001\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 150,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n \n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-011011101010\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-011011101010\",\n \"name\": \"URLLC Slice - V2X Intersection Management\",\n \"description\": \"Intent for V2X intersection management at Hospital Campus with URLLC service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 1,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-000000000001\",\n \"name\": \"URLLC QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-urllc-000001\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"URLLC\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 2,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 150,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"un\n......" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "945c8bbf3b30", + "target_layer": "tmf921", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7692307692307693, + "norm_field_f1": 0.8656716417910447, + "norm_key_f1": 0.9402985074626866, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-1699091a9c7b\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-1699091a9c7b\",\n \"name\": \"eMBB Slice - Cnc Machine Remote Control\",\n \"description\": \"Intent for CNC machine remote control at Hospital Campus with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 4,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-8b042107dba6\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-352a93\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 800,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 200,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n \n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-11b1b1b1b1b1\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-11b1b1b1b1b1\",\n \"name\": \"eMBB Slice - Cnc Machine Remote Control\",\n \"description\": \"Intent for CNC machine remote control at Hospital Campus with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-11b1b1b1b1b1\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-11b1b1\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 800,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 200,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"\n......" + }, + { + "split": "test_template_ood", + "kind": "success", + "id": "ffcc8d47cd63", + "target_layer": "tmf921", + "slice_type": "V2X", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.8860759493670886, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-c09bf08c2c96\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-c09bf08c2c96\",\n \"name\": \"V2X Slice - Cross-Dock Coordination\",\n \"description\": \"Intent for cross-dock coordination at Medical Center District with V2X service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 1,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-2b21f2a05b34\",\n \"name\": \"V2X QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-v2x-b2886e\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"V2X\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 4,\n \"sd\": \"000001\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 25,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 75,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n \n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-011111111111\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-011111111111\",\n \"name\": \"V2X Slice - Cross-Dock Coordination\",\n \"description\": \"Intent for cross-dock coordination at Medical Center District with V2X service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 1,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-011111111111\",\n \"name\": \"V2X QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-v2x-011111\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"V2X\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 4,\n \"sd\": \"000001\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 25,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 75,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n......" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "376fa50e73d1", + "target_layer": "camara", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.43749999999999994, + "norm_field_f1": 0.7142857142857142, + "norm_key_f1": 0.9285714285714286, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"mMTC\",\n \"sliceProfile\": {\n \"latencyTarget\": 500,\n \"reliability\": 99.5,\n \"throughput\": {\n \"downlink\": 0.5,\n \"uplink\": 10\n },\n \"maxNumberOfUEs\": 500000\n },\n \"areaOfService\": {\n \"region\": \"Runway Complex\",\n \"center\": {\n \"latitude\": 25.2532,\n \"longitude\": 55.3657\n },\n \"radius\": 200,\n \"cells\": [\n \"cell-051\",\n \"cell-091\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-04-12T13:00:00Z\",\n \"endTime\": \"2025-04-12T21:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/ce2bc908d0f7\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"mMTC\",\n \"sliceProfile\": {\n \"latencyTarget\": 500,\n \"reliability\": 99.5,\n \"throughput\": {\n \"downlink\": 0.5,\n \"uplink\": 10\n },\n \"maxNumberOfUEs\": 500000\n },\n \"areaOfService\": {\n \"region\": \"Runway Complex\",\n \"center\": {\n \"latitude\": 47.4502,\n \"longitude\": 8.5617\n },\n \"radius\": 500,\n \"cells\": [\n \"cell-081\",\n \"cell-080\",\n \"cell-082\",\n \"cell-083\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-04T03:00:00Z\",\n \"endTime\": \"2025-01-04T04:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/111091190111\"\n }\n}" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "3393971c8ccb", + "target_layer": "camara", + "slice_type": "MPS", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5, + "norm_field_f1": 0.7142857142857143, + "norm_key_f1": 0.8571428571428571, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"MPS\",\n \"sliceProfile\": {\n \"latencyTarget\": 1,\n \"reliability\": 99.9999,\n \"throughput\": {\n \"downlink\": 100,\n \"uplink\": 75\n },\n \"maxNumberOfUEs\": 500\n },\n \"areaOfService\": {\n \"region\": \"Runway Complex\",\n \"center\": {\n \"latitude\": 25.2532,\n \"longitude\": 55.3657\n },\n \"radius\": 500,\n \"cells\": [\n \"cell-100\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-08-20T06:00:00Z\",\n \"endTime\": \"2025-08-20T18:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/1531bb364fd9\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"MPS\",\n \"sliceProfile\": {\n \"latencyTarget\": 1,\n \"reliability\": 99.9999,\n \"throughput\": {\n \"downlink\": 100,\n \"uplink\": 75\n },\n \"maxNumberOfUEs\": 500\n },\n \"areaOfService\": {\n \"region\": \"Runway Complex\",\n \"center\": {\n \"latitude\": 47.4502,\n \"longitude\": 8.5617\n },\n \"radius\": 500,\n \"cells\": [\n \"cell-086\",\n \"cell-080\",\n \"cell-088\",\n \"cell-089\",\n \"cell-085\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-08T03:00:00Z\",\n \"endTime\": \"2025-01-08T05:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/71044401000b\"\n }\n}" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "7042bc34bef1", + "target_layer": "camara", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5, + "norm_field_f1": 0.7142857142857143, + "norm_key_f1": 0.8571428571428571, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"eMBB\",\n \"sliceProfile\": {\n \"latencyTarget\": 30,\n \"reliability\": 99.0,\n \"throughput\": {\n \"downlink\": 300,\n \"uplink\": 25\n },\n \"maxNumberOfUEs\": 5000\n },\n \"areaOfService\": {\n \"region\": \"Power Grid Sector A\",\n \"center\": {\n \"latitude\": 59.3293,\n \"longitude\": 18.0686\n },\n \"radius\": 2000,\n \"cells\": [\n \"cell-072\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-08-22T23:00:00Z\",\n \"endTime\": \"2025-08-23T00:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/67338ea3d0a2\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"eMBB\",\n \"sliceProfile\": {\n \"latencyTarget\": 30,\n \"reliability\": 99.9,\n \"throughput\": {\n \"downlink\": 300,\n \"uplink\": 25\n },\n \"maxNumberOfUEs\": 5000\n },\n \"areaOfService\": {\n \"region\": \"Power Grid Sector A\",\n \"center\": {\n \"latitude\": 59.3293,\n \"longitude\": 18.0686\n },\n \"radius\": 5000,\n \"cells\": [\n \"cell-086\",\n \"cell-080\",\n \"cell-080\",\n \"cell-080\",\n \"cell-080\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-08T06:00:00Z\",\n \"endTime\": \"2025-01-08T14:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/844431119101\"\n }\n}" + }, + { + "split": "test_template_ood", + "kind": "success", + "id": "3ec5d13a3c66", + "target_layer": "camara", + "slice_type": "MPS", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5555555555555556, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"MPS\",\n \"sliceProfile\": {\n \"latencyTarget\": 15,\n \"reliability\": 99.999,\n \"throughput\": {\n \"downlink\": 25,\n \"uplink\": 75\n },\n \"maxNumberOfUEs\": 50\n },\n \"areaOfService\": {\n \"region\": \"Industrial Zone North\",\n \"center\": {\n \"latitude\": 52.52,\n \"longitude\": 13.405\n },\n \"radius\": 500,\n \"cells\": [\n \"cell-080\",\n \"cell-024\",\n \"cell-096\",\n \"cell-050\",\n \"cell-056\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-06-01T18:00:00Z\",\n \"endTime\": \"2025-06-08T18:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/51fb2b19dfd5\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"MPS\",\n \"sliceProfile\": {\n \"latencyTarget\": 15,\n \"reliability\": 99.999,\n \"throughput\": {\n \"downlink\": 25,\n \"uplink\": 75\n },\n \"maxNumberOfUEs\": 50\n },\n \"areaOfService\": {\n \"region\": \"Industrial Zone North\",\n \"center\": {\n \"latitude\": 52.52,\n \"longitude\": 13.405\n },\n \"radius\": 500,\n \"cells\": [\n \"cell-089\",\n \"cell-060\",\n \"cell-080\",\n \"cell-080\",\n \"cell-080\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-19T08:00:00Z\",\n \"endTime\": \"2025-01-19T10:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/100000000000\"\n }\n}" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "1bb88377bb51", + "target_layer": "intent_3gpp", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.775, + "norm_field_f1": 0.6578947368421053, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-mMTC-9f52a80fa133\",\n \"name\": \"Joint Force C2 Network mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-87ee952586d1\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000001\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 50,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 0.5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 500,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 90,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Naval Port Zone\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"joint force\n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-mMTC-0d0444001008\",\n \"name\": \"Joint Force C2 Network mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-0d0444001008\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 0.5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 500,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 90,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Naval Port Zone\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCond\n......" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "e57e5b4bf0f2", + "target_layer": "intent_3gpp", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.775, + "norm_field_f1": 0.6578947368421053, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-mMTC-cac30a9f990f\",\n \"name\": \"Traffic Signal Preemption mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-7d33416f957f\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000001\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 10,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 50,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99.5,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Healthcare Park\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"traffic s\n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-mMTC-110011000008\",\n \"name\": \"Traffic Signal Preemption mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-000000000001\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 99.5,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Healthcare Park\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCo\n......" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "c3ca42fa50ad", + "target_layer": "intent_3gpp", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.775, + "norm_field_f1": 0.6578947368421053, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-mMTC-b563e3be5374\",\n \"name\": \"Agv Fleet Management mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-9a1a93134f17\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000002\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 500,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 95,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 500000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Industrial Zone North\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"AGV fleet\n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-mMTC-091000000000\",\n \"name\": \"Agv Fleet Management mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-000000000000\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 500,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 95,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 500000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Industrial Zone North\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCo\n......" + }, + { + "split": "test_template_ood", + "kind": "success", + "id": "581f62e0a8a8", + "target_layer": "intent_3gpp", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.925, + "norm_field_f1": 0.9736842105263158, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-eMBB-f9bf5c778a43\",\n \"name\": \"Crane Remote Operation eMBB Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-0c144af8205e\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"eMBB\",\n \"sNSSAI\": {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 800,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 50,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 80,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99.9,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"maxNumberOfUEs\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Building Development Zone\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\":\n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-eMBB-100000000001\",\n \"name\": \"Crane Remote Operation eMBB Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-000000000001\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"eMBB\",\n \"sNSSAI\": {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 800,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 50,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 80,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99.9,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"maxNumberOfUEs\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Building Development Zone\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"\n......" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "a54dc4a556cb", + "target_layer": "o1_nrm", + "slice_type": "HMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.2608695652173913, + "norm_field_f1": 0.3, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-006\",\n \"GNBDUFunction\": {\n \"id\": \"DU-033\",\n \"gNBId\": 6,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-033-003\",\n \"cellLocalId\": 3,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n41\",\n \"arfcnDL\": 518533,\n \"bSChannelBwDL\": 20,\n \"nrPCI\": 420,\n \"nrTAC\": 18844,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 5,\n \"sd\": \"000002\"\n },\n \"rrmPolicyMinRatio\": 0.6,\n \"rrmPolicyMaxRatio\": 0.86,\n \"rrmPolicyDedicatedRatio\": 0.73,\n \"rrmPolicyMaxRatioPerSlice\": 0.86\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"234\",\n \"mnc\": \"02\",\n \"sNSSAIList\": [\n {\n \"sst\": 5,\n \"sd\": \"000002\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-016\",\n \"GNBDUFunction\": {\n \"id\": \"DU-008\",\n \"gNBId\": 16,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-008-002\",\n \"cellLocalId\": 2,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n78\",\n \"arfcnDL\": 620126,\n \"bSChannelBwDL\": 40,\n \"nrPCI\": 812,\n \"nrTAC\": 38911,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 5,\n \"sd\": \"000001\"\n },\n \"rrmPolicyMinRatio\": 0.6,\n \"rrmPolicyMaxRatio\": 0.85,\n \"rrmPolicyDedicatedRatio\": 0.72,\n \"rrmPolicyMaxRatioPerSlice\": 0.85\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 5,\n \"sd\": \"000001\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "2c5a416bf2ce", + "target_layer": "o1_nrm", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.2608695652173913, + "norm_field_f1": 0.3, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-036\",\n \"GNBDUFunction\": {\n \"id\": \"DU-042\",\n \"gNBId\": 36,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-042-006\",\n \"cellLocalId\": 6,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n79\",\n \"arfcnDL\": 702159,\n \"bSChannelBwDL\": 60,\n \"nrPCI\": 241,\n \"nrTAC\": 65427,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 1,\n \"sd\": \"000002\"\n },\n \"rrmPolicyMinRatio\": 0.15,\n \"rrmPolicyMaxRatio\": 0.38,\n \"rrmPolicyDedicatedRatio\": 0.27,\n \"rrmPolicyMaxRatioPerSlice\": 0.38\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"234\",\n \"mnc\": \"260\",\n \"sNSSAIList\": [\n {\n \"sst\": 1,\n \"sd\": \"000003\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-016\",\n \"GNBDUFunction\": {\n \"id\": \"DU-008\",\n \"gNBId\": 16,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-008-002\",\n \"cellLocalId\": 2,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n78\",\n \"arfcnDL\": 620012,\n \"bSChannelBwDL\": 40,\n \"nrPCI\": 812,\n \"nrTAC\": 38502,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 1,\n \"sd\": \"000001\"\n },\n \"rrmPolicyMinRatio\": 0.15,\n \"rrmPolicyMaxRatio\": 0.4,\n \"rrmPolicyDedicatedRatio\": 0.28,\n \"rrmPolicyMaxRatioPerSlice\": 0.4\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "05186356f450", + "target_layer": "o1_nrm", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.2608695652173913, + "norm_field_f1": 0.3, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-029\",\n \"GNBDUFunction\": {\n \"id\": \"DU-038\",\n \"gNBId\": 29,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-038-010\",\n \"cellLocalId\": 10,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n79\",\n \"arfcnDL\": 699251,\n \"bSChannelBwDL\": 50,\n \"nrPCI\": 761,\n \"nrTAC\": 26056,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 1,\n \"sd\": \"000002\"\n },\n \"rrmPolicyMinRatio\": 0.25,\n \"rrmPolicyMaxRatio\": 0.55,\n \"rrmPolicyDedicatedRatio\": 0.4,\n \"rrmPolicyMaxRatioPerSlice\": 0.55\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"310\",\n \"mnc\": \"260\",\n \"sNSSAIList\": [\n {\n \"sst\": 1,\n \"sd\": \"000002\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-016\",\n \"GNBDUFunction\": {\n \"id\": \"DU-008\",\n \"gNBId\": 16,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-008-001\",\n \"cellLocalId\": 1,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n78\",\n \"arfcnDL\": 629820,\n \"bSChannelBwDL\": 40,\n \"nrPCI\": 812,\n \"nrTAC\": 58431,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.25,\n \"rrmPolicyMaxRatio\": 0.5,\n \"rrmPolicyDedicatedRatio\": 0.38,\n \"rrmPolicyMaxRatioPerSlice\": 0.5\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "success", + "id": "1366e9589e3d", + "target_layer": "o1_nrm", + "slice_type": "HMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5652173913043478, + "norm_field_f1": 0.65, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-010\",\n \"GNBDUFunction\": {\n \"id\": \"DU-047\",\n \"gNBId\": 10,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-047-009\",\n \"cellLocalId\": 9,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n78\",\n \"arfcnDL\": 632806,\n \"bSChannelBwDL\": 80,\n \"nrPCI\": 964,\n \"nrTAC\": 11290,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 5,\n \"sd\": \"000001\"\n },\n \"rrmPolicyMinRatio\": 0.6,\n \"rrmPolicyMaxRatio\": 0.85,\n \"rrmPolicyDedicatedRatio\": 0.72,\n \"rrmPolicyMaxRatioPerSlice\": 0.85\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"001\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 5,\n \"sd\": \"000001\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-001\",\n \"GNBDUFunction\": {\n \"id\": \"DU-001\",\n \"gNBId\": 1,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-001-001\",\n \"cellLocalId\": 1,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n78\",\n \"arfcnDL\": 629820,\n \"bSChannelBwDL\": 100,\n \"nrPCI\": 812,\n \"nrTAC\": 38921,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 5,\n \"sd\": \"000001\"\n },\n \"rrmPolicyMinRatio\": 0.6,\n \"rrmPolicyMaxRatio\": 0.85,\n \"rrmPolicyDedicatedRatio\": 0.72,\n \"rrmPolicyMaxRatioPerSlice\": 0.85\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 5,\n \"sd\": \"000001\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "23d6129078f6", + "target_layer": "a1_policy", + "slice_type": "MPS", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.42105263157894735, + "norm_field_f1": 0.5, + "norm_key_f1": 1.0, + "error_label": "a1_policy_value_error", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-mps-156efe\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-mps-345235\",\n \"cellId\": \"cell-029\",\n \"sNSSAI\": {\n \"sst\": 5,\n \"sd\": \"000004\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 50,\n \"mfbr\": 60.0,\n \"priorityLevel\": 1,\n \"5qi\": 65,\n \"pdb\": 75,\n \"per\": 1e-05\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 5,\n \"max\": 19\n },\n \"schedulerWeight\": 0.5,\n \"maxUEs\": 10\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-urllc-11000b\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-urllc-11000b\",\n \"cellId\": \"cell-018\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000001\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 50,\n \"mfbr\": 60.0,\n \"priorityLevel\": 3,\n \"5qi\": 84,\n \"pdb\": 30,\n \"per\": 1e-06\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 5,\n \"max\": 23\n },\n \"schedulerWeight\": 0.39,\n \"maxUEs\": 10\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "2be9624ce4ff", + "target_layer": "a1_policy", + "slice_type": "URLLC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.47368421052631576, + "norm_field_f1": 0.5555555555555556, + "norm_key_f1": 1.0, + "error_label": "a1_policy_value_error", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-urllc-dc57c9\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-urllc-d43999\",\n \"cellId\": \"cell-060\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000002\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 20,\n \"mfbr\": 24.0,\n \"priorityLevel\": 1,\n \"5qi\": 82,\n \"pdb\": 10,\n \"per\": 1e-05\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 5,\n \"max\": 33\n },\n \"schedulerWeight\": 0.63,\n \"maxUEs\": 5000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-urllc-0d000d\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-urllc-0d000d\",\n \"cellId\": \"cell-001\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000003\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 20,\n \"mfbr\": 24.0,\n \"priorityLevel\": 3,\n \"5qi\": 84,\n \"pdb\": 30,\n \"per\": 1e-06\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 5,\n \"max\": 23\n },\n \"schedulerWeight\": 0.46,\n \"maxUEs\": 5000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "cdcd70fe4383", + "target_layer": "a1_policy", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.47368421052631576, + "norm_field_f1": 0.5555555555555556, + "norm_key_f1": 1.0, + "error_label": "a1_policy_value_error", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-embb-2fb74c\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-embb-571eb5\",\n \"cellId\": \"cell-051\",\n \"sNSSAI\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 500,\n \"mfbr\": 600.0,\n \"priorityLevel\": 6,\n \"5qi\": 7,\n \"pdb\": 100,\n \"per\": 0.005\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 27,\n \"max\": 49\n },\n \"schedulerWeight\": 0.3,\n \"maxUEs\": 100\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-embb-11410c\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-embb-11410c\",\n \"cellId\": \"cell-061\",\n \"sNSSAI\": {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 500,\n \"mfbr\": 600.0,\n \"priorityLevel\": 5,\n \"5qi\": 6,\n \"pdb\": 300,\n \"per\": 0.0001\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 27,\n \"max\": 43\n },\n \"schedulerWeight\": 0.51,\n \"maxUEs\": 100\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "success", + "id": "891dafff525f", + "target_layer": "a1_policy", + "slice_type": "URLLC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7894736842105263, + "norm_field_f1": 0.8888888888888888, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-urllc-bc7a0d\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-urllc-eaa78a\",\n \"cellId\": \"cell-002\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000002\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 100,\n \"mfbr\": 120.0,\n \"priorityLevel\": 1,\n \"5qi\": 85,\n \"pdb\": 5,\n \"per\": 1e-06\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 5,\n \"max\": 23\n },\n \"schedulerWeight\": 0.46,\n \"maxUEs\": 50\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-urllc-11000d\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-urllc-11000d\",\n \"cellId\": \"cell-086\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000001\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 100,\n \"mfbr\": 120.0,\n \"priorityLevel\": 1,\n \"5qi\": 85,\n \"pdb\": 5,\n \"per\": 1e-06\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 5,\n \"max\": 23\n },\n \"schedulerWeight\": 0.46,\n \"maxUEs\": 50\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "380f2571b07d", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "URLLC", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.23809523809523808, + "norm_field_f1": 0.1111111111111111, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-b1db26b2b3a1\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 1,\n \"measurements\": [\n 0.89,\n 0.99,\n 0.94,\n 0.8,\n 0.74\n ],\n \"average\": 0.872,\n \"min\": 0.74,\n \"max\": 0.99,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 50,\n \"measurements\": [\n 46.8,\n 49.4,\n 54.6,\n 49.4,\n 59.0\n ],\n \"average\": 51.84,\n \"min\": 46.8,\n \"max\": 59.0,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measurements\": [\n 98.0,\n 105.2,\n 91.5,\n 104.2,\n 96.7\n ],\n \"average\": 99.12,\n \"min\": 91.5,\n \"max\": 105.2,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9999,\n \"measurements\": [\n 99.960407,\n 100,\n 99.964983,\n 100,\n 99.95874\n ],\n \"average\": 99.977,\n \"min\": 99.95874,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 50,\n \"measurements\": [\n 36,\n 34,\n 6,\n 24,\n 31\n ],\n \"average\": 26.2,\n \"min\": 6,\n \"max\": 36,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": false,\n \"resourceUtilization\": {\n \"cpuUsage\": 41.3,\n \"memoryUsage\": 23.6,\n \"prbUtilization\": 85.2\n },\n \"@type\": \"IntentAssuranceReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-011011101110\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 10,\n \"measurements\": [\n 6.23,\n 6.82,\n 10.0,\n 9.0,\n 8.48\n ],\n \"average\": 8.106,\n \"min\": 6.23,\n \"max\": 10.0,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 150,\n \"measurements\": [\n 158.4,\n 139.1,\n 168.4,\n 159.4,\n 140.6\n ],\n \"average\": 151.18,\n \"min\": 139.1,\n \"max\": 168.4,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measurements\": [\n 100.0,\n 118.4,\n 99.3,\n 118.6,\n 99.5\n ],\n \"average\": 107.16,\n \"min\": 99.3,\n \"max\": 118.6,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.999,\n \"measurements\": [\n 99.998616,\n 99.998011,\n 99.999011,\n 99.998989,\n 99.998989\n ],\n \"average\": 99.9989,\n \"min\": 99.998011,\n \"max\": 99.999011,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 50,\n \"measurements\": [\n 40,\n 40,\n 44,\n 46,\n 28\n ],\n \"average\": 39.6,\n \"min\": 28,\n \"max\": 46,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 77.7,\n \"memoryUsage\": 28.6,\n \"prbUtilization\": 82.0\n },\n \"@type\n......" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "b90242682a5f", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "HMTC", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.2222222222222222, + "norm_field_f1": 0.1111111111111111, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-30202401179e\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 1,\n \"measurements\": [\n 0.86,\n 0.84,\n 1.03,\n 0.66,\n 0.77\n ],\n \"average\": 0.832,\n \"min\": 0.66,\n \"max\": 1.03,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 1000,\n \"measurements\": [\n 931.3,\n 907.5,\n 1031.7,\n 988.2,\n 1051.7\n ],\n \"average\": 982.08,\n \"min\": 907.5,\n \"max\": 1051.7,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 1000,\n \"measurements\": [\n 1062.8,\n 992.1,\n 1040.9,\n 1060.4,\n 1073.9\n ],\n \"average\": 1046.02,\n \"min\": 992.1,\n \"max\": 1073.9,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9999,\n \"measurements\": [\n 99.908937,\n 100,\n 99.928188,\n 100,\n 99.913298\n ],\n \"average\": 99.95,\n \"min\": 99.908937,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 100000,\n \"measurements\": [\n 38036,\n 6057,\n 46941,\n 22542,\n 80098\n ],\n \"average\": 38734.8,\n \"min\": 6057,\n \"max\": 80098,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 52.4,\n \"memoryUsage\": 21.3,\n \"prbUtilization\": 87.2\n },\n \"@ty\n......", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-30202401179e\",\n \"reportPeriod\": {\n \"start\": \"2025-01-01T00:00:00Z\",\n \"end\": \"2025-01-01T01:00:00Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 3,\n \"measurements\": [\n 2.89,\n 2.86,\n 2.89,\n 2.86,\n 2.86\n ],\n \"average\": 2.872,\n \"min\": 2.86,\n \"max\": 2.89,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 1000,\n \"measurements\": [\n 1012.1,\n 1089.4,\n 1168.8,\n 1096.6,\n 1127.6\n ],\n \"average\": 1098.9,\n \"min\": 1012.1,\n \"max\": 1168.8,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 500,\n \"measurements\": [\n 581.4,\n 558.4,\n 589.1,\n 580.8,\n 589.3\n ],\n \"average\": 579.8,\n \"min\": 558.4,\n \"max\": 589.3,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99999,\n \"measurements\": [\n 99.903672,\n 99.999041,\n 99.999961,\n 99.999966,\n 99.999966\n ],\n \"average\": 99.98,\n \"min\": 99.903672,\n \"max\": 99.999966,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 1000000,\n \"measurements\": [\n 110211,\n 967222,\n 989402,\n 989112,\n 989182\n ],\n \"average\": 989,\n \"min\": 110211,\n \"max\": 989402,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 76.1,\n \"memoryUsage\": 28.6,\n \n......" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "ec5593ba7a48", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "URLLC", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.23809523809523808, + "norm_field_f1": 0.12698412698412698, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-b885cd9a59ae\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 0.5,\n \"measurements\": [\n 0.46,\n 0.52,\n 0.53,\n 0.52,\n 0.51\n ],\n \"average\": 0.508,\n \"min\": 0.46,\n \"max\": 0.53,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 100,\n \"measurements\": [\n 114.0,\n 110.3,\n 102.7,\n 116.9,\n 92.0\n ],\n \"average\": 107.18,\n \"min\": 92.0,\n \"max\": 116.9,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 150,\n \"measurements\": [\n 177.3,\n 142.6,\n 159.7,\n 153.1,\n 172.2\n ],\n \"average\": 160.98,\n \"min\": 142.6,\n \"max\": 177.3,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9999,\n \"measurements\": [\n 99.926031,\n 100,\n 100,\n 99.993946,\n 99.955244\n ],\n \"average\": 99.975,\n \"min\": 99.926031,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 10,\n \"measurements\": [\n 4,\n 7,\n 10,\n 6,\n 8\n ],\n \"average\": 7.0,\n \"min\": 4,\n \"max\": 10,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 50.1,\n \"memoryUsage\": 36.0,\n \"prbUtilization\": 48.4\n },\n \"@type\": \"IntentAssuranceReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-b885cd9a59ae\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 100,\n \"measurements\": [\n 66.06,\n 66.1,\n 90.01,\n 66.0,\n 66.0\n ],\n \"average\": 68.854,\n \"min\": 66.0,\n \"max\": 90.01,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 50,\n \"measurements\": [\n 58.2,\n 58.8,\n 52.8,\n 52.1,\n 58.3\n ],\n \"average\": 54.84,\n \"min\": 52.1,\n \"max\": 58.8,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 25,\n \"measurements\": [\n 29.1,\n 23.1,\n 28.3,\n 23.3,\n 29.8\n ],\n \"average\": 26.72,\n \"min\": 23.1,\n \"max\": 29.8,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9,\n \"measurements\": [\n 99.966176,\n 99.998011,\n 99.998132,\n 99.999831,\n 99.999975\n ],\n \"average\": 99.993,\n \"min\": 99.966176,\n \"max\": 99.999975,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 5000,\n \"measurements\": [\n 1666,\n 1808,\n 4865,\n 1808,\n 4865\n ],\n \"average\": 2880.4,\n \"min\": 1666,\n \"max\": 4865,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 76.1,\n \"memoryUsage\": 28.6,\n \"prbUtilization\": 82.3\n },\n \n......" + }, + { + "split": "test_use_case_ood", + "kind": "success", + "id": "1fc1a9523b74", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "MPS", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.25396825396825395, + "norm_field_f1": 0.2222222222222222, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-1e8c7fc26443\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 1,\n \"measurements\": [\n 0.64,\n 0.93,\n 0.93,\n 1.09,\n 0.82\n ],\n \"average\": 0.882,\n \"min\": 0.64,\n \"max\": 1.09,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 200,\n \"measurements\": [\n 207.7,\n 181.1,\n 207.2,\n 199.1,\n 216.4\n ],\n \"average\": 202.3,\n \"min\": 181.1,\n \"max\": 216.4,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 75,\n \"measurements\": [\n 86.6,\n 87.4,\n 89.5,\n 89.4,\n 69.4\n ],\n \"average\": 84.46,\n \"min\": 69.4,\n \"max\": 89.5,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9999,\n \"measurements\": [\n 100,\n 100,\n 100,\n 99.923407,\n 99.996963\n ],\n \"average\": 99.984,\n \"min\": 99.923407,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 500,\n \"measurements\": [\n 60,\n 425,\n 483,\n 462,\n 10\n ],\n \"average\": 288.0,\n \"min\": 10,\n \"max\": 483,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 55.1,\n \"memoryUsage\": 48.6,\n \"prbUtilization\": 52.7\n },\n \"@type\": \"IntentAssuranceReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-111111111111\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 10,\n \"measurements\": [\n 10.26,\n 6.4,\n 9.5,\n 10.02,\n 8.5\n ],\n \"average\": 8.936,\n \"min\": 6.4,\n \"max\": 10.26,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 150,\n \"measurements\": [\n 158.4,\n 139.1,\n 168.4,\n 158.8,\n 140.6\n ],\n \"average\": 151.26,\n \"min\": 139.1,\n \"max\": 168.4,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measurements\": [\n 100.4,\n 111.6,\n 99.8,\n 100.4,\n 99.9\n ],\n \"average\": 100.42,\n \"min\": 99.8,\n \"max\": 111.6,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9999,\n \"measurements\": [\n 99.903667,\n 99.999617,\n 99.999811,\n 99.999962,\n 100\n ],\n \"average\": 99.98,\n \"min\": 99.903667,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 5000,\n \"measurements\": [\n 1646,\n 4104,\n 4865,\n 4860,\n 4142\n ],\n \"average\": 3923.4,\n \"min\": 1646,\n \"max\": 4865,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 77.7,\n \"memoryUsage\": 28.6,\n \"prbUtilization\": 82.0\n },\n \"\n......" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "1674371747a5", + "target_layer": "tmf921_lifecycle_monitor", + "slice_type": "URLLC", + "lifecycle_operation": "monitor", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5666666666666667, + "norm_field_f1": 0.20689655172413793, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentFulfillmentReport" + ], + "prediction_json_keys": [ + "intentFulfillmentReport" + ], + "gold": "{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-92a95e995de3\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Fulfilled\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 1,\n \"measured\": 0.76,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 150,\n \"measured\": 169.1,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measured\": 107.7,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9999,\n \"measured\": 100,\n \"unit\": \"percent\",\n \"compliant\": true\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 100,\n \"measured\": 12,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": true,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-011011101010\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 10,\n \"measured\": 8.06,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 150,\n \"measured\": 159.8,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measured\": 95.1,\n \"unit\": \"Mbps\",\n \"compliant\": false\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.999,\n \"measured\": 99.901071,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 50,\n \"measured\": 40,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "98ed16395511", + "target_layer": "tmf921_lifecycle_monitor", + "slice_type": "HMTC", + "lifecycle_operation": "monitor", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5666666666666667, + "norm_field_f1": 0.2413793103448276, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentFulfillmentReport" + ], + "prediction_json_keys": [ + "intentFulfillmentReport" + ], + "gold": "{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-c173a11c070c\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Fulfilled\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 0.5,\n \"measured\": 0.36,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 200,\n \"measured\": 220.7,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 200,\n \"measured\": 205.2,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99999,\n \"measured\": 100,\n \"unit\": \"percent\",\n \"compliant\": true\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 100,\n \"measured\": 43,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": true,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-c173a11c070c\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 3,\n \"measured\": 2.89,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 1000,\n \"measured\": 955.1,\n \"unit\": \"Mbps\",\n \"compliant\": false\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 500,\n \"measured\": 529.6,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99999,\n \"measured\": 99.99867,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 100000,\n \"measured\": 87448,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "90bbacd7cde6", + "target_layer": "tmf921_lifecycle_monitor", + "slice_type": "eMBB", + "lifecycle_operation": "monitor", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5666666666666667, + "norm_field_f1": 0.2413793103448276, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentFulfillmentReport" + ], + "prediction_json_keys": [ + "intentFulfillmentReport" + ], + "gold": "{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-c12689b6f1d0\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Fulfilled\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 30,\n \"measured\": 21.27,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 100,\n \"measured\": 114.7,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 50,\n \"measured\": 50.2,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.95,\n \"measured\": 99.966739,\n \"unit\": \"percent\",\n \"compliant\": true\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 10000,\n \"measured\": 5860,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": true,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-111111111111\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 30,\n \"measured\": 28.66,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 1000,\n \"measured\": 955.1,\n \"unit\": \"Mbps\",\n \"compliant\": false\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measured\": 109.2,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9,\n \"measured\": 99.826061,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 10000,\n \"measured\": 8943,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "success", + "id": "436531981b6c", + "target_layer": "tmf921_lifecycle_monitor", + "slice_type": "MPS", + "lifecycle_operation": "monitor", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7666666666666667, + "norm_field_f1": 0.7586206896551724, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "intentFulfillmentReport" + ], + "prediction_json_keys": [ + "intentFulfillmentReport" + ], + "gold": "{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-95f2e40c5e32\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 1,\n \"measured\": 0.81,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 150,\n \"measured\": 154.4,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measured\": 104.6,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9999,\n \"measured\": 99.937449,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 50,\n \"measured\": 20,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-95f2e40c5e32\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 10,\n \"measured\": 8.06,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 150,\n \"measured\": 159.8,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measured\": 109.4,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9999,\n \"measured\": 99.998671,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 100,\n \"measured\": 87,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "bddc54ab839a", + "target_layer": "tmf921", + "slice_type": "URLLC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7692307692307693, + "norm_field_f1": 0.8656716417910447, + "norm_key_f1": 0.9402985074626866, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-8931782be6a9\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-8931782be6a9\",\n \"name\": \"URLLC Slice - Heavy Equipment Telemetry\",\n \"description\": \"Intent for heavy equipment telemetry at Construction Site Gamma with URLLC service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 2,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-6269398c603e\",\n \"name\": \"URLLC QoS Delivery Expectation\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-urllc-f91640\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"URLLC\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 2,\n \"sd\": \"000002\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 300,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 25,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n \n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-11b110b11086\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-11b110b11086\",\n \"name\": \"URLLC Slice - Heavy Equipment Telemetry\",\n \"description\": \"Intent for heavy equipment telemetry at Construction Site Gamma with URLLC service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 1,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-09b110b11086\",\n \"name\": \"URLLC QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-urllc-11b110\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"URLLC\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 2,\n \"sd\": \"000001\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 300,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 25,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \n......" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "749d96c79902", + "target_layer": "tmf921", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7692307692307693, + "norm_field_f1": 0.8656716417910447, + "norm_key_f1": 0.9402985074626866, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-dd34ec4827b1\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-dd34ec4827b1\",\n \"name\": \"eMBB Slice - Marine Environmental Monitoring\",\n \"description\": \"Intent for marine environmental monitoring at Healthcare Park with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 4,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-8ef018f3f905\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-859878\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 500,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 25,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n \n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-10b1b1b1b1b1\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-10b1b1b1b1b1\",\n \"name\": \"eMBB Slice - Marine Environmental Monitoring\",\n \"description\": \"Intent for marine environmental monitoring at Healthcare Park with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-10b1b1b1b1b1\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-10b1b1\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 500,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 25,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \n......" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "981afcea25c0", + "target_layer": "tmf921", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7692307692307693, + "norm_field_f1": 0.8656716417910447, + "norm_key_f1": 0.9402985074626866, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-aa83dfb1c8b4\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-aa83dfb1c8b4\",\n \"name\": \"eMBB Slice - Air Traffic Management Relay\",\n \"description\": \"Intent for air traffic management relay at Healthcare Park with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 5,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-1fdd684f4eb6\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-bbed57\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000002\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1000,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 75,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n \n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-10b1b1b1b1b1\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-10b1b1b1b1b1\",\n \"name\": \"eMBB Slice - Air Traffic Management Relay\",\n \"description\": \"Intent for air traffic management relay at Healthcare Park with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-10b1b1b1b1b1\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-10b1b1\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1000,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 75,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\n......" + }, + { + "split": "test_use_case_ood", + "kind": "success", + "id": "bd89b6f1284a", + "target_layer": "tmf921", + "slice_type": "MPS", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.8734177215189873, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-6baebcfa1da7\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-6baebcfa1da7\",\n \"name\": \"MPS Slice - Dust And Noise Environmental Sensing\",\n \"description\": \"Intent for dust and noise environmental sensing at Industrial Zone North with MPS service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 1,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-b056e728a8e5\",\n \"name\": \"MPS QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-mps-a050fb\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"MPS\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 5,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 25,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-11b0b1b1b1b1\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-11b0b1b1b1b1\",\n \"name\": \"MPS Slice - Dust And Noise Environmental Sensing\",\n \"description\": \"Intent for dust and noise environmental sensing at Industrial Zone North with MPS service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 1,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-11b0b1b1b1b1\",\n \"name\": \"MPS QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-mps-11b0b1\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"MPS\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 5,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 25,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \n......" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "9ae321e930b5", + "target_layer": "camara", + "slice_type": "URLLC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.43750000000000006, + "norm_field_f1": 0.6428571428571429, + "norm_key_f1": 0.8571428571428571, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"URLLC\",\n \"sliceProfile\": {\n \"latencyTarget\": 7,\n \"reliability\": 99.999,\n \"throughput\": {\n \"downlink\": 50,\n \"uplink\": 75\n },\n \"maxNumberOfUEs\": 5000\n },\n \"areaOfService\": {\n \"region\": \"Runway Complex\",\n \"center\": {\n \"latitude\": 25.2532,\n \"longitude\": 55.3657\n },\n \"radius\": 200,\n \"cells\": [\n \"cell-099\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-12-08T01:00:00Z\",\n \"endTime\": \"2025-12-10T01:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/99fac0c4681c\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"URLLC\",\n \"sliceProfile\": {\n \"latencyTarget\": 7,\n \"reliability\": 99.999,\n \"throughput\": {\n \"downlink\": 50,\n \"uplink\": 75\n },\n \"maxNumberOfUEs\": 5000\n },\n \"areaOfService\": {\n \"region\": \"Runway Complex\",\n \"center\": {\n \"latitude\": 47.4502,\n \"longitude\": 8.5617\n },\n \"radius\": 5000,\n \"cells\": [\n \"cell-061\",\n \"cell-080\",\n \"cell-080\",\n \"cell-080\",\n \"cell-080\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-13T06:00:00Z\",\n \"endTime\": \"2025-01-13T10:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/109431191119\"\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "973a1b9e635d", + "target_layer": "camara", + "slice_type": "MPS", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.48, + "norm_field_f1": 0.6666666666666667, + "norm_key_f1": 0.6666666666666667, + "error_label": "structural_mismatch_or_extra_missing_keys", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"MPS\",\n \"sliceProfile\": {\n \"latencyTarget\": 5,\n \"reliability\": 99.999,\n \"throughput\": {\n \"downlink\": 200,\n \"uplink\": 75\n },\n \"maxNumberOfUEs\": 1000\n },\n \"areaOfService\": {\n \"region\": \"Factory Complex A\",\n \"center\": {\n \"latitude\": 48.1351,\n \"longitude\": 11.582\n },\n \"radius\": 2000,\n \"cells\": [\n \"cell-080\",\n \"cell-085\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-22T21:00:00Z\",\n \"endTime\": \"2025-01-23T09:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/733e26a816ae\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"MPS\",\n \"sliceProfile\": {\n \"latencyTarget\": 5,\n \"reliability\": 99.999,\n \"throughput\": {\n \"downlink\": 200,\n \"uplink\": 75\n },\n \"maxNumberOfUEs\": 1000,\n \"serviceArea\": \"Factory Complex A\"\n },\n \"duration\": {\n \"startTime\": \"2025-01-12T08:00:00Z\",\n \"endTime\": \"2025-01-12T10:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/71b0b1b1b101\"\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "f39d8b11e91f", + "target_layer": "camara", + "slice_type": "V2X", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5, + "norm_field_f1": 0.7142857142857143, + "norm_key_f1": 0.8571428571428571, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"V2X\",\n \"sliceProfile\": {\n \"latencyTarget\": 3,\n \"reliability\": 99.99,\n \"throughput\": {\n \"downlink\": 300,\n \"uplink\": 100\n },\n \"maxNumberOfUEs\": 5000\n },\n \"areaOfService\": {\n \"region\": \"Hospital Campus\",\n \"center\": {\n \"latitude\": 40.6301,\n \"longitude\": 22.9521\n },\n \"radius\": 5000,\n \"cells\": [\n \"cell-001\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-07-18T17:00:00Z\",\n \"endTime\": \"2025-07-25T17:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/c95906c68b01\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"V2X\",\n \"sliceProfile\": {\n \"latencyTarget\": 3,\n \"reliability\": 99.999,\n \"throughput\": {\n \"downlink\": 300,\n \"uplink\": 100\n },\n \"maxNumberOfUEs\": 5000\n },\n \"areaOfService\": {\n \"region\": \"Hospital Campus\",\n \"center\": {\n \"latitude\": 40.6301,\n \"longitude\": 22.9521\n },\n \"radius\": 500,\n \"cells\": [\n \"cell-089\",\n \"cell-080\",\n \"cell-080\",\n \"cell-080\",\n \"cell-080\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-01T08:00:00Z\",\n \"endTime\": \"2025-01-01T10:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/100000000000\"\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "success", + "id": "1fed836991bf", + "target_layer": "camara", + "slice_type": "URLLC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5882352941176471, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"URLLC\",\n \"sliceProfile\": {\n \"latencyTarget\": 5,\n \"reliability\": 99.999,\n \"throughput\": {\n \"downlink\": 150,\n \"uplink\": 100\n },\n \"maxNumberOfUEs\": 100\n },\n \"areaOfService\": {\n \"region\": \"Factory Complex A\",\n \"center\": {\n \"latitude\": 48.1351,\n \"longitude\": 11.582\n },\n \"radius\": 500,\n \"cells\": [\n \"cell-046\",\n \"cell-094\",\n \"cell-061\",\n \"cell-065\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-08-12T19:00:00Z\",\n \"endTime\": \"2025-08-12T20:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/f69ac39a2d8b\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"URLLC\",\n \"sliceProfile\": {\n \"latencyTarget\": 5,\n \"reliability\": 99.999,\n \"throughput\": {\n \"downlink\": 150,\n \"uplink\": 100\n },\n \"maxNumberOfUEs\": 100\n },\n \"areaOfService\": {\n \"region\": \"Factory Complex A\",\n \"center\": {\n \"latitude\": 48.1351,\n \"longitude\": 11.582\n },\n \"radius\": 500,\n \"cells\": [\n \"cell-081\",\n \"cell-080\",\n \"cell-082\",\n \"cell-083\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-12T08:00:00Z\",\n \"endTime\": \"2025-01-12T10:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/843041940911\"\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "108cc34d634f", + "target_layer": "intent_3gpp", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.775, + "norm_field_f1": 0.6578947368421053, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-mMTC-a417f9999389\",\n \"name\": \"Underwater Sensor Network mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-910be1619245\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000001\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 10,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 30,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Healthcare Park\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"underwater \n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-mMTC-0d0410b13116\",\n \"name\": \"Underwater Sensor Network mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-1b0b14b1b116\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 30,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 99,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Healthcare Park\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCond\n......" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "255f763bc591", + "target_layer": "intent_3gpp", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.775, + "norm_field_f1": 0.6578947368421053, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-mMTC-ee551691e059\",\n \"name\": \"Smart Shelf Inventory Tracking mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-e121b41b3325\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000001\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 25,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 0.1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 200,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99.9,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 5000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Retail Distribution Hub\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRa\n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-mMTC-11901000000d\",\n \"name\": \"Smart Shelf Inventory Tracking mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-000000000001\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 25,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 0.1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 200,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 99.5,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 5000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Retail Distribution Hub\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \n......" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "ca7dbbf593d5", + "target_layer": "intent_3gpp", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.775, + "norm_field_f1": 0.6578947368421053, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-mMTC-0c0329de6362\",\n \"name\": \"Air Traffic Management Relay mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-4c788ada3508\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000001\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 50,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 25,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 200,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 95,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 500000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Factory Complex A\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"air\n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-mMTC-0d614440b11d\",\n \"name\": \"Air Traffic Management Relay mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-0d614440b11d\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 25,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 200,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 95,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 500000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Factory Complex A\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"con\n......" + }, + { + "split": "test_use_case_ood", + "kind": "success", + "id": "d3c25fcea62e", + "target_layer": "intent_3gpp", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.9500000000000001, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-mMTC-fbe2ce2ef19a\",\n \"name\": \"Isr Data Relay mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-88efa953d21e\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 0.1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 500,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99.5,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 5000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Naval Port Zone\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"ISR data relay\"\n \n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-mMTC-111010101010\",\n \"name\": \"Isr Data Relay mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-000000000001\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 0.1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 500,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99.5,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 5000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Naval Port Zone\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"\n......" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "40463b92ed93", + "target_layer": "o1_nrm", + "slice_type": "MPS", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.17391304347826086, + "norm_field_f1": 0.20000000000000004, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-004\",\n \"GNBDUFunction\": {\n \"id\": \"DU-005\",\n \"gNBId\": 4,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-005-008\",\n \"cellLocalId\": 8,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n258\",\n \"arfcnDL\": 2038010,\n \"bSChannelBwDL\": 200,\n \"nrPCI\": 64,\n \"nrTAC\": 60234,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 5,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.12,\n \"rrmPolicyMaxRatio\": 0.29,\n \"rrmPolicyDedicatedRatio\": 0.2,\n \"rrmPolicyMaxRatioPerSlice\": 0.29\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"001\",\n \"mnc\": \"260\",\n \"sNSSAIList\": [\n {\n \"sst\": 5,\n \"sd\": \"000004\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-001\",\n \"GNBDUFunction\": {\n \"id\": \"DU-001\",\n \"gNBId\": 1,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-001-001\",\n \"cellLocalId\": 1,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n78\",\n \"arfcnDL\": 629819,\n \"bSChannelBwDL\": 40,\n \"nrPCI\": 812,\n \"nrTAC\": 38901,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 2,\n \"sd\": \"000001\"\n },\n \"rrmPolicyMinRatio\": 0.12,\n \"rrmPolicyMaxRatio\": 0.35,\n \"rrmPolicyDedicatedRatio\": 0.23,\n \"rrmPolicyMaxRatioPerSlice\": 0.35\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 2,\n \"sd\": \"000003\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "662ef55ef8a7", + "target_layer": "o1_nrm", + "slice_type": "V2X", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.17391304347826086, + "norm_field_f1": 0.20000000000000004, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-038\",\n \"GNBDUFunction\": {\n \"id\": \"DU-003\",\n \"gNBId\": 38,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-003-002\",\n \"cellLocalId\": 2,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n258\",\n \"arfcnDL\": 2064527,\n \"bSChannelBwDL\": 50,\n \"nrPCI\": 683,\n \"nrTAC\": 41299,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 4,\n \"sd\": \"000002\"\n },\n \"rrmPolicyMinRatio\": 0.14,\n \"rrmPolicyMaxRatio\": 0.3,\n \"rrmPolicyDedicatedRatio\": 0.22,\n \"rrmPolicyMaxRatioPerSlice\": 0.3\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"310\",\n \"mnc\": \"260\",\n \"sNSSAIList\": [\n {\n \"sst\": 4,\n \"sd\": \"000001\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-001\",\n \"GNBDUFunction\": {\n \"id\": \"DU-001\",\n \"gNBId\": 1,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-001-001\",\n \"cellLocalId\": 1,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n78\",\n \"arfcnDL\": 629819,\n \"bSChannelBwDL\": 40,\n \"nrPCI\": 812,\n \"nrTAC\": 3862,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 5,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.14,\n \"rrmPolicyMaxRatio\": 0.31,\n \"rrmPolicyDedicatedRatio\": 0.23,\n \"rrmPolicyMaxRatioPerSlice\": 0.31\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 5,\n \"sd\": \"000003\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "17db82b1a297", + "target_layer": "o1_nrm", + "slice_type": "MPS", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.21739130434782608, + "norm_field_f1": 0.25, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-001\",\n \"GNBDUFunction\": {\n \"id\": \"DU-043\",\n \"gNBId\": 1,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-043-010\",\n \"cellLocalId\": 10,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n41\",\n \"arfcnDL\": 510572,\n \"bSChannelBwDL\": 20,\n \"nrPCI\": 698,\n \"nrTAC\": 28986,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 5,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.09,\n \"rrmPolicyMaxRatio\": 0.4,\n \"rrmPolicyDedicatedRatio\": 0.24,\n \"rrmPolicyMaxRatioPerSlice\": 0.4\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"15\",\n \"sNSSAIList\": [\n {\n \"sst\": 5,\n \"sd\": \"000004\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-016\",\n \"GNBDUFunction\": {\n \"id\": \"DU-001\",\n \"gNBId\": 16,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-001-001\",\n \"cellLocalId\": 1,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n78\",\n \"arfcnDL\": 625620,\n \"bSChannelBwDL\": 40,\n \"nrPCI\": 812,\n \"nrTAC\": 38502,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 2,\n \"sd\": \"000001\"\n },\n \"rrmPolicyMinRatio\": 0.09,\n \"rrmPolicyMaxRatio\": 0.36,\n \"rrmPolicyDedicatedRatio\": 0.22,\n \"rrmPolicyMaxRatioPerSlice\": 0.36\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 2,\n \"sd\": \"000003\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "success", + "id": "65d17918fd84", + "target_layer": "o1_nrm", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5652173913043478, + "norm_field_f1": 0.65, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-005\",\n \"GNBDUFunction\": {\n \"id\": \"DU-046\",\n \"gNBId\": 5,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-046-002\",\n \"cellLocalId\": 2,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n41\",\n \"arfcnDL\": 514306,\n \"bSChannelBwDL\": 100,\n \"nrPCI\": 834,\n \"nrTAC\": 2614,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.23,\n \"rrmPolicyMaxRatio\": 0.48,\n \"rrmPolicyDedicatedRatio\": 0.35,\n \"rrmPolicyMaxRatioPerSlice\": 0.48\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"310\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-006\",\n \"GNBDUFunction\": {\n \"id\": \"DU-008\",\n \"gNBId\": 6,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-008-002\",\n \"cellLocalId\": 2,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n78\",\n \"arfcnDL\": 620896,\n \"bSChannelBwDL\": 40,\n \"nrPCI\": 816,\n \"nrTAC\": 58921,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.23,\n \"rrmPolicyMaxRatio\": 0.48,\n \"rrmPolicyDedicatedRatio\": 0.35,\n \"rrmPolicyMaxRatioPerSlice\": 0.48\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "e76175d7ddac", + "target_layer": "a1_policy", + "slice_type": "HMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.42105263157894735, + "norm_field_f1": 0.5, + "norm_key_f1": 1.0, + "error_label": "a1_policy_value_error", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-hmtc-99efeb\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-hmtc-d13213\",\n \"cellId\": \"cell-083\",\n \"sNSSAI\": {\n \"sst\": 5,\n \"sd\": \"000001\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 200,\n \"mfbr\": 240.0,\n \"priorityLevel\": 1,\n \"5qi\": 2,\n \"pdb\": 150,\n \"per\": 1e-07\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 12,\n \"max\": 42\n },\n \"schedulerWeight\": 0.2,\n \"maxUEs\": 10000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-mmtc-0d0d66\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-mmtc-100000\",\n \"cellId\": \"cell-086\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 200,\n \"mfbr\": 240.0,\n \"priorityLevel\": 8,\n \"5qi\": 70,\n \"pdb\": 200,\n \"per\": 1e-06\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 12,\n \"max\": 32\n },\n \"schedulerWeight\": 0.38,\n \"maxUEs\": 10000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "0b9402e4f6cd", + "target_layer": "a1_policy", + "slice_type": "V2X", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.42105263157894735, + "norm_field_f1": 0.5, + "norm_key_f1": 1.0, + "error_label": "a1_policy_value_error", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-v2x-178d91\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-v2x-d85b31\",\n \"cellId\": \"cell-072\",\n \"sNSSAI\": {\n \"sst\": 4,\n \"sd\": \"000002\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 200,\n \"mfbr\": 240.0,\n \"priorityLevel\": 2,\n \"5qi\": 75,\n \"pdb\": 50,\n \"per\": 1e-05\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 13,\n \"max\": 34\n },\n \"schedulerWeight\": 0.55,\n \"maxUEs\": 10000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-urllc-0d000b\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-urllc-100008\",\n \"cellId\": \"cell-010\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000001\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 200,\n \"mfbr\": 240.0,\n \"priorityLevel\": 1,\n \"5qi\": 85,\n \"pdb\": 5,\n \"per\": 1e-06\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 13,\n \"max\": 33\n },\n \"schedulerWeight\": 0.39,\n \"maxUEs\": 10000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "84f4cc7d2f3d", + "target_layer": "a1_policy", + "slice_type": "V2X", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.42105263157894735, + "norm_field_f1": 0.5, + "norm_key_f1": 1.0, + "error_label": "a1_policy_value_error", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-v2x-90333a\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-v2x-2bddcd\",\n \"cellId\": \"cell-021\",\n \"sNSSAI\": {\n \"sst\": 4,\n \"sd\": \"000002\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 100,\n \"mfbr\": 120.0,\n \"priorityLevel\": 2,\n \"5qi\": 83,\n \"pdb\": 10,\n \"per\": 1e-05\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 6,\n \"max\": 31\n },\n \"schedulerWeight\": 0.37,\n \"maxUEs\": 50\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-v2x-11718b\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-v2x-11718b\",\n \"cellId\": \"cell-086\",\n \"sNSSAI\": {\n \"sst\": 4,\n \"sd\": \"000001\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 100,\n \"mfbr\": 120,\n \"priorityLevel\": 1,\n \"5qi\": 84,\n \"pdb\": 30,\n \"per\": 1e-06\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 6,\n \"max\": 23\n },\n \"schedulerWeight\": 0.39,\n \"maxUEs\": 50\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "success", + "id": "bc08505d37ea", + "target_layer": "a1_policy", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7368421052631579, + "norm_field_f1": 0.8333333333333334, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-embb-53c776\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-embb-10f1ac\",\n \"cellId\": \"cell-051\",\n \"sNSSAI\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 200,\n \"mfbr\": 240.0,\n \"priorityLevel\": 5,\n \"5qi\": 6,\n \"pdb\": 300,\n \"per\": 0.0005\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 13,\n \"max\": 42\n },\n \"schedulerWeight\": 0.45,\n \"maxUEs\": 100\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-embb-11000d\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-embb-0d000d\",\n \"cellId\": \"cell-018\",\n \"sNSSAI\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 200,\n \"mfbr\": 240.0,\n \"priorityLevel\": 5,\n \"5qi\": 6,\n \"pdb\": 300,\n \"per\": 0.0005\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 13,\n \"max\": 33\n },\n \"schedulerWeight\": 0.59,\n \"maxUEs\": 100\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "20f1e24e817b", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "HMTC", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.23809523809523808, + "norm_field_f1": 0.12698412698412698, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-6467092d7703\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 1,\n \"measurements\": [\n 1.06,\n 1.05,\n 0.79,\n 0.99,\n 0.89\n ],\n \"average\": 0.956,\n \"min\": 0.79,\n \"max\": 1.06,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 200,\n \"measurements\": [\n 231.7,\n 232.6,\n 214.1,\n 199.8,\n 206.5\n ],\n \"average\": 216.94,\n \"min\": 199.8,\n \"max\": 232.6,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 1000,\n \"measurements\": [\n 940.5,\n 972.7,\n 1047.4,\n 900.3,\n 1063.3\n ],\n \"average\": 984.84,\n \"min\": 900.3,\n \"max\": 1063.3,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9999,\n \"measurements\": [\n 100,\n 99.951361,\n 99.923609,\n 100,\n 100\n ],\n \"average\": 99.975,\n \"min\": 99.923609,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 100,\n \"measurements\": [\n 22,\n 42,\n 75,\n 94,\n 4\n ],\n \"average\": 47.4,\n \"min\": 4,\n \"max\": 94,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 59.4,\n \"memoryUsage\": 22.4,\n \"prbUtilization\": 47.3\n },\n \"@type\": \"IntentAssuranceReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-6467092d7703\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 3,\n \"measurements\": [\n 2.89,\n 2.86,\n 2.86,\n 2.86,\n 2.86\n ],\n \"average\": 2.866,\n \"min\": 2.86,\n \"max\": 2.89,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 500,\n \"measurements\": [\n 585.1,\n 589.4,\n 521.1,\n 589.4,\n 523.0\n ],\n \"average\": 543.6,\n \"min\": 521.1,\n \"max\": 589.4,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measurements\": [\n 115.6,\n 112.5,\n 99.9,\n 112.1,\n 112.1\n ],\n \"average\": 110.44,\n \"min\": 99.9,\n \"max\": 115.6,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99999,\n \"measurements\": [\n 99.903672,\n 99.999041,\n 99.999995,\n 99.999995,\n 99.999995\n ],\n \"average\": 99.979,\n \"min\": 99.903672,\n \"max\": 99.999995,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 500000,\n \"measurements\": [\n 146442,\n 146442,\n 146442,\n 146442,\n 146442\n ],\n \"average\": 146442.0,\n \"min\": 146442,\n \"max\": 146442,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 76.1,\n \"memoryUsage\": 28.6,\n \"pr\n......" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "76bbd9020b1c", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "eMBB", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.23809523809523808, + "norm_field_f1": 0.12698412698412698, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-83c55e8c3fed\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 80,\n \"measurements\": [\n 72.88,\n 83.85,\n 56.86,\n 53.39,\n 70.19\n ],\n \"average\": 67.434,\n \"min\": 53.39,\n \"max\": 83.85,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 800,\n \"measurements\": [\n 951.7,\n 828.9,\n 926.7,\n 731.3,\n 758.7\n ],\n \"average\": 839.46,\n \"min\": 731.3,\n \"max\": 951.7,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 75,\n \"measurements\": [\n 72.9,\n 82.7,\n 86.7,\n 84.7,\n 84.5\n ],\n \"average\": 82.3,\n \"min\": 72.9,\n \"max\": 86.7,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9,\n \"measurements\": [\n 99.999893,\n 99.874877,\n 99.932637,\n 99.883247,\n 99.971974\n ],\n \"average\": 99.933,\n \"min\": 99.874877,\n \"max\": 99.999893,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 100,\n \"measurements\": [\n 94,\n 7,\n 28,\n 30,\n 73\n ],\n \"average\": 46.4,\n \"min\": 7,\n \"max\": 94,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": false,\n \"resourceUtilization\": {\n \"cpuUsage\": 19.0,\n \"memoryUsage\": 28.3,\n \"prbUtilization\": 31.6\n },\n \"@type\": \"IntentAssuranceR\n......", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-83c55e8c3fed\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 30,\n \"measurements\": [\n 26.66,\n 26.31,\n 23.02,\n 28.26,\n 28.26\n ],\n \"average\": 26.498,\n \"min\": 23.02,\n \"max\": 28.26,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 1000,\n \"measurements\": [\n 1016.1,\n 1118.4,\n 996.8,\n 1086.4,\n 1136.8\n ],\n \"average\": 1070.7,\n \"min\": 996.8,\n \"max\": 1136.8,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measurements\": [\n 115.6,\n 99.4,\n 112.1,\n 112.1,\n 99.9\n ],\n \"average\": 107.82,\n \"min\": 99.4,\n \"max\": 115.6,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9,\n \"measurements\": [\n 99.894556,\n 99.961056,\n 99.990908,\n 99.890851,\n 99.988159\n ],\n \"average\": 99.947,\n \"min\": 99.890851,\n \"max\": 99.990908,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 50000,\n \"measurements\": [\n 14446,\n 28646,\n 28646,\n 28646,\n 28646\n ],\n \"average\": 23830.0,\n \"min\": 14446,\n \"max\": 28646,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 76.1,\n \"memoryUsage\": 28.6,\n \"\n......" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "85fe1586c516", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "mMTC", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.23809523809523808, + "norm_field_f1": 0.14285714285714285, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-5acfd5df040c\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 50,\n \"measurements\": [\n 54.98,\n 31.82,\n 42.37,\n 40.29,\n 45.25\n ],\n \"average\": 42.942,\n \"min\": 31.82,\n \"max\": 54.98,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 50,\n \"measurements\": [\n 58.8,\n 59.0,\n 55.9,\n 49.5,\n 53.6\n ],\n \"average\": 55.36,\n \"min\": 49.5,\n \"max\": 59.0,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 25,\n \"measurements\": [\n 24.1,\n 28.0,\n 22.7,\n 29.2,\n 27.5\n ],\n \"average\": 26.3,\n \"min\": 22.7,\n \"max\": 29.2,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 95,\n \"measurements\": [\n 95.036732,\n 94.942745,\n 95.032511,\n 95.049224,\n 95.075089\n ],\n \"average\": 95.027,\n \"min\": 94.942745,\n \"max\": 95.075089,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 10000,\n \"measurements\": [\n 6646,\n 9177,\n 3412,\n 3906,\n 3052\n ],\n \"average\": 5238.6,\n \"min\": 3052,\n \"max\": 9177,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 39.3,\n \"memoryUsage\": 68.6,\n \"prbUtilization\": 56.8\n },\n \"@type\": \"IntentAs\n......", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-5acfd5df040c\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 10,\n \"measurements\": [\n 6.46,\n 6.7,\n 10.3,\n 10.36,\n 8.5\n ],\n \"average\": 8.384,\n \"min\": 6.46,\n \"max\": 10.36,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 100,\n \"measurements\": [\n 101.1,\n 108.4,\n 112.1,\n 95.2,\n 118.4\n ],\n \"average\": 107.04,\n \"min\": 95.2,\n \"max\": 118.4,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measurements\": [\n 99.4,\n 100.4,\n 99.6,\n 99.6,\n 99.6\n ],\n \"average\": 99.8,\n \"min\": 99.4,\n \"max\": 100.4,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.999,\n \"measurements\": [\n 99.903575,\n 99.998041,\n 99.999611,\n 99.999965,\n 99.999965\n ],\n \"average\": 99.981,\n \"min\": 99.903575,\n \"max\": 99.999965,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 5000,\n \"measurements\": [\n 1802,\n 1802,\n 2802,\n 4553,\n 4443\n ],\n \"average\": 3080.4,\n \"min\": 1802,\n \"max\": 4553,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 77.7,\n \"memoryUsage\": 28.6,\n \"prbUtilization\": 82.0\n }\n......" + }, + { + "split": "test_sector_ood", + "kind": "success", + "id": "f14740af4087", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "V2X", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.2698412698412698, + "norm_field_f1": 0.2857142857142857, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-b100485609db\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 10,\n \"measurements\": [\n 9.04,\n 6.41,\n 7.4,\n 7.96,\n 10.09\n ],\n \"average\": 8.18,\n \"min\": 6.41,\n \"max\": 10.09,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 100,\n \"measurements\": [\n 106.2,\n 91.1,\n 119.0,\n 98.9,\n 93.2\n ],\n \"average\": 101.68,\n \"min\": 91.1,\n \"max\": 119.0,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 150,\n \"measurements\": [\n 137.5,\n 151.4,\n 150.5,\n 138.2,\n 157.0\n ],\n \"average\": 146.92,\n \"min\": 137.5,\n \"max\": 157.0,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99,\n \"measurements\": [\n 99.99674,\n 99.912149,\n 99.896363,\n 99.962708,\n 99.891353\n ],\n \"average\": 99.932,\n \"min\": 99.891353,\n \"max\": 99.99674,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 100,\n \"measurements\": [\n 25,\n 25,\n 89,\n 31,\n 25\n ],\n \"average\": 39.0,\n \"min\": 25,\n \"max\": 89,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 51.1,\n \"memoryUsage\": 36.8,\n \"prbUtilization\": 49.3\n },\n \"@type\": \"IntentAssuranceRep\n......", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-b100485609db\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 10,\n \"measurements\": [\n 6.23,\n 6.86,\n 10.09,\n 8.5,\n 9.5\n ],\n \"average\": 8.036,\n \"min\": 6.23,\n \"max\": 10.09,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 150,\n \"measurements\": [\n 159.8,\n 139.4,\n 168.0,\n 149.4,\n 168.0\n ],\n \"average\": 156.92,\n \"min\": 139.4,\n \"max\": 168.0,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measurements\": [\n 115.1,\n 112.1,\n 99.4,\n 112.1,\n 112.1\n ],\n \"average\": 108.16,\n \"min\": 99.4,\n \"max\": 115.1,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.999,\n \"measurements\": [\n 99.903671,\n 99.998011,\n 99.999622,\n 99.998971,\n 99.999996\n ],\n \"average\": 99.98,\n \"min\": 99.903671,\n \"max\": 99.999996,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 5000,\n \"measurements\": [\n 1802,\n 2820,\n 4565,\n 4865,\n 1802\n ],\n \"average\": 3170.8,\n \"min\": 1802,\n \"max\": 4865,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 77.7,\n \"memoryUsage\": 28.6,\n \"prbUtilization\": 82.0\n......" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "8292fa544c75", + "target_layer": "tmf921_lifecycle_monitor", + "slice_type": "URLLC", + "lifecycle_operation": "monitor", + "parse_json": true, + "exact_match": false, + "field_f1": 0.4666666666666667, + "norm_field_f1": 0.20689655172413793, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentFulfillmentReport" + ], + "prediction_json_keys": [ + "intentFulfillmentReport" + ], + "gold": "{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-4c3988bc1787\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Fulfilled\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 3,\n \"measured\": 2.21,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 200,\n \"measured\": 221.1,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 150,\n \"measured\": 148.5,\n \"unit\": \"Mbps\",\n \"compliant\": false\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9999,\n \"measured\": 100,\n \"unit\": \"percent\",\n \"compliant\": true\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 500,\n \"measured\": 218,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": true,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-111111111111\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 10,\n \"measured\": 10.36,\n \"unit\": \"ms\",\n \"compliant\": false\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 150,\n \"measured\": 159.8,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measured\": 109.0,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.999,\n \"measured\": 99.901071,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 5000,\n \"measured\": 1802,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "196b19287de9", + "target_layer": "tmf921_lifecycle_monitor", + "slice_type": "V2X", + "lifecycle_operation": "monitor", + "parse_json": true, + "exact_match": false, + "field_f1": 0.6, + "norm_field_f1": 0.2413793103448276, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentFulfillmentReport" + ], + "prediction_json_keys": [ + "intentFulfillmentReport" + ], + "gold": "{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-f9d4f84791a4\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Fulfilled\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 20,\n \"measured\": 16.1,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 150,\n \"measured\": 162.6,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measured\": 95.9,\n \"unit\": \"Mbps\",\n \"compliant\": false\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.999,\n \"measured\": 100,\n \"unit\": \"percent\",\n \"compliant\": true\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 100,\n \"measured\": 32,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": true,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-f9d4f84791a4\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 100,\n \"measured\": 75.01,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 100,\n \"measured\": 100.4,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measured\": 100.0,\n \"unit\": \"Mbps\",\n \"compliant\": false\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9,\n \"measured\": 99.894066,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 10000,\n \"measured\": 8103,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "9b5647e0a6cf", + "target_layer": "tmf921_lifecycle_monitor", + "slice_type": "V2X", + "lifecycle_operation": "monitor", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5666666666666667, + "norm_field_f1": 0.2413793103448276, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentFulfillmentReport" + ], + "prediction_json_keys": [ + "intentFulfillmentReport" + ], + "gold": "{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-3772050f9852\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Fulfilled\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 10,\n \"measured\": 9.97,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 25,\n \"measured\": 27.1,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 10,\n \"measured\": 10.0,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99,\n \"measured\": 100,\n \"unit\": \"percent\",\n \"compliant\": true\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 100,\n \"measured\": 40,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": true,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-3772050f9852\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 100,\n \"measured\": 75.01,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 1000,\n \"measured\": 994.3,\n \"unit\": \"Mbps\",\n \"compliant\": false\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measured\": 109.4,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99,\n \"measured\": 99.984066,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 10000,\n \"measured\": 8943,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "success", + "id": "4b96dab3c3fd", + "target_layer": "tmf921_lifecycle_monitor", + "slice_type": "URLLC", + "lifecycle_operation": "monitor", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7, + "norm_field_f1": 0.6896551724137931, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "intentFulfillmentReport" + ], + "prediction_json_keys": [ + "intentFulfillmentReport" + ], + "gold": "{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-b69c2ab78704\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 2,\n \"measured\": 1.73,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 20,\n \"measured\": 20.1,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measured\": 113.2,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.999,\n \"measured\": 99.957143,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 1000,\n \"measured\": 87,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-b69c2ab78704\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 100,\n \"measured\": 75.01,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 1000,\n \"measured\": 1015.8,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measured\": 108.4,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99,\n \"measured\": 99.984066,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 10000,\n \"measured\": 8943,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "b6c6d8a2290b", + "target_layer": "tmf921", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7435897435897437, + "norm_field_f1": 0.8358208955223881, + "norm_key_f1": 0.9402985074626866, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-0223e0f73ebd\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-0223e0f73ebd\",\n \"name\": \"eMBB Slice - Student Collaboration Platform\",\n \"description\": \"Intent for student collaboration platform at University Campus with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 4,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-04c7a83fe977\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-9773e6\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000002\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n \n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-114000000000\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-114000000000\",\n \"name\": \"eMBB Slice - Student Collaboration Platform\",\n \"description\": \"Intent for student collaboration platform at University Campus with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-114000000001\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-114000\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \n......" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "df185cef747c", + "target_layer": "tmf921", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7435897435897437, + "norm_field_f1": 0.8358208955223881, + "norm_key_f1": 0.9402985074626866, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-36128226f673\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-36128226f673\",\n \"name\": \"eMBB Slice - Research Lab Data Transfer\",\n \"description\": \"Intent for research lab data transfer at University Campus with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 4,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-d9e74bee1eea\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-341132\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1000,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-10b110b10b10\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-10b110b10b10\",\n \"name\": \"eMBB Slice - Research Lab Data Transfer\",\n \"description\": \"Intent for research lab data transfer at University Campus with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-0d110b10b10b\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-0d110b\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1000,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\":\n......" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "a818a34c55f1", + "target_layer": "tmf921", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7435897435897437, + "norm_field_f1": 0.8358208955223881, + "norm_key_f1": 0.9402985074626866, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-6502d8fad3db\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-6502d8fad3db\",\n \"name\": \"eMBB Slice - Library Resource Digitization\",\n \"description\": \"Intent for library resource digitization at University Campus with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 4,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-ca322510e221\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-271987\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 800,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 75,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n \n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-011111111111\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-011111111111\",\n \"name\": \"eMBB Slice - Library Resource Digitization\",\n \"description\": \"Intent for library resource digitization at University Campus with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-011111111111\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-011111\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 800,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 75,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"u\n......" + }, + { + "split": "test_sector_ood", + "kind": "success", + "id": "81336d4bb697", + "target_layer": "tmf921", + "slice_type": "V2X", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.8734177215189873, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-bb942bfc8fab\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-bb942bfc8fab\",\n \"name\": \"V2X Slice - Library Resource Digitization\",\n \"description\": \"Intent for library resource digitization at Hospital Campus with V2X service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 1,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-1216b9633a99\",\n \"name\": \"V2X QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-v2x-99ca8c\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"V2X\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 4,\n \"sd\": \"000001\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n\n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-114000111111\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-114000111111\",\n \"name\": \"V2X Slice - Library Resource Digitization\",\n \"description\": \"Intent for library resource digitization at Hospital Campus with V2X service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 1,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-114000111111\",\n \"name\": \"V2X QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-v2x-114000\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"V2X\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 4,\n \"sd\": \"000001\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"\n......" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "61d045f3558d", + "target_layer": "camara", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.375, + "norm_field_f1": 0.5714285714285715, + "norm_key_f1": 0.8571428571428571, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"eMBB\",\n \"sliceProfile\": {\n \"latencyTarget\": 100,\n \"reliability\": 99.0,\n \"throughput\": {\n \"downlink\": 500,\n \"uplink\": 25\n },\n \"maxNumberOfUEs\": 1000\n },\n \"areaOfService\": {\n \"region\": \"Open Pit Zone\",\n \"center\": {\n \"latitude\": 62.0394,\n \"longitude\": 129.7422\n },\n \"radius\": 2000,\n \"cells\": [\n \"cell-064\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-05-14T22:00:00Z\",\n \"endTime\": \"2025-05-15T06:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/3088c4fc1727\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"eMBB\",\n \"sliceProfile\": {\n \"latencyTarget\": 100,\n \"reliability\": 99.9,\n \"throughput\": {\n \"downlink\": 500,\n \"uplink\": 25\n },\n \"maxNumberOfUEs\": 1000\n },\n \"areaOfService\": {\n \"region\": \"Open Pit Zone\",\n \"center\": {\n \"latitude\": 53.5511,\n \"longitude\": 9.9937\n },\n \"radius\": 500,\n \"cells\": [\n \"cell-088\",\n \"cell-080\",\n \"cell-080\",\n \"cell-080\",\n \"cell-080\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-13T08:00:00Z\",\n \"endTime\": \"2025-01-13T10:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/110009111111\"\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "bb39b43ec51b", + "target_layer": "camara", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.375, + "norm_field_f1": 0.5714285714285715, + "norm_key_f1": 0.8571428571428571, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"eMBB\",\n \"sliceProfile\": {\n \"latencyTarget\": 50,\n \"reliability\": 99.95,\n \"throughput\": {\n \"downlink\": 1000,\n \"uplink\": 150\n },\n \"maxNumberOfUEs\": 10000\n },\n \"areaOfService\": {\n \"region\": \"Research Park\",\n \"center\": {\n \"latitude\": 37.4275,\n \"longitude\": -122.1697\n },\n \"radius\": 500,\n \"cells\": [\n \"cell-049\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-10-03T06:00:00Z\",\n \"endTime\": \"2025-10-03T12:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/d6ac8b414857\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"eMBB\",\n \"sliceProfile\": {\n \"latencyTarget\": 50,\n \"reliability\": 99.9,\n \"throughput\": {\n \"downlink\": 1000,\n \"uplink\": 150\n },\n \"maxNumberOfUEs\": 10000\n },\n \"areaOfService\": {\n \"region\": \"Research Park\",\n \"center\": {\n \"latitude\": 37.7749,\n \"longitude\": -122.4194\n },\n \"radius\": 5000,\n \"cells\": [\n \"cell-081\",\n \"cell-080\",\n \"cell-082\",\n \"cell-083\",\n \"cell-084\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-12T08:00:00Z\",\n \"endTime\": \"2025-01-12T10:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/100000000000\"\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "bd7ec7719282", + "target_layer": "camara", + "slice_type": "HMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.3125, + "norm_field_f1": 0.5714285714285715, + "norm_key_f1": 0.9285714285714286, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"HMTC\",\n \"sliceProfile\": {\n \"latencyTarget\": 1,\n \"reliability\": 99.9999,\n \"throughput\": {\n \"downlink\": 1000,\n \"uplink\": 50\n },\n \"maxNumberOfUEs\": 1000\n },\n \"areaOfService\": {\n \"region\": \"Mining Site Alpha\",\n \"center\": {\n \"latitude\": -33.8688,\n \"longitude\": 151.2093\n },\n \"radius\": 1000,\n \"cells\": [\n \"cell-001\",\n \"cell-004\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-11-21T23:00:00Z\",\n \"endTime\": \"2025-11-22T11:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/109daa504948\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"MPS\",\n \"sliceProfile\": {\n \"latencyTarget\": 1,\n \"reliability\": 99.999,\n \"throughput\": {\n \"downlink\": 1000,\n \"uplink\": 50\n },\n \"maxNumberOfUEs\": 1000\n },\n \"areaOfService\": {\n \"region\": \"Mining Site Alpha\",\n \"center\": {\n \"latitude\": -23.4162,\n \"longitude\": 46.4017\n },\n \"radius\": 500,\n \"cells\": [\n \"cell-081\",\n \"cell-080\",\n \"cell-082\",\n \"cell-083\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-18T13:00:00Z\",\n \"endTime\": \"2025-01-18T15:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/110000000000\"\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "success", + "id": "e2c589fd7c09", + "target_layer": "camara", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5882352941176471, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"eMBB\",\n \"sliceProfile\": {\n \"latencyTarget\": 30,\n \"reliability\": 99.9,\n \"throughput\": {\n \"downlink\": 500,\n \"uplink\": 200\n },\n \"maxNumberOfUEs\": 10000\n },\n \"areaOfService\": {\n \"region\": \"Healthcare Park\",\n \"center\": {\n \"latitude\": 37.7749,\n \"longitude\": -122.4194\n },\n \"radius\": 5000,\n \"cells\": [\n \"cell-064\",\n \"cell-037\",\n \"cell-002\",\n \"cell-062\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-06-12T05:00:00Z\",\n \"endTime\": \"2025-06-12T17:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/fcb9f701915d\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"eMBB\",\n \"sliceProfile\": {\n \"latencyTarget\": 30,\n \"reliability\": 99.9,\n \"throughput\": {\n \"downlink\": 500,\n \"uplink\": 200\n },\n \"maxNumberOfUEs\": 10000\n },\n \"areaOfService\": {\n \"region\": \"Healthcare Park\",\n \"center\": {\n \"latitude\": 37.7749,\n \"longitude\": -122.4194\n },\n \"radius\": 5000,\n \"cells\": [\n \"cell-081\",\n \"cell-080\",\n \"cell-082\",\n \"cell-083\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-18T13:00:00Z\",\n \"endTime\": \"2025-01-18T15:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/7d0101001010\"\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "8bb77054da0f", + "target_layer": "intent_3gpp", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7, + "norm_field_f1": 0.5789473684210527, + "norm_key_f1": 0.8947368421052632, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-mMTC-76a2e5e21ca2\",\n \"name\": \"Virtual Classroom Xr Delivery mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-e97dda4b2258\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 200,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 90,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Research Park\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"virtual c\n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-mMTC-11b0b1b1b1b1\",\n \"name\": \"Virtual Classroom Xr Delivery mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-11b0b1b1b1b1\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 200,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 90,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Research Park\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCo\n......" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "36710900c49f", + "target_layer": "intent_3gpp", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.775, + "norm_field_f1": 0.6578947368421053, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-mMTC-0cddd74357ff\",\n \"name\": \"Virtual Classroom Xr Delivery mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-0a58bf066364\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000002\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 0.5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 25,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 100,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99.5,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 5000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Industrial Zone North\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\"\n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-mMTC-110000000000\",\n \"name\": \"Virtual Classroom Xr Delivery mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-000000000000\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 0.5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 25,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 99.5,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 5000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Industrial Zone North\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \n......" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "d8df0122d5e1", + "target_layer": "intent_3gpp", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.775, + "norm_field_f1": 0.6578947368421053, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-mMTC-d0819ce7392d\",\n \"name\": \"Campus Iot Management mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-429a184e74f3\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000001\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 50,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 95,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 5000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Hospital Campus\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"campus IoT manag\n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-mMTC-110110101010\",\n \"name\": \"Campus Iot Management mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-100000000001\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 95,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 5000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Hospital Campus\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\":\n......" + }, + { + "split": "test_sector_ood", + "kind": "success", + "id": "25d1c40596cf", + "target_layer": "intent_3gpp", + "slice_type": "URLLC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.925, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-URLLC-df9d41d88f4c\",\n \"name\": \"Research Lab Data Transfer URLLC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-05434c3a976b\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"URLLC\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000001\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 200,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 100,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 3,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99.999,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"maxNumberOfUEs\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Factory Complex A\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"\n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-URLLC-11b111d01111\",\n \"name\": \"URLLC Slice - Research Lab Data Transfer\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-0d0d0d0d0d0d\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"URLLC\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000001\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 200,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 100,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 3,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99.999,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"maxNumberOfUEs\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Factory Complex A\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"c\n......" + }, + { + "split": "test_adversarial", + "kind": "failure", + "id": "c2c3794511e2", + "target_layer": "adversarial_ambiguous", + "slice_type": "N/A", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": true, + "field_f1": 1.0, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "status", + "message", + "suggestedAction" + ], + "prediction_json_keys": [ + "status", + "message", + "suggestedAction" + ], + "gold": "{\n \"status\": \"CLARIFICATION_REQUIRED\",\n \"message\": \"Intent too vague. Please specify: sliceType, latencyTarget, reliabilityTarget, dlThroughput, ulThroughput, maxUEs, serviceArea.\",\n \"suggestedAction\": \"Reformulate with specific KPI targets.\"\n}", + "prediction": "\n\n\n\n{\n \"status\": \"CLARIFICATION_REQUIRED\",\n \"message\": \"Intent too vague. Please specify: sliceType, latencyTarget, reliabilityTarget, dlThroughput, ulThroughput, maxUEs, serviceArea.\",\n \"suggestedAction\": \"Reformulate with specific KPI targets.\"\n}" + }, + { + "split": "test_adversarial", + "kind": "failure", + "id": "c20e3d39aa40", + "target_layer": "adversarial_ambiguous", + "slice_type": "N/A", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": true, + "field_f1": 1.0, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "status", + "message", + "suggestedAction" + ], + "prediction_json_keys": [ + "status", + "message", + "suggestedAction" + ], + "gold": "{\n \"status\": \"CLARIFICATION_REQUIRED\",\n \"message\": \"Intent too vague. Please specify: sliceType, latencyTarget, reliabilityTarget, dlThroughput, ulThroughput, maxUEs, serviceArea.\",\n \"suggestedAction\": \"Reformulate with specific KPI targets.\"\n}", + "prediction": "\n\n\n\n{\n \"status\": \"CLARIFICATION_REQUIRED\",\n \"message\": \"Intent too vague. Please specify: sliceType, latencyTarget, reliabilityTarget, dlThroughput, ulThroughput, maxUEs, serviceArea.\",\n \"suggestedAction\": \"Reformulate with specific KPI targets.\"\n}" + }, + { + "split": "test_adversarial", + "kind": "failure", + "id": "d2dbc2ecfc38", + "target_layer": "adversarial_ambiguous", + "slice_type": "N/A", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": true, + "field_f1": 1.0, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "status", + "message", + "suggestedAction" + ], + "prediction_json_keys": [ + "status", + "message", + "suggestedAction" + ], + "gold": "{\n \"status\": \"CLARIFICATION_REQUIRED\",\n \"message\": \"Intent too vague. Please specify: sliceType, latencyTarget, reliabilityTarget, dlThroughput, ulThroughput, maxUEs, serviceArea.\",\n \"suggestedAction\": \"Reformulate with specific KPI targets.\"\n}", + "prediction": "\n\n\n\n{\n \"status\": \"CLARIFICATION_REQUIRED\",\n \"message\": \"Intent too vague. Please specify: sliceType, latencyTarget, reliabilityTarget, dlThroughput, ulThroughput, maxUEs, serviceArea.\",\n \"suggestedAction\": \"Reformulate with specific KPI targets.\"\n}" + }, + { + "split": "test_adversarial", + "kind": "success", + "id": "c2c3794511e2", + "target_layer": "adversarial_ambiguous", + "slice_type": "N/A", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": true, + "field_f1": 1.0, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "status", + "message", + "suggestedAction" + ], + "prediction_json_keys": [ + "status", + "message", + "suggestedAction" + ], + "gold": "{\n \"status\": \"CLARIFICATION_REQUIRED\",\n \"message\": \"Intent too vague. Please specify: sliceType, latencyTarget, reliabilityTarget, dlThroughput, ulThroughput, maxUEs, serviceArea.\",\n \"suggestedAction\": \"Reformulate with specific KPI targets.\"\n}", + "prediction": "\n\n\n\n{\n \"status\": \"CLARIFICATION_REQUIRED\",\n \"message\": \"Intent too vague. Please specify: sliceType, latencyTarget, reliabilityTarget, dlThroughput, ulThroughput, maxUEs, serviceArea.\",\n \"suggestedAction\": \"Reformulate with specific KPI targets.\"\n}" + }, + { + "split": "test_adversarial", + "kind": "failure", + "id": "b5e41f85a345", + "target_layer": "adversarial_out_of_scope", + "slice_type": "N/A", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": true, + "field_f1": 1.0, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "status", + "message", + "supportedOperations" + ], + "prediction_json_keys": [ + "status", + "message", + "supportedOperations" + ], + "gold": "{\n \"status\": \"OUT_OF_SCOPE\",\n \"message\": \"Request outside scope. This system handles 5G/6G network slice intent translation only.\",\n \"supportedOperations\": [\n \"slice creation\",\n \"lifecycle management\",\n \"SLA assurance\"\n ]\n}", + "prediction": "\n\n\n\n{\n \"status\": \"OUT_OF_SCOPE\",\n \"message\": \"Request outside scope. This system handles 5G/6G network slice intent translation only.\",\n \"supportedOperations\": [\n \"slice creation\",\n \"lifecycle management\",\n \"SLA assurance\"\n ]\n}" + }, + { + "split": "test_adversarial", + "kind": "failure", + "id": "f5b83d2fb356", + "target_layer": "adversarial_out_of_scope", + "slice_type": "N/A", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": true, + "field_f1": 1.0, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "status", + "message", + "supportedOperations" + ], + "prediction_json_keys": [ + "status", + "message", + "supportedOperations" + ], + "gold": "{\n \"status\": \"OUT_OF_SCOPE\",\n \"message\": \"Request outside scope. This system handles 5G/6G network slice intent translation only.\",\n \"supportedOperations\": [\n \"slice creation\",\n \"lifecycle management\",\n \"SLA assurance\"\n ]\n}", + "prediction": "\n\n\n\n{\n \"status\": \"OUT_OF_SCOPE\",\n \"message\": \"Request outside scope. This system handles 5G/6G network slice intent translation only.\",\n \"supportedOperations\": [\n \"slice creation\",\n \"lifecycle management\",\n \"SLA assurance\"\n ]\n}" + }, + { + "split": "test_adversarial", + "kind": "failure", + "id": "8c2457c3ef17", + "target_layer": "adversarial_out_of_scope", + "slice_type": "N/A", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": true, + "field_f1": 1.0, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "status", + "message", + "supportedOperations" + ], + "prediction_json_keys": [ + "status", + "message", + "supportedOperations" + ], + "gold": "{\n \"status\": \"OUT_OF_SCOPE\",\n \"message\": \"Request outside scope. This system handles 5G/6G network slice intent translation only.\",\n \"supportedOperations\": [\n \"slice creation\",\n \"lifecycle management\",\n \"SLA assurance\"\n ]\n}", + "prediction": "\n\n\n\n{\n \"status\": \"OUT_OF_SCOPE\",\n \"message\": \"Request outside scope. This system handles 5G/6G network slice intent translation only.\",\n \"supportedOperations\": [\n \"slice creation\",\n \"lifecycle management\",\n \"SLA assurance\"\n ]\n}" + }, + { + "split": "test_adversarial", + "kind": "success", + "id": "b5e41f85a345", + "target_layer": "adversarial_out_of_scope", + "slice_type": "N/A", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": true, + "field_f1": 1.0, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "status", + "message", + "supportedOperations" + ], + "prediction_json_keys": [ + "status", + "message", + "supportedOperations" + ], + "gold": "{\n \"status\": \"OUT_OF_SCOPE\",\n \"message\": \"Request outside scope. This system handles 5G/6G network slice intent translation only.\",\n \"supportedOperations\": [\n \"slice creation\",\n \"lifecycle management\",\n \"SLA assurance\"\n ]\n}", + "prediction": "\n\n\n\n{\n \"status\": \"OUT_OF_SCOPE\",\n \"message\": \"Request outside scope. This system handles 5G/6G network slice intent translation only.\",\n \"supportedOperations\": [\n \"slice creation\",\n \"lifecycle management\",\n \"SLA assurance\"\n ]\n}" + } +] \ No newline at end of file