diff --git "a/analysis/stage2_examples/failure_examples.json" "b/analysis/stage2_examples/failure_examples.json" new file mode 100644--- /dev/null +++ "b/analysis/stage2_examples/failure_examples.json" @@ -0,0 +1,2877 @@ +[ + { + "split": "test_in_distribution", + "kind": "failure", + "id": "19b8437132d8", + "target_layer": "o1_nrm", + "slice_type": "V2X", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.21739130434782608, + "norm_field_f1": 0.25, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-035\",\n \"GNBDUFunction\": {\n \"id\": \"DU-011\",\n \"gNBId\": 35,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-011-002\",\n \"cellLocalId\": 2,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n78\",\n \"arfcnDL\": 632207,\n \"bSChannelBwDL\": 50,\n \"nrPCI\": 972,\n \"nrTAC\": 42116,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 4,\n \"sd\": \"000001\"\n },\n \"rrmPolicyMinRatio\": 0.19,\n \"rrmPolicyMaxRatio\": 0.49,\n \"rrmPolicyDedicatedRatio\": 0.34,\n \"rrmPolicyMaxRatioPerSlice\": 0.49\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"15\",\n \"sNSSAIList\": [\n {\n \"sst\": 4,\n \"sd\": \"000001\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-014\",\n \"GNBDUFunction\": {\n \"id\": \"DU-008\",\n \"gNBId\": 14,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-008-002\",\n \"cellLocalId\": 2,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n77\",\n \"arfcnDL\": 629813,\n \"bSChannelBwDL\": 40,\n \"nrPCI\": 812,\n \"nrTAC\": 3864,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 5,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.19,\n \"rrmPolicyMaxRatio\": 0.48,\n \"rrmPolicyDedicatedRatio\": 0.33,\n \"rrmPolicyMaxRatioPerSlice\": 0.48\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"234\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 5,\n \"sd\": \"000003\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "bd82257e1748", + "target_layer": "o1_nrm", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.2608695652173913, + "norm_field_f1": 0.3, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-007\",\n \"GNBDUFunction\": {\n \"id\": \"DU-042\",\n \"gNBId\": 7,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-042-003\",\n \"cellLocalId\": 3,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n79\",\n \"arfcnDL\": 728251,\n \"bSChannelBwDL\": 80,\n \"nrPCI\": 391,\n \"nrTAC\": 51881,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.51,\n \"rrmPolicyMaxRatio\": 0.89,\n \"rrmPolicyDedicatedRatio\": 0.7,\n \"rrmPolicyMaxRatioPerSlice\": 0.89\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"310\",\n \"mnc\": \"01\",\n \"sNSSAIList\": [\n {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-014\",\n \"GNBDUFunction\": {\n \"id\": \"DU-001\",\n \"gNBId\": 14,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-001-001\",\n \"cellLocalId\": 1,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n77\",\n \"arfcnDL\": 621612,\n \"bSChannelBwDL\": 60,\n \"nrPCI\": 81,\n \"nrTAC\": 34561,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 1,\n \"sd\": \"000001\"\n },\n \"rrmPolicyMinRatio\": 0.51,\n \"rrmPolicyMaxRatio\": 0.7,\n \"rrmPolicyDedicatedRatio\": 0.6,\n \"rrmPolicyMaxRatioPerSlice\": 0.7\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"234\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 1,\n \"sd\": \"000003\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "7f4996d61061", + "target_layer": "o1_nrm", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.2608695652173913, + "norm_field_f1": 0.3, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-023\",\n \"GNBDUFunction\": {\n \"id\": \"DU-007\",\n \"gNBId\": 23,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-007-006\",\n \"cellLocalId\": 6,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n79\",\n \"arfcnDL\": 727301,\n \"bSChannelBwDL\": 50,\n \"nrPCI\": 279,\n \"nrTAC\": 56600,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 1,\n \"sd\": \"000002\"\n },\n \"rrmPolicyMinRatio\": 0.12,\n \"rrmPolicyMaxRatio\": 0.27,\n \"rrmPolicyDedicatedRatio\": 0.2,\n \"rrmPolicyMaxRatioPerSlice\": 0.27\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"234\",\n \"mnc\": \"15\",\n \"sNSSAIList\": [\n {\n \"sst\": 1,\n \"sd\": \"000002\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-006\",\n \"GNBDUFunction\": {\n \"id\": \"DU-008\",\n \"gNBId\": 6,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-008-002\",\n \"cellLocalId\": 2,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n77\",\n \"arfcnDL\": 623820,\n \"bSChannelBwDL\": 60,\n \"nrPCI\": 812,\n \"nrTAC\": 34442,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.12,\n \"rrmPolicyMaxRatio\": 0.35,\n \"rrmPolicyDedicatedRatio\": 0.23,\n \"rrmPolicyMaxRatioPerSlice\": 0.35\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "success", + "id": "fdc83e331d62", + "target_layer": "o1_nrm", + "slice_type": "MPS", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5652173913043478, + "norm_field_f1": 0.65, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-037\",\n \"GNBDUFunction\": {\n \"id\": \"DU-003\",\n \"gNBId\": 37,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-003-008\",\n \"cellLocalId\": 8,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n77\",\n \"arfcnDL\": 676355,\n \"bSChannelBwDL\": 60,\n \"nrPCI\": 135,\n \"nrTAC\": 24498,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 5,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.12,\n \"rrmPolicyMaxRatio\": 0.35,\n \"rrmPolicyDedicatedRatio\": 0.23,\n \"rrmPolicyMaxRatioPerSlice\": 0.35\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"234\",\n \"mnc\": \"01\",\n \"sNSSAIList\": [\n {\n \"sst\": 5,\n \"sd\": \"000003\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-001\",\n \"GNBDUFunction\": {\n \"id\": \"DU-001\",\n \"gNBId\": 1,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-001-001\",\n \"cellLocalId\": 1,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n77\",\n \"arfcnDL\": 625612,\n \"bSChannelBwDL\": 40,\n \"nrPCI\": 81,\n \"nrTAC\": 34520,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 5,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.12,\n \"rrmPolicyMaxRatio\": 0.35,\n \"rrmPolicyDedicatedRatio\": 0.23,\n \"rrmPolicyMaxRatioPerSlice\": 0.35\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"234\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 5,\n \"sd\": \"000003\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "79180bddf025", + "target_layer": "a1_policy", + "slice_type": "V2X", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.47368421052631576, + "norm_field_f1": 0.5555555555555556, + "norm_key_f1": 1.0, + "error_label": "a1_policy_value_error", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-v2x-673a3c\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-v2x-1d8937\",\n \"cellId\": \"cell-091\",\n \"sNSSAI\": {\n \"sst\": 4,\n \"sd\": \"000002\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 200,\n \"mfbr\": 240.0,\n \"priorityLevel\": 1,\n \"5qi\": 79,\n \"pdb\": 50,\n \"per\": 1e-05\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 11,\n \"max\": 21\n },\n \"schedulerWeight\": 0.53,\n \"maxUEs\": 10000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-v2x-0d000b\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-v2x-00000d\",\n \"cellId\": \"cell-062\",\n \"sNSSAI\": {\n \"sst\": 4,\n \"sd\": \"000001\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 200,\n \"mfbr\": 240.0,\n \"priorityLevel\": 2,\n \"5qi\": 84,\n \"pdb\": 30,\n \"per\": 1e-06\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 11,\n \"max\": 31\n },\n \"schedulerWeight\": 0.31,\n \"maxUEs\": 10000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "2414e525b358", + "target_layer": "a1_policy", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.47368421052631576, + "norm_field_f1": 0.5555555555555556, + "norm_key_f1": 1.0, + "error_label": "a1_policy_value_error", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-embb-f79e96\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-embb-59c165\",\n \"cellId\": \"cell-094\",\n \"sNSSAI\": {\n \"sst\": 1,\n \"sd\": \"000002\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 300,\n \"mfbr\": 360.0,\n \"priorityLevel\": 6,\n \"5qi\": 7,\n \"pdb\": 100,\n \"per\": 0.005\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 20,\n \"max\": 47\n },\n \"schedulerWeight\": 0.48,\n \"maxUEs\": 100\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-embb-117006\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-embb-117006\",\n \"cellId\": \"cell-081\",\n \"sNSSAI\": {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 300,\n \"mfbr\": 360.0,\n \"priorityLevel\": 5,\n \"5qi\": 6,\n \"pdb\": 300,\n \"per\": 0.0005\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 20,\n \"max\": 31\n },\n \"schedulerWeight\": 0.31,\n \"maxUEs\": 100\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "4f25f461f1f0", + "target_layer": "a1_policy", + "slice_type": "URLLC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.47368421052631576, + "norm_field_f1": 0.5555555555555556, + "norm_key_f1": 1.0, + "error_label": "a1_policy_value_error", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-urllc-d26d08\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-urllc-e3ed0f\",\n \"cellId\": \"cell-049\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000002\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 300,\n \"mfbr\": 360.0,\n \"priorityLevel\": 3,\n \"5qi\": 82,\n \"pdb\": 10,\n \"per\": 1e-05\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 17,\n \"max\": 41\n },\n \"schedulerWeight\": 0.27,\n \"maxUEs\": 50\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-urllc-000001\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-urllc-000001\",\n \"cellId\": \"cell-001\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000001\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 300,\n \"mfbr\": 360.0,\n \"priorityLevel\": 1,\n \"5qi\": 84,\n \"pdb\": 30,\n \"per\": 1e-06\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 17,\n \"max\": 38\n },\n \"schedulerWeight\": 0.35,\n \"maxUEs\": 50\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "success", + "id": "e15f30f1ec77", + "target_layer": "a1_policy", + "slice_type": "HMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7894736842105263, + "norm_field_f1": 0.8888888888888888, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-hmtc-586f4a\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-hmtc-9d8320\",\n \"cellId\": \"cell-010\",\n \"sNSSAI\": {\n \"sst\": 5,\n \"sd\": \"000002\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 2000,\n \"mfbr\": 2400.0,\n \"priorityLevel\": 1,\n \"5qi\": 83,\n \"pdb\": 10,\n \"per\": 1e-06\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 80,\n \"max\": 95\n },\n \"schedulerWeight\": 0.4,\n \"maxUEs\": 10000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-hmtc-000006\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-hmtc-000006\",\n \"cellId\": \"cell-081\",\n \"sNSSAI\": {\n \"sst\": 5,\n \"sd\": \"000002\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 2000,\n \"mfbr\": 2400.0,\n \"priorityLevel\": 1,\n \"5qi\": 83,\n \"pdb\": 10,\n \"per\": 1e-06\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 80,\n \"max\": 95\n },\n \"schedulerWeight\": 0.22,\n \"maxUEs\": 10000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "946b4f8cc48b", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "mMTC", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.23809523809523808, + "norm_field_f1": 0.12698412698412698, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-c16cb4865ac3\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 200,\n \"measurements\": [\n 129.84,\n 194.4,\n 196.94,\n 167.53,\n 129.27\n ],\n \"average\": 163.596,\n \"min\": 129.27,\n \"max\": 196.94,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 10,\n \"measurements\": [\n 10.7,\n 10.7,\n 11.6,\n 11.2,\n 11.9\n ],\n \"average\": 11.22,\n \"min\": 10.7,\n \"max\": 11.9,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 25,\n \"measurements\": [\n 26.3,\n 27.2,\n 29.1,\n 22.5,\n 23.7\n ],\n \"average\": 25.76,\n \"min\": 22.5,\n \"max\": 29.1,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 90,\n \"measurements\": [\n 89.998585,\n 89.914869,\n 89.927755,\n 89.96821,\n 89.984428\n ],\n \"average\": 89.959,\n \"min\": 89.914869,\n \"max\": 89.998585,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 10000,\n \"measurements\": [\n 7645,\n 8673,\n 7210,\n 9086,\n 7839\n ],\n \"average\": 8090.6,\n \"min\": 7210,\n \"max\": 9086,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 50.6,\n \"memoryUsage\": 24.9,\n \"prbUtilization\": 72.0\n },\n \"@type\": \"\n......", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-c16cb4865ac3\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 500,\n \"measurements\": [\n 321.49,\n 302.02,\n 385.22,\n 301.82,\n 340.02\n ],\n \"average\": 330.11,\n \"min\": 301.82,\n \"max\": 385.22,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 50,\n \"measurements\": [\n 58.3,\n 50.4,\n 58.9,\n 55.1,\n 58.6\n ],\n \"average\": 56.26,\n \"min\": 50.4,\n \"max\": 58.9,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 10,\n \"measurements\": [\n 10.4,\n 9.4,\n 9.4,\n 9.1,\n 9.6\n ],\n \"average\": 9.78,\n \"min\": 9.1,\n \"max\": 10.4,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99,\n \"measurements\": [\n 98.907515,\n 98.980229,\n 98.913122,\n 98.966934,\n 98.98327\n ],\n \"average\": 98.95,\n \"min\": 98.907515,\n \"max\": 98.98327,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 1000000,\n \"measurements\": [\n 112211,\n 254222,\n 947123,\n 947123,\n 147409\n ],\n \"average\": 462197.6,\n \"min\": 112211,\n \"max\": 947123,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 67.4,\n \"memoryUsage\": 33.7,\n \"prbUtilization\n......" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "a0dde7740590", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "eMBB", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.2698412698412698, + "norm_field_f1": 0.15873015873015872, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-bb2d09f58a92\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 50,\n \"measurements\": [\n 53.36,\n 38.17,\n 53.78,\n 45.41,\n 47.52\n ],\n \"average\": 47.648,\n \"min\": 38.17,\n \"max\": 53.78,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 1000,\n \"measurements\": [\n 1098.1,\n 1095.5,\n 905.5,\n 1171.4,\n 935.4\n ],\n \"average\": 1041.18,\n \"min\": 905.5,\n \"max\": 1171.4,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 75,\n \"measurements\": [\n 78.5,\n 75.9,\n 79.6,\n 77.9,\n 80.3\n ],\n \"average\": 78.44,\n \"min\": 75.9,\n \"max\": 80.3,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.95,\n \"measurements\": [\n 99.993633,\n 99.904768,\n 100,\n 99.940096,\n 99.980962\n ],\n \"average\": 99.964,\n \"min\": 99.904768,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 10000,\n \"measurements\": [\n 2978,\n 4838,\n 3232,\n 5089,\n 4608\n ],\n \"average\": 4149.0,\n \"min\": 2978,\n \"max\": 5089,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 65.1,\n \"memoryUsage\": 61.2,\n \"prbUtilization\": 48.9\n },\n \"@type\": \"I\n......", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-bb2d09f58a92\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 3,\n \"measurements\": [\n 2.29,\n 2.31,\n 2.31,\n 2.3,\n 2.3\n ],\n \"average\": 2.282,\n \"min\": 2.29,\n \"max\": 2.31,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 20,\n \"measurements\": [\n 21.4,\n 20.4,\n 21.7,\n 21.1,\n 21.1\n ],\n \"average\": 21.14,\n \"min\": 20.4,\n \"max\": 21.7,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measurements\": [\n 115.5,\n 112.1,\n 99.9,\n 112.2,\n 112.6\n ],\n \"average\": 110.46,\n \"min\": 99.9,\n \"max\": 115.5,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.999,\n \"measurements\": [\n 99.946161,\n 100,\n 100,\n 100,\n 100\n ],\n \"average\": 99.989,\n \"min\": 99.946161,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 5000,\n \"measurements\": [\n 1605,\n 1446,\n 2225,\n 2820,\n 2411\n ],\n \"average\": 2101.4,\n \"min\": 1446,\n \"max\": 2820,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 67.3,\n \"memoryUsage\": 30.7,\n \"prbUtilization\": 62.1\n },\n \"@type\": \"IntentAssuranceRepo\n......" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "3d74914329e5", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "V2X", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.25396825396825395, + "norm_field_f1": 0.15873015873015872, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-7894e894dbba\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 3,\n \"measurements\": [\n 1.92,\n 2.24,\n 1.91,\n 2.19,\n 2.71\n ],\n \"average\": 2.194,\n \"min\": 1.91,\n \"max\": 2.71,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 100,\n \"measurements\": [\n 102.9,\n 116.2,\n 107.3,\n 116.8,\n 119.0\n ],\n \"average\": 112.44,\n \"min\": 102.9,\n \"max\": 119.0,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 75,\n \"measurements\": [\n 73.0,\n 76.9,\n 89.8,\n 84.4,\n 68.0\n ],\n \"average\": 78.42,\n \"min\": 68.0,\n \"max\": 89.8,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9999,\n \"measurements\": [\n 100,\n 99.918632,\n 100,\n 99.902147,\n 99.900875\n ],\n \"average\": 99.944,\n \"min\": 99.900875,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 10000,\n \"measurements\": [\n 1638,\n 4771,\n 9409,\n 9747,\n 8579\n ],\n \"average\": 6828.8,\n \"min\": 1638,\n \"max\": 9747,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 46.6,\n \"memoryUsage\": 39.1,\n \"prbUtilization\": 86.1\n },\n \"@type\": \"IntentAssuranceRepor\n......", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-000000000001\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 1,\n \"measurements\": [\n 0.79,\n 0.79,\n 0.79,\n 0.79,\n 0.79\n ],\n \"average\": 0.79,\n \"min\": 0.79,\n \"max\": 0.79,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 200,\n \"measurements\": [\n 229.2,\n 209.4,\n 214.3,\n 221.8,\n 203.6\n ],\n \"average\": 215.66,\n \"min\": 203.6,\n \"max\": 229.2,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 25,\n \"measurements\": [\n 26.6,\n 26.6,\n 26.6,\n 26.6,\n 26.6\n ],\n \"average\": 26.6,\n \"min\": 26.6,\n \"max\": 26.6,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.999,\n \"measurements\": [\n 99.999623,\n 99.999742,\n 100,\n 100,\n 100\n ],\n \"average\": 99.99987,\n \"min\": 99.999623,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 5000,\n \"measurements\": [\n 4656,\n 4656,\n 4656,\n 4656,\n 4656\n ],\n \"average\": 4656.0,\n \"min\": 4656,\n \"max\": 4656,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 75.1,\n \"memoryUsage\": 34.4,\n \"prbUtilization\": 82.3\n },\n \"@type\": \"IntentAss\n......" + }, + { + "split": "test_in_distribution", + "kind": "success", + "id": "9cfbe3c0c88e", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "URLLC", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.3333333333333333, + "norm_field_f1": 0.30158730158730157, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-03ec942da669\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 10,\n \"measurements\": [\n 9.74,\n 9.9,\n 9.39,\n 10.99,\n 7.15\n ],\n \"average\": 9.434,\n \"min\": 7.15,\n \"max\": 10.99,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 20,\n \"measurements\": [\n 23.3,\n 23.9,\n 20.0,\n 18.7,\n 22.8\n ],\n \"average\": 21.74,\n \"min\": 18.7,\n \"max\": 23.9,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 50,\n \"measurements\": [\n 56.7,\n 56.6,\n 49.9,\n 55.0,\n 49.8\n ],\n \"average\": 53.6,\n \"min\": 49.8,\n \"max\": 56.7,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.999,\n \"measurements\": [\n 100,\n 100,\n 100,\n 99.940794,\n 100\n ],\n \"average\": 99.988,\n \"min\": 99.940794,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 20,\n \"measurements\": [\n 9,\n 16,\n 12,\n 7,\n 12\n ],\n \"average\": 11.2,\n \"min\": 7,\n \"max\": 16,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 61.8,\n \"memoryUsage\": 43.2,\n \"prbUtilization\": 72.6\n },\n \"@type\": \"IntentAssuranceReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-03ec942da669\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 10,\n \"measurements\": [\n 6.23,\n 6.48,\n 7.56,\n 10.03,\n 6.5\n ],\n \"average\": 7.36,\n \"min\": 6.23,\n \"max\": 10.03,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 150,\n \"measurements\": [\n 152.2,\n 137.3,\n 168.0,\n 159.2,\n 140.3\n ],\n \"average\": 151.4,\n \"min\": 137.3,\n \"max\": 168.0,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measurements\": [\n 100.0,\n 117.6,\n 94.3,\n 118.5,\n 90.5\n ],\n \"average\": 104.2,\n \"min\": 90.5,\n \"max\": 118.5,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.999,\n \"measurements\": [\n 99.916851,\n 100,\n 100,\n 100,\n 100\n ],\n \"average\": 99.985,\n \"min\": 99.916851,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 50,\n \"measurements\": [\n 45,\n 40,\n 40,\n 41,\n 41\n ],\n \"average\": 41.4,\n \"min\": 40,\n \"max\": 45,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 75.1,\n \"memoryUsage\": 34.4,\n \"prbUtilization\": 82.6\n },\n \"@type\": \"IntentAssuranceReport\"\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "a790d5b62fbc", + "target_layer": "tmf921_lifecycle_monitor", + "slice_type": "eMBB", + "lifecycle_operation": "monitor", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5666666666666667, + "norm_field_f1": 0.3103448275862069, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentFulfillmentReport" + ], + "prediction_json_keys": [ + "intentFulfillmentReport" + ], + "gold": "{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-85ac305b5e9c\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Fulfilled\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 30,\n \"measured\": 22.47,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 500,\n \"measured\": 507.9,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 200,\n \"measured\": 200.2,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99,\n \"measured\": 100,\n \"unit\": \"percent\",\n \"compliant\": true\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 100,\n \"measured\": 95,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": true,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-85ac305b5e9c\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 3,\n \"measured\": 2.89,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 20,\n \"measured\": 20.4,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measured\": 100.4,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.999,\n \"measured\": 99.98495,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 500,\n \"measured\": 144,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "5efed41ac432", + "target_layer": "tmf921_lifecycle_monitor", + "slice_type": "mMTC", + "lifecycle_operation": "monitor", + "parse_json": true, + "exact_match": false, + "field_f1": 0.6666666666666666, + "norm_field_f1": 0.41379310344827586, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentFulfillmentReport" + ], + "prediction_json_keys": [ + "intentFulfillmentReport" + ], + "gold": "{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-54f14eac901c\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 200,\n \"measured\": 197.74,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 25,\n \"measured\": 24.7,\n \"unit\": \"Mbps\",\n \"compliant\": false\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 25,\n \"measured\": 27.4,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.5,\n \"measured\": 99.455458,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 1000000,\n \"measured\": 13690,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-54f14eac901c\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 500,\n \"measured\": 429.21,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 50,\n \"measured\": 50.4,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 10,\n \"measured\": 10.4,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.5,\n \"measured\": 99.495667,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 10000,\n \"measured\": 5842,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "748a95abfa38", + "target_layer": "tmf921_lifecycle_monitor", + "slice_type": "MPS", + "lifecycle_operation": "monitor", + "parse_json": true, + "exact_match": false, + "field_f1": 0.6666666666666666, + "norm_field_f1": 0.41379310344827586, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentFulfillmentReport" + ], + "prediction_json_keys": [ + "intentFulfillmentReport" + ], + "gold": "{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-73d3be81f542\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 20,\n \"measured\": 16.26,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 100,\n \"measured\": 112.4,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measured\": 97.9,\n \"unit\": \"Mbps\",\n \"compliant\": false\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9999,\n \"measured\": 99.999851,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 500,\n \"measured\": 440,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-141011101010\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 10,\n \"measured\": 8.09,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 150,\n \"measured\": 150.6,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measured\": 109.0,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9999,\n \"measured\": 99.946081,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 100,\n \"measured\": 95,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "success", + "id": "9b43b2815c12", + "target_layer": "tmf921_lifecycle_monitor", + "slice_type": "URLLC", + "lifecycle_operation": "monitor", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7, + "norm_field_f1": 0.6896551724137931, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "intentFulfillmentReport" + ], + "prediction_json_keys": [ + "intentFulfillmentReport" + ], + "gold": "{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-72a946c6e2c3\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 0.5,\n \"measured\": 0.36,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 100,\n \"measured\": 108.2,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 50,\n \"measured\": 57.0,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9999,\n \"measured\": 99.904336,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 10,\n \"measured\": 8,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-72a946c6e2c3\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 1,\n \"measured\": 0.86,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 100,\n \"measured\": 100.5,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measured\": 109.0,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.999,\n \"measured\": 99.99464,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 1000,\n \"measured\": 910,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "32c3a5d4c830", + "target_layer": "tmf921", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": false, + "exact_match": false, + "field_f1": 0.0, + "norm_field_f1": 0.0, + "norm_key_f1": 0.0, + "error_label": "invalid_or_unparseable_json", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": null, + "gold": "{\n \"id\": \"intent-e739759aea88\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-e739759aea88\",\n \"name\": \"eMBB Slice - Urban Noise Monitoring\",\n \"description\": \"Intent for urban noise monitoring at Hospital Campus with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 5,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-8032d4ff1385\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-4f8120\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1000,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n \n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-011010101010\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-011010101010\",\n \"name\": \"eMBB Slice - Urban Noise Monitoring\",\n \"description\": \"Intent for urban noise monitoring at Hospital Campus with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-011010101010\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-011010\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1000,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n \n......" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "2e170e5d2f68", + "target_layer": "tmf921", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7692307692307693, + "norm_field_f1": 0.8656716417910447, + "norm_key_f1": 0.9402985074626866, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-6830bb1fb139\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-6830bb1fb139\",\n \"name\": \"eMBB Slice - Electronic Warfare Spectrum Monitoring\",\n \"description\": \"Intent for electronic warfare spectrum monitoring at Forward Operating Area with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 5,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-6b2137891dca\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-b74b1e\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000002\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 25,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-143104301088\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-143104301088\",\n \"name\": \"eMBB Slice - Electronic Warfare Spectrum Monitoring\",\n \"description\": \"Intent for electronic warfare spectrum monitoring at Forward Operating Area with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-333343009908\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-000000\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 25,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NR\n......" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "6a37983cc8b1", + "target_layer": "tmf921", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7692307692307693, + "norm_field_f1": 0.8656716417910447, + "norm_key_f1": 0.9402985074626866, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-0f94edf4e5fd\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-0f94edf4e5fd\",\n \"name\": \"eMBB Slice - Fleet Route Optimization\",\n \"description\": \"Intent for fleet route optimization at Factory Complex A with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 5,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-5ebd3bd193ab\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-08017d\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000002\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1000,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 150,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n \n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-000000000001\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-000000000001\",\n \"name\": \"eMBB Slice - Fleet Route Optimization\",\n \"description\": \"Intent for fleet route optimization at Factory Complex A with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-000000000001\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-000000\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1000,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 150,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"M\n......" + }, + { + "split": "test_in_distribution", + "kind": "success", + "id": "f859b201688c", + "target_layer": "tmf921", + "slice_type": "URLLC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.8734177215189873, + "norm_field_f1": 0.9852941176470589, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-6a7cba7e6ca0\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-6a7cba7e6ca0\",\n \"name\": \"URLLC Slice - Algorithmic Trading Low-Latency\",\n \"description\": \"Intent for algorithmic trading low-latency at Medical Center District with URLLC service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-3e937a70eebb\",\n \"name\": \"URLLC QoS Delivery Expectation\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-urllc-c2b796\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"URLLC\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 2,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 150,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 75,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\":\n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-140007993081\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-140007993081\",\n \"name\": \"URLLC Slice - Algorithmic Trading Low-Latency\",\n \"description\": \"Intent for algorithmic trading low-latency at Medical Center District with URLLC service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-000000000001\",\n \"name\": \"URLLC QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-urllc-000001\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"URLLC\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 2,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 150,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 75,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n......" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "f048c2698152", + "target_layer": "camara", + "slice_type": "MPS", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5, + "norm_field_f1": 0.7142857142857143, + "norm_key_f1": 0.8571428571428571, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"MPS\",\n \"sliceProfile\": {\n \"latencyTarget\": 1,\n \"reliability\": 99.9999,\n \"throughput\": {\n \"downlink\": 100,\n \"uplink\": 75\n },\n \"maxNumberOfUEs\": 500\n },\n \"areaOfService\": {\n \"region\": \"Runway Complex\",\n \"center\": {\n \"latitude\": 25.2532,\n \"longitude\": 55.3657\n },\n \"radius\": 500,\n \"cells\": [\n \"cell-100\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-08-20T06:00:00Z\",\n \"endTime\": \"2025-08-20T18:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/1531bb364fd9\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"MPS\",\n \"sliceProfile\": {\n \"latencyTarget\": 1,\n \"reliability\": 99.999,\n \"throughput\": {\n \"downlink\": 100,\n \"uplink\": 75\n },\n \"maxNumberOfUEs\": 500\n },\n \"areaOfService\": {\n \"region\": \"Runway Complex\",\n \"center\": {\n \"latitude\": 25.2532,\n \"longitude\": 55.3657\n },\n \"radius\": 2000,\n \"cells\": [\n \"cell-010\",\n \"cell-030\",\n \"cell-001\",\n \"cell-002\",\n \"cell-003\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-04T01:00:00Z\",\n \"endTime\": \"2025-01-04T03:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/30000000000e\"\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "213ec6c269ba", + "target_layer": "camara", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5, + "norm_field_f1": 0.7142857142857143, + "norm_key_f1": 0.8571428571428571, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"eMBB\",\n \"sliceProfile\": {\n \"latencyTarget\": 100,\n \"reliability\": 99.5,\n \"throughput\": {\n \"downlink\": 1000,\n \"uplink\": 150\n },\n \"maxNumberOfUEs\": 500\n },\n \"areaOfService\": {\n \"region\": \"Industrial Zone North\",\n \"center\": {\n \"latitude\": 52.52,\n \"longitude\": 13.405\n },\n \"radius\": 200,\n \"cells\": [\n \"cell-005\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-11-05T14:00:00Z\",\n \"endTime\": \"2025-11-05T16:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/a299a3f67120\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"eMBB\",\n \"sliceProfile\": {\n \"latencyTarget\": 100,\n \"reliability\": 99.9,\n \"throughput\": {\n \"downlink\": 1000,\n \"uplink\": 150\n },\n \"maxNumberOfUEs\": 500\n },\n \"areaOfService\": {\n \"region\": \"Industrial Zone North\",\n \"center\": {\n \"latitude\": 52.52,\n \"longitude\": 13.405\n },\n \"radius\": 2000,\n \"cells\": [\n \"cell-010\",\n \"cell-060\",\n \"cell-001\",\n \"cell-010\",\n \"cell-010\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-14T01:00:00Z\",\n \"endTime\": \"2025-01-14T03:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/34333310000e\"\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "0a173c1d64c5", + "target_layer": "camara", + "slice_type": "URLLC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5, + "norm_field_f1": 0.7142857142857143, + "norm_key_f1": 0.8571428571428571, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"URLLC\",\n \"sliceProfile\": {\n \"latencyTarget\": 0.5,\n \"reliability\": 99.9999,\n \"throughput\": {\n \"downlink\": 100,\n \"uplink\": 50\n },\n \"maxNumberOfUEs\": 50\n },\n \"areaOfService\": {\n \"region\": \"Smart Farm Sector\",\n \"center\": {\n \"latitude\": 41.8781,\n \"longitude\": -93.0977\n },\n \"radius\": 2000,\n \"cells\": [\n \"cell-001\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-07-16T15:00:00Z\",\n \"endTime\": \"2025-07-18T15:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/1bffeee545da\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"URLLC\",\n \"sliceProfile\": {\n \"latencyTarget\": 0.5,\n \"reliability\": 99.999,\n \"throughput\": {\n \"downlink\": 100,\n \"uplink\": 50\n },\n \"maxNumberOfUEs\": 50\n },\n \"areaOfService\": {\n \"region\": \"Smart Farm Sector\",\n \"center\": {\n \"latitude\": 41.8781,\n \"longitude\": -93.0977\n },\n \"radius\": 5000,\n \"cells\": [\n \"cell-010\",\n \"cell-060\",\n \"cell-080\",\n \"cell-010\",\n \"cell-020\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-14T01:00:00Z\",\n \"endTime\": \"2025-01-14T03:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/334334301006\"\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "success", + "id": "6b2998fbd4c4", + "target_layer": "camara", + "slice_type": "URLLC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5555555555555556, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"URLLC\",\n \"sliceProfile\": {\n \"latencyTarget\": 10,\n \"reliability\": 99.9999,\n \"throughput\": {\n \"downlink\": 50,\n \"uplink\": 50\n },\n \"maxNumberOfUEs\": 5000\n },\n \"areaOfService\": {\n \"region\": \"Offshore Platform Sector\",\n \"center\": {\n \"latitude\": 56.1304,\n \"longitude\": 3.2112\n },\n \"radius\": 5000,\n \"cells\": [\n \"cell-081\",\n \"cell-027\",\n \"cell-076\",\n \"cell-048\",\n \"cell-086\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-04-18T04:00:00Z\",\n \"endTime\": \"2025-04-18T12:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/c649a56781a2\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"URLLC\",\n \"sliceProfile\": {\n \"latencyTarget\": 10,\n \"reliability\": 99.9999,\n \"throughput\": {\n \"downlink\": 50,\n \"uplink\": 50\n },\n \"maxNumberOfUEs\": 5000\n },\n \"areaOfService\": {\n \"region\": \"Offshore Platform Sector\",\n \"center\": {\n \"latitude\": 56.1304,\n \"longitude\": 3.2112\n },\n \"radius\": 5000,\n \"cells\": [\n \"cell-010\",\n \"cell-080\",\n \"cell-020\",\n \"cell-019\",\n \"cell-032\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-04T01:00:00Z\",\n \"endTime\": \"2025-01-04T03:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/73343407301e\"\n }\n}" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "b66b35b1a043", + "target_layer": "intent_3gpp", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.775, + "norm_field_f1": 0.6578947368421053, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-mMTC-00014d3c0cdd\",\n \"name\": \"Baggage Tracking Rfid mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-fde9acfa922f\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000001\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 50,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 30,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 95,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Medical Center District\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"baggag\n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-mMTC-081030881014\",\n \"name\": \"Baggage Tracking Rfid mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-000000000001\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 30,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 95,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Medical Center District\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contex\n......" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "a66263d4f62c", + "target_layer": "intent_3gpp", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.775, + "norm_field_f1": 0.6578947368421053, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-mMTC-0b01052bc4ce\",\n \"name\": \"Hd Map Real-Time Update mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-7f2fe9e11859\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000002\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 0.5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 0.1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 1000,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 95,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Autonomous Test Track\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"HD\n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-mMTC-00000100000b\",\n \"name\": \"Hd Map Real-Time Update mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-000001000006\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 0.5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 0.1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1000,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 95,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Autonomous Test Track\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"co\n......" + }, + { + "split": "test_in_distribution", + "kind": "failure", + "id": "1365f89d3e33", + "target_layer": "intent_3gpp", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.775, + "norm_field_f1": 0.6578947368421053, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-mMTC-3e24f2da03ac\",\n \"name\": \"Soil Moisture Sensor Mesh mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-ae8ba2fb4584\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000002\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 0.1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 30,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Rural Coverage Area\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"soil \n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-mMTC-0d0e04303008\",\n \"name\": \"Soil Moisture Sensor Mesh mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-0d0e04303008\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 0.1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 30,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 99,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Rural Coverage Area\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"conte\n......" + }, + { + "split": "test_in_distribution", + "kind": "success", + "id": "32ddbb98e457", + "target_layer": "intent_3gpp", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.9500000000000001, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-eMBB-a5540fd9c553\",\n \"name\": \"Concrete Pour Monitoring eMBB Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-6b4bf57e7434\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"eMBB\",\n \"sNSSAI\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 1000,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 200,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 80,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99.95,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"maxNumberOfUEs\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Construction Site Gamma\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-eMBB-144030873084\",\n \"name\": \"Concrete Pour Monitoring eMBB Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-000000000001\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"eMBB\",\n \"sNSSAI\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 1000,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 200,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 80,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99.95,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"maxNumberOfUEs\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Construction Site Gamma\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \n......" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "e76a129ef0e9", + "target_layer": "o1_nrm", + "slice_type": "URLLC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.17391304347826086, + "norm_field_f1": 0.20000000000000004, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-009\",\n \"GNBDUFunction\": {\n \"id\": \"DU-004\",\n \"gNBId\": 9,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-004-003\",\n \"cellLocalId\": 3,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n79\",\n \"arfcnDL\": 708059,\n \"bSChannelBwDL\": 50,\n \"nrPCI\": 176,\n \"nrTAC\": 14794,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 2,\n \"sd\": \"000002\"\n },\n \"rrmPolicyMinRatio\": 0.05,\n \"rrmPolicyMaxRatio\": 0.25,\n \"rrmPolicyDedicatedRatio\": 0.15,\n \"rrmPolicyMaxRatioPerSlice\": 0.25\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"001\",\n \"mnc\": \"260\",\n \"sNSSAIList\": [\n {\n \"sst\": 2,\n \"sd\": \"000002\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-014\",\n \"GNBDUFunction\": {\n \"id\": \"DU-008\",\n \"gNBId\": 14,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-008-002\",\n \"cellLocalId\": 2,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n77\",\n \"arfcnDL\": 629813,\n \"bSChannelBwDL\": 40,\n \"nrPCI\": 812,\n \"nrTAC\": 3862,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 5,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.05,\n \"rrmPolicyMaxRatio\": 0.23,\n \"rrmPolicyDedicatedRatio\": 0.14,\n \"rrmPolicyMaxRatioPerSlice\": 0.23\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 5,\n \"sd\": \"000003\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "60c525414a65", + "target_layer": "o1_nrm", + "slice_type": "V2X", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.21739130434782608, + "norm_field_f1": 0.25, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-015\",\n \"GNBDUFunction\": {\n \"id\": \"DU-007\",\n \"gNBId\": 15,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-007-005\",\n \"cellLocalId\": 5,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n77\",\n \"arfcnDL\": 669944,\n \"bSChannelBwDL\": 20,\n \"nrPCI\": 815,\n \"nrTAC\": 61118,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 4,\n \"sd\": \"000001\"\n },\n \"rrmPolicyMinRatio\": 0.06,\n \"rrmPolicyMaxRatio\": 0.4,\n \"rrmPolicyDedicatedRatio\": 0.23,\n \"rrmPolicyMaxRatioPerSlice\": 0.4\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"15\",\n \"sNSSAIList\": [\n {\n \"sst\": 4,\n \"sd\": \"000002\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-014\",\n \"GNBDUFunction\": {\n \"id\": \"DU-001\",\n \"gNBId\": 14,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-001-001\",\n \"cellLocalId\": 1,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n77\",\n \"arfcnDL\": 629813,\n \"bSChannelBwDL\": 40,\n \"nrPCI\": 812,\n \"nrTAC\": 34211,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 5,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.06,\n \"rrmPolicyMaxRatio\": 0.22,\n \"rrmPolicyDedicatedRatio\": 0.14,\n \"rrmPolicyMaxRatioPerSlice\": 0.22\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"234\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 5,\n \"sd\": \"000003\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "3ab70dc309fc", + "target_layer": "o1_nrm", + "slice_type": "V2X", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.21739130434782608, + "norm_field_f1": 0.25, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-027\",\n \"GNBDUFunction\": {\n \"id\": \"DU-013\",\n \"gNBId\": 27,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-013-007\",\n \"cellLocalId\": 7,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n41\",\n \"arfcnDL\": 533808,\n \"bSChannelBwDL\": 40,\n \"nrPCI\": 347,\n \"nrTAC\": 28252,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 4,\n \"sd\": \"000001\"\n },\n \"rrmPolicyMinRatio\": 0.15,\n \"rrmPolicyMaxRatio\": 0.31,\n \"rrmPolicyDedicatedRatio\": 0.23,\n \"rrmPolicyMaxRatioPerSlice\": 0.31\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"001\",\n \"mnc\": \"01\",\n \"sNSSAIList\": [\n {\n \"sst\": 4,\n \"sd\": \"000001\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-006\",\n \"GNBDUFunction\": {\n \"id\": \"DU-001\",\n \"gNBId\": 6,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-001-001\",\n \"cellLocalId\": 1,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n77\",\n \"arfcnDL\": 623680,\n \"bSChannelBwDL\": 20,\n \"nrPCI\": 81,\n \"nrTAC\": 3866,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 2,\n \"sd\": \"000001\"\n },\n \"rrmPolicyMinRatio\": 0.15,\n \"rrmPolicyMaxRatio\": 0.4,\n \"rrmPolicyDedicatedRatio\": 0.28,\n \"rrmPolicyMaxRatioPerSlice\": 0.4\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 2,\n \"sd\": \"000003\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_template_ood", + "kind": "success", + "id": "864fefe5f0df", + "target_layer": "o1_nrm", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5217391304347826, + "norm_field_f1": 0.6, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-038\",\n \"GNBDUFunction\": {\n \"id\": \"DU-045\",\n \"gNBId\": 38,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-045-009\",\n \"cellLocalId\": 9,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n77\",\n \"arfcnDL\": 636814,\n \"bSChannelBwDL\": 60,\n \"nrPCI\": 324,\n \"nrTAC\": 57299,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.05,\n \"rrmPolicyMaxRatio\": 0.23,\n \"rrmPolicyDedicatedRatio\": 0.14,\n \"rrmPolicyMaxRatioPerSlice\": 0.23\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"02\",\n \"sNSSAIList\": [\n {\n \"sst\": 3,\n \"sd\": \"000002\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-014\",\n \"GNBDUFunction\": {\n \"id\": \"DU-008\",\n \"gNBId\": 14,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-008-002\",\n \"cellLocalId\": 2,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n77\",\n \"arfcnDL\": 629813,\n \"bSChannelBwDL\": 40,\n \"nrPCI\": 1006,\n \"nrTAC\": 3862,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.05,\n \"rrmPolicyMaxRatio\": 0.23,\n \"rrmPolicyDedicatedRatio\": 0.14,\n \"rrmPolicyMaxRatioPerSlice\": 0.23\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "14348f59be66", + "target_layer": "a1_policy", + "slice_type": "MPS", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.42105263157894735, + "norm_field_f1": 0.5, + "norm_key_f1": 1.0, + "error_label": "a1_policy_value_error", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-mps-eff6d6\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-mps-59d80a\",\n \"cellId\": \"cell-079\",\n \"sNSSAI\": {\n \"sst\": 5,\n \"sd\": \"000003\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 200,\n \"mfbr\": 240.0,\n \"priorityLevel\": 1,\n \"5qi\": 69,\n \"pdb\": 60,\n \"per\": 1e-05\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 15,\n \"max\": 41\n },\n \"schedulerWeight\": 0.71,\n \"maxUEs\": 5000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-urllc-114386\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-urllc-08900b\",\n \"cellId\": \"cell-062\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000001\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 200,\n \"mfbr\": 240.0,\n \"priorityLevel\": 3,\n \"5qi\": 85,\n \"pdb\": 5,\n \"per\": 1e-06\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 15,\n \"max\": 34\n },\n \"schedulerWeight\": 0.35,\n \"maxUEs\": 5000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "2a4c961b861b", + "target_layer": "a1_policy", + "slice_type": "V2X", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.42105263157894735, + "norm_field_f1": 0.5, + "norm_key_f1": 1.0, + "error_label": "a1_policy_value_error", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-v2x-afe37b\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-v2x-f87cb8\",\n \"cellId\": \"cell-047\",\n \"sNSSAI\": {\n \"sst\": 4,\n \"sd\": \"000002\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 150,\n \"mfbr\": 180.0,\n \"priorityLevel\": 2,\n \"5qi\": 84,\n \"pdb\": 30,\n \"per\": 0.0001\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 11,\n \"max\": 39\n },\n \"schedulerWeight\": 0.48,\n \"maxUEs\": 10000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-mps-14408b\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-mps-093086\",\n \"cellId\": \"cell-082\",\n \"sNSSAI\": {\n \"sst\": 5,\n \"sd\": \"000003\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 150,\n \"mfbr\": 180.0,\n \"priorityLevel\": 1,\n \"5qi\": 65,\n \"pdb\": 75,\n \"per\": 1e-06\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 11,\n \"max\": 31\n },\n \"schedulerWeight\": 0.31,\n \"maxUEs\": 10000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "5b87f6f26d99", + "target_layer": "a1_policy", + "slice_type": "V2X", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.42105263157894735, + "norm_field_f1": 0.5, + "norm_key_f1": 1.0, + "error_label": "a1_policy_value_error", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-v2x-3afdde\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-v2x-e2230f\",\n \"cellId\": \"cell-054\",\n \"sNSSAI\": {\n \"sst\": 4,\n \"sd\": \"000001\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 150,\n \"mfbr\": 180.0,\n \"priorityLevel\": 2,\n \"5qi\": 3,\n \"pdb\": 50,\n \"per\": 1e-05\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 8,\n \"max\": 29\n },\n \"schedulerWeight\": 0.51,\n \"maxUEs\": 1000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-urllc-000001\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-urllc-000001\",\n \"cellId\": \"cell-001\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000003\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 150,\n \"mfbr\": 180.0,\n \"priorityLevel\": 3,\n \"5qi\": 84,\n \"pdb\": 30,\n \"per\": 1e-06\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 8,\n \"max\": 28\n },\n \"schedulerWeight\": 0.31,\n \"maxUEs\": 1000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_template_ood", + "kind": "success", + "id": "664611f7e9df", + "target_layer": "a1_policy", + "slice_type": "MPS", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7368421052631579, + "norm_field_f1": 0.8333333333333334, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-mps-2fa195\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-mps-f57284\",\n \"cellId\": \"cell-043\",\n \"sNSSAI\": {\n \"sst\": 5,\n \"sd\": \"000003\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 50,\n \"mfbr\": 60.0,\n \"priorityLevel\": 1,\n \"5qi\": 69,\n \"pdb\": 60,\n \"per\": 1e-05\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 5,\n \"max\": 25\n },\n \"schedulerWeight\": 0.43,\n \"maxUEs\": 1000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-mps-14400b\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-mps-08000b\",\n \"cellId\": \"cell-080\",\n \"sNSSAI\": {\n \"sst\": 5,\n \"sd\": \"000003\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 50,\n \"mfbr\": 60.0,\n \"priorityLevel\": 1,\n \"5qi\": 69,\n \"pdb\": 60,\n \"per\": 1e-05\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 5,\n \"max\": 23\n },\n \"schedulerWeight\": 0.41,\n \"maxUEs\": 1000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "7f4c7e3c8d4e", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "HMTC", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.31746031746031744, + "norm_field_f1": 0.14285714285714285, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-676527391acc\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 3,\n \"measurements\": [\n 1.8,\n 2.04,\n 2.99,\n 2.57,\n 2.84\n ],\n \"average\": 2.448,\n \"min\": 1.8,\n \"max\": 2.99,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 100,\n \"measurements\": [\n 95.7,\n 108.9,\n 98.5,\n 104.4,\n 118.3\n ],\n \"average\": 105.16,\n \"min\": 95.7,\n \"max\": 118.3,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 1000,\n \"measurements\": [\n 931.2,\n 912.2,\n 1194.4,\n 1192.6,\n 946.9\n ],\n \"average\": 1035.46,\n \"min\": 912.2,\n \"max\": 1194.4,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99999,\n \"measurements\": [\n 100,\n 100,\n 99.95766,\n 99.988027,\n 99.93028\n ],\n \"average\": 99.975,\n \"min\": 99.93028,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 100000,\n \"measurements\": [\n 60415,\n 88288,\n 41560,\n 77445,\n 94786\n ],\n \"average\": 72498.8,\n \"min\": 41560,\n \"max\": 94786,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 56.7,\n \"memoryUsage\": 69.1,\n \"prbUtilization\": 59.8\n },\n \"@type\": \"Inte\n......", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-114010883084\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 3,\n \"measurements\": [\n 2.29,\n 2.31,\n 2.31,\n 2.3,\n 2.3\n ],\n \"average\": 2.282,\n \"min\": 2.29,\n \"max\": 2.31,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 2000,\n \"measurements\": [\n 2112.9,\n 2212.1,\n 2233.4,\n 2131.6,\n 2272.8\n ],\n \"average\": 2192.56,\n \"min\": 2112.9,\n \"max\": 2272.8,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 1000,\n \"measurements\": [\n 954.1,\n 1092.1,\n 1135.3,\n 921.2,\n 1110.3\n ],\n \"average\": 1042.6,\n \"min\": 921.2,\n \"max\": 1135.3,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99999,\n \"measurements\": [\n 100,\n 100,\n 100,\n 100,\n 100\n ],\n \"average\": 100.0,\n \"min\": 100,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 1000,\n \"measurements\": [\n 128,\n 140,\n 912,\n 550,\n 215\n ],\n \"average\": 389.0,\n \"min\": 128,\n \"max\": 912,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 67.4,\n \"memoryUsage\": 30.4,\n \"prbUtilization\": 82.3\n },\n \"@type\": \"IntentAssuran\n......" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "b4ae6c9d9034", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "HMTC", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.2698412698412698, + "norm_field_f1": 0.1746031746031746, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-a1c5e0cd2107\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 1,\n \"measurements\": [\n 0.77,\n 1.07,\n 0.92,\n 0.82,\n 0.88\n ],\n \"average\": 0.892,\n \"min\": 0.77,\n \"max\": 1.07,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 100,\n \"measurements\": [\n 102.5,\n 114.4,\n 112.1,\n 106.4,\n 109.8\n ],\n \"average\": 109.04,\n \"min\": 102.5,\n \"max\": 114.4,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 500,\n \"measurements\": [\n 515.7,\n 460.6,\n 538.9,\n 552.5,\n 593.1\n ],\n \"average\": 532.16,\n \"min\": 460.6,\n \"max\": 593.1,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99999,\n \"measurements\": [\n 99.949115,\n 99.978169,\n 99.934188,\n 99.978159,\n 100\n ],\n \"average\": 99.968,\n \"min\": 99.934188,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 500,\n \"measurements\": [\n 393,\n 192,\n 246,\n 45,\n 280\n ],\n \"average\": 231.2,\n \"min\": 45,\n \"max\": 393,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 56.7,\n \"memoryUsage\": 39.4,\n \"prbUtilization\": 69.3\n },\n \"@type\": \"IntentAssuranceR\n......", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-000000000001\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 3,\n \"measurements\": [\n 2.29,\n 2.31,\n 2.31,\n 2.3,\n 2.29\n ],\n \"average\": 2.28,\n \"min\": 2.29,\n \"max\": 2.31,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 2000,\n \"measurements\": [\n 2112.9,\n 2212.1,\n 2137.7,\n 2295.5,\n 2231.7\n ],\n \"average\": 2198.0,\n \"min\": 2112.9,\n \"max\": 2295.5,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 1000,\n \"measurements\": [\n 954.1,\n 1011.1,\n 983.0,\n 921.0,\n 1117.3\n ],\n \"average\": 997.3,\n \"min\": 921.0,\n \"max\": 1117.3,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99999,\n \"measurements\": [\n 99.903696,\n 100,\n 99.916972,\n 100,\n 100\n ],\n \"average\": 99.964,\n \"min\": 99.903696,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 5000,\n \"measurements\": [\n 4656,\n 4656,\n 4656,\n 4656,\n 4656\n ],\n \"average\": 4656.0,\n \"min\": 4656,\n \"max\": 4656,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 77.7,\n \"memoryUsage\": 33.2,\n \"prbUtilization\": 82.3\n },\n \n......" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "a43297d5ddee", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "HMTC", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.2698412698412698, + "norm_field_f1": 0.25396825396825395, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-ed3943082134\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 5,\n \"measurements\": [\n 3.79,\n 4.98,\n 3.83,\n 4.78,\n 5.43\n ],\n \"average\": 4.562,\n \"min\": 3.79,\n \"max\": 5.43,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 100,\n \"measurements\": [\n 110.9,\n 99.1,\n 119.2,\n 119.3,\n 110.7\n ],\n \"average\": 111.84,\n \"min\": 99.1,\n \"max\": 119.3,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 500,\n \"measurements\": [\n 508.6,\n 506.8,\n 503.3,\n 541.3,\n 559.8\n ],\n \"average\": 523.96,\n \"min\": 503.3,\n \"max\": 559.8,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99999,\n \"measurements\": [\n 99.923372,\n 100,\n 99.961583,\n 99.913077,\n 99.998417\n ],\n \"average\": 99.959,\n \"min\": 99.913077,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 500,\n \"measurements\": [\n 73,\n 119,\n 268,\n 421,\n 487\n ],\n \"average\": 273.6,\n \"min\": 73,\n \"max\": 487,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 72.9,\n \"memoryUsage\": 47.5,\n \"prbUtilization\": 32.4\n },\n \"@type\": \"IntentAssuranceRep\n......", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-201063833084\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 3,\n \"measurements\": [\n 2.29,\n 2.31,\n 2.31,\n 2.3,\n 2.3\n ],\n \"average\": 2.282,\n \"min\": 2.29,\n \"max\": 2.31,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 2000,\n \"measurements\": [\n 2112.9,\n 2212.1,\n 2233.4,\n 2191.7,\n 2295.6\n ],\n \"average\": 2209.14,\n \"min\": 2112.9,\n \"max\": 2295.6,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 1000,\n \"measurements\": [\n 954.1,\n 1011.1,\n 999.4,\n 921.2,\n 1113.3\n ],\n \"average\": 1001.82,\n \"min\": 921.2,\n \"max\": 1113.3,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99999,\n \"measurements\": [\n 99.90465,\n 100,\n 99.916942,\n 100,\n 99.945342\n ],\n \"average\": 99.953,\n \"min\": 99.90465,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 1000,\n \"measurements\": [\n 128,\n 140,\n 55,\n 550,\n 55\n ],\n \"average\": 173.6,\n \"min\": 55,\n \"max\": 550,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 67.4,\n \"memoryUsage\": 30.7,\n \"prbUtilization\": 82.3\n },\n \"@t\n......" + }, + { + "split": "test_template_ood", + "kind": "success", + "id": "a43297d5ddee", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "HMTC", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.2698412698412698, + "norm_field_f1": 0.25396825396825395, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-ed3943082134\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 5,\n \"measurements\": [\n 3.79,\n 4.98,\n 3.83,\n 4.78,\n 5.43\n ],\n \"average\": 4.562,\n \"min\": 3.79,\n \"max\": 5.43,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 100,\n \"measurements\": [\n 110.9,\n 99.1,\n 119.2,\n 119.3,\n 110.7\n ],\n \"average\": 111.84,\n \"min\": 99.1,\n \"max\": 119.3,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 500,\n \"measurements\": [\n 508.6,\n 506.8,\n 503.3,\n 541.3,\n 559.8\n ],\n \"average\": 523.96,\n \"min\": 503.3,\n \"max\": 559.8,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99999,\n \"measurements\": [\n 99.923372,\n 100,\n 99.961583,\n 99.913077,\n 99.998417\n ],\n \"average\": 99.959,\n \"min\": 99.913077,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 500,\n \"measurements\": [\n 73,\n 119,\n 268,\n 421,\n 487\n ],\n \"average\": 273.6,\n \"min\": 73,\n \"max\": 487,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 72.9,\n \"memoryUsage\": 47.5,\n \"prbUtilization\": 32.4\n },\n \"@type\": \"IntentAssuranceRep\n......", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-201063833084\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 3,\n \"measurements\": [\n 2.29,\n 2.31,\n 2.31,\n 2.3,\n 2.3\n ],\n \"average\": 2.282,\n \"min\": 2.29,\n \"max\": 2.31,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 2000,\n \"measurements\": [\n 2112.9,\n 2212.1,\n 2233.4,\n 2191.7,\n 2295.6\n ],\n \"average\": 2209.14,\n \"min\": 2112.9,\n \"max\": 2295.6,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 1000,\n \"measurements\": [\n 954.1,\n 1011.1,\n 999.4,\n 921.2,\n 1113.3\n ],\n \"average\": 1001.82,\n \"min\": 921.2,\n \"max\": 1113.3,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99999,\n \"measurements\": [\n 99.90465,\n 100,\n 99.916942,\n 100,\n 99.945342\n ],\n \"average\": 99.953,\n \"min\": 99.90465,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 1000,\n \"measurements\": [\n 128,\n 140,\n 55,\n 550,\n 55\n ],\n \"average\": 173.6,\n \"min\": 55,\n \"max\": 550,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 67.4,\n \"memoryUsage\": 30.7,\n \"prbUtilization\": 82.3\n },\n \"@t\n......" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "a0fe060ff40a", + "target_layer": "tmf921", + "slice_type": "MPS", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7692307692307693, + "norm_field_f1": 0.8656716417910447, + "norm_key_f1": 0.9402985074626866, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-efe789e21dd7\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-efe789e21dd7\",\n \"name\": \"MPS Slice - Cashierless Checkout System\",\n \"description\": \"Intent for cashierless checkout system at Retail Distribution Hub with MPS service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 2,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-41b07a58b7e6\",\n \"name\": \"MPS QoS Delivery Expectation\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-mps-53c380\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"MPS\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 5,\n \"sd\": \"000004\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 150,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n \n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-14331430000e\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-14331430000e\",\n \"name\": \"MPS Slice - Cashierless Checkout System\",\n \"description\": \"Intent for cashierless checkout system at Retail Distribution Hub with MPS service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-0e0e0e0e0e0e\",\n \"name\": \"MPS QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-mps-0e0e0e\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"MPS\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 5,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 150,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\n......" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "5564e4e38681", + "target_layer": "tmf921", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7692307692307693, + "norm_field_f1": 0.8656716417910447, + "norm_key_f1": 0.9402985074626866, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-c993726567f4\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-c993726567f4\",\n \"name\": \"eMBB Slice - Electronic Warfare Spectrum Monitoring\",\n \"description\": \"Intent for electronic warfare spectrum monitoring at Factory Complex A with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 4,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-c384185090be\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-25a21d\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 800,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 150,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"un\n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-330434301088\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-330434301088\",\n \"name\": \"eMBB Slice - Electronic Warfare Spectrum Monitoring\",\n \"description\": \"Intent for electronic warfare spectrum monitoring at Factory Complex A with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-090300000006\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-000006\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 800,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 150,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCell\n......" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "75ff5712aae2", + "target_layer": "tmf921", + "slice_type": "MPS", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7307692307692307, + "norm_field_f1": 0.8656716417910447, + "norm_key_f1": 0.9402985074626866, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-57fe3dc7da91\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-57fe3dc7da91\",\n \"name\": \"MPS Slice - Public Wifi Mesh Backhaul\",\n \"description\": \"Intent for public WiFi mesh backhaul at Urban Innovation District with MPS service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-c2805cb10e14\",\n \"name\": \"MPS QoS Delivery Expectation\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-mps-a4d53b\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"MPS\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 5,\n \"sd\": \"000004\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-011010000000\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-011010000000\",\n \"name\": \"URLLC Slice - Public Wifi Mesh Backhaul\",\n \"description\": \"Intent for public WiFi mesh backhaul at Urban Innovation District with URLLC service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-011010000000\",\n \"name\": \"URLLC QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-urllc-011010\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"URLLC\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 2,\n \"sd\": \"000001\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \n......" + }, + { + "split": "test_template_ood", + "kind": "success", + "id": "2fb175ff25ac", + "target_layer": "tmf921", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.8860759493670886, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-9d24819ca906\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-9d24819ca906\",\n \"name\": \"mMTC Slice - Hospital Ar Surgical Guidance\",\n \"description\": \"Intent for hospital AR surgical guidance at Hospital Campus with mMTC service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-0d7036133363\",\n \"name\": \"mMTC QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-mmtc-be797a\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"mMTC\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 0.1,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n \n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-011000000000\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-011000000000\",\n \"name\": \"mMTC Slice - Hospital Ar Surgical Guidance\",\n \"description\": \"Intent for hospital AR surgical guidance at Hospital Campus with mMTC service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-011000000001\",\n \"name\": \"mMTC QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-mmtc-011000\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"mMTC\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 0.1,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"uni\n......" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "75e8c403e863", + "target_layer": "camara", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.42424242424242425, + "norm_field_f1": 0.6896551724137931, + "norm_key_f1": 0.896551724137931, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"eMBB\",\n \"sliceProfile\": {\n \"latencyTarget\": 20,\n \"reliability\": 99.5,\n \"throughput\": {\n \"downlink\": 100,\n \"uplink\": 200\n },\n \"maxNumberOfUEs\": 50000\n },\n \"areaOfService\": {\n \"region\": \"Highway Corridor M1\",\n \"center\": {\n \"latitude\": 52.2053,\n \"longitude\": -0.1218\n },\n \"radius\": 200,\n \"cells\": [\n \"cell-091\",\n \"cell-055\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-04-03T10:00:00Z\",\n \"endTime\": \"2025-04-03T14:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/4d13aea978ab\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"eMBB\",\n \"sliceProfile\": {\n \"latencyTarget\": 20,\n \"reliability\": 99.9,\n \"throughput\": {\n \"downlink\": 100,\n \"uplink\": 200\n },\n \"maxNumberOfUEs\": 50000\n },\n \"areaOfService\": {\n \"region\": \"Highway Corridor M1\",\n \"center\": {\n \"latitude\": 52.2053,\n \"longitude\": -0.1278\n },\n \"radius\": 5000,\n \"cells\": [\n \"cell-081\",\n \"cell-080\",\n \"cell-079\",\n \"cell-078\",\n \"cell-077\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-04T01:00:00Z\",\n \"endTime\": \"2025-01-04T03:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/71333309338e\"\n }\n}" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "ed910912d032", + "target_layer": "camara", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.39999999999999997, + "norm_field_f1": 0.7096774193548386, + "norm_key_f1": 0.967741935483871, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"eMBB\",\n \"sliceProfile\": {\n \"latencyTarget\": 30,\n \"reliability\": 99.0,\n \"throughput\": {\n \"downlink\": 500,\n \"uplink\": 100\n },\n \"maxNumberOfUEs\": 500\n },\n \"areaOfService\": {\n \"region\": \"Financial District\",\n \"center\": {\n \"latitude\": 40.7075,\n \"longitude\": -74.0021\n },\n \"radius\": 2000,\n \"cells\": [\n \"cell-026\",\n \"cell-086\",\n \"cell-030\",\n \"cell-031\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-05-13T20:00:00Z\",\n \"endTime\": \"2025-05-13T21:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/e972b23295e3\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"eMBB\",\n \"sliceProfile\": {\n \"latencyTarget\": 30,\n \"reliability\": 99.9,\n \"throughput\": {\n \"downlink\": 500,\n \"uplink\": 100\n },\n \"maxNumberOfUEs\": 500\n },\n \"areaOfService\": {\n \"region\": \"Financial District\",\n \"center\": {\n \"latitude\": 40.706,\n \"longitude\": -74.006\n },\n \"radius\": 500,\n \"cells\": [\n \"cell-008\",\n \"cell-030\",\n \"cell-030\",\n \"cell-030\",\n \"cell-030\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-13T01:00:00Z\",\n \"endTime\": \"2025-01-13T03:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/31330990000e\"\n }\n}" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "fb15d091a9db", + "target_layer": "camara", + "slice_type": "URLLC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5, + "norm_field_f1": 0.7142857142857143, + "norm_key_f1": 0.8571428571428571, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"URLLC\",\n \"sliceProfile\": {\n \"latencyTarget\": 0.5,\n \"reliability\": 99.9999,\n \"throughput\": {\n \"downlink\": 200,\n \"uplink\": 100\n },\n \"maxNumberOfUEs\": 50\n },\n \"areaOfService\": {\n \"region\": \"Border Sector C\",\n \"center\": {\n \"latitude\": 31.9686,\n \"longitude\": -99.9018\n },\n \"radius\": 2000,\n \"cells\": [\n \"cell-083\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-26T21:00:00Z\",\n \"endTime\": \"2025-01-26T23:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/a48317062195\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"URLLC\",\n \"sliceProfile\": {\n \"latencyTarget\": 0.5,\n \"reliability\": 99.999,\n \"throughput\": {\n \"downlink\": 200,\n \"uplink\": 100\n },\n \"maxNumberOfUEs\": 50\n },\n \"areaOfService\": {\n \"region\": \"Border Sector C\",\n \"center\": {\n \"latitude\": 31.9686,\n \"longitude\": -99.9018\n },\n \"radius\": 5000,\n \"cells\": [\n \"cell-010\",\n \"cell-030\",\n \"cell-030\",\n \"cell-030\",\n \"cell-030\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-14T11:00:00Z\",\n \"endTime\": \"2025-01-14T12:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/334304300101\"\n }\n}" + }, + { + "split": "test_template_ood", + "kind": "success", + "id": "b7684821e891", + "target_layer": "camara", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5555555555555556, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"mMTC\",\n \"sliceProfile\": {\n \"latencyTarget\": 50,\n \"reliability\": 99,\n \"throughput\": {\n \"downlink\": 5,\n \"uplink\": 25\n },\n \"maxNumberOfUEs\": 10000\n },\n \"areaOfService\": {\n \"region\": \"Industrial Zone North\",\n \"center\": {\n \"latitude\": 52.52,\n \"longitude\": 13.405\n },\n \"radius\": 2000,\n \"cells\": [\n \"cell-041\",\n \"cell-002\",\n \"cell-026\",\n \"cell-095\",\n \"cell-015\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-11-11T05:00:00Z\",\n \"endTime\": \"2025-11-11T07:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/920c4415aa01\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"mMTC\",\n \"sliceProfile\": {\n \"latencyTarget\": 50,\n \"reliability\": 99,\n \"throughput\": {\n \"downlink\": 5,\n \"uplink\": 25\n },\n \"maxNumberOfUEs\": 10000\n },\n \"areaOfService\": {\n \"region\": \"Industrial Zone North\",\n \"center\": {\n \"latitude\": 52.52,\n \"longitude\": 13.405\n },\n \"radius\": 2000,\n \"cells\": [\n \"cell-016\",\n \"cell-080\",\n \"cell-020\",\n \"cell-020\",\n \"cell-010\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-04T01:00:00Z\",\n \"endTime\": \"2025-01-04T03:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/33300499000e\"\n }\n}" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "1bb88377bb51", + "target_layer": "intent_3gpp", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.775, + "norm_field_f1": 0.6578947368421053, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-mMTC-9f52a80fa133\",\n \"name\": \"Joint Force C2 Network mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-87ee952586d1\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000001\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 50,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 0.5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 500,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 90,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Naval Port Zone\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"joint force\n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-mMTC-0d0744403006\",\n \"name\": \"Joint Force C2 Network mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-0e0744403006\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 0.5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 500,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 90,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Naval Port Zone\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCond\n......" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "e57e5b4bf0f2", + "target_layer": "intent_3gpp", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.775, + "norm_field_f1": 0.6578947368421053, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-mMTC-cac30a9f990f\",\n \"name\": \"Traffic Signal Preemption mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-7d33416f957f\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000001\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 10,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 50,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99.5,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Healthcare Park\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"traffic s\n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-mMTC-083334801906\",\n \"name\": \"Traffic Signal Preemption mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-000000000001\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 99.5,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Healthcare Park\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCo\n......" + }, + { + "split": "test_template_ood", + "kind": "failure", + "id": "c3ca42fa50ad", + "target_layer": "intent_3gpp", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.775, + "norm_field_f1": 0.6578947368421053, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-mMTC-b563e3be5374\",\n \"name\": \"Agv Fleet Management mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-9a1a93134f17\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000002\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 500,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 95,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 500000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Industrial Zone North\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"AGV fleet\n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-mMTC-08433010900a\",\n \"name\": \"Agv Fleet Management mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-000000000001\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 500,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 95,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 500000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Industrial Zone North\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCo\n......" + }, + { + "split": "test_template_ood", + "kind": "success", + "id": "d9b8b720b04d", + "target_layer": "intent_3gpp", + "slice_type": "URLLC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.9500000000000001, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-URLLC-643a0a6f7df4\",\n \"name\": \"Logistics Convoy Tracking URLLC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-f71ffc010a7f\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"URLLC\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 20,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 10,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 7,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99.999,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"maxNumberOfUEs\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Medical Center District\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\"\n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-URLLC-0900e0e0e000\",\n \"name\": \"Logistics Convoy Tracking URLLC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-000000000000\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"URLLC\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 20,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 10,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 7,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99.999,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"maxNumberOfUEs\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Medical Center District\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \n......" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "4f7fc2453ba3", + "target_layer": "o1_nrm", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.2608695652173913, + "norm_field_f1": 0.3, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-041\",\n \"GNBDUFunction\": {\n \"id\": \"DU-013\",\n \"gNBId\": 41,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-013-007\",\n \"cellLocalId\": 7,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n258\",\n \"arfcnDL\": 2053623,\n \"bSChannelBwDL\": 50,\n \"nrPCI\": 709,\n \"nrTAC\": 26848,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.25,\n \"rrmPolicyMaxRatio\": 0.54,\n \"rrmPolicyDedicatedRatio\": 0.4,\n \"rrmPolicyMaxRatioPerSlice\": 0.54\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"234\",\n \"mnc\": \"15\",\n \"sNSSAIList\": [\n {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-006\",\n \"GNBDUFunction\": {\n \"id\": \"DU-008\",\n \"gNBId\": 6,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-008-002\",\n \"cellLocalId\": 2,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n77\",\n \"arfcnDL\": 623610,\n \"bSChannelBwDL\": 40,\n \"nrPCI\": 812,\n \"nrTAC\": 3405,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 1,\n \"sd\": \"000001\"\n },\n \"rrmPolicyMinRatio\": 0.25,\n \"rrmPolicyMaxRatio\": 0.5,\n \"rrmPolicyDedicatedRatio\": 0.38,\n \"rrmPolicyMaxRatioPerSlice\": 0.5\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 1,\n \"sd\": \"000003\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "30dde4cc787e", + "target_layer": "o1_nrm", + "slice_type": "MPS", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.2608695652173913, + "norm_field_f1": 0.3, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-016\",\n \"GNBDUFunction\": {\n \"id\": \"DU-050\",\n \"gNBId\": 16,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-050-003\",\n \"cellLocalId\": 3,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n41\",\n \"arfcnDL\": 514241,\n \"bSChannelBwDL\": 50,\n \"nrPCI\": 551,\n \"nrTAC\": 36166,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 5,\n \"sd\": \"000004\"\n },\n \"rrmPolicyMinRatio\": 0.05,\n \"rrmPolicyMaxRatio\": 0.27,\n \"rrmPolicyDedicatedRatio\": 0.16,\n \"rrmPolicyMaxRatioPerSlice\": 0.27\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"460\",\n \"mnc\": \"260\",\n \"sNSSAIList\": [\n {\n \"sst\": 5,\n \"sd\": \"000004\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-006\",\n \"GNBDUFunction\": {\n \"id\": \"DU-001\",\n \"gNBId\": 6,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-001-001\",\n \"cellLocalId\": 1,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n78\",\n \"arfcnDL\": 620312,\n \"bSChannelBwDL\": 40,\n \"nrPCI\": 81,\n \"nrTAC\": 34640,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 5,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.05,\n \"rrmPolicyMaxRatio\": 0.3,\n \"rrmPolicyDedicatedRatio\": 0.17,\n \"rrmPolicyMaxRatioPerSlice\": 0.3\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 5,\n \"sd\": \"000003\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "a54dc4a556cb", + "target_layer": "o1_nrm", + "slice_type": "HMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.2608695652173913, + "norm_field_f1": 0.3, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-006\",\n \"GNBDUFunction\": {\n \"id\": \"DU-033\",\n \"gNBId\": 6,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-033-003\",\n \"cellLocalId\": 3,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n41\",\n \"arfcnDL\": 518533,\n \"bSChannelBwDL\": 20,\n \"nrPCI\": 420,\n \"nrTAC\": 18844,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 5,\n \"sd\": \"000002\"\n },\n \"rrmPolicyMinRatio\": 0.6,\n \"rrmPolicyMaxRatio\": 0.86,\n \"rrmPolicyDedicatedRatio\": 0.73,\n \"rrmPolicyMaxRatioPerSlice\": 0.86\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"234\",\n \"mnc\": \"02\",\n \"sNSSAIList\": [\n {\n \"sst\": 5,\n \"sd\": \"000002\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-016\",\n \"GNBDUFunction\": {\n \"id\": \"DU-001\",\n \"gNBId\": 16,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-001-001\",\n \"cellLocalId\": 1,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n77\",\n \"arfcnDL\": 629813,\n \"bSChannelBwDL\": 80,\n \"nrPCI\": 81,\n \"nrTAC\": 3862,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 5,\n \"sd\": \"000001\"\n },\n \"rrmPolicyMinRatio\": 0.6,\n \"rrmPolicyMaxRatio\": 0.85,\n \"rrmPolicyDedicatedRatio\": 0.72,\n \"rrmPolicyMaxRatioPerSlice\": 0.85\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 5,\n \"sd\": \"000001\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "success", + "id": "1366e9589e3d", + "target_layer": "o1_nrm", + "slice_type": "HMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5652173913043478, + "norm_field_f1": 0.65, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-010\",\n \"GNBDUFunction\": {\n \"id\": \"DU-047\",\n \"gNBId\": 10,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-047-009\",\n \"cellLocalId\": 9,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n78\",\n \"arfcnDL\": 632806,\n \"bSChannelBwDL\": 80,\n \"nrPCI\": 964,\n \"nrTAC\": 11290,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 5,\n \"sd\": \"000001\"\n },\n \"rrmPolicyMinRatio\": 0.6,\n \"rrmPolicyMaxRatio\": 0.85,\n \"rrmPolicyDedicatedRatio\": 0.72,\n \"rrmPolicyMaxRatioPerSlice\": 0.85\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"001\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 5,\n \"sd\": \"000001\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-006\",\n \"GNBDUFunction\": {\n \"id\": \"DU-008\",\n \"gNBId\": 6,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-008-002\",\n \"cellLocalId\": 2,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n77\",\n \"arfcnDL\": 629813,\n \"bSChannelBwDL\": 80,\n \"nrPCI\": 812,\n \"nrTAC\": 38420,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 5,\n \"sd\": \"000001\"\n },\n \"rrmPolicyMinRatio\": 0.6,\n \"rrmPolicyMaxRatio\": 0.85,\n \"rrmPolicyDedicatedRatio\": 0.72,\n \"rrmPolicyMaxRatioPerSlice\": 0.85\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 5,\n \"sd\": \"000001\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "d20551b861b5", + "target_layer": "a1_policy", + "slice_type": "URLLC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.42105263157894735, + "norm_field_f1": 0.5, + "norm_key_f1": 1.0, + "error_label": "a1_policy_value_error", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-urllc-466eea\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-urllc-8cab9c\",\n \"cellId\": \"cell-090\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000002\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 200,\n \"mfbr\": 240.0,\n \"priorityLevel\": 1,\n \"5qi\": 1,\n \"pdb\": 100,\n \"per\": 1e-05\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 10,\n \"max\": 22\n },\n \"schedulerWeight\": 0.39,\n \"maxUEs\": 50\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-urllc-14430c\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-urllc-000007\",\n \"cellId\": \"cell-081\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000003\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 200,\n \"mfbr\": 220.0,\n \"priorityLevel\": 3,\n \"5qi\": 85,\n \"pdb\": 5,\n \"per\": 1e-06\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 10,\n \"max\": 34\n },\n \"schedulerWeight\": 0.46,\n \"maxUEs\": 50\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "af237004fb36", + "target_layer": "a1_policy", + "slice_type": "MPS", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.42105263157894735, + "norm_field_f1": 0.5, + "norm_key_f1": 1.0, + "error_label": "a1_policy_value_error", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-mps-00df4b\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-mps-6a5536\",\n \"cellId\": \"cell-097\",\n \"sNSSAI\": {\n \"sst\": 5,\n \"sd\": \"000004\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 25,\n \"mfbr\": 30.0,\n \"priorityLevel\": 1,\n \"5qi\": 1,\n \"pdb\": 100,\n \"per\": 1e-05\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 6,\n \"max\": 27\n },\n \"schedulerWeight\": 0.3,\n \"maxUEs\": 5000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-urllc-00000d\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-urllc-00000d\",\n \"cellId\": \"cell-084\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000003\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 25,\n \"mfbr\": 30.0,\n \"priorityLevel\": 3,\n \"5qi\": 84,\n \"pdb\": 30,\n \"per\": 1e-06\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 6,\n \"max\": 20\n },\n \"schedulerWeight\": 0.35,\n \"maxUEs\": 5000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "3bde530beffd", + "target_layer": "a1_policy", + "slice_type": "URLLC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.42105263157894735, + "norm_field_f1": 0.5, + "norm_key_f1": 1.0, + "error_label": "a1_policy_value_error", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-urllc-f9b163\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-urllc-d88b40\",\n \"cellId\": \"cell-097\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000001\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 200,\n \"mfbr\": 240.0,\n \"priorityLevel\": 1,\n \"5qi\": 2,\n \"pdb\": 150,\n \"per\": 1e-05\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 15,\n \"max\": 29\n },\n \"schedulerWeight\": 0.67,\n \"maxUEs\": 5000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-urllc-0d000b\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-urllc-00000b\",\n \"cellId\": \"cell-081\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000003\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 200,\n \"mfbr\": 220.0,\n \"priorityLevel\": 3,\n \"5qi\": 84,\n \"pdb\": 30,\n \"per\": 1e-06\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 15,\n \"max\": 35\n },\n \"schedulerWeight\": 0.38,\n \"maxUEs\": 5000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "success", + "id": "19640a255869", + "target_layer": "a1_policy", + "slice_type": "HMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7894736842105263, + "norm_field_f1": 0.8888888888888888, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-hmtc-cbdbe8\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-hmtc-f1668f\",\n \"cellId\": \"cell-063\",\n \"sNSSAI\": {\n \"sst\": 5,\n \"sd\": \"000001\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 100,\n \"mfbr\": 120.0,\n \"priorityLevel\": 1,\n \"5qi\": 83,\n \"pdb\": 10,\n \"per\": 1e-07\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 10,\n \"max\": 37\n },\n \"schedulerWeight\": 0.5,\n \"maxUEs\": 500\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-hmtc-000001\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-hmtc-000001\",\n \"cellId\": \"cell-084\",\n \"sNSSAI\": {\n \"sst\": 5,\n \"sd\": \"000001\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 100,\n \"mfbr\": 120.0,\n \"priorityLevel\": 1,\n \"5qi\": 83,\n \"pdb\": 10,\n \"per\": 1e-07\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 10,\n \"max\": 38\n },\n \"schedulerWeight\": 0.5,\n \"maxUEs\": 500\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "b39af03e36d5", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "mMTC", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.23809523809523808, + "norm_field_f1": 0.1111111111111111, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-47f071f77e4b\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 50,\n \"measurements\": [\n 42.31,\n 33.59,\n 38.86,\n 49.74,\n 30.3\n ],\n \"average\": 38.96,\n \"min\": 30.3,\n \"max\": 49.74,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 25,\n \"measurements\": [\n 30.0,\n 27.7,\n 26.1,\n 28.0,\n 25.8\n ],\n \"average\": 27.52,\n \"min\": 25.8,\n \"max\": 30.0,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 1,\n \"measurements\": [\n 1.2,\n 1.0,\n 1.1,\n 1.0,\n 1.1\n ],\n \"average\": 1.08,\n \"min\": 1.0,\n \"max\": 1.2,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.5,\n \"measurements\": [\n 99.413025,\n 99.476689,\n 99.439216,\n 99.540217,\n 99.422296\n ],\n \"average\": 99.458,\n \"min\": 99.413025,\n \"max\": 99.540217,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 500000,\n \"measurements\": [\n 429726,\n 46704,\n 290479,\n 321895,\n 387783\n ],\n \"average\": 295317.4,\n \"min\": 46704,\n \"max\": 429726,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 20.8,\n \"memoryUsage\": 29.6,\n \"prbUtilization\": 77.4\n },\n \"@type\": \"In\n......", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-47f071f77e4b\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 30,\n \"measurements\": [\n 26.09,\n 26.02,\n 23.33,\n 23.8,\n 26.8\n ],\n \"average\": 25.008,\n \"min\": 23.33,\n \"max\": 26.8,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 50,\n \"measurements\": [\n 58.3,\n 51.9,\n 58.8,\n 58.2,\n 55.1\n ],\n \"average\": 56.46,\n \"min\": 51.9,\n \"max\": 58.8,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 75,\n \"measurements\": [\n 79.9,\n 79.4,\n 74.6,\n 85.0,\n 78.2\n ],\n \"average\": 81.62,\n \"min\": 74.6,\n \"max\": 85.0,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9,\n \"measurements\": [\n 99.824202,\n 99.974421,\n 99.833912,\n 99.893222,\n 99.898445\n ],\n \"average\": 99.884,\n \"min\": 99.824202,\n \"max\": 99.974421,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 10000,\n \"measurements\": [\n 1446,\n 5203,\n 2244,\n 5440,\n 4243\n ],\n \"average\": 3727.2,\n \"min\": 1446,\n \"max\": 5440,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 67.4,\n \"memoryUsage\": 30.7,\n \"prbUtilization\": 62.3\n },\n \n......" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "694271213dd3", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "eMBB", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.2222222222222222, + "norm_field_f1": 0.1111111111111111, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-5de27c39c987\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 20,\n \"measurements\": [\n 19.05,\n 20.74,\n 17.1,\n 14.67,\n 14.99\n ],\n \"average\": 17.31,\n \"min\": 14.67,\n \"max\": 20.74,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 800,\n \"measurements\": [\n 913.9,\n 799.1,\n 906.2,\n 945.8,\n 883.8\n ],\n \"average\": 889.76,\n \"min\": 799.1,\n \"max\": 945.8,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 50,\n \"measurements\": [\n 58.0,\n 52.2,\n 47.8,\n 51.4,\n 52.5\n ],\n \"average\": 52.38,\n \"min\": 47.8,\n \"max\": 58.0,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.95,\n \"measurements\": [\n 100,\n 99.922209,\n 99.976677,\n 100,\n 99.852026\n ],\n \"average\": 99.95,\n \"min\": 99.852026,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 500,\n \"measurements\": [\n 5,\n 41,\n 353,\n 121,\n 334\n ],\n \"average\": 170.8,\n \"min\": 5,\n \"max\": 353,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 33.2,\n \"memoryUsage\": 22.7,\n \"prbUtilization\": 65.4\n },\n \"@type\": \"IntentAssuranceReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-144009894882\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 80,\n \"measurements\": [\n 62.0,\n 55.41,\n 52.06,\n 55.1,\n 68.8\n ],\n \"average\": 58.674,\n \"min\": 52.06,\n \"max\": 68.8,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 200,\n \"measurements\": [\n 221.1,\n 209.6,\n 226.0,\n 219.4,\n 212.2\n ],\n \"average\": 217.46,\n \"min\": 209.6,\n \"max\": 226.0,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measurements\": [\n 104.0,\n 108.4,\n 94.5,\n 99.4,\n 99.6\n ],\n \"average\": 101.38,\n \"min\": 94.5,\n \"max\": 108.4,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9,\n \"measurements\": [\n 99.894262,\n 99.961221,\n 99.893242,\n 99.948135,\n 99.892102\n ],\n \"average\": 99.918,\n \"min\": 99.892102,\n \"max\": 99.961221,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 10000,\n \"measurements\": [\n 1446,\n 4232,\n 2724,\n 5461,\n 6842\n ],\n \"average\": 4141.0,\n \"min\": 1446,\n \"max\": 6842,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 67.9,\n \"memoryUsage\": 30.7,\n \"prbUtilization\": 62\n......" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "380f2571b07d", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "URLLC", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.2698412698412698, + "norm_field_f1": 0.15873015873015872, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-b1db26b2b3a1\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 1,\n \"measurements\": [\n 0.89,\n 0.99,\n 0.94,\n 0.8,\n 0.74\n ],\n \"average\": 0.872,\n \"min\": 0.74,\n \"max\": 0.99,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 50,\n \"measurements\": [\n 46.8,\n 49.4,\n 54.6,\n 49.4,\n 59.0\n ],\n \"average\": 51.84,\n \"min\": 46.8,\n \"max\": 59.0,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measurements\": [\n 98.0,\n 105.2,\n 91.5,\n 104.2,\n 96.7\n ],\n \"average\": 99.12,\n \"min\": 91.5,\n \"max\": 105.2,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9999,\n \"measurements\": [\n 99.960407,\n 100,\n 99.964983,\n 100,\n 99.95874\n ],\n \"average\": 99.977,\n \"min\": 99.95874,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 50,\n \"measurements\": [\n 36,\n 34,\n 6,\n 24,\n 31\n ],\n \"average\": 26.2,\n \"min\": 6,\n \"max\": 36,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": false,\n \"resourceUtilization\": {\n \"cpuUsage\": 41.3,\n \"memoryUsage\": 23.6,\n \"prbUtilization\": 85.2\n },\n \"@type\": \"IntentAssuranceReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-23433010998b\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 10,\n \"measurements\": [\n 6.23,\n 6.42,\n 6.3,\n 6.89,\n 6.5\n ],\n \"average\": 6.488,\n \"min\": 6.23,\n \"max\": 6.89,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 150,\n \"measurements\": [\n 152.2,\n 147.2,\n 146.0,\n 157.8,\n 157.2\n ],\n \"average\": 152.08,\n \"min\": 146.0,\n \"max\": 157.8,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measurements\": [\n 100.4,\n 100.0,\n 104.3,\n 94.3,\n 107.0\n ],\n \"average\": 101.2,\n \"min\": 94.3,\n \"max\": 107.0,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.999,\n \"measurements\": [\n 99.914081,\n 99.944642,\n 100,\n 100,\n 99.938371\n ],\n \"average\": 99.959,\n \"min\": 99.914081,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 50,\n \"measurements\": [\n 40,\n 41,\n 10,\n 40,\n 45\n ],\n \"average\": 37.2,\n \"min\": 10,\n \"max\": 45,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 75.1,\n \"memoryUsage\": 34.9,\n \"prbUtilization\": 82.3\n },\n \"@type\": \"IntentAssuranceR\n......" + }, + { + "split": "test_use_case_ood", + "kind": "success", + "id": "31371ca89f55", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "V2X", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.3333333333333333, + "norm_field_f1": 0.3333333333333333, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-3489c7b83422\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 20,\n \"measurements\": [\n 14.26,\n 21.45,\n 17.22,\n 13.33,\n 16.83\n ],\n \"average\": 16.618,\n \"min\": 13.33,\n \"max\": 21.45,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 300,\n \"measurements\": [\n 277.9,\n 278.4,\n 330.7,\n 272.6,\n 311.8\n ],\n \"average\": 294.28,\n \"min\": 272.6,\n \"max\": 330.7,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measurements\": [\n 115.6,\n 103.7,\n 98.2,\n 116.7,\n 91.5\n ],\n \"average\": 105.14,\n \"min\": 91.5,\n \"max\": 116.7,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.999,\n \"measurements\": [\n 99.935814,\n 100,\n 99.985915,\n 100,\n 99.927923\n ],\n \"average\": 99.97,\n \"min\": 99.927923,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 5000,\n \"measurements\": [\n 4417,\n 3180,\n 2848,\n 4558,\n 3289\n ],\n \"average\": 3658.4,\n \"min\": 2848,\n \"max\": 4558,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 62.4,\n \"memoryUsage\": 34.8,\n \"prbUtilization\": 83.0\n },\n \"@type\": \"IntentAs\n......", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-3489c7b83422\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 3,\n \"measurements\": [\n 2.06,\n 2.9,\n 2.89,\n 2.86,\n 2.89\n ],\n \"average\": 2.92,\n \"min\": 2.06,\n \"max\": 2.9,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 20,\n \"measurements\": [\n 21.4,\n 20.5,\n 22.1,\n 21.1,\n 21.2\n ],\n \"average\": 21.26,\n \"min\": 20.5,\n \"max\": 22.1,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measurements\": [\n 112.5,\n 112.1,\n 99.6,\n 100.6,\n 115.3\n ],\n \"average\": 108.02,\n \"min\": 99.6,\n \"max\": 115.3,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.999,\n \"measurements\": [\n 99.94617,\n 100,\n 100,\n 100,\n 100\n ],\n \"average\": 99.989,\n \"min\": 99.94617,\n \"max\": 100,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 5000,\n \"measurements\": [\n 1606,\n 1442,\n 2225,\n 4052,\n 4543\n ],\n \"average\": 2773.6,\n \"min\": 1442,\n \"max\": 4543,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 65.8,\n \"memoryUsage\": 30.4,\n \"prbUtilization\": 62.3\n },\n \"@type\": \"IntentAssuranceReport\"\n......" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "e6705984f81d", + "target_layer": "tmf921_lifecycle_monitor", + "slice_type": "V2X", + "lifecycle_operation": "monitor", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5666666666666667, + "norm_field_f1": 0.2413793103448276, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentFulfillmentReport" + ], + "prediction_json_keys": [ + "intentFulfillmentReport" + ], + "gold": "{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-a53ad458bf38\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Fulfilled\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 5,\n \"measured\": 3.65,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 200,\n \"measured\": 210.3,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 25,\n \"measured\": 26.2,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99,\n \"measured\": 100,\n \"unit\": \"percent\",\n \"compliant\": true\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 5000,\n \"measured\": 2874,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": true,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-a53ad458bf38\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 10,\n \"measured\": 10.39,\n \"unit\": \"ms\",\n \"compliant\": false\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 150,\n \"measured\": 150.6,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 75,\n \"measured\": 79.9,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.999,\n \"measured\": 99.996667,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 5000,\n \"measured\": 1343,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "98ed16395511", + "target_layer": "tmf921_lifecycle_monitor", + "slice_type": "HMTC", + "lifecycle_operation": "monitor", + "parse_json": true, + "exact_match": false, + "field_f1": 0.6333333333333333, + "norm_field_f1": 0.27586206896551724, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentFulfillmentReport" + ], + "prediction_json_keys": [ + "intentFulfillmentReport" + ], + "gold": "{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-c173a11c070c\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Fulfilled\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 0.5,\n \"measured\": 0.36,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 200,\n \"measured\": 220.7,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 200,\n \"measured\": 205.2,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99999,\n \"measured\": 100,\n \"unit\": \"percent\",\n \"compliant\": true\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 100,\n \"measured\": 43,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": true,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-c173a11c070c\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 3,\n \"measured\": 2.89,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 2000,\n \"measured\": 2111.7,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 1000,\n \"measured\": 1074.6,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99999,\n \"measured\": 99.913156,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 100,\n \"measured\": 14,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "90bbacd7cde6", + "target_layer": "tmf921_lifecycle_monitor", + "slice_type": "eMBB", + "lifecycle_operation": "monitor", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5333333333333333, + "norm_field_f1": 0.27586206896551724, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentFulfillmentReport" + ], + "prediction_json_keys": [ + "intentFulfillmentReport" + ], + "gold": "{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-c12689b6f1d0\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Fulfilled\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 30,\n \"measured\": 21.27,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 100,\n \"measured\": 114.7,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 50,\n \"measured\": 50.2,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.95,\n \"measured\": 99.966739,\n \"unit\": \"percent\",\n \"compliant\": true\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 10000,\n \"measured\": 5860,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": true,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-1e014980300e\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 80,\n \"measured\": 65.06,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 200,\n \"measured\": 219.6,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measured\": 100.4,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9,\n \"measured\": 99.824045,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 1000,\n \"measured\": 141,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "success", + "id": "436531981b6c", + "target_layer": "tmf921_lifecycle_monitor", + "slice_type": "MPS", + "lifecycle_operation": "monitor", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7666666666666667, + "norm_field_f1": 0.7586206896551724, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "intentFulfillmentReport" + ], + "prediction_json_keys": [ + "intentFulfillmentReport" + ], + "gold": "{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-95f2e40c5e32\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 1,\n \"measured\": 0.81,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 150,\n \"measured\": 154.4,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measured\": 104.6,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9999,\n \"measured\": 99.937449,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 50,\n \"measured\": 20,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-95f2e40c5e32\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 10,\n \"measured\": 8.09,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 150,\n \"measured\": 150.6,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measured\": 109.0,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9999,\n \"measured\": 99.94607,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 100,\n \"measured\": 14,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "812160e34595", + "target_layer": "tmf921", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": false, + "exact_match": false, + "field_f1": 0.0, + "norm_field_f1": 0.0, + "norm_key_f1": 0.0, + "error_label": "invalid_or_unparseable_json", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": null, + "gold": "{\n \"id\": \"intent-0a38a7d9057e\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-0a38a7d9057e\",\n \"name\": \"eMBB Slice - Isr Data Relay\",\n \"description\": \"Intent for ISR data relay at Healthcare Park with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 5,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-77dd8b4ae650\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-7ea765\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 800,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 150,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-01101000000a\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-01101000000a\",\n \"name\": \"eMBB Slice - Isr Data Relay\",\n \"description\": \"Intent for ISR data relay at Healthcare Park with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-011010000006\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-011010\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 800,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 150,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n \n......" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "8499daec9a2c", + "target_layer": "tmf921", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": false, + "exact_match": false, + "field_f1": 0.0, + "norm_field_f1": 0.0, + "norm_key_f1": 0.0, + "error_label": "invalid_or_unparseable_json", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": null, + "gold": "{\n \"id\": \"intent-0e77c43b02e3\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-0e77c43b02e3\",\n \"name\": \"eMBB Slice - Heavy Equipment Telemetry\",\n \"description\": \"Intent for heavy equipment telemetry at Infrastructure Corridor with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 5,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-60cce9493714\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-f30461\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1000,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 75,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n \n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-31303030-3030-4135-8130-303030303030\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-31303030-3030-4135-8130-303030303030\",\n \"name\": \"eMBB Slice - Heavy Equipment Telemetry\",\n \"description\": \"Intent for heavy equipment telemetry at Infrastructure Corridor with eMBB service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-30303030-3030-4632-8130-303030303030\",\n \"name\": \"eMBB QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-embb-303030-3030-4135-8130-303030303030\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"eMBB\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1000,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \n......" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "bddc54ab839a", + "target_layer": "tmf921", + "slice_type": "URLLC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7692307692307693, + "norm_field_f1": 0.8656716417910447, + "norm_key_f1": 0.9402985074626866, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-8931782be6a9\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-8931782be6a9\",\n \"name\": \"URLLC Slice - Heavy Equipment Telemetry\",\n \"description\": \"Intent for heavy equipment telemetry at Construction Site Gamma with URLLC service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 2,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-6269398c603e\",\n \"name\": \"URLLC QoS Delivery Expectation\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-urllc-f91640\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"URLLC\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 2,\n \"sd\": \"000002\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 300,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 25,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n \n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-000000000001\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-000000000001\",\n \"name\": \"URLLC Slice - Heavy Equipment Telemetry\",\n \"description\": \"Intent for heavy equipment telemetry at Construction Site Gamma with URLLC service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 1,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-000000000001\",\n \"name\": \"URLLC QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-urllc-000000\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"URLLC\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 2,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 300,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 25,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \n......" + }, + { + "split": "test_use_case_ood", + "kind": "success", + "id": "220528448a4a", + "target_layer": "tmf921", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.8860759493670886, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-d8137d5145a5\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-d8137d5145a5\",\n \"name\": \"mMTC Slice - Passenger Connectivity Gateway\",\n \"description\": \"Intent for passenger connectivity gateway at Airport Terminal Area with mMTC service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-456b67b884bd\",\n \"name\": \"mMTC QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-mmtc-f04e16\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"mMTC\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n \n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-140334300088\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-140334300088\",\n \"name\": \"mMTC Slice - Passenger Connectivity Gateway\",\n \"description\": \"Intent for passenger connectivity gateway at Airport Terminal Area with mMTC service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-330073000008\",\n \"name\": \"mMTC QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-mmtc-000008\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"mMTC\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \n......" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "576adb0036d3", + "target_layer": "camara", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.4848484848484848, + "norm_field_f1": 0.6896551724137931, + "norm_key_f1": 0.896551724137931, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"eMBB\",\n \"sliceProfile\": {\n \"latencyTarget\": 50,\n \"reliability\": 99.99,\n \"throughput\": {\n \"downlink\": 100,\n \"uplink\": 25\n },\n \"maxNumberOfUEs\": 100\n },\n \"areaOfService\": {\n \"region\": \"Hospital Campus\",\n \"center\": {\n \"latitude\": 40.6301,\n \"longitude\": 22.9521\n },\n \"radius\": 200,\n \"cells\": [\n \"cell-031\",\n \"cell-082\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-19T23:00:00Z\",\n \"endTime\": \"2025-01-20T05:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/061dc30d797a\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"eMBB\",\n \"sliceProfile\": {\n \"latencyTarget\": 50,\n \"reliability\": 99.9,\n \"throughput\": {\n \"downlink\": 100,\n \"uplink\": 25\n },\n \"maxNumberOfUEs\": 100\n },\n \"areaOfService\": {\n \"region\": \"Hospital Campus\",\n \"center\": {\n \"latitude\": 40.6301,\n \"longitude\": 22.9521\n },\n \"radius\": 500,\n \"cells\": [\n \"cell-010\",\n \"cell-080\",\n \"cell-020\",\n \"cell-019\",\n \"cell-024\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-04T01:00:00Z\",\n \"endTime\": \"2025-01-04T03:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/33303030-3030-3030-3030-303030303030\"\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "300120f67c71", + "target_layer": "camara", + "slice_type": "V2X", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5, + "norm_field_f1": 0.7142857142857143, + "norm_key_f1": 0.8571428571428571, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"V2X\",\n \"sliceProfile\": {\n \"latencyTarget\": 1,\n \"reliability\": 99.9999,\n \"throughput\": {\n \"downlink\": 200,\n \"uplink\": 100\n },\n \"maxNumberOfUEs\": 50\n },\n \"areaOfService\": {\n \"region\": \"Medical Center District\",\n \"center\": {\n \"latitude\": 51.5074,\n \"longitude\": -0.1278\n },\n \"radius\": 500,\n \"cells\": [\n \"cell-048\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-03-23T02:00:00Z\",\n \"endTime\": \"2025-03-25T02:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/92619812b944\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"V2X\",\n \"sliceProfile\": {\n \"latencyTarget\": 1,\n \"reliability\": 99.999,\n \"throughput\": {\n \"downlink\": 200,\n \"uplink\": 100\n },\n \"maxNumberOfUEs\": 50\n },\n \"areaOfService\": {\n \"region\": \"Medical Center District\",\n \"center\": {\n \"latitude\": 51.5074,\n \"longitude\": -0.1278\n },\n \"radius\": 5000,\n \"cells\": [\n \"cell-010\",\n \"cell-080\",\n \"cell-020\",\n \"cell-010\",\n \"cell-020\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-01T01:00:00Z\",\n \"endTime\": \"2025-01-01T03:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/343434343434\"\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "f39d8b11e91f", + "target_layer": "camara", + "slice_type": "V2X", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5, + "norm_field_f1": 0.7142857142857143, + "norm_key_f1": 0.8571428571428571, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"V2X\",\n \"sliceProfile\": {\n \"latencyTarget\": 3,\n \"reliability\": 99.99,\n \"throughput\": {\n \"downlink\": 300,\n \"uplink\": 100\n },\n \"maxNumberOfUEs\": 5000\n },\n \"areaOfService\": {\n \"region\": \"Hospital Campus\",\n \"center\": {\n \"latitude\": 40.6301,\n \"longitude\": 22.9521\n },\n \"radius\": 5000,\n \"cells\": [\n \"cell-001\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-07-18T17:00:00Z\",\n \"endTime\": \"2025-07-25T17:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/c95906c68b01\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"V2X\",\n \"sliceProfile\": {\n \"latencyTarget\": 3,\n \"reliability\": 99.999,\n \"throughput\": {\n \"downlink\": 300,\n \"uplink\": 100\n },\n \"maxNumberOfUEs\": 5000\n },\n \"areaOfService\": {\n \"region\": \"Hospital Campus\",\n \"center\": {\n \"latitude\": 40.6301,\n \"longitude\": 22.9521\n },\n \"radius\": 500,\n \"cells\": [\n \"cell-008\",\n \"cell-080\",\n \"cell-081\",\n \"cell-082\",\n \"cell-083\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-14T01:00:00Z\",\n \"endTime\": \"2025-01-14T03:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/343334301010\"\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "success", + "id": "496bc3d7d1b0", + "target_layer": "camara", + "slice_type": "URLLC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5555555555555556, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"URLLC\",\n \"sliceProfile\": {\n \"latencyTarget\": 0.5,\n \"reliability\": 99.999,\n \"throughput\": {\n \"downlink\": 200,\n \"uplink\": 25\n },\n \"maxNumberOfUEs\": 10\n },\n \"areaOfService\": {\n \"region\": \"Forward Operating Area\",\n \"center\": {\n \"latitude\": 33.3152,\n \"longitude\": 44.3661\n },\n \"radius\": 5000,\n \"cells\": [\n \"cell-043\",\n \"cell-021\",\n \"cell-050\",\n \"cell-009\",\n \"cell-069\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-12-05T01:00:00Z\",\n \"endTime\": \"2025-12-05T07:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/4aa281cc449c\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"URLLC\",\n \"sliceProfile\": {\n \"latencyTarget\": 0.5,\n \"reliability\": 99.999,\n \"throughput\": {\n \"downlink\": 200,\n \"uplink\": 25\n },\n \"maxNumberOfUEs\": 10\n },\n \"areaOfService\": {\n \"region\": \"Forward Operating Area\",\n \"center\": {\n \"latitude\": 33.3152,\n \"longitude\": 44.3661\n },\n \"radius\": 5000,\n \"cells\": [\n \"cell-010\",\n \"cell-080\",\n \"cell-080\",\n \"cell-020\",\n \"cell-020\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-04T01:00:00Z\",\n \"endTime\": \"2025-01-04T03:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/343434343434\"\n }\n}" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "108cc34d634f", + "target_layer": "intent_3gpp", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.775, + "norm_field_f1": 0.6578947368421053, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-mMTC-a417f9999389\",\n \"name\": \"Underwater Sensor Network mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-910be1619245\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000001\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 10,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 30,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Healthcare Park\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"underwater \n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-mMTC-000000000001\",\n \"name\": \"Underwater Sensor Network mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-000000000001\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 1,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 30,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 99,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Healthcare Park\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCond\n......" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "ca7dbbf593d5", + "target_layer": "intent_3gpp", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.775, + "norm_field_f1": 0.6578947368421053, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-mMTC-0c0329de6362\",\n \"name\": \"Air Traffic Management Relay mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-4c788ada3508\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000001\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 50,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 25,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 200,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 95,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 500000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Factory Complex A\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"air\n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-mMTC-0d3443403406\",\n \"name\": \"Air Traffic Management Relay mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-0e0700707006\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 25,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 200,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 95,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 500000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Factory Complex A\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"con\n......" + }, + { + "split": "test_use_case_ood", + "kind": "failure", + "id": "913e2d391f08", + "target_layer": "intent_3gpp", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.775, + "norm_field_f1": 0.6578947368421053, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-mMTC-1ac629575ab0\",\n \"name\": \"Heavy Equipment Telemetry mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-ae71ed7b7bdb\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000001\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 10,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 50,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 95,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Construction Site Gamma\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"he\n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-mMTC-083010181018\",\n \"name\": \"Heavy Equipment Telemetry mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-000000000001\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 95,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Construction Site Gamma\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"co\n......" + }, + { + "split": "test_use_case_ood", + "kind": "success", + "id": "bc288badd1c3", + "target_layer": "intent_3gpp", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.9500000000000001, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-eMBB-a9619bf34370\",\n \"name\": \"Isr Data Relay eMBB Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-c0c49b41daee\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"eMBB\",\n \"sNSSAI\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 500,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 50,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 30,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99.0,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"maxNumberOfUEs\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Factory Complex A\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"ISR data relay\"\n\n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-eMBB-04303080908e\",\n \"name\": \"Isr Data Relay eMBB Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-09000000000a\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"eMBB\",\n \"sNSSAI\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 500,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 50,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 30,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99.0,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"maxNumberOfUEs\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Factory Complex A\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\"\n......" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "b3ffccc04041", + "target_layer": "o1_nrm", + "slice_type": "V2X", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.17391304347826086, + "norm_field_f1": 0.20000000000000004, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-016\",\n \"GNBDUFunction\": {\n \"id\": \"DU-008\",\n \"gNBId\": 16,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-008-010\",\n \"cellLocalId\": 10,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n41\",\n \"arfcnDL\": 508427,\n \"bSChannelBwDL\": 50,\n \"nrPCI\": 56,\n \"nrTAC\": 64532,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 4,\n \"sd\": \"000001\"\n },\n \"rrmPolicyMinRatio\": 0.09,\n \"rrmPolicyMaxRatio\": 0.37,\n \"rrmPolicyDedicatedRatio\": 0.23,\n \"rrmPolicyMaxRatioPerSlice\": 0.37\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"001\",\n \"mnc\": \"01\",\n \"sNSSAIList\": [\n {\n \"sst\": 4,\n \"sd\": \"000002\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-014\",\n \"GNBDUFunction\": {\n \"id\": \"DU-001\",\n \"gNBId\": 14,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-001-001\",\n \"cellLocalId\": 1,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n78\",\n \"arfcnDL\": 620192,\n \"bSChannelBwDL\": 40,\n \"nrPCI\": 812,\n \"nrTAC\": 38602,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 5,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.09,\n \"rrmPolicyMaxRatio\": 0.31,\n \"rrmPolicyDedicatedRatio\": 0.2,\n \"rrmPolicyMaxRatioPerSlice\": 0.31\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 5,\n \"sd\": \"000003\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "662ef55ef8a7", + "target_layer": "o1_nrm", + "slice_type": "V2X", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.17391304347826086, + "norm_field_f1": 0.20000000000000004, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-038\",\n \"GNBDUFunction\": {\n \"id\": \"DU-003\",\n \"gNBId\": 38,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-003-002\",\n \"cellLocalId\": 2,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n258\",\n \"arfcnDL\": 2064527,\n \"bSChannelBwDL\": 50,\n \"nrPCI\": 683,\n \"nrTAC\": 41299,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 4,\n \"sd\": \"000002\"\n },\n \"rrmPolicyMinRatio\": 0.14,\n \"rrmPolicyMaxRatio\": 0.3,\n \"rrmPolicyDedicatedRatio\": 0.22,\n \"rrmPolicyMaxRatioPerSlice\": 0.3\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"310\",\n \"mnc\": \"260\",\n \"sNSSAIList\": [\n {\n \"sst\": 4,\n \"sd\": \"000001\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-006\",\n \"GNBDUFunction\": {\n \"id\": \"DU-001\",\n \"gNBId\": 6,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-001-001\",\n \"cellLocalId\": 1,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n77\",\n \"arfcnDL\": 629813,\n \"bSChannelBwDL\": 40,\n \"nrPCI\": 81,\n \"nrTAC\": 3404,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 5,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.14,\n \"rrmPolicyMaxRatio\": 0.42,\n \"rrmPolicyDedicatedRatio\": 0.28,\n \"rrmPolicyMaxRatioPerSlice\": 0.42\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"234\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 5,\n \"sd\": \"000003\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "17db82b1a297", + "target_layer": "o1_nrm", + "slice_type": "MPS", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.21739130434782608, + "norm_field_f1": 0.25, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-001\",\n \"GNBDUFunction\": {\n \"id\": \"DU-043\",\n \"gNBId\": 1,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-043-010\",\n \"cellLocalId\": 10,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n41\",\n \"arfcnDL\": 510572,\n \"bSChannelBwDL\": 20,\n \"nrPCI\": 698,\n \"nrTAC\": 28986,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 5,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.09,\n \"rrmPolicyMaxRatio\": 0.4,\n \"rrmPolicyDedicatedRatio\": 0.24,\n \"rrmPolicyMaxRatioPerSlice\": 0.4\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"15\",\n \"sNSSAIList\": [\n {\n \"sst\": 5,\n \"sd\": \"000004\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-014\",\n \"GNBDUFunction\": {\n \"id\": \"DU-001\",\n \"gNBId\": 14,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-001-001\",\n \"cellLocalId\": 1,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n78\",\n \"arfcnDL\": 629619,\n \"bSChannelBwDL\": 40,\n \"nrPCI\": 812,\n \"nrTAC\": 3862,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 2,\n \"sd\": \"000001\"\n },\n \"rrmPolicyMinRatio\": 0.09,\n \"rrmPolicyMaxRatio\": 0.3,\n \"rrmPolicyDedicatedRatio\": 0.2,\n \"rrmPolicyMaxRatioPerSlice\": 0.3\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"262\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 2,\n \"sd\": \"000003\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "success", + "id": "2500b8c8441c", + "target_layer": "o1_nrm", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.6086956521739131, + "norm_field_f1": 0.7, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "ManagedElement" + ], + "prediction_json_keys": [ + "ManagedElement" + ], + "gold": "{\n \"ManagedElement\": {\n \"id\": \"gNB-048\",\n \"GNBDUFunction\": {\n \"id\": \"DU-034\",\n \"gNBId\": 48,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-034-001\",\n \"cellLocalId\": 1,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n78\",\n \"arfcnDL\": 644994,\n \"bSChannelBwDL\": 50,\n \"nrPCI\": 32,\n \"nrTAC\": 11190,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.42,\n \"rrmPolicyMaxRatio\": 0.66,\n \"rrmPolicyDedicatedRatio\": 0.54,\n \"rrmPolicyMaxRatioPerSlice\": 0.66\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"234\",\n \"mnc\": \"15\",\n \"sNSSAIList\": [\n {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n ]\n }\n ]\n }\n }\n }\n}", + "prediction": "\n\n\n\n{\n \"ManagedElement\": {\n \"id\": \"gNB-001\",\n \"GNBDUFunction\": {\n \"id\": \"DU-001\",\n \"gNBId\": 1,\n \"gNBIdLength\": 22,\n \"NRCellDU\": {\n \"id\": \"NRCellDU-001-001\",\n \"cellLocalId\": 1,\n \"operationalState\": \"ENABLED\",\n \"administrativeState\": \"UNLOCKED\",\n \"nrFrequencyRef\": \"NRFrequency=n78\",\n \"arfcnDL\": 620190,\n \"bSChannelBwDL\": 40,\n \"nrPCI\": 812,\n \"nrTAC\": 3864,\n \"rrmPolicyMemberList\": [\n {\n \"snssai\": {\n \"sst\": 1,\n \"sd\": \"000003\"\n },\n \"rrmPolicyMinRatio\": 0.42,\n \"rrmPolicyMaxRatio\": 0.66,\n \"rrmPolicyDedicatedRatio\": 0.54,\n \"rrmPolicyMaxRatioPerSlice\": 0.66\n }\n ],\n \"pLMNInfoList\": [\n {\n \"mcc\": \"234\",\n \"mnc\": \"00\",\n \"sNSSAIList\": [\n {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n ]\n }\n ]\n }\n }\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "0b9402e4f6cd", + "target_layer": "a1_policy", + "slice_type": "V2X", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.42105263157894735, + "norm_field_f1": 0.5, + "norm_key_f1": 1.0, + "error_label": "a1_policy_value_error", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-v2x-178d91\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-v2x-d85b31\",\n \"cellId\": \"cell-072\",\n \"sNSSAI\": {\n \"sst\": 4,\n \"sd\": \"000002\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 200,\n \"mfbr\": 240.0,\n \"priorityLevel\": 2,\n \"5qi\": 75,\n \"pdb\": 50,\n \"per\": 1e-05\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 13,\n \"max\": 34\n },\n \"schedulerWeight\": 0.55,\n \"maxUEs\": 10000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-urllc-00000d\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-urllc-00000d\",\n \"cellId\": \"cell-001\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000003\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 200,\n \"mfbr\": 240.0,\n \"priorityLevel\": 3,\n \"5qi\": 85,\n \"pdb\": 5,\n \"per\": 1e-06\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 13,\n \"max\": 33\n },\n \"schedulerWeight\": 0.39,\n \"maxUEs\": 10000\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "67b27dd7b5c0", + "target_layer": "a1_policy", + "slice_type": "MPS", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.42105263157894735, + "norm_field_f1": 0.5, + "norm_key_f1": 1.0, + "error_label": "a1_policy_value_error", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-mps-58a47a\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-mps-2e12a7\",\n \"cellId\": \"cell-082\",\n \"sNSSAI\": {\n \"sst\": 5,\n \"sd\": \"000004\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 200,\n \"mfbr\": 240.0,\n \"priorityLevel\": 1,\n \"5qi\": 1,\n \"pdb\": 100,\n \"per\": 1e-05\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 12,\n \"max\": 41\n },\n \"schedulerWeight\": 0.71,\n \"maxUEs\": 100\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-urllc-000001\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-urllc-000001\",\n \"cellId\": \"cell-081\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000001\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 200,\n \"mfbr\": 240.0,\n \"priorityLevel\": 3,\n \"5qi\": 84,\n \"pdb\": 30,\n \"per\": 1e-06\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 12,\n \"max\": 32\n },\n \"schedulerWeight\": 0.39,\n \"maxUEs\": 100\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "0314c9a2416b", + "target_layer": "a1_policy", + "slice_type": "MPS", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.42105263157894735, + "norm_field_f1": 0.5, + "norm_key_f1": 1.0, + "error_label": "a1_policy_value_error", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-mps-58a47a\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-mps-2e12a7\",\n \"cellId\": \"cell-082\",\n \"sNSSAI\": {\n \"sst\": 5,\n \"sd\": \"000004\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 200,\n \"mfbr\": 240.0,\n \"priorityLevel\": 1,\n \"5qi\": 1,\n \"pdb\": 100,\n \"per\": 1e-05\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 12,\n \"max\": 41\n },\n \"schedulerWeight\": 0.71,\n \"maxUEs\": 100\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-urllc-000001\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-urllc-000001\",\n \"cellId\": \"cell-001\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000001\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 200,\n \"mfbr\": 240.0,\n \"priorityLevel\": 3,\n \"5qi\": 84,\n \"pdb\": 30,\n \"per\": 1e-06\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 12,\n \"max\": 32\n },\n \"schedulerWeight\": 0.46,\n \"maxUEs\": 100\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "success", + "id": "6295016ae15b", + "target_layer": "a1_policy", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7894736842105263, + "norm_field_f1": 0.8888888888888888, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "a1Policy" + ], + "prediction_json_keys": [ + "a1Policy" + ], + "gold": "{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-embb-a065c0\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-embb-35ef49\",\n \"cellId\": \"cell-069\",\n \"sNSSAI\": {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 800,\n \"mfbr\": 960.0,\n \"priorityLevel\": 5,\n \"5qi\": 6,\n \"pdb\": 300,\n \"per\": 0.0001\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 43,\n \"max\": 63\n },\n \"schedulerWeight\": 0.33,\n \"maxUEs\": 500\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}", + "prediction": "\n\n\n\n{\n \"a1Policy\": {\n \"policyId\": \"a1-pol-embb-00000d\",\n \"policyTypeId\": \"QoSTarget\",\n \"scope\": {\n \"ueId\": null,\n \"groupId\": \"slice-embb-00000d\",\n \"cellId\": \"cell-081\",\n \"sNSSAI\": {\n \"sst\": 1,\n \"sd\": \"000001\"\n }\n },\n \"qosObjectives\": {\n \"gfbr\": 800,\n \"mfbr\": 960.0,\n \"priorityLevel\": 5,\n \"5qi\": 6,\n \"pdb\": 300,\n \"per\": 0.0001\n },\n \"resourceAllocation\": {\n \"prbQuota\": {\n \"min\": 43,\n \"max\": 63\n },\n \"schedulerWeight\": 0.3,\n \"maxUEs\": 500\n },\n \"enforcementAction\": \"APPLY\",\n \"enforcementScope\": \"RAN_SLICE\"\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "85fe1586c516", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "mMTC", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.25396825396825395, + "norm_field_f1": 0.12698412698412698, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-5acfd5df040c\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 50,\n \"measurements\": [\n 54.98,\n 31.82,\n 42.37,\n 40.29,\n 45.25\n ],\n \"average\": 42.942,\n \"min\": 31.82,\n \"max\": 54.98,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 50,\n \"measurements\": [\n 58.8,\n 59.0,\n 55.9,\n 49.5,\n 53.6\n ],\n \"average\": 55.36,\n \"min\": 49.5,\n \"max\": 59.0,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 25,\n \"measurements\": [\n 24.1,\n 28.0,\n 22.7,\n 29.2,\n 27.5\n ],\n \"average\": 26.3,\n \"min\": 22.7,\n \"max\": 29.2,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 95,\n \"measurements\": [\n 95.036732,\n 94.942745,\n 95.032511,\n 95.049224,\n 95.075089\n ],\n \"average\": 95.027,\n \"min\": 94.942745,\n \"max\": 95.075089,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 10000,\n \"measurements\": [\n 6646,\n 9177,\n 3412,\n 3906,\n 3052\n ],\n \"average\": 5238.6,\n \"min\": 3052,\n \"max\": 9177,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 39.3,\n \"memoryUsage\": 68.6,\n \"prbUtilization\": 56.8\n },\n \"@type\": \"IntentAs\n......", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-5acfd5df040c\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 100,\n \"measurements\": [\n 66.09,\n 65.82,\n 69.85,\n 65.1,\n 66.3\n ],\n \"average\": 66.632,\n \"min\": 65.1,\n \"max\": 69.85,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 50,\n \"measurements\": [\n 58.3,\n 51.9,\n 50.9,\n 58.6,\n 52.2\n ],\n \"average\": 54.38,\n \"min\": 50.9,\n \"max\": 58.6,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 50,\n \"measurements\": [\n 55.9,\n 50.3,\n 59.8,\n 58.0,\n 58.4\n ],\n \"average\": 56.48,\n \"min\": 50.3,\n \"max\": 59.8,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9,\n \"measurements\": [\n 99.814563,\n 99.961522,\n 99.893011,\n 99.893221,\n 99.893945\n ],\n \"average\": 99.893,\n \"min\": 99.814563,\n \"max\": 99.961522,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 5000,\n \"measurements\": [\n 1866,\n 1402,\n 1807,\n 4052,\n 4444\n ],\n \"average\": 2914.2,\n \"min\": 1402,\n \"max\": 4444,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 67.6,\n \"memoryUsage\": 33.7,\n \"prbUtilization\": 82.3\n },\n \n......" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "76bbd9020b1c", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "eMBB", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.25396825396825395, + "norm_field_f1": 0.12698412698412698, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-83c55e8c3fed\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 80,\n \"measurements\": [\n 72.88,\n 83.85,\n 56.86,\n 53.39,\n 70.19\n ],\n \"average\": 67.434,\n \"min\": 53.39,\n \"max\": 83.85,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 800,\n \"measurements\": [\n 951.7,\n 828.9,\n 926.7,\n 731.3,\n 758.7\n ],\n \"average\": 839.46,\n \"min\": 731.3,\n \"max\": 951.7,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 75,\n \"measurements\": [\n 72.9,\n 82.7,\n 86.7,\n 84.7,\n 84.5\n ],\n \"average\": 82.3,\n \"min\": 72.9,\n \"max\": 86.7,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9,\n \"measurements\": [\n 99.999893,\n 99.874877,\n 99.932637,\n 99.883247,\n 99.971974\n ],\n \"average\": 99.933,\n \"min\": 99.874877,\n \"max\": 99.999893,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 100,\n \"measurements\": [\n 94,\n 7,\n 28,\n 30,\n 73\n ],\n \"average\": 46.4,\n \"min\": 7,\n \"max\": 94,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": false,\n \"resourceUtilization\": {\n \"cpuUsage\": 19.0,\n \"memoryUsage\": 28.3,\n \"prbUtilization\": 31.6\n },\n \"@type\": \"IntentAssuranceR\n......", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-83c55e8c3fed\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 80,\n \"measurements\": [\n 65.01,\n 50.32,\n 56.02,\n 52.87,\n 68.0\n ],\n \"average\": 58.444,\n \"min\": 50.32,\n \"max\": 68.0,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 1000,\n \"measurements\": [\n 1012.1,\n 995.2,\n 1043.8,\n 1137.6,\n 1127.3\n ],\n \"average\": 1063.2,\n \"min\": 995.2,\n \"max\": 1137.6,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measurements\": [\n 108.4,\n 99.0,\n 100.3,\n 94.6,\n 100.5\n ],\n \"average\": 100.56,\n \"min\": 94.6,\n \"max\": 108.4,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9,\n \"measurements\": [\n 99.824172,\n 99.966442,\n 99.898235,\n 99.898382,\n 99.978134\n ],\n \"average\": 99.914,\n \"min\": 99.824172,\n \"max\": 99.978134,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 10000,\n \"measurements\": [\n 1446,\n 4224,\n 2841,\n 4872,\n 4443\n ],\n \"average\": 3787.2,\n \"min\": 1446,\n \"max\": 4872,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 67.4,\n \"memoryUsage\": 30.7,\n \"prbUtiliza\n......" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "f9bc89206a87", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "eMBB", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.20634920634920634, + "norm_field_f1": 0.14285714285714285, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-e05be29d7f06\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 20,\n \"measurements\": [\n 20.85,\n 20.74,\n 18.94,\n 19.06,\n 12.47\n ],\n \"average\": 18.412,\n \"min\": 12.47,\n \"max\": 20.85,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 500,\n \"measurements\": [\n 469.1,\n 477.7,\n 577.4,\n 535.2,\n 574.0\n ],\n \"average\": 526.68,\n \"min\": 469.1,\n \"max\": 577.4,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 150,\n \"measurements\": [\n 150.9,\n 143.3,\n 155.5,\n 151.7,\n 161.0\n ],\n \"average\": 152.48,\n \"min\": 143.3,\n \"max\": 161.0,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.0,\n \"measurements\": [\n 99.030183,\n 99.083069,\n 99.035761,\n 99.041719,\n 99.087613\n ],\n \"average\": 99.056,\n \"min\": 99.030183,\n \"max\": 99.087613,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 1000,\n \"measurements\": [\n 130,\n 685,\n 566,\n 87,\n 461\n ],\n \"average\": 385.8,\n \"min\": 87,\n \"max\": 685,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": false,\n \"resourceUtilization\": {\n \"cpuUsage\": 17.8,\n \"memoryUsage\": 67.8,\n \"prbUtilization\": 83.2\n },\n \"@type\"\n......", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-110018001006\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 80,\n \"measurements\": [\n 65.01,\n 53.02,\n 54.33,\n 50.86,\n 68.0\n ],\n \"average\": 58.244,\n \"min\": 50.86,\n \"max\": 68.0,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 1000,\n \"measurements\": [\n 1012.1,\n 995.2,\n 1043.8,\n 1137.6,\n 1127.3\n ],\n \"average\": 1063.2,\n \"min\": 995.2,\n \"max\": 1137.6,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measurements\": [\n 108.4,\n 99.0,\n 109.6,\n 99.1,\n 113.0\n ],\n \"average\": 107.82,\n \"min\": 99.0,\n \"max\": 113.0,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9,\n \"measurements\": [\n 99.894262,\n 99.968231,\n 99.997872,\n 99.893221,\n 99.94918\n ],\n \"average\": 99.941,\n \"min\": 99.893221,\n \"max\": 99.997872,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 10000,\n \"measurements\": [\n 1446,\n 2270,\n 2721,\n 4262,\n 6436\n ],\n \"average\": 3429.0,\n \"min\": 1446,\n \"max\": 6436,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 67.4,\n \"memoryUsage\": 30.7,\n \"prbUtilizat\n......" + }, + { + "split": "test_sector_ood", + "kind": "success", + "id": "c51794fa8935", + "target_layer": "tmf921_lifecycle_report", + "slice_type": "eMBB", + "lifecycle_operation": "report", + "parse_json": true, + "exact_match": false, + "field_f1": 0.2222222222222222, + "norm_field_f1": 0.23809523809523808, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "intentAssuranceReport" + ], + "prediction_json_keys": [ + "intentAssuranceReport" + ], + "gold": "{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-1b356c618234\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 80,\n \"measurements\": [\n 71.25,\n 85.75,\n 70.29,\n 73.34,\n 58.83\n ],\n \"average\": 71.892,\n \"min\": 58.83,\n \"max\": 85.75,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 100,\n \"measurements\": [\n 112.0,\n 114.2,\n 99.1,\n 106.0,\n 92.8\n ],\n \"average\": 104.82,\n \"min\": 92.8,\n \"max\": 114.2,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 75,\n \"measurements\": [\n 89.3,\n 81.2,\n 81.2,\n 89.3,\n 75.1\n ],\n \"average\": 83.22,\n \"min\": 75.1,\n \"max\": 89.3,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.5,\n \"measurements\": [\n 99.527262,\n 99.47924,\n 99.408221,\n 99.498615,\n 99.427245\n ],\n \"average\": 99.468,\n \"min\": 99.408221,\n \"max\": 99.527262,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 1000,\n \"measurements\": [\n 419,\n 836,\n 445,\n 516,\n 663\n ],\n \"average\": 575.8,\n \"min\": 419,\n \"max\": 836,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": false,\n \"resourceUtilization\": {\n \"cpuUsage\": 63.8,\n \"memoryUsage\": 24.1,\n \"prbUtilization\": 33.8\n },\n \"@type\": \"IntentAs\n......", + "prediction": "\n\n\n\n{\n \"intentAssuranceReport\": {\n \"intentId\": \"intent-000000000001\",\n \"reportPeriod\": {\n \"start\": \"2026-04-24T08:11:31Z\",\n \"end\": \"2026-04-24T09:11:31Z\"\n },\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 80,\n \"measurements\": [\n 62.0,\n 55.59,\n 50.15,\n 59.31,\n 68.45\n ],\n \"average\": 59.1,\n \"min\": 50.15,\n \"max\": 68.45,\n \"unit\": \"ms\"\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 1000,\n \"measurements\": [\n 1012.1,\n 999.4,\n 1057.6,\n 1136.5,\n 1157.3\n ],\n \"average\": 1072.6,\n \"min\": 999.4,\n \"max\": 1157.3,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measurements\": [\n 108.5,\n 99.4,\n 100.4,\n 95.9,\n 94.1\n ],\n \"average\": 99.66,\n \"min\": 94.1,\n \"max\": 108.5,\n \"unit\": \"Mbps\"\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9,\n \"measurements\": [\n 99.824072,\n 99.974421,\n 99.893912,\n 99.974135,\n 99.944474\n ],\n \"average\": 99.922,\n \"min\": 99.824072,\n \"max\": 99.974421,\n \"unit\": \"percent\"\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 10000,\n \"measurements\": [\n 1446,\n 9805,\n 2824,\n 5704,\n 2802\n ],\n \"average\": 4576.2,\n \"min\": 1446,\n \"max\": 9805,\n \"unit\": \"UEs\"\n }\n ],\n \"overallCompliance\": true,\n \"resourceUtilization\": {\n \"cpuUsage\": 67.4,\n \"memoryUsage\": 30.7,\n \"prbUtilizatio\n......" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "3339306cb82f", + "target_layer": "tmf921_lifecycle_monitor", + "slice_type": "eMBB", + "lifecycle_operation": "monitor", + "parse_json": true, + "exact_match": false, + "field_f1": 0.6, + "norm_field_f1": 0.20689655172413793, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentFulfillmentReport" + ], + "prediction_json_keys": [ + "intentFulfillmentReport" + ], + "gold": "{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-54ddbb2fb84e\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Fulfilled\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 30,\n \"measured\": 23.42,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 100,\n \"measured\": 103.9,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 75,\n \"measured\": 73.6,\n \"unit\": \"Mbps\",\n \"compliant\": false\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.0,\n \"measured\": 99.07832,\n \"unit\": \"percent\",\n \"compliant\": true\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 500,\n \"measured\": 259,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": true,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-54ddbb2fb84e\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 80,\n \"measured\": 65.03,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 100,\n \"measured\": 105.7,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measured\": 99.4,\n \"unit\": \"Mbps\",\n \"compliant\": false\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9,\n \"measured\": 99.824045,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 1000,\n \"measured\": 141,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "4ca220109eee", + "target_layer": "tmf921_lifecycle_monitor", + "slice_type": "mMTC", + "lifecycle_operation": "monitor", + "parse_json": true, + "exact_match": false, + "field_f1": 0.6333333333333333, + "norm_field_f1": 0.27586206896551724, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentFulfillmentReport" + ], + "prediction_json_keys": [ + "intentFulfillmentReport" + ], + "gold": "{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-fca74a7cf825\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Fulfilled\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 1000,\n \"measured\": 858.12,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 1,\n \"measured\": 1.0,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 0.1,\n \"measured\": 0.1,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 90,\n \"measured\": 90.005011,\n \"unit\": \"percent\",\n \"compliant\": true\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 1000000,\n \"measured\": 351789,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": true,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-fca74a7cf825\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 500,\n \"measured\": 429.21,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 1,\n \"measured\": 1.0,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 1,\n \"measured\": 1.0,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.9,\n \"measured\": 99.893526,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 10000,\n \"measured\": 1434,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "f46aca552e6b", + "target_layer": "tmf921_lifecycle_monitor", + "slice_type": "HMTC", + "lifecycle_operation": "monitor", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5666666666666667, + "norm_field_f1": 0.27586206896551724, + "norm_key_f1": 1.0, + "error_label": "correct_structure_wrong_values", + "gold_json_keys": [ + "intentFulfillmentReport" + ], + "prediction_json_keys": [ + "intentFulfillmentReport" + ], + "gold": "{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-067e7d57a02c\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Fulfilled\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 5,\n \"measured\": 4.84,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 1000,\n \"measured\": 1064.9,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 500,\n \"measured\": 504.7,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99999,\n \"measured\": 100,\n \"unit\": \"percent\",\n \"compliant\": true\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 1000,\n \"measured\": 78,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": true,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-000000000001\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 3,\n \"measured\": 2.24,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 2000,\n \"measured\": 2119.9,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 50,\n \"measured\": 52.1,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.99999,\n \"measured\": 99.913156,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 100,\n \"measured\": 96,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "success", + "id": "4b96dab3c3fd", + "target_layer": "tmf921_lifecycle_monitor", + "slice_type": "URLLC", + "lifecycle_operation": "monitor", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7666666666666667, + "norm_field_f1": 0.7586206896551724, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "intentFulfillmentReport" + ], + "prediction_json_keys": [ + "intentFulfillmentReport" + ], + "gold": "{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-b69c2ab78704\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 2,\n \"measured\": 1.73,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 20,\n \"measured\": 20.1,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 100,\n \"measured\": 113.2,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.999,\n \"measured\": 99.957143,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 1000,\n \"measured\": 87,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}", + "prediction": "\n\n\n\n{\n \"intentFulfillmentReport\": {\n \"intentId\": \"intent-b69c2ab78704\",\n \"reportTimestamp\": \"2026-04-24T09:11:31Z\",\n \"fulfillmentStatus\": \"Degraded\",\n \"kpiMeasurements\": [\n {\n \"kpi\": \"latency\",\n \"target\": 3,\n \"measured\": 2.89,\n \"unit\": \"ms\",\n \"compliant\": true\n },\n {\n \"kpi\": \"dlThroughput\",\n \"target\": 20,\n \"measured\": 20.1,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"ulThroughput\",\n \"target\": 50,\n \"measured\": 52.2,\n \"unit\": \"Mbps\",\n \"compliant\": true\n },\n {\n \"kpi\": \"reliability\",\n \"target\": 99.999,\n \"measured\": 99.984363,\n \"unit\": \"percent\",\n \"compliant\": false\n },\n {\n \"kpi\": \"connectedUEs\",\n \"target\": 500,\n \"measured\": 147,\n \"unit\": \"UEs\",\n \"compliant\": true\n }\n ],\n \"overallSlaCompliance\": false,\n \"@type\": \"IntentFulfillmentReport\"\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "af14a312ef12", + "target_layer": "tmf921", + "slice_type": "URLLC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7435897435897437, + "norm_field_f1": 0.8358208955223881, + "norm_key_f1": 0.9402985074626866, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-a5769252fdba\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-a5769252fdba\",\n \"name\": \"URLLC Slice - Digital Examination Proctoring\",\n \"description\": \"Intent for digital examination proctoring at Education District with URLLC service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 2,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-f0fe2980f0c2\",\n \"name\": \"URLLC QoS Delivery Expectation\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-urllc-4de34e\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"URLLC\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 2,\n \"sd\": \"000001\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 20,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 150,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-011000000000\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-011000000000\",\n \"name\": \"URLLC Slice - Digital Examination Proctoring\",\n \"description\": \"Intent for digital examination proctoring at Education District with URLLC service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-011000000001\",\n \"name\": \"URLLC QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-urllc-011000\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"URLLC\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 2,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 20,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 150,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \n......" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "849b574b28a0", + "target_layer": "tmf921", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.7435897435897437, + "norm_field_f1": 0.8358208955223881, + "norm_key_f1": 0.9402985074626866, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-e19d7501c8cb\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-e19d7501c8cb\",\n \"name\": \"mMTC Slice - Blast Area Safety Monitoring\",\n \"description\": \"Intent for blast area safety monitoring at Mining Site Alpha with mMTC service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 4,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-ad5d195c7641\",\n \"name\": \"mMTC QoS Delivery Expectation\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-mmtc-b1a7f4\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"mMTC\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 3,\n \"sd\": \"000001\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 0.1,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n \n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-01101000000a\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-01101000000a\",\n \"name\": \"mMTC Slice - Blast Area Safety Monitoring\",\n \"description\": \"Intent for blast area safety monitoring at Mining Site Alpha with mMTC service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-011010000006\",\n \"name\": \"mMTC QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-mmtc-011010\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"mMTC\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 10,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 0.1,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"uni\n......" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "b930a105870d", + "target_layer": "tmf921", + "slice_type": "URLLC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.751592356687898, + "norm_field_f1": 0.8444444444444444, + "norm_key_f1": 0.9333333333333335, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-7ba4a49d599f\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-7ba4a49d599f\",\n \"name\": \"URLLC Slice - Underground Vehicle Teleoperation\",\n \"description\": \"Intent for underground vehicle teleoperation at Underground Level 3 with URLLC service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-09d137955138\",\n \"name\": \"URLLC QoS Delivery Expectation\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-urllc-0dc4e3\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"URLLC\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 2,\n \"sd\": \"000001\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 75,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\":\n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-011011101010\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-011011101010\",\n \"name\": \"URLLC Slice - Underground Vehicle Teleoperation\",\n \"description\": \"Intent for underground vehicle teleoperation at Underground Level 3 with URLLC service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 1,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-000000000001\",\n \"name\": \"URLLC QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-urllc-000001\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"URLLC\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 2,\n \"sd\": \"000003\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 75,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n......" + }, + { + "split": "test_sector_ood", + "kind": "success", + "id": "b765dd0651e9", + "target_layer": "tmf921", + "slice_type": "V2X", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.8860759493670886, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "prediction_json_keys": [ + "id", + "href", + "name", + "description", + "lifecycleState", + "priority", + "intentExpression", + "relatedParty", + "fulfillmentInfo", + "@type", + "@baseType", + "@schemaLocation" + ], + "gold": "{\n \"id\": \"intent-80c9c5f1d79a\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-80c9c5f1d79a\",\n \"name\": \"V2X Slice - Crusher Plant Remote Control\",\n \"description\": \"Intent for crusher plant remote control at Medical Center District with V2X service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-105b2339c5be\",\n \"name\": \"V2X QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-v2x-0bee59\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"V2X\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 4,\n \"sd\": \"000001\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n \n......", + "prediction": "\n\n\n\n{\n \"id\": \"intent-011111111111\",\n \"href\": \"/tmf-api/intentManagement/v4/intent/intent-011111111111\",\n \"name\": \"V2X Slice - Crusher Plant Remote Control\",\n \"description\": \"Intent for crusher plant remote control at Medical Center District with V2X service level\",\n \"lifecycleState\": \"Active\",\n \"priority\": 3,\n \"intentExpression\": {\n \"@type\": \"IntentExpression\",\n \"intentExpectation\": [\n {\n \"@type\": \"DeliveryExpectation\",\n \"id\": \"exp-011111111111\",\n \"name\": \"V2X QoS Delivery Expectation\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"@type\": \"ExpectationObject\",\n \"objectType\": \"NetworkSlice\",\n \"objectInstance\": \"slice-v2x-011111\",\n \"objectContexts\": [\n {\n \"contextAttribute\": \"sliceServiceType\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"V2X\"\n },\n {\n \"contextAttribute\": \"sNSSAI\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": {\n \"sst\": 4,\n \"sd\": \"000001\"\n }\n }\n ]\n },\n \"expectationTargets\": [\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"targetMetric\": \"PLMNID.dl.ThpVol.NRCellDU\",\n \"unit\": \"Mbps\"\n },\n {\n \"@type\": \"ExpectationTarget\",\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"targetMetric\": \"PLMNID.ul.ThpVol.NRCellDU\",\n \"uni\n......" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "82b916decdce", + "target_layer": "camara", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.375, + "norm_field_f1": 0.5, + "norm_key_f1": 0.8571428571428571, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"mMTC\",\n \"sliceProfile\": {\n \"latencyTarget\": 1000,\n \"reliability\": 99.5,\n \"throughput\": {\n \"downlink\": 25,\n \"uplink\": 0.5\n },\n \"maxNumberOfUEs\": 50000\n },\n \"areaOfService\": {\n \"region\": \"Research Park\",\n \"center\": {\n \"latitude\": 37.4275,\n \"longitude\": -122.1697\n },\n \"radius\": 500,\n \"cells\": [\n \"cell-050\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-11-03T10:00:00Z\",\n \"endTime\": \"2025-11-03T11:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/1b295daa4133\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"mMTC\",\n \"sliceProfile\": {\n \"latencyTarget\": 1000,\n \"reliability\": 99.9,\n \"throughput\": {\n \"downlink\": 25,\n \"uplink\": 0.5\n },\n \"maxNumberOfUEs\": 50000\n },\n \"areaOfService\": {\n \"region\": \"Research Park\",\n \"center\": {\n \"latitude\": 37.4979,\n \"longitude\": -122.1630\n },\n \"radius\": 2000,\n \"cells\": [\n \"cell-010\",\n \"cell-080\",\n \"cell-020\",\n \"cell-031\",\n \"cell-030\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-04T01:00:00Z\",\n \"endTime\": \"2025-01-04T03:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/33303030-1000-4000-8000-000000000000\"\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "f75010a927f2", + "target_layer": "camara", + "slice_type": "MPS", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.375, + "norm_field_f1": 0.5, + "norm_key_f1": 0.8571428571428571, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"MPS\",\n \"sliceProfile\": {\n \"latencyTarget\": 1,\n \"reliability\": 99.9999,\n \"throughput\": {\n \"downlink\": 25,\n \"uplink\": 10\n },\n \"maxNumberOfUEs\": 500\n },\n \"areaOfService\": {\n \"region\": \"Open Pit Zone\",\n \"center\": {\n \"latitude\": 62.0394,\n \"longitude\": 129.7422\n },\n \"radius\": 5000,\n \"cells\": [\n \"cell-015\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-08-15T08:00:00Z\",\n \"endTime\": \"2025-08-17T08:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/d3dfd3c17c33\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"MPS\",\n \"sliceProfile\": {\n \"latencyTarget\": 1,\n \"reliability\": 99.999,\n \"throughput\": {\n \"downlink\": 25,\n \"uplink\": 10\n },\n \"maxNumberOfUEs\": 500\n },\n \"areaOfService\": {\n \"region\": \"Open Pit Zone\",\n \"center\": {\n \"latitude\": 33.4484,\n \"longitude\": -112.074\n },\n \"radius\": 500,\n \"cells\": [\n \"cell-001\",\n \"cell-010\",\n \"cell-030\",\n \"cell-031\",\n \"cell-032\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-04T01:00:00Z\",\n \"endTime\": \"2025-01-04T03:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/33303030-3030-3030-3030-303030303030\"\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "bd7ec7719282", + "target_layer": "camara", + "slice_type": "HMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.303030303030303, + "norm_field_f1": 0.5517241379310345, + "norm_key_f1": 0.896551724137931, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"HMTC\",\n \"sliceProfile\": {\n \"latencyTarget\": 1,\n \"reliability\": 99.9999,\n \"throughput\": {\n \"downlink\": 1000,\n \"uplink\": 50\n },\n \"maxNumberOfUEs\": 1000\n },\n \"areaOfService\": {\n \"region\": \"Mining Site Alpha\",\n \"center\": {\n \"latitude\": -33.8688,\n \"longitude\": 151.2093\n },\n \"radius\": 1000,\n \"cells\": [\n \"cell-001\",\n \"cell-004\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-11-21T23:00:00Z\",\n \"endTime\": \"2025-11-22T11:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/109daa504948\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"MPS\",\n \"sliceProfile\": {\n \"latencyTarget\": 1,\n \"reliability\": 99.999,\n \"throughput\": {\n \"downlink\": 1000,\n \"uplink\": 50\n },\n \"maxNumberOfUEs\": 1000\n },\n \"areaOfService\": {\n \"region\": \"Mining Site Alpha\",\n \"center\": {\n \"latitude\": -23.4162,\n \"longitude\": 46.4233\n },\n \"radius\": 2000,\n \"cells\": [\n \"cell-010\",\n \"cell-080\",\n \"cell-080\",\n \"cell-030\",\n \"cell-010\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-04T03:00:00Z\",\n \"endTime\": \"2025-01-04T05:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/343434343434\"\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "success", + "id": "3205bc882211", + "target_layer": "camara", + "slice_type": "eMBB", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.5882352941176471, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "networkSliceBooking" + ], + "prediction_json_keys": [ + "networkSliceBooking" + ], + "gold": "{\n \"networkSliceBooking\": {\n \"sliceType\": \"eMBB\",\n \"sliceProfile\": {\n \"latencyTarget\": 30,\n \"reliability\": 99.9,\n \"throughput\": {\n \"downlink\": 500,\n \"uplink\": 200\n },\n \"maxNumberOfUEs\": 10000\n },\n \"areaOfService\": {\n \"region\": \"Healthcare Park\",\n \"center\": {\n \"latitude\": 37.7749,\n \"longitude\": -122.4194\n },\n \"radius\": 5000,\n \"cells\": [\n \"cell-064\",\n \"cell-037\",\n \"cell-002\",\n \"cell-062\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-06-12T05:00:00Z\",\n \"endTime\": \"2025-06-12T17:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/fcb9f701915d\"\n }\n}", + "prediction": "\n\n\n\n{\n \"networkSliceBooking\": {\n \"sliceType\": \"eMBB\",\n \"sliceProfile\": {\n \"latencyTarget\": 30,\n \"reliability\": 99.9,\n \"throughput\": {\n \"downlink\": 500,\n \"uplink\": 200\n },\n \"maxNumberOfUEs\": 10000\n },\n \"areaOfService\": {\n \"region\": \"Healthcare Park\",\n \"center\": {\n \"latitude\": 37.7749,\n \"longitude\": -122.4194\n },\n \"radius\": 5000,\n \"cells\": [\n \"cell-081\",\n \"cell-080\",\n \"cell-082\",\n \"cell-083\"\n ]\n },\n \"duration\": {\n \"startTime\": \"2025-01-01T08:00:00Z\",\n \"endTime\": \"2025-01-01T14:00:00Z\"\n },\n \"notificationUrl\": \"https://callback.example.com/slice-events/74044470340e\"\n }\n}" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "6b49800b2f85", + "target_layer": "intent_3gpp", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.75, + "norm_field_f1": 0.631578947368421, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-mMTC-0cddd74357ff\",\n \"name\": \"Virtual Classroom Xr Delivery mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-0a58bf066364\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000002\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 0.5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 25,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 100,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99.5,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 5000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Industrial Zone North\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\"\n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-mMTC-08301010000a\",\n \"name\": \"Virtual Classroom Xr Delivery mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-000000000001\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 0.5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 25,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 99.9,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 5000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Industrial Zone North\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \n......" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "3f1be0deef70", + "target_layer": "intent_3gpp", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.75, + "norm_field_f1": 0.631578947368421, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-mMTC-0cddd74357ff\",\n \"name\": \"Virtual Classroom Xr Delivery mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-0a58bf066364\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000002\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 0.5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 25,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 100,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99.5,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 5000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Industrial Zone North\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\"\n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-mMTC-34333000000a\",\n \"name\": \"Virtual Classroom Xr Delivery mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-343330000006\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 0.5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 25,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 99.9,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 5000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Industrial Zone North\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \n......" + }, + { + "split": "test_sector_ood", + "kind": "failure", + "id": "36710900c49f", + "target_layer": "intent_3gpp", + "slice_type": "mMTC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.775, + "norm_field_f1": 0.6578947368421053, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-mMTC-0cddd74357ff\",\n \"name\": \"Virtual Classroom Xr Delivery mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-0a58bf066364\",\n \"expectationVerb\": \"ENSURE\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000002\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 0.5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 25,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 100,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99.5,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 5000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Industrial Zone North\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\"\n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-mMTC-083300893086\",\n \"name\": \"Virtual Classroom Xr Delivery mMTC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-000000000001\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"mMTC\",\n \"sNSSAI\": {\n \"sst\": 3,\n \"sd\": \"000003\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 0.5,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 25,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 100,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 99.5,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"deviceDensity\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 5000,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Industrial Zone North\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \n......" + }, + { + "split": "test_sector_ood", + "kind": "success", + "id": "25d1c40596cf", + "target_layer": "intent_3gpp", + "slice_type": "URLLC", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.9, + "norm_field_f1": 0.9736842105263158, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "intent" + ], + "prediction_json_keys": [ + "intent" + ], + "gold": "{\n \"intent\": {\n \"id\": \"intent-URLLC-df9d41d88f4c\",\n \"name\": \"Research Lab Data Transfer URLLC Intent\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-05434c3a976b\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"URLLC\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000001\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 200,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 100,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 3,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99.999,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"maxNumberOfUEs\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Factory Complex A\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"contextValueRange\": \"\n......", + "prediction": "\n\n\n\n{\n \"intent\": {\n \"id\": \"intent-URLLC-011010101010\",\n \"name\": \"URLLC Slice - Research Lab Data Transfer\",\n \"lifecycleStatus\": \"active\",\n \"intentExpectation\": [\n {\n \"expectationId\": \"exp-000000000001\",\n \"expectationVerb\": \"DELIVER\",\n \"expectationObject\": {\n \"objectType\": \"NW_SLICE\",\n \"objectInstance\": {\n \"sliceType\": \"URLLC\",\n \"sNSSAI\": {\n \"sst\": 2,\n \"sd\": \"000001\"\n }\n }\n },\n \"expectationTargets\": [\n {\n \"targetName\": \"downlinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 200,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"uplinkThroughput\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 100,\n \"unit\": \"Mbps\"\n },\n {\n \"targetName\": \"latency\",\n \"targetCondition\": \"IS_LESS_THAN\",\n \"targetValueRange\": 3,\n \"unit\": \"ms\"\n },\n {\n \"targetName\": \"reliability\",\n \"targetCondition\": \"IS_GREATER_THAN\",\n \"targetValueRange\": 99.999,\n \"unit\": \"percent\"\n },\n {\n \"targetName\": \"maxNumberOfUEs\",\n \"targetCondition\": \"IS_LESS_THAN_OR_EQUAL_TO\",\n \"targetValueRange\": 50,\n \"unit\": \"UEs\"\n }\n ],\n \"expectationContexts\": [\n {\n \"contextAttribute\": \"serviceArea\",\n \"contextCondition\": \"IS_IN\",\n \"contextValueRange\": \"Factory Complex A\"\n },\n {\n \"contextAttribute\": \"useCase\",\n \"contextCondition\": \"IS_EQUAL_TO\",\n \"c\n......" + }, + { + "split": "test_adversarial", + "kind": "failure", + "id": "c2c3794511e2", + "target_layer": "adversarial_ambiguous", + "slice_type": "N/A", + "lifecycle_operation": "create", + "parse_json": false, + "exact_match": false, + "field_f1": 0.0, + "norm_field_f1": 0.0, + "norm_key_f1": 0.0, + "error_label": "invalid_or_unparseable_json", + "gold_json_keys": [ + "status", + "message", + "suggestedAction" + ], + "prediction_json_keys": null, + "gold": "{\n \"status\": \"CLARIFICATION_REQUIRED\",\n \"message\": \"Intent too vague. Please specify: sliceType, latencyTarget, reliabilityTarget, dlThroughput, ulThroughput, maxUEs, serviceArea.\",\n \"suggestedAction\": \"Reformulate with specific KPI targets.\"\n}", + "prediction": "\n\n\n\nPlease provide a specific network intent so I can assist with provisioning." + }, + { + "split": "test_adversarial", + "kind": "failure", + "id": "d2dbc2ecfc38", + "target_layer": "adversarial_ambiguous", + "slice_type": "N/A", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": false, + "field_f1": 0.6666666666666666, + "norm_field_f1": 0.6666666666666666, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "status", + "message", + "suggestedAction" + ], + "prediction_json_keys": [ + "status", + "message", + "suggestedAction" + ], + "gold": "{\n \"status\": \"CLARIFICATION_REQUIRED\",\n \"message\": \"Intent too vague. Please specify: sliceType, latencyTarget, reliabilityTarget, dlThroughput, ulThroughput, maxUEs, serviceArea.\",\n \"suggestedAction\": \"Reformulate with specific KPI targets.\"\n}", + "prediction": "\n\n\n\n{\n \"status\": \"CLARIFICATION_REQUIRED\",\n \"message\": \"Intent too vague. Please specify: sliceType, serviceArea, qtiRequirements, reliabilityTarget, latencyTarget, dlThroughput, ulThroughput, maxUEs, startTime, endTime.\",\n \"suggestedAction\": \"Reformulate with specific KPI targets.\"\n}" + }, + { + "split": "test_adversarial", + "kind": "failure", + "id": "c20e3d39aa40", + "target_layer": "adversarial_ambiguous", + "slice_type": "N/A", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": true, + "field_f1": 1.0, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "status", + "message", + "suggestedAction" + ], + "prediction_json_keys": [ + "status", + "message", + "suggestedAction" + ], + "gold": "{\n \"status\": \"CLARIFICATION_REQUIRED\",\n \"message\": \"Intent too vague. Please specify: sliceType, latencyTarget, reliabilityTarget, dlThroughput, ulThroughput, maxUEs, serviceArea.\",\n \"suggestedAction\": \"Reformulate with specific KPI targets.\"\n}", + "prediction": "\n\n\n\n{\n \"status\": \"CLARIFICATION_REQUIRED\",\n \"message\": \"Intent too vague. Please specify: sliceType, latencyTarget, reliabilityTarget, dlThroughput, ulThroughput, maxUEs, serviceArea.\",\n \"suggestedAction\": \"Reformulate with specific KPI targets.\"\n}" + }, + { + "split": "test_adversarial", + "kind": "success", + "id": "c20e3d39aa40", + "target_layer": "adversarial_ambiguous", + "slice_type": "N/A", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": true, + "field_f1": 1.0, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "status", + "message", + "suggestedAction" + ], + "prediction_json_keys": [ + "status", + "message", + "suggestedAction" + ], + "gold": "{\n \"status\": \"CLARIFICATION_REQUIRED\",\n \"message\": \"Intent too vague. Please specify: sliceType, latencyTarget, reliabilityTarget, dlThroughput, ulThroughput, maxUEs, serviceArea.\",\n \"suggestedAction\": \"Reformulate with specific KPI targets.\"\n}", + "prediction": "\n\n\n\n{\n \"status\": \"CLARIFICATION_REQUIRED\",\n \"message\": \"Intent too vague. Please specify: sliceType, latencyTarget, reliabilityTarget, dlThroughput, ulThroughput, maxUEs, serviceArea.\",\n \"suggestedAction\": \"Reformulate with specific KPI targets.\"\n}" + }, + { + "split": "test_adversarial", + "kind": "failure", + "id": "b5e41f85a345", + "target_layer": "adversarial_out_of_scope", + "slice_type": "N/A", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": true, + "field_f1": 1.0, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "status", + "message", + "supportedOperations" + ], + "prediction_json_keys": [ + "status", + "message", + "supportedOperations" + ], + "gold": "{\n \"status\": \"OUT_OF_SCOPE\",\n \"message\": \"Request outside scope. This system handles 5G/6G network slice intent translation only.\",\n \"supportedOperations\": [\n \"slice creation\",\n \"lifecycle management\",\n \"SLA assurance\"\n ]\n}", + "prediction": "\n\n\n\n{\n \"status\": \"OUT_OF_SCOPE\",\n \"message\": \"Request outside scope. This system handles 5G/6G network slice intent translation only.\",\n \"supportedOperations\": [\n \"slice creation\",\n \"lifecycle management\",\n \"SLA assurance\"\n ]\n}" + }, + { + "split": "test_adversarial", + "kind": "failure", + "id": "f5b83d2fb356", + "target_layer": "adversarial_out_of_scope", + "slice_type": "N/A", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": true, + "field_f1": 1.0, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "status", + "message", + "supportedOperations" + ], + "prediction_json_keys": [ + "status", + "message", + "supportedOperations" + ], + "gold": "{\n \"status\": \"OUT_OF_SCOPE\",\n \"message\": \"Request outside scope. This system handles 5G/6G network slice intent translation only.\",\n \"supportedOperations\": [\n \"slice creation\",\n \"lifecycle management\",\n \"SLA assurance\"\n ]\n}", + "prediction": "\n\n\n\n{\n \"status\": \"OUT_OF_SCOPE\",\n \"message\": \"Request outside scope. This system handles 5G/6G network slice intent translation only.\",\n \"supportedOperations\": [\n \"slice creation\",\n \"lifecycle management\",\n \"SLA assurance\"\n ]\n}" + }, + { + "split": "test_adversarial", + "kind": "failure", + "id": "8c2457c3ef17", + "target_layer": "adversarial_out_of_scope", + "slice_type": "N/A", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": true, + "field_f1": 1.0, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "value_level_mismatch", + "gold_json_keys": [ + "status", + "message", + "supportedOperations" + ], + "prediction_json_keys": [ + "status", + "message", + "supportedOperations" + ], + "gold": "{\n \"status\": \"OUT_OF_SCOPE\",\n \"message\": \"Request outside scope. This system handles 5G/6G network slice intent translation only.\",\n \"supportedOperations\": [\n \"slice creation\",\n \"lifecycle management\",\n \"SLA assurance\"\n ]\n}", + "prediction": "\n\n\n\n{\n \"status\": \"OUT_OF_SCOPE\",\n \"message\": \"Request outside scope. This system handles 5G/6G network slice intent translation only.\",\n \"supportedOperations\": [\n \"slice creation\",\n \"lifecycle management\",\n \"SLA assurance\"\n ]\n}" + }, + { + "split": "test_adversarial", + "kind": "success", + "id": "b5e41f85a345", + "target_layer": "adversarial_out_of_scope", + "slice_type": "N/A", + "lifecycle_operation": "create", + "parse_json": true, + "exact_match": true, + "field_f1": 1.0, + "norm_field_f1": 1.0, + "norm_key_f1": 1.0, + "error_label": "success_or_high_scoring_example", + "gold_json_keys": [ + "status", + "message", + "supportedOperations" + ], + "prediction_json_keys": [ + "status", + "message", + "supportedOperations" + ], + "gold": "{\n \"status\": \"OUT_OF_SCOPE\",\n \"message\": \"Request outside scope. This system handles 5G/6G network slice intent translation only.\",\n \"supportedOperations\": [\n \"slice creation\",\n \"lifecycle management\",\n \"SLA assurance\"\n ]\n}", + "prediction": "\n\n\n\n{\n \"status\": \"OUT_OF_SCOPE\",\n \"message\": \"Request outside scope. This system handles 5G/6G network slice intent translation only.\",\n \"supportedOperations\": [\n \"slice creation\",\n \"lifecycle management\",\n \"SLA assurance\"\n ]\n}" + } +] \ No newline at end of file