Spaces:
Sleeping
Sleeping
File size: 2,709 Bytes
67c8aca | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 | import os
import sys
from dotenv import load_dotenv
from openai import OpenAI
# Add project root to path
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from services.llm_advisor import generate_explanation, LLM_MODELS, SYSTEM_PROMPT
load_dotenv()
def test_single_model(model_name, packet):
print(f"\n--- Testing Model: {model_name} ---")
api_key = os.getenv("NVIDIA_API_KEY")
client = OpenAI(base_url="https://integrate.api.nvidia.com/v1", api_key=api_key)
user_content = f"Result: {packet['prediction']}, Confidence: {packet['confidence']}, DTI: {packet['dti_ratio']}"
try:
completion = client.chat.completions.create(
model=model_name,
messages=[
{"role": "system", "content": SYSTEM_PROMPT},
{"role": "user", "content": user_content}
],
temperature=0.1,
max_tokens=512
)
content = completion.choices[0].message.content
print(f"Response received ({len(content)} chars)")
if "@@POINT@@" in content:
print("SUCCESS: Output contains required @@POINT@@ delimiters.")
else:
print("FAILURE: Output missing @@POINT@@ delimiters.")
return True
except Exception as e:
print(f"ERROR: Model {model_name} failed: {e}")
return False
def test_fallback_logic(packet):
print("\n--- Testing Full Fallback Workflow ---")
# Forcing a failure by temporarily corrupting the model list in memory
original_models = list(LLM_MODELS)
# Inject a fake non-existent model at the start
import services.llm_advisor as advisor
advisor.LLM_MODELS = ["invalid/non-existent-model-1234"] + original_models
print("Injected invalid model. Expecting fail-over to the first valid model...")
response = advisor.generate_explanation(packet)
if "@@POINT@@" in response:
print("SUCCESS: Fallback logic correctly skipped the invalid model and used a secondary model.")
else:
print("FAILURE: Fallback logic did not return a valid AI summary.")
# Reset models
advisor.LLM_MODELS = original_models
if __name__ == "__main__":
sample_packet = {
"prediction": "Y",
"confidence": 0.88,
"dti_ratio": 32.5,
"optimized_suggestion": "Excellent profile."
}
print("STARTING COMPREHENSIVE LLM VERIFICATION")
# 1. Test every model in the list
results = []
for m in LLM_MODELS:
results.append(test_single_model(m, sample_packet))
# 2. Test the fallback mechanism
test_fallback_logic(sample_packet)
print("\nVERIFICATION COMPLETE")
|