Spaces:
Sleeping
Sleeping
| import os | |
| import sys | |
| from dotenv import load_dotenv | |
| from openai import OpenAI | |
| # Add project root to path | |
| sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) | |
| from services.llm_advisor import generate_explanation, LLM_MODELS, SYSTEM_PROMPT | |
| load_dotenv() | |
| def test_single_model(model_name, packet): | |
| print(f"\n--- Testing Model: {model_name} ---") | |
| api_key = os.getenv("NVIDIA_API_KEY") | |
| client = OpenAI(base_url="https://integrate.api.nvidia.com/v1", api_key=api_key) | |
| user_content = f"Result: {packet['prediction']}, Confidence: {packet['confidence']}, DTI: {packet['dti_ratio']}" | |
| try: | |
| completion = client.chat.completions.create( | |
| model=model_name, | |
| messages=[ | |
| {"role": "system", "content": SYSTEM_PROMPT}, | |
| {"role": "user", "content": user_content} | |
| ], | |
| temperature=0.1, | |
| max_tokens=512 | |
| ) | |
| content = completion.choices[0].message.content | |
| print(f"Response received ({len(content)} chars)") | |
| if "@@POINT@@" in content: | |
| print("SUCCESS: Output contains required @@POINT@@ delimiters.") | |
| else: | |
| print("FAILURE: Output missing @@POINT@@ delimiters.") | |
| return True | |
| except Exception as e: | |
| print(f"ERROR: Model {model_name} failed: {e}") | |
| return False | |
| def test_fallback_logic(packet): | |
| print("\n--- Testing Full Fallback Workflow ---") | |
| # Forcing a failure by temporarily corrupting the model list in memory | |
| original_models = list(LLM_MODELS) | |
| # Inject a fake non-existent model at the start | |
| import services.llm_advisor as advisor | |
| advisor.LLM_MODELS = ["invalid/non-existent-model-1234"] + original_models | |
| print("Injected invalid model. Expecting fail-over to the first valid model...") | |
| response = advisor.generate_explanation(packet) | |
| if "@@POINT@@" in response: | |
| print("SUCCESS: Fallback logic correctly skipped the invalid model and used a secondary model.") | |
| else: | |
| print("FAILURE: Fallback logic did not return a valid AI summary.") | |
| # Reset models | |
| advisor.LLM_MODELS = original_models | |
| if __name__ == "__main__": | |
| sample_packet = { | |
| "prediction": "Y", | |
| "confidence": 0.88, | |
| "dti_ratio": 32.5, | |
| "optimized_suggestion": "Excellent profile." | |
| } | |
| print("STARTING COMPREHENSIVE LLM VERIFICATION") | |
| # 1. Test every model in the list | |
| results = [] | |
| for m in LLM_MODELS: | |
| results.append(test_single_model(m, sample_packet)) | |
| # 2. Test the fallback mechanism | |
| test_fallback_logic(sample_packet) | |
| print("\nVERIFICATION COMPLETE") | |