""" Scenario data for Stack Doctor. Each scenario encodes a hidden root cause, the correct fix, an incident ticket, hardware/model/backend context, log and code snippets, and specialist opinions (some of which may be wrong). """ from __future__ import annotations import json import os import random from dataclasses import dataclass, field ROOT_CAUSES = [ "arch_guard", "backend_whitelist", "runtime_loader", "backend_selector", "model_config", "weight_layout", "memory_oom", "quantization_error", "distributed_comm", "driver_compat", ] FIXES = [ "relax_arch_check", "add_whitelist_entry", "fix_runtime_path", "switch_backend", "update_model_config", "fix_weight_mapping", "tune_memory_config", "fix_quantization", "fix_comm_config", "update_driver_config", ] # 1:1 mapping ROOT_CAUSE_TO_FIX = dict(zip(ROOT_CAUSES, FIXES)) FIX_TO_ROOT_CAUSE = {v: k for k, v in ROOT_CAUSE_TO_FIX.items()} SPECIALISTS = ["runtime", "dispatch", "kernel", "loader"] HARDWARE_OPTIONS = [ "NVIDIA SM121 (DGX Spark)", "NVIDIA SM120 (GeForce RTX 5090)", "AMD MI300X", "AMD MI355X", "NVIDIA H100", "NVIDIA B200", ] MODEL_OPTIONS = [ "DeepSeek-V3-671B", "Llama-4-Maverick-17Bx128E", "Qwen3-235B-A22B", "Mistral-Large-2", "DeepSeek-R1-Distill-70B", "Llama-3.3-70B-Instruct", ] BACKEND_OPTIONS = [ "vLLM 0.8.x", "SGLang 0.5.x", "TensorRT-LLM 0.18", "FlashInfer 0.4", "Triton Inference Server", ] @dataclass class SpecialistOpinion: opinion: str confidence: float is_correct: bool @dataclass class InspectResult: logs: str config: str snippet: str metrics: str @dataclass class Scenario: id: str root_cause: str correct_fix: str incident_ticket: str hardware: str model_name: str backend: str initial_log: str initial_snippet: str specialist_opinions: dict[str, SpecialistOpinion] inspect_results: InspectResult # For ask_specialist follow-ups specialist_followups: dict[str, str] = field(default_factory=dict) # --------------------------------------------------------------------------- # Seed scenarios # --------------------------------------------------------------------------- def _make_scenarios() -> list[Scenario]: scenarios = [] # --- arch_guard scenarios --- scenarios.append(Scenario( id="arch_guard_01", root_cause="arch_guard", correct_fix="relax_arch_check", incident_ticket=( "INCIDENT: FlashInfer attention kernel fails to launch on newly provisioned " "DGX Spark nodes. Error: 'Unsupported GPU architecture sm_121'. " "Identical model config works on H100 nodes." ), hardware="NVIDIA SM121 (DGX Spark)", model_name="DeepSeek-V3-671B", backend="FlashInfer 0.4", initial_log=( "[FlashInfer] Checking GPU capability... sm_121 detected\n" "[FlashInfer] ERROR: is_supported_arch() returned False for sm_121\n" "[FlashInfer] Falling back to... no fallback available\n" "RuntimeError: No compatible attention kernel for architecture sm_121" ), initial_snippet=( "# flashinfer/arch_check.py\n" "SUPPORTED_ARCHS = {70, 75, 80, 86, 89, 90}\n" "\n" "def is_supported_arch(cc: int) -> bool:\n" " return cc in SUPPORTED_ARCHS" ), specialist_opinions={ "runtime": SpecialistOpinion( "CUDA runtime loaded successfully. No runtime issues detected.", 0.85, False ), "dispatch": SpecialistOpinion( "Architecture check is blocking kernel dispatch. The SM121 architecture " "is not in the supported set despite being SM90-compatible at the instruction level.", 0.92, True ), "kernel": SpecialistOpinion( "The HMMA m16n8k16 instructions used by the attention kernel are available on SM121. " "This looks like a capability check issue, not a kernel issue.", 0.88, True ), "loader": SpecialistOpinion( "Model weights loaded correctly. Weight layout is standard.", 0.80, False ), }, inspect_results=InspectResult( logs=( "[FlashInfer] GPU: NVIDIA GH200 (sm_121)\n" "[FlashInfer] CUDA version: 13.0\n" "[FlashInfer] is_supported_arch(121) = False\n" "[FlashInfer] Architecture check FAILED\n" "[CUDA] All CUDA operations nominal\n" "[System] GPU memory: 96GB available" ), config=( "gpu_architecture: sm_121\n" "cuda_version: 13.0\n" "flashinfer_version: 0.4.1\n" "attention_backend: flashinfer\n" "supported_archs: [70, 75, 80, 86, 89, 90]" ), snippet=( "# The arch check function uses an exact match:\n" "def is_supported_arch(cc):\n" " return cc in SUPPORTED_ARCHS # misses sm_12x family\n\n" "# SM121 supports HMMA m16n8k16 (same as SM90)\n" "# but is not in the allowlist" ), metrics=( "kernel_launch_attempts: 47\n" "kernel_launch_failures: 47\n" "fallback_attempts: 47\n" "fallback_failures: 47\n" "gpu_utilization: 0%" ), ), specialist_followups={ "runtime": "I confirmed CUDA 13.0 runtime is functional. All driver calls succeed. This isn't a runtime issue.", "dispatch": "The dispatch table maps arch -> kernel. SM121 has no entry. Adding sm_12x family to the arch check should fix it.", "kernel": "I inspected the PTX. The kernel only needs HMMA m16n8k16 which SM121 supports. The kernel itself is fine.", "loader": "Weights are in the expected layout. No loader issues.", }, )) scenarios.append(Scenario( id="arch_guard_02", root_cause="arch_guard", correct_fix="relax_arch_check", incident_ticket=( "INCIDENT: MLA attention fails on GeForce RTX 5090. Error: " "'compute capability 120 not supported'. Customer reports RTX 4090 works fine." ), hardware="NVIDIA SM120 (GeForce RTX 5090)", model_name="DeepSeek-R1-Distill-70B", backend="vLLM 0.8.x", initial_log=( "[vLLM] Detecting GPU... GeForce RTX 5090 (sm_120)\n" "[vLLM] FlashAttention: compute capability 120 not in supported list\n" "[vLLM] ERROR: Cannot initialize attention backend" ), initial_snippet=( "# vllm/attention/backends/flash_attn.py\n" "MIN_CC = 80\n" "MAX_CC = 90\n" "\n" "def is_supported(cc: int) -> bool:\n" " return MIN_CC <= cc <= MAX_CC" ), specialist_opinions={ "runtime": SpecialistOpinion("Runtime is fine. CUDA 13 loaded.", 0.75, False), "dispatch": SpecialistOpinion( "The capability range check excludes SM120. Needs to include SM12x family.", 0.90, True ), "kernel": SpecialistOpinion( "Possible kernel incompatibility — SM120 lacks tcgen05 MMA.", 0.60, False ), "loader": SpecialistOpinion("Weights look fine.", 0.70, False), }, inspect_results=InspectResult( logs="[vLLM] GPU cc=120 rejected by range [80,90]\n[vLLM] No fallback attention backend", config="compute_capability: 120\nmax_supported_cc: 90\nattention_backend: flash_attn", snippet="# Range check: MIN_CC(80) <= cc <= MAX_CC(90)\n# SM120 = 120 > 90, so rejected\n# Fix: add sm_12x family check", metrics="attention_init_failures: 1\nmodel_load_time: 0s (blocked at init)", ), specialist_followups={ "runtime": "CUDA 13.0 runtime is healthy. Driver version matches.", "dispatch": "SM120 uses HMMA path (no warp specialization), same code path as SM86. Just need to update the arch range.", "kernel": "On closer inspection, SM120 does support the needed HMMA instructions. My earlier concern about tcgen05 was wrong — that's only needed for Hopper-style warp specialization.", "loader": "No weight issues detected.", }, )) # --- backend_whitelist scenarios --- scenarios.append(Scenario( id="backend_whitelist_01", root_cause="backend_whitelist", correct_fix="add_whitelist_entry", incident_ticket=( "INCIDENT: Marlin quantized inference crashes on SM121 nodes. " "Error: 'Marlin kernel not available for current GPU'. " "FP16 inference works, only quantized (GPTQ/AWQ) path fails." ), hardware="NVIDIA SM121 (DGX Spark)", model_name="Llama-3.3-70B-Instruct", backend="vLLM 0.8.x", initial_log=( "[vLLM] Loading GPTQ-quantized model...\n" "[vLLM] Checking Marlin kernel availability for sm_121\n" "[vLLM] WARNING: GPU sm_121 not in Marlin whitelist\n" "[vLLM] ERROR: No quantization kernel available" ), initial_snippet=( "# vllm/model_executor/layers/quantization/marlin.py\n" "MARLIN_SUPPORTED_GPUS = [\n" " 'A100', 'A10', 'H100', 'L40', 'RTX 4090',\n" "]\n" ), specialist_opinions={ "runtime": SpecialistOpinion("CUDA runtime OK. Libraries loaded.", 0.80, False), "dispatch": SpecialistOpinion( "Marlin whitelist doesn't include SM121 GPU names. Need to add the entry.", 0.91, True ), "kernel": SpecialistOpinion( "Marlin kernels use standard HMMA ops that SM121 supports. It's just not whitelisted.", 0.85, True ), "loader": SpecialistOpinion( "Quantized weights loaded but kernel never launches. Might be a weight format issue.", 0.55, False ), }, inspect_results=InspectResult( logs="[Marlin] GPU name 'NVIDIA GH200' not in whitelist\n[Marlin] Whitelist: ['A100','A10','H100','L40','RTX 4090']", config="quantization: gptq\nmarlin_whitelist: [A100, A10, H100, L40, RTX 4090]\ngpu_name: NVIDIA GH200", snippet="# Whitelist check uses GPU product name string matching\n# GH200 / DGX Spark not in the list\n# Should use arch family check instead of name matching", metrics="quantized_kernel_attempts: 1\nquantized_kernel_failures: 1\nfp16_fallback: not_attempted", ), specialist_followups={ "runtime": "All good on the runtime side.", "dispatch": "The whitelist is name-based, not arch-based. Adding 'GH200' or switching to family-level arch checks fixes this.", "kernel": "The Marlin FP8 GEMM dispatch works with SM121's MMA units. It's purely a whitelist gap.", "loader": "Actually, the weights loaded fine. I retract my earlier concern.", }, )) scenarios.append(Scenario( id="backend_whitelist_02", root_cause="backend_whitelist", correct_fix="add_whitelist_entry", incident_ticket=( "INCIDENT: AWQ quantization backend refuses to initialize on MI300X. " "Error: 'GPU not supported for AWQ acceleration'. " "Other backends work fine on the same hardware." ), hardware="AMD MI300X", model_name="Qwen3-235B-A22B", backend="vLLM 0.8.x", initial_log=( "[vLLM] Initializing AWQ backend...\n" "[vLLM] GPU: AMD Instinct MI300X\n" "[vLLM] AWQ: GPU not in supported devices list\n" "[vLLM] ERROR: AWQ acceleration unavailable" ), initial_snippet=( "# vllm/model_executor/layers/quantization/awq.py\n" "AWQ_SUPPORTED = {'A100', 'H100', 'RTX 4090', 'L40S'}\n" ), specialist_opinions={ "runtime": SpecialistOpinion("ROCm runtime healthy. HIP version matches.", 0.82, False), "dispatch": SpecialistOpinion( "AWQ whitelist is NVIDIA-only. MI300X needs to be added.", 0.93, True ), "kernel": SpecialistOpinion( "MI300X has MFMA instructions that can handle the AWQ GEMM. Not a kernel issue.", 0.87, True ), "loader": SpecialistOpinion("Weight format might not match AMD layout expectations.", 0.50, False), }, inspect_results=InspectResult( logs="[AWQ] Device 'AMD Instinct MI300X' not in AWQ_SUPPORTED\n[AWQ] Supported: A100, H100, RTX 4090, L40S", config="quantization: awq\nawq_supported: [A100, H100, RTX 4090, L40S]\ngpu: AMD Instinct MI300X", snippet="# AWQ_SUPPORTED only lists NVIDIA GPUs\n# MI300X MFMA f32_32x32x8_f16 can handle AWQ ops\n# Need to add MI300X to whitelist", metrics="awq_init_failures: 1\nfallback_to_fp16: pending", ), specialist_followups={ "runtime": "ROCm 6.3 loaded successfully. No runtime concerns.", "dispatch": "Simple whitelist gap. Adding MI300X resolves the issue.", "kernel": "Confirmed: MFMA ops on MI300X handle the AWQ GEMM pattern.", "loader": "I was wrong earlier — weights are fine. It's the whitelist.", }, )) # --- runtime_loader scenarios --- scenarios.append(Scenario( id="runtime_loader_01", root_cause="runtime_loader", correct_fix="fix_runtime_path", incident_ticket=( "INCIDENT: SGLang server crashes on startup with CUDA 13 on DGX Spark. " "Error: 'libcudart.so.13: cannot open shared object file'. " "System has CUDA 13 installed but SGLang can't find it." ), hardware="NVIDIA SM121 (DGX Spark)", model_name="Llama-4-Maverick-17Bx128E", backend="SGLang 0.5.x", initial_log=( "[SGLang] Starting server...\n" "[SGLang] Loading CUDA runtime...\n" "[SGLang] ERROR: libcudart.so.13: cannot open shared object file\n" "[SGLang] LD_LIBRARY_PATH=/usr/local/cuda-12/lib64\n" "ImportError: CUDA runtime not found" ), initial_snippet=( "# sglang/startup.py\n" "CUDA_LIB_PATH = os.environ.get(\n" " 'CUDA_HOME', '/usr/local/cuda'\n" ") + '/lib64'\n" "# Hardcoded to cuda, not cuda-13\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "CUDA 13 is installed at /usr/local/cuda-13 but LD_LIBRARY_PATH points to cuda-12. " "The runtime path needs to be updated.", 0.95, True ), "dispatch": SpecialistOpinion("Can't tell — server never gets to dispatch phase.", 0.40, False), "kernel": SpecialistOpinion("No kernel issue — server crashes before kernel init.", 0.60, False), "loader": SpecialistOpinion( "The CUDA shared library loader can't find libcudart.so.13. Path issue.", 0.88, True ), }, inspect_results=InspectResult( logs=( "[System] CUDA installations:\n" " /usr/local/cuda-12 -> CUDA 12.4\n" " /usr/local/cuda-13 -> CUDA 13.0\n" " /usr/local/cuda -> symlink to cuda-12\n" "[SGLang] Trying to load libcudart.so.13 from /usr/local/cuda/lib64 -> NOT FOUND" ), config="CUDA_HOME=/usr/local/cuda\nLD_LIBRARY_PATH=/usr/local/cuda-12/lib64\ncuda_13_path=/usr/local/cuda-13", snippet="# /usr/local/cuda symlinks to cuda-12\n# Need: export CUDA_HOME=/usr/local/cuda-13\n# Or: update symlink", metrics="server_start_attempts: 3\nserver_start_failures: 3\nuptime: 0s", ), specialist_followups={ "runtime": "Confirmed: /usr/local/cuda symlink targets cuda-12. CUDA 13 is at /usr/local/cuda-13. Fix the path.", "dispatch": "Server never started, so I can't diagnose dispatch.", "kernel": "Same — no kernel loaded.", "loader": "The dynamic linker searches LD_LIBRARY_PATH first. It needs /usr/local/cuda-13/lib64.", }, )) scenarios.append(Scenario( id="runtime_loader_02", root_cause="runtime_loader", correct_fix="fix_runtime_path", incident_ticket=( "INCIDENT: ROCm HIP runtime fails to initialize on MI300X cluster. " "Error: 'hipErrorNoDevice' despite GPUs being visible in lspci. " "Worked yesterday before system update." ), hardware="AMD MI300X", model_name="DeepSeek-V3-671B", backend="vLLM 0.8.x", initial_log=( "[HIP] Initializing runtime...\n" "[HIP] ERROR: hipErrorNoDevice (code 100)\n" "[System] lspci shows 8x AMD Instinct MI300X\n" "[System] /opt/rocm -> /opt/rocm-6.2 (outdated symlink)" ), initial_snippet=( "# environment setup\n" "ROCM_PATH=/opt/rocm # symlinks to rocm-6.2\n" "# But rocm-6.3 installed at /opt/rocm-6.3\n" "# Driver expects rocm-6.3 runtime\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "ROCm path mismatch. /opt/rocm points to 6.2 but driver needs 6.3 runtime.", 0.94, True ), "dispatch": SpecialistOpinion("Not a dispatch issue — runtime doesn't initialize.", 0.70, False), "kernel": SpecialistOpinion("Might be a kernel module issue with the GPU driver.", 0.45, False), "loader": SpecialistOpinion("ROCm shared libraries at wrong version.", 0.80, True), }, inspect_results=InspectResult( logs="[System] /opt/rocm -> /opt/rocm-6.2\n[System] Driver version: 6.3.0\n[HIP] Runtime version mismatch: expected 6.3, found 6.2", config="ROCM_PATH=/opt/rocm\nrocm_symlink_target=/opt/rocm-6.2\ninstalled_versions: [6.2, 6.3]\ndriver_version: 6.3.0", snippet="# The system was updated and ROCm 6.3 driver installed\n# But /opt/rocm symlink still points to 6.2\n# Fix: ln -sf /opt/rocm-6.3 /opt/rocm", metrics="gpu_init_failures: 8\ndriver_version: 6.3.0\nruntime_version: 6.2.0", ), specialist_followups={ "runtime": "Classic version mismatch after system update. Fix the symlink to point to rocm-6.3.", "dispatch": "Can't assess dispatch without a working runtime.", "kernel": "I was wrong — it's not a kernel module issue. The GPU driver is fine, it's the userspace runtime path.", "loader": "The shared library loader finds rocm-6.2 libs but driver expects 6.3. Path fix needed.", }, )) # --- backend_selector scenarios --- scenarios.append(Scenario( id="backend_selector_01", root_cause="backend_selector", correct_fix="switch_backend", incident_ticket=( "INCIDENT: Extreme latency (10x expected) on H100 serving Llama-3.3-70B. " "No errors, just very slow. GPU utilization looks low. " "Other models on the same node are fast." ), hardware="NVIDIA H100", model_name="Llama-3.3-70B-Instruct", backend="vLLM 0.8.x", initial_log=( "[vLLM] Selected attention backend: xformers\n" "[vLLM] WARNING: FlashAttention v2 not selected (override with VLLM_ATTENTION_BACKEND)\n" "[vLLM] Serving Llama-3.3-70B-Instruct...\n" "[vLLM] p99 latency: 4200ms (expected: ~400ms)" ), initial_snippet=( "# vllm/attention/selector.py\n" "def get_attention_backend(model_config):\n" " if model_config.head_dim not in [64, 128]:\n" " return 'xformers' # fallback\n" " return 'flash_attn'\n" ), specialist_opinions={ "runtime": SpecialistOpinion("CUDA runtime is fine. No errors.", 0.75, False), "dispatch": SpecialistOpinion( "Wrong attention backend selected. xformers is much slower than FlashAttention on H100. " "The backend selector has a bug in head_dim detection.", 0.94, True ), "kernel": SpecialistOpinion( "The xformers kernel is correct but suboptimal for H100. Should use flash_attn.", 0.82, True ), "loader": SpecialistOpinion("Model loaded correctly. Not a weight issue.", 0.80, False), }, inspect_results=InspectResult( logs="[vLLM] head_dim=128, num_heads=64\n[vLLM] Backend selection: model reports head_dim=None (config missing) -> fallback to xformers", config="attention_backend: xformers (auto-selected)\nmodel_head_dim: null\nactual_head_dim: 128\ngpu: H100", snippet="# The model config doesn't explicitly set head_dim\n# Selector falls back to xformers when head_dim is None\n# Should infer head_dim from hidden_size / num_heads", metrics="p50_latency_ms: 3100\np99_latency_ms: 4200\ngpu_utilization: 12%\nexpected_gpu_util: 85%", ), specialist_followups={ "runtime": "No runtime issues. The server is running, just slow.", "dispatch": "Backend selector bug: head_dim is None in model config, causing xformers fallback. Switch to flash_attn.", "kernel": "xformers works but doesn't use H100 TMA/warp specialization. flash_attn v2 would be 8-10x faster.", "loader": "Weights loaded correctly.", }, )) scenarios.append(Scenario( id="backend_selector_02", root_cause="backend_selector", correct_fix="switch_backend", incident_ticket=( "INCIDENT: FP8 inference on MI300X producing garbage output. " "Model loads, tokens generate, but output is nonsensical. " "BF16 inference on same hardware works perfectly." ), hardware="AMD MI300X", model_name="Mistral-Large-2", backend="vLLM 0.8.x", initial_log=( "[vLLM] FP8 quantization: e4m3fn format selected\n" "[vLLM] WARNING: MI300X uses e4m3fnuz format, not e4m3fn\n" "[vLLM] Serving with FP8...\n" "[vLLM] Output quality check: FAIL (perplexity 847.3, expected <15)" ), initial_snippet=( "# vllm/quantization/fp8.py\n" "FP8_FORMAT = 'e4m3fn' # NVIDIA default\n" "# AMD MI300X needs e4m3fnuz (no NaN, unsigned zero)\n" ), specialist_opinions={ "runtime": SpecialistOpinion("ROCm runtime is healthy.", 0.80, False), "dispatch": SpecialistOpinion( "Wrong FP8 format selected. MI300X uses e4m3fnuz, not e4m3fn. " "The backend selector should detect AMD and switch format.", 0.93, True ), "kernel": SpecialistOpinion( "The GEMM kernel runs but produces wrong results due to format mismatch.", 0.85, True ), "loader": SpecialistOpinion( "Weight dequantization might be wrong for AMD FP8 format.", 0.65, False ), }, inspect_results=InspectResult( logs="[FP8] Using e4m3fn format\n[FP8] AMD GPU detected but format not switched\n[FP8] Numerical errors in first GEMM", config="fp8_format: e4m3fn\ngpu_vendor: AMD\nexpected_format: e4m3fnuz\nformat_mismatch: true", snippet="# e4m3fn: 1 sign, 4 exp, 3 mantissa, has NaN encoding\n# e4m3fnuz: 1 sign, 4 exp, 3 mantissa, NO NaN, unsigned zero\n# Bit patterns interpreted differently -> garbage output", metrics="output_perplexity: 847.3\nexpected_perplexity: 12.5\ngemm_numerical_errors: 100%", ), specialist_followups={ "runtime": "ROCm fine. This is a numerical issue, not runtime.", "dispatch": "Switch the FP8 format selector to use e4m3fnuz for AMD GPUs. Clear fix.", "kernel": "The kernel math is correct for the format it's given — the problem is the format itself.", "loader": "Actually, weights are fine. The issue is at the GEMM dispatch level.", }, )) # --- model_config scenarios --- scenarios.append(Scenario( id="model_config_01", root_cause="model_config", correct_fix="update_model_config", incident_ticket=( "INCIDENT: DeepSeek-V3 MoE routing crashes with shape mismatch. " "Error: 'Expected expert count 256, got 160'. " "Model just updated to new checkpoint, was working before." ), hardware="NVIDIA H100", model_name="DeepSeek-V3-671B", backend="SGLang 0.5.x", initial_log=( "[SGLang] Loading DeepSeek-V3-671B...\n" "[SGLang] MoE config: num_experts=256 (from config.json)\n" "[SGLang] Actual weight shape: experts.0-159\n" "[SGLang] ERROR: Shape mismatch in MoE layer: expected 256 experts, found 160" ), initial_snippet=( "# config.json (model repo)\n" '{\n' ' "num_local_experts": 256,\n' ' "num_experts_per_tok": 8,\n' ' "intermediate_size": 2048\n' '}\n' "# But actual checkpoint has 160 experts\n" ), specialist_opinions={ "runtime": SpecialistOpinion("Runtime is fine. Model loading proceeds until shape error.", 0.75, False), "dispatch": SpecialistOpinion("Not a dispatch bug — the model config is wrong.", 0.70, False), "kernel": SpecialistOpinion( "MoE kernel expects expert count from config. Config says 256 but weights have 160. " "Config needs to be updated to match the new checkpoint.", 0.90, True ), "loader": SpecialistOpinion( "The model config doesn't match the checkpoint. num_local_experts should be 160.", 0.92, True ), }, inspect_results=InspectResult( logs="[SGLang] config.json: num_local_experts=256\n[SGLang] checkpoint expert layers: 160\n[SGLang] Mismatch detected at layer 0", config="num_local_experts: 256 (config)\nactual_experts: 160 (checkpoint)\nnum_experts_per_tok: 8\ncheckpoint_version: v3.1", snippet="# New checkpoint v3.1 reduced experts from 256 to 160\n# But config.json wasn't updated\n# Fix: set num_local_experts=160 in config.json", metrics="model_load_progress: 15%\nlayers_loaded: 0/60\nerror_at: moe_layer_0", ), specialist_followups={ "runtime": "No runtime issue. Pure config mismatch.", "dispatch": "Dispatch looks fine. The error is before dispatch even runs.", "kernel": "The grouped GEMM kernel allocates buffers based on config expert count. Fix the config.", "loader": "Config.json says 256 experts but the v3.1 checkpoint only has 160. Update the config.", }, )) scenarios.append(Scenario( id="model_config_02", root_cause="model_config", correct_fix="update_model_config", incident_ticket=( "INCIDENT: Qwen3 MoE model gives wrong results after hardware migration. " "Output is coherent but factually wrong. " "Same model on old cluster was correct." ), hardware="NVIDIA B200", model_name="Qwen3-235B-A22B", backend="vLLM 0.8.x", initial_log=( "[vLLM] Loading Qwen3-235B-A22B...\n" "[vLLM] Config: rope_theta=1000000.0\n" "[vLLM] WARNING: RoPE scaling config missing for extended context\n" "[vLLM] Serving... output quality degraded at positions > 4096" ), initial_snippet=( "# config.json\n" '{\n' ' "rope_theta": 1000000.0,\n' ' "max_position_embeddings": 32768\n' ' // Missing: rope_scaling config for YaRN\n' '}\n' ), specialist_opinions={ "runtime": SpecialistOpinion("Runtime fine. No crashes.", 0.80, False), "dispatch": SpecialistOpinion("Backend selected correctly.", 0.65, False), "kernel": SpecialistOpinion( "RoPE computation looks standard. Config might be missing the scaling parameters.", 0.78, True ), "loader": SpecialistOpinion( "Model config is incomplete — missing rope_scaling section for YaRN. " "Old cluster had a patched config.", 0.91, True ), }, inspect_results=InspectResult( logs="[vLLM] RoPE: theta=1e6, no scaling applied\n[vLLM] Quality degrades > 4096 tokens\n[vLLM] Old cluster config had rope_scaling: {type: yarn, factor: 4}", config="rope_theta: 1000000.0\nrope_scaling: null\nmax_position_embeddings: 32768\nold_config_had: {rope_scaling: {type: yarn, factor: 4}}", snippet="# Missing rope_scaling config:\n# rope_scaling: {type: 'yarn', factor: 4, ...}\n# Without it, positions > 4096 are garbage", metrics="quality_0_4k: 95%\nquality_4k_8k: 43%\nquality_8k_plus: 12%", ), specialist_followups={ "runtime": "No runtime issues.", "dispatch": "Backend is correct. Not a dispatch issue.", "kernel": "The RoPE kernel is fine — it just doesn't have the scaling config to apply YaRN.", "loader": "The config.json from the model repo is missing rope_scaling. Add it back.", }, )) # --- weight_layout scenarios --- scenarios.append(Scenario( id="weight_layout_01", root_cause="weight_layout", correct_fix="fix_weight_mapping", incident_ticket=( "INCIDENT: Model produces random output after converting weights from " "HuggingFace format to TensorRT-LLM format. Conversion reported success " "but inference output is gibberish." ), hardware="NVIDIA H100", model_name="Llama-3.3-70B-Instruct", backend="TensorRT-LLM 0.18", initial_log=( "[TRT-LLM] Loading converted weights...\n" "[TRT-LLM] Weight shapes match expected layout\n" "[TRT-LLM] Running inference...\n" "[TRT-LLM] Output: 'asdfjkl; the the the purple 2847...'\n" "[TRT-LLM] Perplexity: 2341.7 (expected < 10)" ), initial_snippet=( "# convert_weights.py\n" "# gate_proj and up_proj were swapped during conversion\n" "mapping = {\n" " 'gate_proj': 'linear_fc1_gate',\n" " 'up_proj': 'linear_fc1_up',\n" "}\n" "# TRT-LLM expects opposite order\n" ), specialist_opinions={ "runtime": SpecialistOpinion("Runtime and engine init successful. No errors.", 0.80, False), "dispatch": SpecialistOpinion("Backend dispatch is correct. TRT engine built fine.", 0.70, False), "kernel": SpecialistOpinion( "Kernels execute without error. This is a data issue, not compute.", 0.75, False ), "loader": SpecialistOpinion( "Weight mapping is wrong. gate_proj and up_proj are swapped in the conversion script. " "TRT-LLM expects the opposite order.", 0.94, True ), }, inspect_results=InspectResult( logs="[TRT-LLM] Weight conversion: gate_proj -> linear_fc1_gate, up_proj -> linear_fc1_up\n[TRT-LLM] Expected: gate_proj -> linear_fc1_up, up_proj -> linear_fc1_gate", config="weight_mapping:\n gate_proj: linear_fc1_gate # WRONG\n up_proj: linear_fc1_up # WRONG\n # Should be swapped", snippet="# TRT-LLM MLP layout: [up_proj; gate_proj] concatenated\n# But converter wrote [gate_proj; up_proj]\n# Result: SiLU applied to wrong half", metrics="output_perplexity: 2341.7\nexpected_perplexity: 8.2\nweight_shapes: correct\nweight_values: misaligned", ), specialist_followups={ "runtime": "Engine runs fine. Not a runtime issue.", "dispatch": "TRT engine dispatch is correct.", "kernel": "Compute is correct for the data it gets. Fix the data (weights).", "loader": "Classic weight mapping bug. Swap gate_proj and up_proj in the conversion mapping.", }, )) scenarios.append(Scenario( id="weight_layout_02", root_cause="weight_layout", correct_fix="fix_weight_mapping", incident_ticket=( "INCIDENT: QKV attention weights transposed incorrectly for GQA model. " "Attention scores are wrong — model generates repetitive text. " "Happened after switching from MHA to GQA config." ), hardware="AMD MI300X", model_name="Llama-4-Maverick-17Bx128E", backend="FlashInfer 0.4", initial_log=( "[FlashInfer] GQA mode: 64 query heads, 8 KV heads\n" "[FlashInfer] WARNING: QKV projection weight shape unexpected\n" "[FlashInfer] Expected Q:[8192,8192] K:[8192,1024] V:[8192,1024]\n" "[FlashInfer] Got Q:[8192,8192] K:[8192,8192] V:[8192,1024]\n" "[FlashInfer] Repetitive output detected" ), initial_snippet=( "# weight_converter.py\n" "# GQA: Q has num_heads, K/V have num_kv_heads\n" "q_proj = weights['q_proj'] # [8192, 8192] correct\n" "k_proj = weights['q_proj'] # BUG: should be 'k_proj'\n" "v_proj = weights['v_proj'] # [8192, 1024] correct\n" ), specialist_opinions={ "runtime": SpecialistOpinion("ROCm runtime fine.", 0.75, False), "dispatch": SpecialistOpinion("FlashInfer dispatch selected GQA path correctly.", 0.70, False), "kernel": SpecialistOpinion( "GQA attention kernel is correct but K weights are wrong shape. " "Looks like Q weights loaded twice instead of K.", 0.88, True ), "loader": SpecialistOpinion( "Weight mapping bug: k_proj loaded from q_proj key. Copy-paste error in converter.", 0.95, True ), }, inspect_results=InspectResult( logs="[FlashInfer] K weight shape [8192,8192] != expected [8192,1024]\n[FlashInfer] K weights appear identical to Q weights\n[FlashInfer] This causes attention to compute Q*Q^T instead of Q*K^T", config="num_query_heads: 64\nnum_kv_heads: 8\nhead_dim: 128\nq_shape: [8192,8192]\nk_shape: [8192,8192] # WRONG\nv_shape: [8192,1024]", snippet="# Bug in weight_converter.py line 47:\n# k_proj = weights['q_proj'] # should be weights['k_proj']\n# Result: K = Q, so attention = softmax(Q @ Q^T) -> repetitive", metrics="attention_entropy: 0.03 (expected > 2.0)\nrepetition_rate: 94%\nperplexity: 567.8", ), specialist_followups={ "runtime": "No runtime problems.", "dispatch": "GQA dispatch path is correct for this model.", "kernel": "Attention kernel computes correctly for the data given. K weights are just wrong.", "loader": "Line 47 has `weights['q_proj']` instead of `weights['k_proj']`. Classic copy-paste bug.", }, )) # --- arch_guard additional scenarios --- scenarios.append(Scenario( id="arch_guard_03", root_cause="arch_guard", correct_fix="relax_arch_check", incident_ticket=( "INCIDENT: TensorRT-LLM refuses to build engine for B200 GPU. " "Error: 'Unsupported compute capability 120'. " "Same model builds fine targeting H100." ), hardware="NVIDIA B200", model_name="Qwen3-235B-A22B", backend="TensorRT-LLM 0.18", initial_log=( "[TRT-LLM] Building engine for gpu_arch=sm_120...\n" "[TRT-LLM] ERROR: Compute capability 120 not in supported set\n" "[TRT-LLM] Supported: {70, 75, 80, 86, 89, 90}" ), initial_snippet=( "# tensorrt_llm/builder.py\n" "SUPPORTED_SM = {70, 75, 80, 86, 89, 90}\n" "if sm not in SUPPORTED_SM:\n" " raise UnsupportedGPU(f'sm_{sm}')" ), specialist_opinions={ "runtime": SpecialistOpinion("CUDA 13 runtime loaded fine.", 0.78, False), "dispatch": SpecialistOpinion( "Architecture guard rejects sm_120. B200 uses Blackwell arch not in the allowlist.", 0.91, True ), "kernel": SpecialistOpinion( "Try switching to a different quantization scheme for B200.", 0.45, False ), "loader": SpecialistOpinion("No weight loading attempted yet — blocked at engine build.", 0.72, False), }, inspect_results=InspectResult( logs="[TRT-LLM] sm_120 not in {70,75,80,86,89,90}\n[TRT-LLM] Engine build aborted before weight conversion", config="target_gpu: sm_120\nsupported_sm: [70,75,80,86,89,90]\nbuilder_version: 0.18.0", snippet="# B200 (sm_120) supports FP8 MMA, BF16 HMMA\n# Same instruction set as H100 for inference\n# Just not in the allowlist", metrics="engine_build_attempts: 1\nengine_build_failures: 1\nmodel_loaded: false", ), specialist_followups={ "runtime": "Runtime is fine. Engine builder is the blocker.", "dispatch": "Add sm_120 (and sm_12x family) to SUPPORTED_SM. The instructions are compatible.", "kernel": "On reflection, quantization scheme isn't the issue. It's the arch check.", "loader": "Can't load weights until engine builds.", }, )) scenarios.append(Scenario( id="arch_guard_04", root_cause="arch_guard", correct_fix="relax_arch_check", incident_ticket=( "INCIDENT: Flash-Attention fwd pass returns CUDA error on MI355X. " "Error: 'Unsupported AMD GPU architecture'. " "MI300X works fine with same code." ), hardware="AMD MI355X", model_name="Llama-3.3-70B-Instruct", backend="vLLM 0.8.x", initial_log=( "[Flash-Attn] Checking GPU: AMD Instinct MI355X (gfx950)\n" "[Flash-Attn] Supported AMD archs: [gfx90a, gfx942]\n" "[Flash-Attn] ERROR: gfx950 not supported" ), initial_snippet=( "# flash_attn/amd_check.py\n" "AMD_SUPPORTED = ['gfx90a', 'gfx942']\n" "if gpu_arch not in AMD_SUPPORTED:\n" " raise RuntimeError(f'{gpu_arch} not supported')" ), specialist_opinions={ "runtime": SpecialistOpinion("ROCm 6.4 runtime operational.", 0.80, False), "dispatch": SpecialistOpinion( "gfx950 (MI355X/CDNA4) isn't in the AMD arch allowlist. Needs to be added.", 0.92, True ), "kernel": SpecialistOpinion( "MI355X has different MFMA tile sizes — kernel might actually be incompatible.", 0.55, False ), "loader": SpecialistOpinion("Can't assess — kernel never launched.", 0.60, False), }, inspect_results=InspectResult( logs="[Flash-Attn] gfx950 not in [gfx90a, gfx942]\n[Flash-Attn] MI355X CDNA4 arch check failed", config="gpu_arch: gfx950\namd_supported: [gfx90a, gfx942]\nrocm_version: 6.4", snippet="# MI355X (gfx950/CDNA4) extends gfx942 instruction set\n# MFMA f32_32x32x16_fp8 available\n# Just missing from allowlist", metrics="kernel_launch_failures: 1\ngpu_utilization: 0%", ), specialist_followups={ "runtime": "ROCm works. Not a runtime issue.", "dispatch": "Add gfx950 to AMD_SUPPORTED. CDNA4 is backwards-compatible with gfx942 kernels.", "kernel": "I was wrong — gfx950 does support the needed MFMA instructions. It's just the allowlist.", "loader": "No weight issues.", }, )) scenarios.append(Scenario( id="arch_guard_05", root_cause="arch_guard", correct_fix="relax_arch_check", incident_ticket=( "INCIDENT: Triton kernel compilation fails on RTX 5090 for custom MoE layer. " "Error: 'target sm_120 not recognized'. Compiled fine for sm_90." ), hardware="NVIDIA SM120 (GeForce RTX 5090)", model_name="DeepSeek-V3-671B", backend="SGLang 0.5.x", initial_log=( "[Triton] Compiling MoE routing kernel for sm_120...\n" "[Triton] ERROR: Unknown target 'sm_120'\n" "[Triton] Known targets: sm_70, sm_75, sm_80, sm_86, sm_89, sm_90" ), initial_snippet=( "# triton/compiler/target.py\n" "KNOWN_TARGETS = ['sm_70','sm_75','sm_80','sm_86','sm_89','sm_90']\n" ), specialist_opinions={ "runtime": SpecialistOpinion("CUDA and Triton installed correctly.", 0.78, False), "dispatch": SpecialistOpinion( "Triton's target list doesn't include sm_120. Need to add Blackwell family.", 0.90, True ), "kernel": SpecialistOpinion( "The MoE kernel uses standard tl.dot which works on any SM >= 70.", 0.82, True ), "loader": SpecialistOpinion( "Weights load fine. Error is at JIT compilation stage.", 0.70, False ), }, inspect_results=InspectResult( logs="[Triton] JIT target 'sm_120' not recognized\n[Triton] Compilation aborted before PTX generation", config="triton_target: sm_120\nknown_targets: [sm_70..sm_90]\ntriton_version: 3.2", snippet="# Triton target registry doesn't know sm_120\n# sm_120 can use sm_90 codegen path\n# Add sm_120 to target list or use family mapping", metrics="jit_compile_failures: 1\nkernel_cache_hits: 0", ), specialist_followups={ "runtime": "No runtime issue. Triton JIT compiler is the blocker.", "dispatch": "Triton target registry needs sm_120. Can map to sm_90 codegen path since instruction set overlaps.", "kernel": "The kernel code is fine — it's the compiler target check, not the kernel logic.", "loader": "No weight involvement at this stage.", }, )) # --- backend_whitelist additional scenarios --- scenarios.append(Scenario( id="backend_whitelist_03", root_cause="backend_whitelist", correct_fix="add_whitelist_entry", incident_ticket=( "INCIDENT: GPTQ quantization fails on B200 with 'GPU not whitelisted for Marlin'. " "Same quantized model serves fine on H100. B200 has FP16 working." ), hardware="NVIDIA B200", model_name="Mistral-Large-2", backend="vLLM 0.8.x", initial_log=( "[vLLM] Loading GPTQ model on B200...\n" "[vLLM] Marlin check: GPU 'NVIDIA B200' not whitelisted\n" "[vLLM] Available kernels for non-whitelisted: none\n" "[vLLM] ERROR: Cannot serve quantized model" ), initial_snippet=( "# vllm/quantization/marlin.py\n" "WHITELIST = {'A100','H100','A10G','L40S','RTX 4090'}\n" "if gpu_name not in WHITELIST:\n" " raise RuntimeError('GPU not whitelisted')\n" ), specialist_opinions={ "runtime": SpecialistOpinion("CUDA runtime healthy on B200.", 0.80, False), "dispatch": SpecialistOpinion( "Whitelist check is string-based. 'B200' not in the set. Add it.", 0.93, True ), "kernel": SpecialistOpinion( "B200 FP8 is different from H100. Might need a different quantization kernel.", 0.50, False ), "loader": SpecialistOpinion("Quantized weights loaded correctly.", 0.75, False), }, inspect_results=InspectResult( logs="[Marlin] GPU 'NVIDIA B200' not in whitelist\n[Marlin] Whitelist: {A100,H100,A10G,L40S,RTX 4090}", config="gpu_name: NVIDIA B200\nmarlin_whitelist: [A100,H100,A10G,L40S,RTX 4090]\nquant_method: gptq", snippet="# B200 supports all Marlin GEMM ops (INT4 deq + FP16 MMA)\n# Name-based whitelist just doesn't include it\n# Fix: add 'B200' or switch to arch-based check", metrics="quant_init_failures: 1\nfp16_serving: available\nquant_serving: blocked", ), specialist_followups={ "runtime": "Runtime fine.", "dispatch": "Simple whitelist gap. Add 'B200' to WHITELIST set.", "kernel": "I was wrong — B200 Marlin kernels use same INT4 deq + MMA path as H100. Whitelist issue only.", "loader": "Weights are fine.", }, )) scenarios.append(Scenario( id="backend_whitelist_04", root_cause="backend_whitelist", correct_fix="add_whitelist_entry", incident_ticket=( "INCIDENT: FlashInfer FP8 GEMM blocked on DGX Spark. " "Error: 'FP8 dispatch not available for this GPU'. " "SM121 should support FP8 natively." ), hardware="NVIDIA SM121 (DGX Spark)", model_name="DeepSeek-R1-Distill-70B", backend="FlashInfer 0.4", initial_log=( "[FlashInfer] FP8 GEMM dispatch...\n" "[FlashInfer] GPU family check: sm_121\n" "[FlashInfer] FP8 whitelist: [sm_89, sm_90]\n" "[FlashInfer] ERROR: FP8 not available for sm_121" ), initial_snippet=( "# flashinfer/gemm/fp8_dispatch.py\n" "FP8_ENABLED_SM = {89, 90} # Ada, Hopper\n" "# Missing SM12x which has FP8 MMA\n" ), specialist_opinions={ "runtime": SpecialistOpinion("CUDA 13 runtime fine.", 0.78, False), "dispatch": SpecialistOpinion( "FP8 dispatch whitelist only has Ada/Hopper. SM121 supports FP8 MMA natively but isn't listed.", 0.94, True ), "kernel": SpecialistOpinion( "SM121 FP8 might use different MMA instruction encoding.", 0.48, False ), "loader": SpecialistOpinion("FP8 weights loaded. Dispatch is the blocker.", 0.82, True), }, inspect_results=InspectResult( logs="[FlashInfer] sm_121 not in FP8_ENABLED_SM {89, 90}\n[FlashInfer] FP8 GEMM dispatch blocked", config="gpu_sm: 121\nfp8_whitelist: [89, 90]\nfp8_hw_support: true", snippet="# SM121 uses m16n8k32 FP8 MMA (same encoding as SM90)\n# Just not in FP8_ENABLED_SM set\n# Add 120, 121 to enable FP8 dispatch", metrics="fp8_dispatch_blocked: true\nfp8_hw_capable: true\nfallback_to_bf16: not_attempted", ), specialist_followups={ "runtime": "Runtime is fine.", "dispatch": "Add SM12x to FP8_ENABLED_SM. SM121 uses identical FP8 MMA to SM90.", "kernel": "I checked — SM121 uses the same m16n8k32 encoding as SM90. My concern was unfounded.", "loader": "FP8 weights are ready. Just need dispatch to be unblocked.", }, )) scenarios.append(Scenario( id="backend_whitelist_05", root_cause="backend_whitelist", correct_fix="add_whitelist_entry", incident_ticket=( "INCIDENT: SGLang refuses to enable speculative decoding on RTX 5090. " "Error: 'Speculative decoding not supported for consumer GPUs'. " "Feature works on A100." ), hardware="NVIDIA SM120 (GeForce RTX 5090)", model_name="Llama-3.3-70B-Instruct", backend="SGLang 0.5.x", initial_log=( "[SGLang] Speculative decoding requested...\n" "[SGLang] GPU: GeForce RTX 5090\n" "[SGLang] Spec decode whitelist: [A100, H100, A10G]\n" "[SGLang] ERROR: Consumer GPU not in spec-decode whitelist" ), initial_snippet=( "# sglang/server/spec_decode.py\n" "SPEC_DECODE_GPUS = ['A100', 'H100', 'A10G']\n" "# Only data center GPUs whitelisted\n" ), specialist_opinions={ "runtime": SpecialistOpinion("Runtime fine. GPU has 24GB VRAM.", 0.78, False), "dispatch": SpecialistOpinion( "RTX 5090 not in spec-decode whitelist. Datacenter-only check is too restrictive.", 0.91, True ), "kernel": SpecialistOpinion( "RTX 5090 might not have enough VRAM for speculative decoding with 70B.", 0.60, False ), "loader": SpecialistOpinion("Model weights fine.", 0.72, False), }, inspect_results=InspectResult( logs="[SGLang] GPU 'GeForce RTX 5090' not in SPEC_DECODE_GPUS\n[SGLang] Whitelist is datacenter-only", config="gpu_name: GeForce RTX 5090\nspec_decode_whitelist: [A100,H100,A10G]\nvram: 32GB", snippet="# RTX 5090 has 32GB VRAM, sufficient for spec decode\n# Whitelist artificially restricts to datacenter GPUs\n# Add RTX 5090 or use VRAM-based check", metrics="spec_decode_attempts: 1\nspec_decode_blocked: true\nvram_available: 32GB", ), specialist_followups={ "runtime": "No runtime issue.", "dispatch": "Add RTX 5090 to whitelist. 32GB VRAM is plenty for spec decode.", "kernel": "32GB is sufficient for speculative decoding with 70B quantized. VRAM isn't the issue.", "loader": "Weights loaded. Dispatch blocker only.", }, )) scenarios.append(Scenario( id="backend_whitelist_06", root_cause="backend_whitelist", correct_fix="add_whitelist_entry", incident_ticket=( "INCIDENT: H100 SXM nodes cannot run INT8 SmoothQuant models via vLLM. " "Error: 'SmoothQuant kernel not available for current device'. " "Same model runs on A100 with SmoothQuant. H100 FP16/BF16 paths work fine." ), hardware="NVIDIA H100 SXM", model_name="Qwen3-235B-A22B", backend="vLLM 0.8.x", initial_log=( "[vLLM] Loading SmoothQuant INT8 model...\n" "[vLLM] SmoothQuant device check: 'NVIDIA H100 SXM'\n" "[vLLM] SQ_SUPPORTED_DEVICES = ['A100-SXM4-80GB', 'A100-SXM4-40GB', 'A10G']\n" "[vLLM] ERROR: SmoothQuant kernel not available for current device" ), initial_snippet=( "# vllm/model_executor/layers/quantization/smoothquant.py\n" "SQ_SUPPORTED_DEVICES = [\n" " 'A100-SXM4-80GB', 'A100-SXM4-40GB', 'A10G',\n" "]\n" "if torch.cuda.get_device_name() not in SQ_SUPPORTED_DEVICES:\n" " raise RuntimeError('SmoothQuant kernel not available')\n" ), specialist_opinions={ "runtime": SpecialistOpinion("CUDA runtime healthy. H100 recognized with 80GB HBM3.", 0.82, False), "dispatch": SpecialistOpinion( "SmoothQuant whitelist only has Ampere GPUs. H100 device name doesn't match any entry.", 0.94, True ), "kernel": SpecialistOpinion( "H100 INT8 tensor core layout differs from A100. Might need a dedicated kernel.", 0.45, False ), "loader": SpecialistOpinion("INT8 weights loaded correctly into GPU memory.", 0.76, False), }, inspect_results=InspectResult( logs=( "[SmoothQuant] torch.cuda.get_device_name() = 'NVIDIA H100 80GB HBM3'\n" "[SmoothQuant] Not in SQ_SUPPORTED_DEVICES\n" "[SmoothQuant] Supported: ['A100-SXM4-80GB', 'A100-SXM4-40GB', 'A10G']" ), config=( "gpu_name: NVIDIA H100 80GB HBM3\n" "sq_supported: [A100-SXM4-80GB, A100-SXM4-40GB, A10G]\n" "quantization: smoothquant\n" "int8_hw_support: true" ), snippet=( "# H100 has full INT8 tensor core support via IMMA instructions\n" "# SQ_SUPPORTED_DEVICES was written pre-Hopper, only lists Ampere\n" "# Fix: add H100 device names or switch to capability-based check" ), metrics=( "sq_init_failures: 1\n" "fp16_fallback: available\n" "int8_hw_capable: true\n" "expected_speedup_from_int8: 1.8x" ), ), specialist_followups={ "runtime": "Runtime is fine. H100 fully operational.", "dispatch": "Whitelist is Ampere-only. Adding 'NVIDIA H100 80GB HBM3' (and other H100 SKUs) resolves it.", "kernel": "I was wrong — H100 INT8 IMMA ops are backward-compatible with A100 SmoothQuant kernels. Whitelist issue only.", "loader": "Weights loaded fine. It's the device name check blocking init.", }, )) scenarios.append(Scenario( id="backend_whitelist_07", root_cause="backend_whitelist", correct_fix="add_whitelist_entry", incident_ticket=( "INCIDENT: B200 cluster fails to launch TensorRT-LLM engine with FP8 quantization. " "Error: 'FP8 post-training quantization not supported on this GPU'. " "B200 has native FP8 support. Same engine builds fine targeting H100." ), hardware="NVIDIA B200", model_name="DeepSeek-V3-671B", backend="TensorRT-LLM 0.18", initial_log=( "[TRT-LLM] Building FP8 engine for B200...\n" "[TRT-LLM] GPU detected: NVIDIA B200 192GB\n" "[TRT-LLM] FP8 PTQ supported GPUs: ['H100', 'H200', 'L40S']\n" "[TRT-LLM] ERROR: FP8 post-training quantization not supported on this GPU" ), initial_snippet=( "# tensorrt_llm/quantization/fp8.py\n" "FP8_PTQ_GPUS = {'H100', 'H200', 'L40S'}\n" "gpu_name = get_gpu_short_name()\n" "if gpu_name not in FP8_PTQ_GPUS:\n" " raise ValueError('FP8 PTQ not supported on this GPU')\n" ), specialist_opinions={ "runtime": SpecialistOpinion("CUDA 13.1 runtime fine. B200 fully visible.", 0.80, False), "dispatch": SpecialistOpinion( "FP8 PTQ whitelist was written for Hopper generation. B200 (Blackwell) not added yet.", 0.92, True ), "kernel": SpecialistOpinion( "B200 FP8 tensor cores use a new micro-architecture. The FP8 GEMM kernels may not be compatible.", 0.42, False ), "loader": SpecialistOpinion( "Engine build fails before weight loading. It's a pre-build validation check.", 0.83, True ), }, inspect_results=InspectResult( logs=( "[TRT-LLM] get_gpu_short_name() = 'B200'\n" "[TRT-LLM] 'B200' not in FP8_PTQ_GPUS {'H100', 'H200', 'L40S'}\n" "[TRT-LLM] FP8 engine build aborted at validation stage" ), config=( "gpu: NVIDIA B200 192GB\n" "fp8_ptq_gpus: [H100, H200, L40S]\n" "target_quant: fp8\n" "engine_build_stage: pre-validation" ), snippet=( "# B200 Blackwell architecture has 5th-gen tensor cores with FP8\n" "# Fully backward-compatible with Hopper FP8 GEMM kernels\n" "# Whitelist only has Hopper-era GPUs\n" "# Fix: add 'B200' and 'B100' to FP8_PTQ_GPUS" ), metrics=( "engine_build_attempts: 1\n" "engine_build_failures: 1\n" "fp8_hw_capable: true\n" "fp16_engine_build: success" ), ), specialist_followups={ "runtime": "Runtime healthy. B200 fully recognized.", "dispatch": "Add 'B200' to FP8_PTQ_GPUS. Blackwell FP8 tensor cores are Hopper-compatible.", "kernel": "I checked the ISA — B200 FP8 MMA instructions are a superset of H100. Fully compatible. It's just the whitelist.", "loader": "Correct — engine build fails before weights are even touched. Whitelist check is the blocker.", }, )) scenarios.append(Scenario( id="backend_whitelist_08", root_cause="backend_whitelist", correct_fix="add_whitelist_entry", incident_ticket=( "INCIDENT: DGX Spark (SM121) cannot serve models via SGLang with RadixAttention. " "Error: 'RadixAttention not supported on this compute capability'. " "Standard attention works. RadixAttention is required for prefix caching workloads." ), hardware="NVIDIA SM121 (DGX Spark)", model_name="Llama-4-Maverick-17Bx128E", backend="SGLang 0.5.x", initial_log=( "[SGLang] Initializing RadixAttention backend...\n" "[SGLang] Compute capability: sm_121\n" "[SGLang] RadixAttention supported SM: [80, 86, 89, 90]\n" "[SGLang] ERROR: RadixAttention not supported on this compute capability" ), initial_snippet=( "# sglang/srt/layers/radix_attention.py\n" "RADIX_ATTN_SM = {80, 86, 89, 90}\n" "cc = torch.cuda.get_device_capability()\n" "sm = cc[0] * 10 + cc[1]\n" "if sm not in RADIX_ATTN_SM:\n" " raise RuntimeError('RadixAttention not supported')\n" ), specialist_opinions={ "runtime": SpecialistOpinion("CUDA 13 runtime loaded. SM121 detected correctly.", 0.79, False), "dispatch": SpecialistOpinion( "RadixAttention SM whitelist was authored for Ampere/Ada/Hopper. SM121 not listed.", 0.93, True ), "kernel": SpecialistOpinion( "SM121's shared memory layout is different from SM90. RadixAttention kernels might not tile correctly.", 0.47, False ), "loader": SpecialistOpinion("Model loaded to GPU. Attention backend selection is the blocker.", 0.81, True), }, inspect_results=InspectResult( logs=( "[SGLang] torch.cuda.get_device_capability() = (12, 1)\n" "[SGLang] sm = 121, not in RADIX_ATTN_SM {80, 86, 89, 90}\n" "[SGLang] Falling back to standard attention (no prefix caching)" ), config=( "gpu_sm: 121\n" "radix_attn_sm: [80, 86, 89, 90]\n" "prefix_caching: required\n" "attention_backend: blocked" ), snippet=( "# SM121 supports all warp-level primitives needed by RadixAttention\n" "# Shared memory per SM: 228KB (more than SM90's 228KB)\n" "# Tiling strategy is SM-agnostic; only needs cooperative groups\n" "# Fix: add 120 and 121 to RADIX_ATTN_SM" ), metrics=( "radix_attn_init_failures: 1\n" "prefix_cache_hit_rate: 0% (feature disabled)\n" "standard_attn_fallback: active\n" "ttft_regression_vs_radix: 3.2x" ), ), specialist_followups={ "runtime": "Runtime fine. No driver or library issues.", "dispatch": "Add SM12x to RADIX_ATTN_SM. The kernel uses standard cooperative groups that SM121 fully supports.", "kernel": "I rechecked — SM121 has 228KB shared mem per SM, identical to SM90. Tiling works. My concern was unfounded.", "loader": "Model loaded correctly. RadixAttention whitelist is the only blocker for prefix caching.", }, )) scenarios.append(Scenario( id="backend_whitelist_09", root_cause="backend_whitelist", correct_fix="add_whitelist_entry", incident_ticket=( "INCIDENT: A100 nodes fail to enable chunked prefill in vLLM. " "Error: 'Chunked prefill only supported on Hopper+'. " "A100 has sufficient HBM and SM count. Feature needed for long-context workloads." ), hardware="NVIDIA A100 SXM 80GB", model_name="Llama-3.3-70B-Instruct", backend="vLLM 0.8.x", initial_log=( "[vLLM] Enabling chunked prefill for long-context serving...\n" "[vLLM] GPU: NVIDIA A100-SXM4-80GB (sm_80)\n" "[vLLM] Chunked prefill min SM: 90 (Hopper)\n" "[vLLM] ERROR: Chunked prefill only supported on Hopper+" ), initial_snippet=( "# vllm/core/chunked_prefill.py\n" "MIN_SM_CHUNKED_PREFILL = 90 # Hopper\n" "device_sm = torch.cuda.get_device_capability()\n" "sm = device_sm[0] * 10 + device_sm[1]\n" "if sm < MIN_SM_CHUNKED_PREFILL:\n" " raise RuntimeError('Chunked prefill only supported on Hopper+')\n" ), specialist_opinions={ "runtime": SpecialistOpinion("CUDA 12.4 runtime loaded. A100 fully functional.", 0.81, False), "dispatch": SpecialistOpinion( "Chunked prefill gate uses SM >= 90 but A100 is SM80. The feature doesn't actually need Hopper-specific instructions.", 0.92, True ), "kernel": SpecialistOpinion( "A100 might not have enough L2 cache bandwidth for chunked prefill at scale.", 0.48, False ), "loader": SpecialistOpinion("Model loaded. Scheduler configuration is the issue.", 0.77, False), }, inspect_results=InspectResult( logs=( "[vLLM] sm_80 < MIN_SM_CHUNKED_PREFILL (90)\n" "[vLLM] Chunked prefill disabled\n" "[vLLM] Long sequences processed monolithically -> OOM risk" ), config=( "gpu_sm: 80\n" "min_sm_chunked_prefill: 90\n" "max_context_len: 131072\n" "chunked_prefill: blocked" ), snippet=( "# Chunked prefill splits long prefills into chunks processed incrementally\n" "# Uses standard FlashAttention-2 kernels available on SM80+\n" "# MIN_SM gate was over-restrictive; A100 fully supports the feature\n" "# Fix: lower MIN_SM_CHUNKED_PREFILL to 80 or add A100 to whitelist" ), metrics=( "chunked_prefill_blocked: true\n" "max_prefill_len_without_chunking: 32768\n" "oom_on_long_sequences: true\n" "a100_hbm_available: 80GB" ), ), specialist_followups={ "runtime": "Runtime fine. A100 fully recognized.", "dispatch": "MIN_SM gate is too restrictive. Chunked prefill uses FlashAttention-2 which works on SM80. Lower the gate or add A100 to an explicit allow-list.", "kernel": "L2 bandwidth on A100 is fine for chunked prefill. The 40MB L2 on A100 handles chunk boundaries well. Not a hardware limitation.", "loader": "Model loaded correctly. It's the scheduler guard blocking chunked prefill.", }, )) scenarios.append(Scenario( id="backend_whitelist_10", root_cause="backend_whitelist", correct_fix="add_whitelist_entry", incident_ticket=( "INCIDENT: L40S GPUs blocked from running FP8 KV-cache quantization in SGLang. " "Error: 'FP8 KV-cache requires datacenter H-class GPU'. " "L40S is a datacenter Ada Lovelace GPU with FP8 support. BF16 KV-cache works." ), hardware="NVIDIA L40S", model_name="DeepSeek-R1-Distill-70B", backend="SGLang 0.5.x", initial_log=( "[SGLang] Enabling FP8 KV-cache quantization...\n" "[SGLang] GPU: NVIDIA L40S (sm_89)\n" "[SGLang] FP8 KV whitelist: ['H100', 'H200', 'GH200']\n" "[SGLang] ERROR: FP8 KV-cache requires datacenter H-class GPU" ), initial_snippet=( "# sglang/srt/mem_cache/fp8_kv.py\n" "FP8_KV_GPUS = ['H100', 'H200', 'GH200']\n" "gpu_name = torch.cuda.get_device_name()\n" "if not any(g in gpu_name for g in FP8_KV_GPUS):\n" " raise RuntimeError('FP8 KV-cache requires datacenter H-class GPU')\n" ), specialist_opinions={ "runtime": SpecialistOpinion("CUDA 12.4 runtime fine. L40S detected with 48GB GDDR6.", 0.80, False), "dispatch": SpecialistOpinion( "FP8 KV-cache whitelist only includes Hopper GPUs. L40S (Ada SM89) has FP8 tensor cores but isn't listed.", 0.93, True ), "kernel": SpecialistOpinion( "L40S FP8 uses different cache line sizes than H100 HBM3. KV-cache quantization might produce wrong results.", 0.44, False ), "loader": SpecialistOpinion("Model and initial KV-cache allocated. FP8 conversion is blocked at runtime.", 0.79, True), }, inspect_results=InspectResult( logs=( "[SGLang] torch.cuda.get_device_name() = 'NVIDIA L40S'\n" "[SGLang] Substring match against FP8_KV_GPUS: no match\n" "[SGLang] FP8 KV-cache quantization disabled" ), config=( "gpu_name: NVIDIA L40S\n" "fp8_kv_gpus: [H100, H200, GH200]\n" "kv_cache_dtype: bf16 (fallback)\n" "fp8_hw_support: true" ), snippet=( "# L40S (SM89) supports FP8 via Ada Lovelace tensor cores\n" "# FP8 E4M3 format is identical across Ada and Hopper\n" "# KV-cache quantization only needs FP8 <-> BF16 conversion kernels\n" "# Fix: add 'L40S' to FP8_KV_GPUS or use SM-based capability check" ), metrics=( "fp8_kv_blocked: true\n" "kv_cache_memory_bf16: 38.2GB\n" "kv_cache_memory_fp8_projected: 19.1GB\n" "memory_savings_blocked: 50%" ), ), specialist_followups={ "runtime": "Runtime fine. L40S fully operational.", "dispatch": "Add 'L40S' to FP8_KV_GPUS. Ada SM89 FP8 tensor cores handle E4M3 KV-cache quantization identically to Hopper.", "kernel": "I was wrong — FP8 E4M3 format is the same on Ada and Hopper. Cache line size doesn't affect the quantization math. Whitelist issue only.", "loader": "Model loaded fine. FP8 KV-cache conversion is blocked by the name-based whitelist check.", }, )) # --- runtime_loader additional scenarios --- scenarios.append(Scenario( id="runtime_loader_03", root_cause="runtime_loader", correct_fix="fix_runtime_path", incident_ticket=( "INCIDENT: vLLM fails with 'libcublas.so.13 not found' on freshly provisioned node. " "nvidia-smi shows GPU. CUDA toolkit installed. Other CUDA apps work." ), hardware="NVIDIA H100", model_name="Llama-4-Maverick-17Bx128E", backend="vLLM 0.8.x", initial_log=( "[vLLM] Initializing CUDA...\n" "[vLLM] ERROR: libcublas.so.13: cannot open shared object file\n" "[vLLM] LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu\n" "[vLLM] Note: /usr/local/cuda-13/lib64 not in path" ), initial_snippet=( "# /etc/environment\n" "LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu\n" "# Missing: /usr/local/cuda-13/lib64\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "CUDA 13 is installed but its lib64 directory isn't in LD_LIBRARY_PATH. Path fix needed.", 0.95, True ), "dispatch": SpecialistOpinion("Server crashes before any dispatch.", 0.65, False), "kernel": SpecialistOpinion("Not a kernel issue — can't load CUDA libraries.", 0.70, False), "loader": SpecialistOpinion( "Dynamic linker can't find libcublas.so.13. Add CUDA 13 lib path.", 0.90, True ), }, inspect_results=InspectResult( logs="[ldconfig] libcublas.so.13 not in cache\n[System] /usr/local/cuda-13/lib64/libcublas.so.13 EXISTS but not in path", config="LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu\ncuda_13_libs=/usr/local/cuda-13/lib64\nldconfig_cache: stale", snippet="# libcublas.so.13 exists at /usr/local/cuda-13/lib64/\n# But LD_LIBRARY_PATH doesn't include it\n# Fix: add /usr/local/cuda-13/lib64 to LD_LIBRARY_PATH", metrics="import_failures: 1\ncuda_available: false (library missing)", ), specialist_followups={ "runtime": "Classic provisioning issue. CUDA installed but path not configured. Add to LD_LIBRARY_PATH.", "dispatch": "Nothing to dispatch — server won't start.", "kernel": "No kernel involvement.", "loader": "Add /usr/local/cuda-13/lib64 to LD_LIBRARY_PATH or run ldconfig.", }, )) scenarios.append(Scenario( id="runtime_loader_04", root_cause="runtime_loader", correct_fix="fix_runtime_path", incident_ticket=( "INCIDENT: FlashInfer JIT compilation fails with 'nvcc not found'. " "GPU inference should work but JIT kernels can't compile. " "nvidia-smi works fine." ), hardware="NVIDIA SM121 (DGX Spark)", model_name="Qwen3-235B-A22B", backend="FlashInfer 0.4", initial_log=( "[FlashInfer] JIT compiling attention kernel for sm_121...\n" "[FlashInfer] Searching for nvcc...\n" "[FlashInfer] ERROR: nvcc not found in PATH\n" "[FlashInfer] CUDA_HOME not set" ), initial_snippet=( "# Container environment\n" "PATH=/usr/local/bin:/usr/bin:/bin\n" "# Missing: /usr/local/cuda-13/bin (where nvcc lives)\n" "CUDA_HOME= # not set\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "CUDA toolkit is installed but nvcc isn't in PATH and CUDA_HOME isn't set.", 0.93, True ), "dispatch": SpecialistOpinion("Dispatch can't run without JIT-compiled kernels.", 0.60, False), "kernel": SpecialistOpinion( "SM121 needs JIT compilation for attention kernels. Without nvcc, it can't compile.", 0.80, True ), "loader": SpecialistOpinion("Try using pre-compiled AOT kernels instead.", 0.45, False), }, inspect_results=InspectResult( logs="[System] which nvcc -> not found\n[System] ls /usr/local/cuda-13/bin/nvcc -> EXISTS\n[System] CUDA_HOME unset", config="PATH=/usr/local/bin:/usr/bin:/bin\nCUDA_HOME=(unset)\nnvcc_location=/usr/local/cuda-13/bin/nvcc", snippet="# nvcc exists at /usr/local/cuda-13/bin/ but not in PATH\n# Fix: export CUDA_HOME=/usr/local/cuda-13\n# Fix: export PATH=$CUDA_HOME/bin:$PATH", metrics="jit_compile_attempts: 3\njit_compile_failures: 3\naot_kernels_available: false", ), specialist_followups={ "runtime": "Set CUDA_HOME=/usr/local/cuda-13 and add its bin/ to PATH.", "dispatch": "Once nvcc is found, JIT compilation will work and dispatch proceeds normally.", "kernel": "The kernel code is ready to compile. Just need the compiler to be findable.", "loader": "AOT kernels aren't available for SM121 yet. JIT path is needed.", }, )) scenarios.append(Scenario( id="runtime_loader_05", root_cause="runtime_loader", correct_fix="fix_runtime_path", incident_ticket=( "INCIDENT: Python can't import torch on MI300X node. " "Error: 'libtorch_hip.so: cannot open shared object'. " "PyTorch ROCm wheel installed but missing HIP libs." ), hardware="AMD MI300X", model_name="Mistral-Large-2", backend="vLLM 0.8.x", initial_log=( "[Python] import torch\n" "[Python] ERROR: libtorch_hip.so: cannot open shared object file\n" "[System] ROCm installed at /opt/rocm-6.3\n" "[System] LD_LIBRARY_PATH does not include /opt/rocm-6.3/lib" ), initial_snippet=( "# Container env\n" "LD_LIBRARY_PATH=/usr/local/lib\n" "# Needs: /opt/rocm-6.3/lib:/opt/rocm-6.3/hip/lib\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "ROCm 6.3 installed but libs not in LD_LIBRARY_PATH. Classic path issue.", 0.94, True ), "dispatch": SpecialistOpinion("Can't assess — Python crashes on import.", 0.50, False), "kernel": SpecialistOpinion("Maybe PyTorch ROCm wheel is for wrong ROCm version.", 0.55, False), "loader": SpecialistOpinion( "Dynamic linker needs /opt/rocm-6.3/lib in LD_LIBRARY_PATH.", 0.90, True ), }, inspect_results=InspectResult( logs="[System] /opt/rocm-6.3/lib/libtorch_hip.so EXISTS\n[System] ldd: libtorch_hip.so => not found\n[System] LD_LIBRARY_PATH=/usr/local/lib only", config="LD_LIBRARY_PATH=/usr/local/lib\nrocm_path=/opt/rocm-6.3\nrocm_lib=/opt/rocm-6.3/lib", snippet="# ROCm libs at /opt/rocm-6.3/lib/ and /opt/rocm-6.3/hip/lib/\n# Not in LD_LIBRARY_PATH\n# Fix: export LD_LIBRARY_PATH=/opt/rocm-6.3/lib:/opt/rocm-6.3/hip/lib:$LD_LIBRARY_PATH", metrics="import_failures: 1\ntorch_available: false", ), specialist_followups={ "runtime": "Add ROCm lib paths to LD_LIBRARY_PATH. Standard post-install issue.", "dispatch": "Can't run without PyTorch importing.", "kernel": "The ROCm version matches the wheel. It's just a path issue.", "loader": "Add /opt/rocm-6.3/lib to LD_LIBRARY_PATH.", }, )) # --- backend_selector additional scenarios --- scenarios.append(Scenario( id="backend_selector_03", root_cause="backend_selector", correct_fix="switch_backend", incident_ticket=( "INCIDENT: SGLang MoE expert parallelism selecting wrong GEMM backend. " "Using generic GEMM instead of grouped GEMM for MoE layers. " "Throughput is 5x lower than expected." ), hardware="NVIDIA H100", model_name="DeepSeek-V3-671B", backend="SGLang 0.5.x", initial_log=( "[SGLang] MoE layer: 256 experts, top-8 routing\n" "[SGLang] GEMM backend: generic (cublas)\n" "[SGLang] WARNING: Grouped GEMM backend not selected\n" "[SGLang] Throughput: 15 tok/s (expected: 80 tok/s)" ), initial_snippet=( "# sglang/moe/dispatch.py\n" "def select_moe_backend(num_experts, gpu):\n" " if num_experts <= 64:\n" " return 'grouped_gemm'\n" " return 'generic' # Wrong fallback for large expert count\n" ), specialist_opinions={ "runtime": SpecialistOpinion("CUDA runtime fine. No errors.", 0.75, False), "dispatch": SpecialistOpinion( "MoE backend selector falls back to generic GEMM when experts > 64. " "Should use grouped GEMM for any expert count on H100.", 0.95, True ), "kernel": SpecialistOpinion( "Generic cuBLAS GEMM launches one kernel per expert. Grouped GEMM batches them. " "Switch to grouped GEMM backend.", 0.88, True ), "loader": SpecialistOpinion("Weights loaded. Not a loading issue.", 0.72, False), }, inspect_results=InspectResult( logs="[SGLang] 256 experts > 64 threshold -> generic backend\n[SGLang] Each expert: separate cuBLAS call\n[SGLang] Kernel launch overhead: 256 launches/layer", config="num_experts: 256\nmoe_backend: generic\nthreshold: 64\ngpu: H100", snippet="# Backend selector has wrong threshold logic\n# Should use grouped_gemm for ALL expert counts on H100\n# Current: only grouped_gemm when experts <= 64", metrics="throughput_tok_s: 15\nexpected_throughput: 80\nkernel_launches_per_step: 256\ngpu_utilization: 18%", ), specialist_followups={ "runtime": "No runtime issues.", "dispatch": "Switch to grouped_gemm backend. The 64-expert threshold is a bug.", "kernel": "Grouped GEMM would batch all 256 experts into one kernel launch. 10-15x fewer launches.", "loader": "Not a weight issue.", }, )) scenarios.append(Scenario( id="backend_selector_04", root_cause="backend_selector", correct_fix="switch_backend", incident_ticket=( "INCIDENT: Attention on B200 using FlashAttention v1 path instead of v2. " "Memory usage 3x higher than expected. OOM on large batch sizes. " "Same model fits in memory on H100." ), hardware="NVIDIA B200", model_name="Llama-4-Maverick-17Bx128E", backend="vLLM 0.8.x", initial_log=( "[vLLM] Attention backend: flash_attn_v1\n" "[vLLM] WARNING: v2 backend not selected (GPU not in v2 list)\n" "[vLLM] Memory: attention uses O(n^2) instead of O(n)\n" "[vLLM] OOM at batch_size=32 (expected to fit at batch_size=128)" ), initial_snippet=( "# vllm/attention/selector.py\n" "def select_flash_version(gpu_sm):\n" " if gpu_sm in {80, 86, 89, 90}:\n" " return 'v2'\n" " return 'v1' # B200 (sm_120) falls here\n" ), specialist_opinions={ "runtime": SpecialistOpinion("CUDA runtime OK. Memory allocation works.", 0.75, False), "dispatch": SpecialistOpinion( "Backend selector picks FA v1 for sm_120. B200 supports v2 — selector needs updating.", 0.93, True ), "kernel": SpecialistOpinion( "FA v1 uses O(n^2) memory. v2 uses O(n). That explains the OOM.", 0.85, True ), "loader": SpecialistOpinion( "Maybe model weights are larger than expected for this architecture.", 0.45, False ), }, inspect_results=InspectResult( logs="[vLLM] sm_120 not in {80,86,89,90} -> flash_attn_v1\n[vLLM] FA v1 attention memory: O(seq_len^2)\n[vLLM] OOM threshold hit at 32 batch", config="gpu_sm: 120\nflash_attn_version: v1\nv2_supported_sm: [80,86,89,90]\nmemory_profile: quadratic", snippet="# B200 (sm_120) supports FlashAttention v2\n# Selector only checks old SM list\n# Fix: add sm_120 to v2 supported set or switch to v2 backend", metrics="attention_memory_gb: 24.5\nexpected_attention_memory_gb: 2.1\nbatch_size_limit: 32\nexpected_batch_limit: 128", ), specialist_followups={ "runtime": "Memory system works. Problem is FA v1's quadratic memory.", "dispatch": "Add sm_120 to v2 supported set. B200 has full v2 support.", "kernel": "FA v1 materializes full attention matrix. v2 uses tiling. Fix the selector.", "loader": "Weight size is correct. It's the attention memory that's excessive.", }, )) scenarios.append(Scenario( id="backend_selector_05", root_cause="backend_selector", correct_fix="switch_backend", incident_ticket=( "INCIDENT: MI300X inference using CK (Composable Kernel) attention but should use Triton. " "CK path has a known bug with GQA + variable-length sequences. " "Random crashes during batched inference." ), hardware="AMD MI300X", model_name="Qwen3-235B-A22B", backend="vLLM 0.8.x", initial_log=( "[vLLM] AMD GPU detected -> Composable Kernel attention\n" "[vLLM] GQA + varlen: CK backend selected\n" "[vLLM] CRASH: segfault in ck_attention_varlen_gqa\n" "[vLLM] This is a known CK bug. Use Triton backend instead." ), initial_snippet=( "# vllm/attention/backends/rocm.py\n" "def get_rocm_backend(config):\n" " return 'composable_kernel' # Always uses CK\n" " # Should check for known CK bugs and use Triton\n" ), specialist_opinions={ "runtime": SpecialistOpinion("ROCm runtime fine before the segfault.", 0.72, False), "dispatch": SpecialistOpinion( "Backend selector always picks CK on AMD. Should use Triton for GQA+varlen due to known CK bug.", 0.94, True ), "kernel": SpecialistOpinion( "Known CK bug with GQA + varlen sequences. Triton attention works correctly.", 0.90, True ), "loader": SpecialistOpinion("Might be a weight alignment issue for AMD.", 0.40, False), }, inspect_results=InspectResult( logs="[CK] ck_attention_varlen_gqa: SIGSEGV\n[CK] Known issue: GQA + variable-length triggers OOB access\n[Triton] Triton attention works for this config", config="rocm_attention: composable_kernel\ngqa_enabled: true\nvarlen: true\nknown_ck_bugs: [gqa_varlen]", snippet="# CK has a bug in GQA + varlen attention (OOB memory access)\n# Triton backend handles this correctly\n# Fix: route GQA+varlen to Triton on AMD", metrics="crashes: 3/10 requests\nsegfaults: 3\ntriton_fallback: not_configured", ), specialist_followups={ "runtime": "The segfault is in CK library code, not a runtime issue.", "dispatch": "Switch to Triton attention for GQA+varlen on AMD. CK bug is known and not yet fixed upstream.", "kernel": "CK varlen GQA kernel has off-by-one in tile boundary. Triton implementation doesn't have this bug.", "loader": "Not a weight issue. The crash is in the attention computation.", }, )) # --- model_config additional scenarios --- scenarios.append(Scenario( id="model_config_03", root_cause="model_config", correct_fix="update_model_config", incident_ticket=( "INCIDENT: DeepSeek MLA attention produces wrong KV cache size. " "OOM on sequences that should fit. Config shows standard MHA dimensions " "but model uses MLA with compressed KV." ), hardware="NVIDIA SM121 (DGX Spark)", model_name="DeepSeek-V3-671B", backend="FlashInfer 0.4", initial_log=( "[FlashInfer] KV cache: allocating for 64 KV heads x 128 dim = 8192 per token\n" "[FlashInfer] Expected MLA: kv_lora_rank=512, much smaller KV cache\n" "[FlashInfer] OOM: KV cache exceeds 80GB at seq_len=4096" ), initial_snippet=( "# config.json\n" '{\n' ' "num_key_value_heads": 64,\n' ' "head_dim": 128\n' ' // Missing: kv_lora_rank, qk_rope_head_dim for MLA\n' '}\n' ), specialist_opinions={ "runtime": SpecialistOpinion("Memory allocation works. Just allocating too much.", 0.72, False), "dispatch": SpecialistOpinion("FlashInfer correctly reading config. Config is the problem.", 0.68, False), "kernel": SpecialistOpinion( "MLA attention needs kv_lora_rank in config to use compressed KV. " "Without it, falls back to full MHA KV cache sizing.", 0.92, True ), "loader": SpecialistOpinion( "Config.json doesn't have MLA parameters. Need kv_lora_rank=512 and qk_rope_head_dim=64.", 0.93, True ), }, inspect_results=InspectResult( logs="[FlashInfer] No kv_lora_rank in config -> full MHA KV\n[FlashInfer] KV per token: 64*128*2=16384 (should be 512*2=1024 with MLA)\n[FlashInfer] 16x memory overhead", config="num_kv_heads: 64\nhead_dim: 128\nkv_lora_rank: (missing)\nqk_rope_head_dim: (missing)\nattention_type: inferred as MHA", snippet="# DeepSeek MLA config needs:\n# kv_lora_rank: 512\n# qk_rope_head_dim: 64\n# Without these, system allocates full MHA KV cache", metrics="kv_cache_per_token_bytes: 16384\nexpected_bytes: 1024\nmemory_overhead: 16x\noom_at_seq_len: 4096", ), specialist_followups={ "runtime": "No runtime issue. Memory allocation succeeds until OOM.", "dispatch": "Config drives the dispatch. Fix the config.", "kernel": "MLA kernel exists but won't activate without kv_lora_rank in config.", "loader": "Add kv_lora_rank=512 and qk_rope_head_dim=64 to config.json.", }, )) scenarios.append(Scenario( id="model_config_04", root_cause="model_config", correct_fix="update_model_config", incident_ticket=( "INCIDENT: Llama-4 Maverick MoE model failing with 'Expected 128 experts'. " "Config lists num_local_experts=128 but actual checkpoint uses sparse layout " "with 16 active experts per token from 128 total, stored differently." ), hardware="NVIDIA H100", model_name="Llama-4-Maverick-17Bx128E", backend="vLLM 0.8.x", initial_log=( "[vLLM] MoE init: 128 experts, 2 active per token\n" "[vLLM] Loading expert weights...\n" "[vLLM] WARNING: Expert weight tensor shape doesn't match config\n" "[vLLM] Expected: [128, hidden, ffn] Got: [128, ffn//4, hidden]" ), initial_snippet=( "# config.json\n" '{\n' ' "num_local_experts": 128,\n' ' "num_experts_per_tok": 2,\n' ' "expert_layout": "dense"\n' ' // Should be "interleaved" for Maverick architecture\n' '}\n' ), specialist_opinions={ "runtime": SpecialistOpinion("Runtime OK.", 0.75, False), "dispatch": SpecialistOpinion("MoE dispatch looks correct for the config.", 0.60, False), "kernel": SpecialistOpinion( "Expert weight tensor shape is transposed vs config expectation. " "Config says dense layout but weights are in interleaved format.", 0.85, True ), "loader": SpecialistOpinion( "Config expert_layout should be 'interleaved' not 'dense'. " "Maverick uses interleaved expert storage.", 0.93, True ), }, inspect_results=InspectResult( logs="[vLLM] Config: expert_layout=dense\n[vLLM] Actual weights: interleaved layout\n[vLLM] Shape mismatch in MoE layer 0", config="expert_layout: dense (wrong)\nactual_layout: interleaved\nnum_experts: 128\nexperts_per_token: 2", snippet="# Maverick checkpoint uses interleaved expert layout:\n# experts stored as [expert_idx, ffn_chunk, hidden]\n# Config says 'dense' which expects [expert_idx, hidden, ffn]\n# Fix: set expert_layout='interleaved'", metrics="model_load_progress: 5%\nshape_mismatches: 128\nerror_at: expert_layer_0", ), specialist_followups={ "runtime": "Not a runtime issue.", "dispatch": "Dispatch follows config. Fix the config first.", "kernel": "Weight shapes don't match the layout assumption. Config needs updating.", "loader": "Set expert_layout to 'interleaved' in config.json. Maverick stores experts interleaved.", }, )) scenarios.append(Scenario( id="model_config_05", root_cause="model_config", correct_fix="update_model_config", incident_ticket=( "INCIDENT: Sliding window attention not activating for Mistral model. " "Memory usage growing linearly with sequence length. " "Should plateau after window size." ), hardware="NVIDIA B200", model_name="Mistral-Large-2", backend="SGLang 0.5.x", initial_log=( "[SGLang] Attention config: full attention (no sliding window)\n" "[SGLang] KV cache growing linearly with seq_len\n" "[SGLang] Memory at 32k tokens: 40GB (expected: 12GB with sliding window)\n" "[SGLang] sliding_window not found in config.json" ), initial_snippet=( "# config.json\n" '{\n' ' "max_position_embeddings": 32768,\n' ' "num_attention_heads": 96\n' ' // Missing: "sliding_window": 4096\n' '}\n' ), specialist_opinions={ "runtime": SpecialistOpinion("Runtime fine. Memory growing as expected for full attention.", 0.78, False), "dispatch": SpecialistOpinion( "Backend correctly doing full attention because config doesn't specify sliding window.", 0.70, True ), "kernel": SpecialistOpinion( "Kernel supports sliding window. Config just needs the parameter.", 0.82, True ), "loader": SpecialistOpinion( "Config.json missing sliding_window=4096. Mistral models use 4096-token sliding window.", 0.92, True ), }, inspect_results=InspectResult( logs="[SGLang] No sliding_window in config -> full attention\n[SGLang] KV cache: 32k * 96 heads * 128 dim * 2 = 40GB", config="sliding_window: null\nmax_position_embeddings: 32768\nexpected_sliding_window: 4096", snippet="# Mistral-Large-2 uses 4096-token sliding window\n# Config missing: sliding_window: 4096\n# Without it, full O(n) KV cache used", metrics="kv_cache_32k_gb: 40\nexpected_kv_cache_gb: 12\nmemory_overhead: 3.3x", ), specialist_followups={ "runtime": "Memory growth is correct for the config given. Fix the config.", "dispatch": "Backend reads config. Add sliding_window=4096.", "kernel": "Sliding window attention kernel exists. Just needs the config parameter to activate.", "loader": "Add sliding_window: 4096 to config.json.", }, )) # --- weight_layout additional scenarios --- scenarios.append(Scenario( id="weight_layout_03", root_cause="weight_layout", correct_fix="fix_weight_mapping", incident_ticket=( "INCIDENT: Model outputs garbage after quantization with GPTQ. " "Original FP16 model is fine. GPTQ quantization reports success " "but group indices are misaligned." ), hardware="NVIDIA H100", model_name="Qwen3-235B-A22B", backend="vLLM 0.8.x", initial_log=( "[vLLM] Loading GPTQ-quantized Qwen3...\n" "[vLLM] Quantization: 4-bit, group_size=128\n" "[vLLM] WARNING: g_idx tensor shape mismatch in layer 0\n" "[vLLM] Output: incoherent (perplexity 1247)" ), initial_snippet=( "# GPTQ packing\n" "# g_idx maps each weight column to its quantization group\n" "# Expected shape: [in_features]\n" "# Got shape: [in_features // group_size] (wrong!)\n" ), specialist_opinions={ "runtime": SpecialistOpinion("CUDA fine. Kernels launch.", 0.78, False), "dispatch": SpecialistOpinion("GPTQ backend selected correctly.", 0.65, False), "kernel": SpecialistOpinion( "Dequantization kernel gets wrong group assignments because g_idx is wrong shape.", 0.82, True ), "loader": SpecialistOpinion( "GPTQ group index (g_idx) tensor has wrong shape. The quantization script packed it incorrectly. " "Needs regeneration with correct per-column group mapping.", 0.94, True ), }, inspect_results=InspectResult( logs="[GPTQ] g_idx shape: [128] (wrong) vs expected [16384]\n[GPTQ] Each column needs its own group index\n[GPTQ] Wrong g_idx causes random dequant scale selection", config="group_size: 128\nin_features: 16384\ng_idx_shape: [128]\nexpected_g_idx_shape: [16384]", snippet="# g_idx should be per-column: shape [in_features]\n# But quantizer produced per-group: shape [in_features//group_size]\n# This assigns wrong scales during dequantization", metrics="perplexity: 1247\nexpected_perplexity: 10.2\nlayers_affected: all\ng_idx_misaligned: true", ), specialist_followups={ "runtime": "No runtime issues.", "dispatch": "Backend selection is fine.", "kernel": "Kernel dequantizes correctly when given right g_idx. Fix the mapping.", "loader": "Regenerate g_idx with per-column mapping (shape [in_features], not [in_features//group_size]).", }, )) scenarios.append(Scenario( id="weight_layout_04", root_cause="weight_layout", correct_fix="fix_weight_mapping", incident_ticket=( "INCIDENT: FP8 model on MI300X gives NaN after first layer. " "Dequantization scales appear transposed. " "Same checkpoint works on NVIDIA with e4m3fn format." ), hardware="AMD MI300X", model_name="DeepSeek-R1-Distill-70B", backend="vLLM 0.8.x", initial_log=( "[vLLM] FP8 dequant: loading scales...\n" "[vLLM] Scale tensor shape: [out_features, 1] — expected [1, out_features] for AMD\n" "[vLLM] Layer 0 output: NaN (scale applied to wrong dimension)\n" "[vLLM] All subsequent layers: NaN" ), initial_snippet=( "# fp8_weights.py\n" "# NVIDIA: scales are per-output-channel [out, 1]\n" "# AMD: scales are per-input-channel [1, in]\n" "# Converter didn't transpose for AMD\n" ), specialist_opinions={ "runtime": SpecialistOpinion("ROCm runtime fine.", 0.78, False), "dispatch": SpecialistOpinion("FP8 backend selected. Format mismatch possible.", 0.65, False), "kernel": SpecialistOpinion( "FP8 GEMM applies scale in wrong dimension due to transposed scale tensor.", 0.85, True ), "loader": SpecialistOpinion( "FP8 scale tensors need transposing for AMD. NVIDIA uses [out,1], AMD uses [1,in]. " "Weight converter didn't handle this.", 0.95, True ), }, inspect_results=InspectResult( logs="[FP8] Scale shape [4096,1] but AMD MFMA expects [1,4096]\n[FP8] Dequant: scale broadcast on wrong axis -> NaN\n[FP8] First non-NaN result never produced", config="fp8_scale_shape: [out_features, 1]\namd_expected: [1, in_features]\nscale_transpose_needed: true", snippet="# NVIDIA layout: W_fp8 * scale[out,1] -> per-output-channel\n# AMD layout: W_fp8 * scale[1,in] -> per-input-channel\n# Converter assumed NVIDIA layout\n# Fix: transpose scales for AMD", metrics="nan_outputs: 100%\nlayers_producing_nan: all\nfirst_nan_at: layer_0", ), specialist_followups={ "runtime": "Not a runtime issue.", "dispatch": "FP8 selected correctly. Scale orientation is the issue.", "kernel": "GEMM kernel applies scale along wrong dimension. Transpose the scales.", "loader": "Transpose FP8 scale tensors from [out,1] to [1,in] for AMD.", }, )) scenarios.append(Scenario( id="weight_layout_05", root_cause="weight_layout", correct_fix="fix_weight_mapping", incident_ticket=( "INCIDENT: Embedding layer produces identical vectors for all tokens. " "After checkpoint conversion, embedding weights appear row-shuffled. " "Tokenizer maps to wrong rows." ), hardware="NVIDIA SM121 (DGX Spark)", model_name="Llama-4-Maverick-17Bx128E", backend="SGLang 0.5.x", initial_log=( "[SGLang] Embedding layer: 128256 tokens x 4096 dim\n" "[SGLang] Token 'Hello' -> embedding row 85432 (expected: row 9906)\n" "[SGLang] All outputs identical — embeddings mapped to wrong rows\n" "[SGLang] Suspect: tokenizer vocab offset not applied during conversion" ), initial_snippet=( "# convert_checkpoint.py\n" "embed = original_weights['embed_tokens.weight'] # [128256, 4096]\n" "# BUG: added_tokens offset not applied\n" "# Tokenizer expects base_vocab at rows 0-127999\n" "# Converter put added_tokens at rows 0-255\n" ), specialist_opinions={ "runtime": SpecialistOpinion("Runtime fine. Model loads.", 0.75, False), "dispatch": SpecialistOpinion("Backend dispatch correct.", 0.68, False), "kernel": SpecialistOpinion( "Embedding lookup works mechanically but returns wrong vectors. Data issue.", 0.78, True ), "loader": SpecialistOpinion( "Embedding weight rows are misaligned after conversion. Tokenizer indices map to wrong rows. " "Converter needs to preserve original row ordering.", 0.94, True ), }, inspect_results=InspectResult( logs="[SGLang] Token 'Hello' (id=9906) -> embedding from original row 85432\n[SGLang] Row mapping offset: 75526\n[SGLang] Converter applied wrong row permutation", config="vocab_size: 128256\nembed_dim: 4096\nrow_offset_error: 75526", snippet="# Converter reordered rows: put added_tokens (256) first, then base vocab\n# Tokenizer expects base vocab at row 0\n# Fix: preserve original row order in embedding conversion", metrics="embedding_cosine_sim_to_expected: 0.02\nall_outputs_identical: true\nperplexity: infinity", ), specialist_followups={ "runtime": "No runtime issue.", "dispatch": "Dispatch is correct.", "kernel": "Embedding lookup returns whatever is at the indexed row. The rows are just wrong.", "loader": "Converter put added_tokens at index 0. Fix: keep original row order.", }, )) # --- Additional eval scenarios (_06 suffix) --- scenarios.append(Scenario( id="arch_guard_06", root_cause="arch_guard", correct_fix="relax_arch_check", incident_ticket=( "INCIDENT: CUTLASS GEMM kernel rejects SM121 with 'unsupported architecture'. " "is_family_of() check fails because SM121 not in family table. " "FP8 inference completely blocked." ), hardware="NVIDIA SM121 (DGX Spark)", model_name="Mistral-Large-2", backend="TensorRT-LLM 0.18", initial_log=( "[CUTLASS] is_family_of(sm_121, sm_90) = false\n" "[CUTLASS] SM121 not registered in family hierarchy\n" "[CUTLASS] FP8 GEMM dispatch: BLOCKED" ), initial_snippet=( "# cutlass/arch/family.py\n" "FAMILY_MAP = {90: [90], 89: [89], 86: [86], 80: [80]}\n" "# SM121 not in any family\n" ), specialist_opinions={ "runtime": SpecialistOpinion("CUDA 13 fine.", 0.78, False), "dispatch": SpecialistOpinion( "CUTLASS family map doesn't include SM12x. Need to register SM120/121 family.", 0.93, True ), "kernel": SpecialistOpinion( "The kernel weight format might be wrong for SM121.", 0.40, False ), "loader": SpecialistOpinion("Engine built. Weights loaded. GEMM dispatch blocked.", 0.70, False), }, inspect_results=InspectResult( logs="[CUTLASS] FAMILY_MAP has no entry for 121\n[CUTLASS] is_family_of(121, 90) -> False\n[CUTLASS] FP8 GEMM requires family >= 90", config="gpu_sm: 121\nfamily_map: {90:[90],89:[89],...}\nsm121_family: undefined", snippet="# SM12x is its own family but shares FP8 MMA with SM90\n# Fix: add 120: [120, 121] and 121: [120, 121] to FAMILY_MAP\n# Or: register SM12x as SM90-compatible for GEMM", metrics="fp8_gemm_blocked: true\nbf16_gemm: functional", ), specialist_followups={ "runtime": "Runtime fine.", "dispatch": "Register SM12x family in CUTLASS. SM121 FP8 MMA is SM90-compatible.", "kernel": "Weight format is fine. It's the arch family check blocking dispatch.", "loader": "Weights loaded correctly. GEMM dispatch is the issue.", }, )) scenarios.append(Scenario( id="backend_selector_06", root_cause="backend_selector", correct_fix="switch_backend", incident_ticket=( "INCIDENT: DGX Spark running PagedAttention v1 instead of v2. " "Prefix caching not working. Cache hit rate near 0%. " "Same prompts re-computed every request." ), hardware="NVIDIA SM121 (DGX Spark)", model_name="DeepSeek-V3-671B", backend="vLLM 0.8.x", initial_log=( "[vLLM] PagedAttention version: v1\n" "[vLLM] Prefix caching: disabled (requires PA v2)\n" "[vLLM] Cache hit rate: 0.1% (expected: 60%+ with repeated prefixes)\n" "[vLLM] TTFT p99: 2100ms (expected: 400ms with caching)" ), initial_snippet=( "# vllm/core/scheduler.py\n" "def select_paged_attention(gpu_sm):\n" " if gpu_sm >= 80 and gpu_sm <= 90:\n" " return 'v2' # with prefix caching\n" " return 'v1' # SM121 > 90, falls here\n" ), specialist_opinions={ "runtime": SpecialistOpinion("CUDA runtime fine. Server runs.", 0.75, False), "dispatch": SpecialistOpinion( "PagedAttention version selector has range bug. SM121 > 90 so gets v1 without prefix caching.", 0.94, True ), "kernel": SpecialistOpinion( "PA v2 kernel works on SM121. It's the selector that's wrong.", 0.85, True ), "loader": SpecialistOpinion("Model loaded fine. Not a weight issue.", 0.72, False), }, inspect_results=InspectResult( logs="[vLLM] sm_121 not in range [80,90] -> PA v1\n[vLLM] PA v1 doesn't support prefix caching\n[vLLM] Every prefix re-computed from scratch", config="paged_attention: v1\nprefix_caching: disabled\ngpu_sm: 121\nv2_range: [80, 90]", snippet="# PA v2 supports prefix caching, reducing TTFT 3-5x\n# Selector range [80,90] excludes SM121\n# Fix: include SM12x in v2-eligible set", metrics="cache_hit_rate: 0.1%\nexpected_cache_hit_rate: 62%\nttft_p99_ms: 2100\nexpected_ttft_ms: 400", ), specialist_followups={ "runtime": "Server runs fine. Performance issue only.", "dispatch": "Fix the range check to include SM12x. PA v2 works on SM121.", "kernel": "PA v2 kernel is compatible. Just need the selector to pick it.", "loader": "Not a loading issue.", }, )) scenarios.append(Scenario( id="runtime_loader_06", root_cause="runtime_loader", correct_fix="fix_runtime_path", incident_ticket=( "INCIDENT: Container on B200 node fails with 'CUDA driver version insufficient'. " "Host has driver 565 but container sees driver 535. " "nvidia-smi inside container shows old driver." ), hardware="NVIDIA B200", model_name="Llama-3.3-70B-Instruct", backend="vLLM 0.8.x", initial_log=( "[Container] nvidia-smi: Driver Version: 535.183.01\n" "[Host] nvidia-smi: Driver Version: 565.57.01\n" "[vLLM] CUDA 13 requires driver >= 560\n" "[vLLM] ERROR: CUDA driver version insufficient for CUDA runtime" ), initial_snippet=( "# Docker run command\n" "docker run --gpus all \\\n" " -e NVIDIA_DRIVER_CAPABILITIES=compute,utility \\\n" " -e NVIDIA_VISIBLE_DEVICES=all \\\n" " # Missing: --runtime=nvidia or proper CDI config\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "Container seeing old driver. Docker GPU passthrough not configured correctly. " "Need proper nvidia-container-runtime setup.", 0.94, True ), "dispatch": SpecialistOpinion("Server never starts. Can't assess dispatch.", 0.50, False), "kernel": SpecialistOpinion( "Maybe the B200 needs a newer CUDA toolkit version.", 0.45, False ), "loader": SpecialistOpinion( "Container's nvidia driver libs are stale. Bind mount is pointing to wrong driver version.", 0.88, True ), }, inspect_results=InspectResult( logs="[Container] /usr/lib/x86_64-linux-gnu/libnvidia-ml.so -> driver 535\n[Host] /usr/lib/x86_64-linux-gnu/libnvidia-ml.so -> driver 565\n[Docker] nvidia-container-runtime not in daemon.json", config="host_driver: 565.57.01\ncontainer_driver: 535.183.01\nnvidia_runtime: not_configured", snippet="# Docker daemon.json missing nvidia runtime\n# Container bundles old driver libs instead of using host driver\n# Fix: configure nvidia-container-runtime or CDI", metrics="container_start_failures: 1\ndriver_mismatch: true\ncuda_init: failed", ), specialist_followups={ "runtime": "nvidia-container-toolkit needs to be configured to pass host driver into container.", "dispatch": "Can't run without CUDA init.", "kernel": "The toolkit version is fine. It's the driver passthrough that's broken.", "loader": "Container needs host's driver libs mounted. Fix Docker runtime config.", }, )) scenarios.append(Scenario( id="model_config_06", root_cause="model_config", correct_fix="update_model_config", incident_ticket=( "INCIDENT: BF16 model serving on MI300X has 2x expected memory usage. " "Config says float16 dtype but model should use bfloat16. " "Unnecessary fp16->bf16 conversion happening at runtime." ), hardware="AMD MI300X", model_name="DeepSeek-R1-Distill-70B", backend="vLLM 0.8.x", initial_log=( "[vLLM] Config dtype: float16\n" "[vLLM] Actual weights: bfloat16\n" "[vLLM] Runtime conversion float16 config -> bfloat16 weights\n" "[vLLM] Extra memory for conversion buffers: 35GB" ), initial_snippet=( "# config.json\n" '{\n' ' "torch_dtype": "float16"\n' ' // Actual checkpoint is bfloat16\n' ' // Mismatch causes runtime conversion overhead\n' '}\n' ), specialist_opinions={ "runtime": SpecialistOpinion("ROCm runtime healthy. Memory available.", 0.78, False), "dispatch": SpecialistOpinion("Backend dispatch fine.", 0.65, False), "kernel": SpecialistOpinion( "Kernels running with dtype conversion overhead. " "Config says fp16 but weights are bf16, so vLLM converts at load time.", 0.82, True ), "loader": SpecialistOpinion( "Config torch_dtype=float16 doesn't match checkpoint dtype=bfloat16. " "Fix config to say bfloat16 to avoid conversion overhead.", 0.93, True ), }, inspect_results=InspectResult( logs="[vLLM] Config: float16, Checkpoint: bfloat16\n[vLLM] Allocating conversion buffers: 35GB\n[vLLM] Total memory: model(35GB) + conversion(35GB) = 70GB", config="torch_dtype: float16\ncheckpoint_dtype: bfloat16\nmismatch: true", snippet="# Config says float16 but checkpoint is bfloat16\n# vLLM allocates both versions during conversion\n# Fix: set torch_dtype='bfloat16' in config.json", metrics="memory_used_gb: 70\nexpected_memory_gb: 35\nconversion_overhead_gb: 35", ), specialist_followups={ "runtime": "Memory subsystem fine. Just using too much.", "dispatch": "Dispatch fine after conversion.", "kernel": "Conversion overhead is the issue. Fix config to match checkpoint dtype.", "loader": "Set torch_dtype to bfloat16 in config.json.", }, )) scenarios.append(Scenario( id="weight_layout_06", root_cause="weight_layout", correct_fix="fix_weight_mapping", incident_ticket=( "INCIDENT: Rotary position encoding giving wrong angles after checkpoint merge. " "Two LoRA adapters merged into base model, but RoPE inv_freq tensor " "accidentally overwritten with adapter values. Outputs degrade past position 128." ), hardware="NVIDIA H100", model_name="Mistral-Large-2", backend="vLLM 0.8.x", initial_log=( "[vLLM] Loading merged checkpoint...\n" "[vLLM] RoPE inv_freq shape: [64] (correct)\n" "[vLLM] RoPE inv_freq values: [0.001, 0.001, ...] (all same — WRONG)\n" "[vLLM] Expected: geometric sequence 1/10000^(2i/d)" ), initial_snippet=( "# merge_lora.py\n" "# BUG: LoRA merge accidentally overwrote inv_freq\n" "merged['inv_freq'] = adapter_state['inv_freq'] # adapter had dummy values\n" "# Should have kept base model's inv_freq\n" ), specialist_opinions={ "runtime": SpecialistOpinion("Runtime fine.", 0.78, False), "dispatch": SpecialistOpinion("Backend dispatch correct.", 0.65, False), "kernel": SpecialistOpinion( "RoPE kernel computes correct rotations for the freq values given. But freq values are wrong.", 0.80, True ), "loader": SpecialistOpinion( "LoRA merge script overwrote inv_freq with adapter's dummy values. " "Need to restore base model's inv_freq or regenerate from formula.", 0.95, True ), }, inspect_results=InspectResult( logs="[RoPE] inv_freq: all values = 0.001 (constant)\n[RoPE] Expected: geometric decay from 1.0 to 1e-4\n[RoPE] Position encoding essentially constant -> no position info after ~128 tokens", config="inv_freq_values: [0.001]*64\nexpected: geometric_series(1/10000, dim=128)\nrope_theta: 10000", snippet="# inv_freq should be: 1 / (theta ** (torch.arange(0, dim, 2) / dim))\n# Instead: all 0.001 from LoRA adapter dummy init\n# Fix: regenerate inv_freq from formula or restore from base model", metrics="quality_0_128: 90%\nquality_128_1k: 25%\nquality_1k_plus: 5%", ), specialist_followups={ "runtime": "No runtime issue.", "dispatch": "Dispatch correct.", "kernel": "RoPE kernel works. Just getting wrong frequencies.", "loader": "Restore inv_freq from base model. LoRA merge script has a bug that overwrites non-LoRA tensors.", }, )) # --- memory_oom scenarios --- scenarios.append(Scenario( id="memory_oom_01", root_cause="memory_oom", correct_fix="tune_memory_config", incident_ticket=( "INCIDENT: vLLM OOM crash serving DeepSeek-V3-671B on 8xH100. " "Model loads successfully but crashes after ~50 concurrent requests. " "GPU memory fragmentation suspected. KV cache allocation fails." ), hardware="NVIDIA H100", model_name="DeepSeek-V3-671B", backend="vLLM 0.8.x", initial_log=( "[vLLM] Model loaded: 671B params across 8 GPUs (tensor parallel=8)\n" "[vLLM] KV cache: allocated 45GB per GPU (gpu_memory_utilization=0.95)\n" "[vLLM] Serving... 48 concurrent requests OK\n" "[vLLM] Request 51: torch.cuda.OutOfMemoryError: CUDA out of memory. " "Tried to allocate 2.1 GiB. GPU 3 has 0.8 GiB free." ), initial_snippet=( "# vllm serve config\n" "gpu_memory_utilization: 0.95\n" "max_num_seqs: 256\n" "max_model_len: 32768\n" "# No swap space configured\n" "# No memory headroom for activation spikes\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "CUDA runtime is healthy. OOM is a memory planning issue, not runtime.", 0.80, False ), "dispatch": SpecialistOpinion( "Backend dispatch is fine. This is a memory capacity issue.", 0.65, False ), "kernel": SpecialistOpinion( "MoE expert activation creates memory spikes. With 256 experts and " "dynamic routing, peak memory is unpredictable. Need to lower " "gpu_memory_utilization to leave headroom.", 0.91, True ), "loader": SpecialistOpinion( "gpu_memory_utilization=0.95 leaves no headroom. With MoE models, " "activation memory varies per-request depending on expert routing. " "Reduce to 0.85 and set max_num_seqs to 128.", 0.93, True ), }, inspect_results=InspectResult( logs=( "[vLLM] Per-GPU memory: 80GB total, 76GB allocated (95%)\n" "[vLLM] Model weights: 28GB per GPU\n" "[vLLM] KV cache: 45GB per GPU\n" "[vLLM] Remaining: 3GB (insufficient for MoE activation spikes)\n" "[vLLM] Peak activation for DeepSeek MoE: ~5GB per GPU" ), config=( "gpu_memory_utilization: 0.95\n" "max_num_seqs: 256\n" "max_model_len: 32768\n" "swap_space_gb: 0\n" "tensor_parallel_size: 8\n" "model_weights_per_gpu_gb: 28\n" "kv_cache_per_gpu_gb: 45" ), snippet=( "# Memory budget too tight for MoE model\n" "# Model weights: 28GB + KV cache: 45GB = 73GB\n" "# Remaining: 3GB for activations\n" "# MoE expert routing can spike to 5GB+ activation\n" "# Fix: gpu_memory_utilization=0.85, max_num_seqs=128" ), metrics=( "oom_crashes: 14 in 1 hour\n" "avg_concurrent_at_crash: 52\n" "peak_activation_gb: 5.2\n" "available_at_crash_gb: 0.8\n" "gpu_memory_utilization: 0.95" ), ), specialist_followups={ "runtime": "CUDA runtime is fine. The OOM is caused by overcommitted memory planning.", "dispatch": "Backend is dispatching correctly. Memory budget is the issue.", "kernel": "MoE activation spikes are the trigger. Reduce gpu_memory_utilization to 0.85.", "loader": "Lower gpu_memory_utilization to 0.85 and cap max_num_seqs at 128 for MoE headroom.", }, )) scenarios.append(Scenario( id="memory_oom_02", root_cause="memory_oom", correct_fix="tune_memory_config", incident_ticket=( "INCIDENT: SGLang server on B200 crashes with OOM during long-context requests. " "Short prompts (<4K tokens) work fine. Any request over 16K tokens causes crash. " "Serving Llama-4-Maverick with 128 experts." ), hardware="NVIDIA B200", model_name="Llama-4-Maverick-17Bx128E", backend="SGLang 0.5.x", initial_log=( "[SGLang] Model loaded: Llama-4-Maverick-17Bx128E\n" "[SGLang] KV cache pre-allocated for max_total_tokens=65536\n" "[SGLang] Request (len=18432): allocating KV blocks...\n" "[SGLang] FATAL: torch.cuda.OutOfMemoryError during KV cache expansion\n" "[SGLang] GPU memory: 189GB used / 192GB total" ), initial_snippet=( "# sglang_config.yaml\n" "max_total_tokens: 65536\n" "chunked_prefill_size: 8192\n" "mem_fraction_static: 0.90\n" "# KV cache over-allocated for 128-expert MoE model\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "CUDA runtime healthy. Memory fragmentation from large contiguous allocations.", 0.72, False ), "dispatch": SpecialistOpinion( "Chunked prefill is working but chunk size may be too large for this context length.", 0.55, False ), "kernel": SpecialistOpinion( "KV cache expansion tries to allocate contiguous memory. " "With 128 experts, the KV cache per-token is much larger than dense models. " "max_total_tokens needs to be reduced for MoE.", 0.90, True ), "loader": SpecialistOpinion( "mem_fraction_static=0.90 is too high for MoE. With 128 experts, " "the shared expert and routing tensors need dynamic memory. " "Lower to 0.80 and reduce max_total_tokens to 32768.", 0.94, True ), }, inspect_results=InspectResult( logs=( "[SGLang] Per-token KV size: 2.4MB (128 experts x shared KV)\n" "[SGLang] 65536 tokens * 2.4MB = 157GB KV cache requested\n" "[SGLang] Model weights: 35GB\n" "[SGLang] Total needed: 192GB (exceeds 192GB GPU memory)\n" "[SGLang] OOM at KV block allocation for long sequences" ), config=( "max_total_tokens: 65536\n" "mem_fraction_static: 0.90\n" "kv_per_token_mb: 2.4\n" "model_weights_gb: 35\n" "gpu_memory_gb: 192" ), snippet=( "# MoE KV cache is much larger than dense model estimate\n" "# Dense model: ~0.5MB per token KV\n" "# 128-expert MoE: ~2.4MB per token KV (shared + expert KV)\n" "# Fix: max_total_tokens=32768, mem_fraction_static=0.80" ), metrics=( "oom_count: 23\n" "avg_sequence_len_at_oom: 17500\n" "max_successful_len: 15200\n" "kv_cache_gb_at_crash: 157\n" "available_gb: 192" ), ), specialist_followups={ "runtime": "Not a runtime issue. Memory planning doesn't account for MoE KV overhead.", "dispatch": "Chunked prefill helps but doesn't solve the KV cache over-allocation.", "kernel": "KV cache per-token is 4.8x larger than estimated. Reduce max_total_tokens.", "loader": "Set max_total_tokens=32768 and mem_fraction_static=0.80 for MoE headroom.", }, )) scenarios.append(Scenario( id="memory_oom_03", root_cause="memory_oom", correct_fix="tune_memory_config", incident_ticket=( "INCIDENT: TensorRT-LLM engine build OOM on RTX 5090 for Qwen3-235B. " "Engine build phase exhausts 32GB VRAM during weight conversion. " "Same model builds fine on 80GB A100." ), hardware="NVIDIA SM120 (GeForce RTX 5090)", model_name="Qwen3-235B-A22B", backend="TensorRT-LLM 0.18", initial_log=( "[TensorRT-LLM] Building engine for Qwen3-235B-A22B...\n" "[TensorRT-LLM] Weight conversion phase: loading FP16 weights into GPU memory\n" "[TensorRT-LLM] ERROR: torch.cuda.OutOfMemoryError during weight quantization\n" "[TensorRT-LLM] Attempted to allocate 28GB for layer conversion buffer\n" "[TensorRT-LLM] GPU memory: 31.5GB used / 32GB total" ), initial_snippet=( "# trtllm_build_config.py\n" "build_config = {\n" " 'max_batch_size': 64,\n" " 'max_input_len': 8192,\n" " 'weight_streaming': False, # loads all weights to GPU\n" " 'use_paged_context_fmha': True,\n" "}\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "CUDA 13 runtime is fine. GPU memory is just 32GB — too small for full model load.", 0.82, True ), "dispatch": SpecialistOpinion( "TensorRT-LLM engine build shouldn't need all weights on GPU at once. " "Enable weight_streaming to convert layer-by-layer.", 0.91, True ), "kernel": SpecialistOpinion( "Maybe try a different quantization algorithm that uses less memory.", 0.45, False ), "loader": SpecialistOpinion( "The model is too large for this GPU. Need a bigger GPU.", 0.40, False ), }, inspect_results=InspectResult( logs=( "[TensorRT-LLM] Total FP16 weight size: 470GB\n" "[TensorRT-LLM] Per-layer max weight: 3.2GB\n" "[TensorRT-LLM] Conversion buffer: 28GB (loading 9 layers at once)\n" "[TensorRT-LLM] weight_streaming: disabled -> all layers loaded to GPU\n" "[TensorRT-LLM] GPU VRAM: 32GB" ), config=( "weight_streaming: false\n" "layers_loaded_simultaneously: 9\n" "per_layer_weight_gb: 3.2\n" "conversion_buffer_gb: 28\n" "gpu_vram_gb: 32" ), snippet=( "# weight_streaming=False loads multiple layers to GPU simultaneously\n" "# 9 layers * 3.2GB = 28GB conversion buffer\n" "# + engine workspace + CUDA context = >32GB\n" "# Fix: enable weight_streaming for layer-by-layer conversion" ), metrics=( "build_oom_count: 1\n" "conversion_buffer_gb: 28\n" "gpu_vram_gb: 32\n" "layers_loaded: 9\n" "max_layers_for_32gb: 3" ), ), specialist_followups={ "runtime": "32GB VRAM is sufficient if weight streaming is enabled for layer-by-layer conversion.", "dispatch": "Enable weight_streaming=True so the builder converts one layer at a time instead of loading 9.", "kernel": "Quantization algorithm is fine. The issue is the build-time memory strategy.", "loader": "The model can serve on 32GB with streaming. It's the build phase that's misconfigured.", }, )) scenarios.append(Scenario( id="memory_oom_04", root_cause="memory_oom", correct_fix="tune_memory_config", incident_ticket=( "INCIDENT: MI355X serving Llama-3.3-70B runs out of memory when beam search " "is enabled (num_beams=8). Greedy decoding works fine. " "OOM occurs during the first beam expansion step." ), hardware="AMD MI355X", model_name="Llama-3.3-70B-Instruct", backend="vLLM 0.8.x", initial_log=( "[vLLM] Serving Llama-3.3-70B-Instruct on MI355X\n" "[vLLM] Greedy decoding: OK (memory stable at 88%)\n" "[vLLM] Beam search (num_beams=8): allocating beam KV caches...\n" "[vLLM] ERROR: OOM during beam expansion. Each beam duplicates KV cache.\n" "[vLLM] Attempted: 8 * 14GB = 112GB KV for single request" ), initial_snippet=( "# vllm serve params\n" "--gpu-memory-utilization 0.92\n" "--max-num-seqs 64\n" "--max-model-len 16384\n" "# Beam search multiplies KV cache by num_beams\n" "# No per-request memory limit configured\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "ROCm runtime healthy. OOM is a memory allocation issue.", 0.75, False ), "dispatch": SpecialistOpinion( "Beam search KV duplication is the root cause. With 8 beams, each beam " "needs its own KV cache copy. Need to limit beam count or reserve memory.", 0.92, True ), "kernel": SpecialistOpinion( "The beam search kernel should share KV prefix across beams using copy-on-write. " "But vLLM's beam search implementation does full copies. This is a known issue.", 0.88, True ), "loader": SpecialistOpinion( "Maybe the model weights are in the wrong dtype causing extra memory.", 0.40, False ), }, inspect_results=InspectResult( logs=( "[vLLM] Greedy: 1 KV cache per request = 14GB\n" "[vLLM] Beam search: 8 KV caches per request = 112GB\n" "[vLLM] GPU memory: 128GB (MI355X)\n" "[vLLM] Model weights: 35GB\n" "[vLLM] Available for KV: 83GB (insufficient for 112GB beam KV)" ), config=( "gpu_memory_utilization: 0.92\n" "max_num_seqs: 64\n" "num_beams: 8\n" "kv_per_request_gb: 14\n" "gpu_total_gb: 128" ), snippet=( "# Beam search duplicates full KV cache per beam\n" "# 8 beams * 14GB = 112GB for single request\n" "# Available after model weights: 83GB\n" "# Fix: reduce num_beams to 4, lower max_model_len, or use\n" "# gpu_memory_utilization=0.85 with max_num_seqs=8 for beam mode" ), metrics=( "oom_on_beam_search: true\n" "kv_per_beam_gb: 14\n" "total_beam_kv_gb: 112\n" "available_gb: 83\n" "greedy_oom: false" ), ), specialist_followups={ "runtime": "ROCm runtime is fine. Memory budget doesn't account for beam duplication.", "dispatch": "Limit beams to 4 and reduce max_model_len to 8192 for beam search mode.", "kernel": "vLLM beam search copies full KV per beam. Use fewer beams or enable prefix sharing.", "loader": "Model weights are correct dtype. The issue is beam search KV duplication.", }, )) scenarios.append(Scenario( id="memory_oom_05", root_cause="memory_oom", correct_fix="tune_memory_config", incident_ticket=( "INCIDENT: Triton Inference Server on DGX Spark OOMs when loading second model. " "First model (DeepSeek-R1-Distill-70B) loads fine. Loading Mistral-Large-2 " "concurrently causes OOM. Both models fit individually but not together." ), hardware="NVIDIA SM121 (DGX Spark)", model_name="DeepSeek-R1-Distill-70B", backend="Triton Inference Server", initial_log=( "[Triton] Model 1 (DeepSeek-R1-Distill-70B): loaded, 65GB VRAM\n" "[Triton] Model 2 (Mistral-Large-2): loading...\n" "[Triton] ERROR: CUDA OOM allocating 48GB for model 2\n" "[Triton] GPU memory: 91GB used / 96GB total\n" "[Triton] Model instance groups both set to GPU 0" ), initial_snippet=( "# model_repository/config.pbtxt (both models)\n" "instance_group [\n" " { count: 1, kind: KIND_GPU, gpus: [0] }\n" "]\n" "# Both models pinned to GPU 0 (96GB)\n" "# No model loading policy configured\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "CUDA runtime fine. Both models are pinned to GPU 0 instead of being spread " "across available GPUs.", 0.87, True ), "dispatch": SpecialistOpinion( "Triton model placement is wrong. Both models on GPU 0 but 96GB isn't enough " "for both. Need to set model_loading_policy or spread across GPUs.", 0.92, True ), "kernel": SpecialistOpinion( "Maybe one of the models has a memory leak in its kernels.", 0.35, False ), "loader": SpecialistOpinion( "Both models are loaded eagerly at startup. Try lazy loading with " "model_loading_policy=on_demand.", 0.60, False ), }, inspect_results=InspectResult( logs=( "[Triton] GPU 0: 65GB (model 1) + 5GB (CUDA ctx) = 70GB used\n" "[Triton] GPU 0: 96GB total, 26GB free\n" "[Triton] Model 2 needs 48GB — doesn't fit on GPU 0\n" "[Triton] GPU 1: 96GB free (unused!)\n" "[Triton] Both models pinned to gpus: [0]" ), config=( "model_1_gpu: [0]\n" "model_2_gpu: [0]\n" "available_gpus: [0, 1, 2, 3]\n" "model_1_vram_gb: 65\n" "model_2_vram_gb: 48\n" "gpu_0_total_gb: 96" ), snippet=( "# Both models pinned to GPU 0 via instance_group config\n" "# GPU 0: 96GB (insufficient for 65+48=113GB)\n" "# GPUs 1-3: completely idle\n" "# Fix: assign model 2 to GPU 1, or use auto-placement" ), metrics=( "gpu_0_used_gb: 70\n" "gpu_0_free_gb: 26\n" "gpu_1_used_gb: 0\n" "model_2_needed_gb: 48\n" "total_gpus: 4" ), ), specialist_followups={ "runtime": "GPU 1-3 are idle. Assign model 2 to GPU 1 in instance_group config.", "dispatch": "Fix the instance_group config to spread models across GPUs. Or use auto-placement.", "kernel": "No memory leak. Both models just don't fit on one 96GB GPU.", "loader": "Lazy loading doesn't help — both models are needed concurrently. Spread across GPUs.", }, )) scenarios.append(Scenario( id="memory_oom_06", root_cause="memory_oom", correct_fix="tune_memory_config", incident_ticket=( "INCIDENT: vLLM on MI300X crashes with OOM when prefix caching is enabled. " "Without prefix caching, Qwen3-235B serves fine. With prefix caching, " "memory grows unbounded until OOM after ~200 requests." ), hardware="AMD MI300X", model_name="Qwen3-235B-A22B", backend="vLLM 0.8.x", initial_log=( "[vLLM] Prefix caching: enabled\n" "[vLLM] Initial GPU memory usage: 142GB / 192GB\n" "[vLLM] After 50 requests: 165GB / 192GB (prefix cache growing)\n" "[vLLM] After 150 requests: 188GB / 192GB\n" "[vLLM] Request 203: OOM — prefix cache consumed all free memory" ), initial_snippet=( "# vllm serve\n" "--enable-prefix-caching\n" "--gpu-memory-utilization 0.90\n" "--max-num-seqs 32\n" "# No prefix cache eviction policy\n" "# No max prefix cache size configured\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "ROCm runtime is fine. The prefix cache is never evicted and grows forever.", 0.83, True ), "dispatch": SpecialistOpinion( "Try disabling prefix caching entirely. It doesn't work with MoE models.", 0.40, False ), "kernel": SpecialistOpinion( "Prefix cache blocks are never freed. vLLM's evictor isn't respecting the " "memory budget. Need to set max_prefix_cache_tokens or lower gpu_memory_utilization.", 0.92, True ), "loader": SpecialistOpinion( "Model loaded correctly. Memory issue is in the KV cache subsystem.", 0.70, False ), }, inspect_results=InspectResult( logs=( "[vLLM] Prefix cache entries: 847 (never evicted)\n" "[vLLM] Prefix cache size: 46GB\n" "[vLLM] Cache eviction policy: NONE (default)\n" "[vLLM] gpu_memory_utilization=0.90 doesn't cap prefix cache\n" "[vLLM] OOM when active KV + prefix cache > available memory" ), config=( "enable_prefix_caching: true\n" "prefix_cache_eviction: none\n" "max_prefix_cache_tokens: unlimited\n" "gpu_memory_utilization: 0.90\n" "prefix_cache_gb: 46" ), snippet=( "# Prefix cache grows without bound\n" "# gpu_memory_utilization only limits initial KV cache allocation\n" "# Prefix cache is separate and has no eviction\n" "# Fix: set gpu_memory_utilization=0.80 and enable LRU eviction" ), metrics=( "prefix_cache_entries: 847\n" "prefix_cache_gb: 46\n" "time_to_oom_minutes: 22\n" "requests_before_oom: 203\n" "eviction_events: 0" ), ), specialist_followups={ "runtime": "Prefix cache grows without eviction. Set a size limit or enable LRU eviction.", "dispatch": "Prefix caching works fine with MoE. The issue is unbounded cache growth.", "kernel": "Set gpu_memory_utilization=0.80 and configure prefix cache eviction policy to LRU.", "loader": "Weights loaded fine. Tune the prefix cache memory budget.", }, )) # --- quantization_error scenarios --- scenarios.append(Scenario( id="quantization_error_01", root_cause="quantization_error", correct_fix="fix_quantization", incident_ticket=( "INCIDENT: GPTQ 4-bit quantized Llama-3.3-70B produces garbled output on H100. " "Perplexity is 450+ (expected <10). The quantization was done with an older " "AutoGPTQ version. Suspected calibration data mismatch." ), hardware="NVIDIA H100", model_name="Llama-3.3-70B-Instruct", backend="vLLM 0.8.x", initial_log=( "[vLLM] Loading GPTQ model: 4-bit, group_size=128\n" "[vLLM] Quantization config: bits=4, sym=True, desc_act=True\n" "[vLLM] WARNING: qweight shape unexpected for layer 0: [4096, 1024]\n" "[vLLM] Output sample: 'the the the the the the...'\n" "[vLLM] Perplexity check: 458.7 (FAIL, threshold: 15)" ), initial_snippet=( "# quantize.py (AutoGPTQ v0.4)\n" "quantize_config = BaseQuantizeConfig(\n" " bits=4, group_size=128, desc_act=True,\n" " sym=True, true_sequential=True\n" ")\n" "# Calibration dataset: 128 samples of random tokens\n" "# Should use representative text from target domain\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "CUDA runtime is fine. Model loads and runs. Output quality is the issue.", 0.78, False ), "dispatch": SpecialistOpinion( "GPTQ dequantization kernel dispatched correctly for H100.", 0.65, False ), "kernel": SpecialistOpinion( "The dequantization kernel is correct. qweight tensor packing format changed " "between AutoGPTQ v0.4 and v0.7. Old format is being read with new unpacker.", 0.93, True ), "loader": SpecialistOpinion( "GPTQ checkpoint was quantized with AutoGPTQ v0.4 which uses a different " "packing order (column-major) than vLLM expects (row-major). " "Re-quantize with compatible version or convert packing format.", 0.91, True ), }, inspect_results=InspectResult( logs=( "[GPTQ] qweight packing: column-major (AutoGPTQ v0.4)\n" "[GPTQ] Expected packing: row-major (vLLM/AutoGPTQ v0.7+)\n" "[GPTQ] Bit unpacking yields wrong int4 values\n" "[GPTQ] Layer 0 dequantized weights: mean=47.3, std=812 (expected mean~0, std~0.02)" ), config=( "autogptq_version: 0.4.2\n" "vllm_expected_version: 0.7+\n" "packing_format: column_major\n" "expected_packing: row_major\n" "bits: 4\n" "group_size: 128" ), snippet=( "# AutoGPTQ v0.4: packs 8 int4 values column-major into int32\n" "# AutoGPTQ v0.7+: packs row-major (matches Marlin/Exllama kernels)\n" "# vLLM's Marlin kernel expects row-major packing\n" "# Fix: re-quantize with AutoGPTQ>=0.7 or use repack script" ), metrics=( "perplexity: 458.7\n" "expected_perplexity: 8.2\n" "dequant_mean_error: 47.3\n" "dequant_std_error: 812\n" "affected_layers: all" ), ), specialist_followups={ "runtime": "Runtime is fine. Quantized weights are just unpacked incorrectly.", "dispatch": "Marlin kernel dispatched. But input data is in wrong packing format.", "kernel": "Marlin kernel unpacks row-major. Checkpoint is column-major. Re-quantize or repack.", "loader": "Re-quantize with AutoGPTQ >= 0.7 to get row-major packing compatible with vLLM.", }, )) scenarios.append(Scenario( id="quantization_error_02", root_cause="quantization_error", correct_fix="fix_quantization", incident_ticket=( "INCIDENT: AWQ 4-bit Mistral-Large-2 on RTX 5090 has quality degradation " "only in the last 20 layers. First 40 layers produce correct activations. " "Full-precision model works perfectly. Suspected quantization calibration issue." ), hardware="NVIDIA SM120 (GeForce RTX 5090)", model_name="Mistral-Large-2", backend="vLLM 0.8.x", initial_log=( "[vLLM] AWQ model loaded: 4-bit, group_size=128\n" "[vLLM] Layer 0-39: activations normal (cosine sim > 0.99 vs FP16)\n" "[vLLM] Layer 40+: activations diverging (cosine sim < 0.3 vs FP16)\n" "[vLLM] Output: partially coherent, degrades mid-sentence\n" "[vLLM] Perplexity: 87.3 (expected: 9.1)" ), initial_snippet=( "# awq_calibration.py\n" "calib_data = load_dataset('wikitext', split='train[:128]')\n" "# Calibration only runs 128 samples with max_length=512\n" "# Mistral-Large-2 has 60 layers\n" "# Short calibration doesn't capture activation ranges in deep layers\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "CUDA 13 runtime is fine. No errors during inference.", 0.75, False ), "dispatch": SpecialistOpinion( "AWQ backend selected and dispatching correctly.", 0.60, False ), "kernel": SpecialistOpinion( "The AWQ dequantization kernels are mathematically correct. " "The issue is the quantization scales themselves — they were computed from " "insufficient calibration data. Deep layers got poor scale estimates.", 0.90, True ), "loader": SpecialistOpinion( "AWQ calibration used only 128 samples with max_length=512. " "Deep layers (40-59) never saw diverse activation ranges. " "Re-quantize with 512+ samples at max_length=2048.", 0.94, True ), }, inspect_results=InspectResult( logs=( "[AWQ] Calibration stats: 128 samples, max_len=512\n" "[AWQ] Layer 0-39 scale variance: 0.012 (good)\n" "[AWQ] Layer 40-59 scale variance: 0.0001 (flat — under-calibrated)\n" "[AWQ] Deep layer scales almost identical — poor quantization" ), config=( "awq_bits: 4\n" "group_size: 128\n" "calib_samples: 128\n" "calib_max_length: 512\n" "model_layers: 60\n" "under_calibrated_layers: 40-59" ), snippet=( "# AWQ scales for layers 40-59 are nearly flat\n" "# Calibration data too short to activate deep layers meaningfully\n" "# Fix: re-quantize with 512+ samples, max_length=2048\n" "# This ensures all layers see representative activations" ), metrics=( "perplexity: 87.3\n" "expected_perplexity: 9.1\n" "cosine_sim_layer0_39: 0.993\n" "cosine_sim_layer40_59: 0.27\n" "under_calibrated_layers: 20" ), ), specialist_followups={ "runtime": "No runtime issue. Quantization quality is the problem.", "dispatch": "AWQ backend is correct. Scale values are the issue.", "kernel": "Dequant kernel is correct. The scales for deep layers are flat. Re-calibrate.", "loader": "Re-quantize with more calibration data (512 samples, max_length=2048) to fix deep layers.", }, )) scenarios.append(Scenario( id="quantization_error_03", root_cause="quantization_error", correct_fix="fix_quantization", incident_ticket=( "INCIDENT: FP8 quantized DeepSeek-V3-671B on B200 produces NaN in MoE expert " "layers. Dense layers work fine. Suspected FP8 scale overflow in expert weights " "due to outlier channels in MoE FFN." ), hardware="NVIDIA B200", model_name="DeepSeek-V3-671B", backend="TensorRT-LLM 0.18", initial_log=( "[TensorRT-LLM] FP8 inference: DeepSeek-V3-671B\n" "[TensorRT-LLM] Dense layers 0-3: output OK\n" "[TensorRT-LLM] MoE layer 4, expert 37: NaN detected\n" "[TensorRT-LLM] FP8 dequant scale for expert 37: 1847.5 (extreme)\n" "[TensorRT-LLM] Overflow during dequantization: scale * FP8_max > FP16_max" ), initial_snippet=( "# fp8_quantize.py\n" "# Per-tensor FP8 quantization\n" "scale = weight.abs().max() / FP8_E4M3_MAX\n" "# MoE expert FFN layers have outlier channels with |w| > 500\n" "# Per-tensor scale too coarse for these layers\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "CUDA runtime fine. NaN propagation from dequantization.", 0.78, False ), "dispatch": SpecialistOpinion( "FP8 backend dispatched correctly. The quantization scheme is the issue.", 0.70, False ), "kernel": SpecialistOpinion( "FP8 GEMM kernel produces NaN because dequant scale overflows FP16 range. " "Expert 37 has outlier weight channels that make per-tensor scaling too coarse. " "Need per-channel FP8 scaling for MoE experts.", 0.94, True ), "loader": SpecialistOpinion( "MoE expert weights have extreme outliers in a few channels. " "Per-tensor FP8 quantization can't handle this — scale is dominated by outliers " "while other channels get crushed to zero. Switch to per-channel quantization.", 0.91, True ), }, inspect_results=InspectResult( logs=( "[FP8] Expert 37 weight stats: max=547.2, mean=0.03, outlier_ratio=0.001\n" "[FP8] Per-tensor scale: 547.2 / 448.0 = 1.22\n" "[FP8] Dequant: 448.0 * 1.22 = 546.6 (close to FP16 overflow with accumulation)\n" "[FP8] Non-outlier channels quantized to 0 (lost information)" ), config=( "fp8_format: e4m3fn\n" "quantization_scheme: per_tensor\n" "expert_37_weight_max: 547.2\n" "fp8_e4m3_max: 448.0\n" "fp16_max: 65504.0" ), snippet=( "# Per-tensor scale for expert 37: max(|w|)=547.2\n" "# FP8 max representable: 448.0\n" "# Scale = 547.2/448 = 1.22\n" "# Dequant of max values: 448*1.22 = 546.6\n" "# Accumulated in FP16 GEMM -> overflow with large sequences\n" "# Fix: use per-channel FP8 scaling for MoE expert layers" ), metrics=( "nan_experts: [37, 91, 203]\n" "total_experts: 256\n" "outlier_ratio: 0.001\n" "per_tensor_scale_max: 1847.5\n" "non_outlier_precision_loss: 95%" ), ), specialist_followups={ "runtime": "NaN is from dequant overflow, not runtime issue.", "dispatch": "FP8 dispatch is correct. Quantization granularity needs to change.", "kernel": "Switch to per-channel FP8 scaling for MoE expert FFN layers to handle outliers.", "loader": "Re-quantize MoE experts with per-channel scaling. Dense layers can stay per-tensor.", }, )) scenarios.append(Scenario( id="quantization_error_04", root_cause="quantization_error", correct_fix="fix_quantization", incident_ticket=( "INCIDENT: SmoothQuant INT8 on H100 for Qwen3-235B has severe accuracy loss. " "Perplexity 130+ vs expected 8.5. SmoothQuant migration factor alpha=0.5 " "was tuned for a different model architecture." ), hardware="NVIDIA H100", model_name="Qwen3-235B-A22B", backend="vLLM 0.8.x", initial_log=( "[vLLM] SmoothQuant INT8: alpha=0.5\n" "[vLLM] Activation smoothing applied...\n" "[vLLM] WARNING: post-smooth activation range still large: [-127, 127] maps to [-847, 923]\n" "[vLLM] INT8 GEMM output quality: FAIL\n" "[vLLM] Perplexity: 134.2 (expected: 8.5)" ), initial_snippet=( "# smooth_quant_config.yaml\n" "quantization: smoothquant\n" "smoothquant_alpha: 0.5\n" "# alpha=0.5 was tuned for Llama-2-70B\n" "# Qwen3 MoE has different activation distribution\n" "# Needs per-layer alpha tuning for MoE gating layers\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "Runtime is fine. INT8 GEMM kernels launch correctly.", 0.75, False ), "dispatch": SpecialistOpinion( "INT8 backend dispatched. But the quantization configuration is wrong for this model.", 0.68, True ), "kernel": SpecialistOpinion( "The INT8 GEMM kernel is correct. SmoothQuant alpha=0.5 doesn't properly balance " "the activation/weight difficulty for Qwen3's MoE architecture. " "MoE gating layers need alpha=0.75+.", 0.93, True ), "loader": SpecialistOpinion( "Weights loaded fine. Try increasing batch size to improve throughput.", 0.35, False ), }, inspect_results=InspectResult( logs=( "[SmoothQuant] alpha=0.5 (global)\n" "[SmoothQuant] Dense layer activation range after smooth: [-23, 31] (OK)\n" "[SmoothQuant] MoE gating layer activation range after smooth: [-847, 923] (BAD)\n" "[SmoothQuant] MoE gating layers need higher alpha to push more to weights" ), config=( "smoothquant_alpha: 0.5\n" "optimal_alpha_dense: 0.5\n" "optimal_alpha_moe_gate: 0.80\n" "optimal_alpha_moe_ffn: 0.70\n" "per_layer_alpha: false" ), snippet=( "# SmoothQuant: X_smooth = X / diag(s^alpha), W_smooth = W * diag(s^alpha)\n" "# alpha=0.5 works for dense Llama but not for Qwen3 MoE\n" "# MoE gating/FFN layers have spikier activations needing alpha=0.7-0.8\n" "# Fix: enable per-layer alpha tuning or use alpha=0.75 globally" ), metrics=( "perplexity: 134.2\n" "expected_perplexity: 8.5\n" "dense_layer_accuracy: 97%\n" "moe_layer_accuracy: 34%\n" "activation_clipping_rate: 42%" ), ), specialist_followups={ "runtime": "No runtime issue. Quantization accuracy is the problem.", "dispatch": "INT8 dispatch is fine. The quantization parameters need adjustment.", "kernel": "Re-run SmoothQuant calibration with per-layer alpha or alpha=0.75 for Qwen3 MoE.", "loader": "Batch size is irrelevant. The issue is SmoothQuant alpha mismatch.", }, )) scenarios.append(Scenario( id="quantization_error_05", root_cause="quantization_error", correct_fix="fix_quantization", incident_ticket=( "INCIDENT: GGUF Q4_K_M quantized DeepSeek-R1-Distill-70B fails to load in " "SGLang on MI355X. Error: 'unsupported quant type Q4_K_M for ROCm'. " "Same GGUF file works on NVIDIA hardware." ), hardware="AMD MI355X", model_name="DeepSeek-R1-Distill-70B", backend="SGLang 0.5.x", initial_log=( "[SGLang] Loading GGUF: DeepSeek-R1-Distill-70B-Q4_K_M.gguf\n" "[SGLang] Detected quant type: Q4_K_M (k-quant mixed)\n" "[SGLang] ROCm backend: checking dequant kernel support...\n" "[SGLang] ERROR: Q4_K_M dequantization kernel not available for ROCm/HIP\n" "[SGLang] Supported ROCm quant types: Q4_0, Q8_0, FP16" ), initial_snippet=( "# sglang/quantization/gguf_loader.py\n" "ROCM_SUPPORTED_QUANTS = {'Q4_0', 'Q8_0', 'FP16'}\n" "# K-quant types (Q4_K_M, Q5_K_M, etc.) have CUDA kernels only\n" "# ROCm/HIP dequant kernels not yet implemented for k-quants\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "ROCm runtime loads fine. The dequantization kernel is missing for this quant type.", 0.82, True ), "dispatch": SpecialistOpinion( "Dispatch correctly identifies the quant type but has no ROCm kernel to call.", 0.80, True ), "kernel": SpecialistOpinion( "K-quant dequantization uses CUDA-specific warp shuffle instructions. " "Need to re-quantize to Q4_0 or Q8_0 for ROCm, or convert to a supported format.", 0.92, True ), "loader": SpecialistOpinion( "The GGUF file is corrupted. Try re-downloading.", 0.30, False ), }, inspect_results=InspectResult( logs=( "[SGLang] GGUF header: Q4_K_M, 70B params, 80 layers\n" "[SGLang] ROCm dequant kernel registry: Q4_0, Q8_0, FP16\n" "[SGLang] Q4_K_M not in registry\n" "[SGLang] K-quant kernels use __shfl_xor_sync (CUDA only)" ), config=( "gguf_quant_type: Q4_K_M\n" "rocm_supported: [Q4_0, Q8_0, FP16]\n" "k_quant_rocm_support: false\n" "gpu_vendor: AMD" ), snippet=( "# Q4_K_M uses k-quant mixed precision (4-bit with 6-bit super-blocks)\n" "# CUDA kernel uses __shfl_xor_sync for dequant\n" "# No HIP equivalent implemented in SGLang\n" "# Fix: re-quantize model to Q8_0 or Q4_0 for ROCm\n" "# Or: convert GGUF to safetensors with AWQ/GPTQ quantization" ), metrics=( "load_failures: 1\n" "quant_type: Q4_K_M\n" "rocm_kernel_available: false\n" "nvidia_kernel_available: true" ), ), specialist_followups={ "runtime": "ROCm works. K-quant dequant kernels are CUDA-only in SGLang.", "dispatch": "Dispatch is correct in rejecting unsupported quant type. Need compatible quant format.", "kernel": "Re-quantize to Q4_0 or Q8_0 which have HIP/ROCm kernels. Or use AWQ instead of GGUF.", "loader": "File is fine. The quant format just isn't supported on ROCm.", }, )) scenarios.append(Scenario( id="quantization_error_06", root_cause="quantization_error", correct_fix="fix_quantization", incident_ticket=( "INCIDENT: Marlin INT4 kernel on DGX Spark gives wrong results for " "Llama-4-Maverick MoE model. Dense layers quantize fine but expert FFN " "weights have incorrect permutation after Marlin repacking." ), hardware="NVIDIA SM121 (DGX Spark)", model_name="Llama-4-Maverick-17Bx128E", backend="vLLM 0.8.x", initial_log=( "[vLLM] Marlin INT4 kernel loading...\n" "[vLLM] Repacking GPTQ weights to Marlin format...\n" "[vLLM] Dense layers: repack OK (verified against reference)\n" "[vLLM] Expert 0 FFN: repack MISMATCH (output differs from GPTQ reference)\n" "[vLLM] 73 of 128 experts have repack errors" ), initial_snippet=( "# vllm/quantization/marlin_repack.py\n" "def repack_gptq_to_marlin(qweight, scales, g_idx, num_experts=None):\n" " # BUG: expert FFN weight shape is [E, N, K//8]\n" " # Repacker assumes [N, K//8] (no expert dim)\n" " perm = get_marlin_permutation(qweight.shape[-2], qweight.shape[-1])\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "CUDA runtime fine. Marlin kernel launches. Output is wrong.", 0.75, False ), "dispatch": SpecialistOpinion( "Marlin kernel dispatched. But the weight repacking has a shape handling bug.", 0.72, True ), "kernel": SpecialistOpinion( "Marlin repacker doesn't handle the 3D expert weight tensor [E, N, K//8]. " "It treats the expert dimension as part of N, creating wrong permutation. " "Need to repack each expert slice independently.", 0.95, True ), "loader": SpecialistOpinion( "The model has too many experts. Try reducing the number of active experts.", 0.30, False ), }, inspect_results=InspectResult( logs=( "[Marlin] Expert weight shape: [128, 4096, 512]\n" "[Marlin] Repacker expected: [4096, 512] per expert\n" "[Marlin] Repacker got: [128*4096, 512] (flattened expert dim into N)\n" "[Marlin] Permutation computed for 524288 rows instead of 4096 rows" ), config=( "num_experts: 128\n" "expert_ffn_shape: [128, 4096, 512]\n" "expected_per_expert: [4096, 512]\n" "marlin_repack_dim: 2D_only\n" "repack_bug: expert_dim_flattened" ), snippet=( "# Marlin repacker assumes 2D weight matrix [N, K//8]\n" "# MoE expert weights are 3D [E, N, K//8]\n" "# Repacker flattens to [E*N, K//8] -> wrong permutation\n" "# Fix: iterate over expert dim and repack each [N, K//8] slice" ), metrics=( "experts_with_repack_error: 73\n" "total_experts: 128\n" "dense_layers_correct: true\n" "output_cosine_sim: 0.12\n" "expected_cosine_sim: 0.99" ), ), specialist_followups={ "runtime": "Runtime fine. Weight repacking is wrong for 3D expert tensors.", "dispatch": "Marlin dispatch is OK. Fix the repack function to handle expert dimension.", "kernel": "Repack each expert slice [N, K//8] independently. Don't flatten the expert dim.", "loader": "Number of experts is fine. The Marlin repacker has a shape bug for MoE weights.", }, )) # --- distributed_comm scenarios --- scenarios.append(Scenario( id="distributed_comm_01", root_cause="distributed_comm", correct_fix="fix_comm_config", incident_ticket=( "INCIDENT: 8-GPU tensor parallel DeepSeek-V3-671B hangs during all-reduce on H100. " "Model loads, first forward pass completes, second forward pass hangs indefinitely. " "NCCL timeout after 300 seconds. Single-GPU works fine." ), hardware="NVIDIA H100", model_name="DeepSeek-V3-671B", backend="vLLM 0.8.x", initial_log=( "[vLLM] Tensor parallel size: 8\n" "[vLLM] NCCL version: 2.21.5\n" "[vLLM] Forward pass 1: OK (12.3s)\n" "[vLLM] Forward pass 2: all-reduce HANG at layer 3\n" "[NCCL] WARN Timeout after 300000ms on rank 5" ), initial_snippet=( "# Environment\n" "NCCL_P2P_DISABLE=0\n" "NCCL_IB_DISABLE=0\n" "CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7\n" "# GPUs span two NVSwitch domains (0-3 and 4-7)\n" "# NCCL topology detection may be incorrect\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "CUDA runtime is fine on all 8 GPUs individually. The hang is in NCCL collective.", 0.82, True ), "dispatch": SpecialistOpinion( "Backend dispatches correctly. Tensor parallel scatter/gather is the issue.", 0.70, False ), "kernel": SpecialistOpinion( "NCCL all-reduce hangs because rank 5 (GPU 5) is using a different communicator " "than ranks 0-4. Two NVSwitch domains need NCCL_CROSS_NIC=1 and proper " "NCCL_SOCKET_IFNAME configuration.", 0.93, True ), "loader": SpecialistOpinion( "Weights sharded correctly across 8 GPUs. Not a loading issue.", 0.65, False ), }, inspect_results=InspectResult( logs=( "[NCCL] Topology: 2 NVSwitch domains [0-3] [4-7]\n" "[NCCL] Ring 0: 0->1->2->3->4 (crosses domain boundary)\n" "[NCCL] Rank 5 timeout: peer 4 not responding on NVLink\n" "[NCCL] NVLink 4->5: bandwidth 0 (link not established across domains)\n" "[NCCL] NCCL_CROSS_NIC not set, defaults to 0" ), config=( "tensor_parallel_size: 8\n" "nccl_version: 2.21.5\n" "nvswitch_domains: 2\n" "domain_0_gpus: [0,1,2,3]\n" "domain_1_gpus: [4,5,6,7]\n" "nccl_cross_nic: 0" ), snippet=( "# 8 GPUs span 2 NVSwitch domains\n" "# NCCL ring tries to route 4->5 via NVLink but no direct link\n" "# Cross-domain traffic needs to go via PCIe/InfiniBand\n" "# Fix: set NCCL_CROSS_NIC=1, NCCL_NET_GDR_LEVEL=5" ), metrics=( "successful_allreduce: 1\n" "hung_allreduce: 1\n" "timeout_rank: 5\n" "nvlink_4_5_bandwidth: 0\n" "nccl_timeout_ms: 300000" ), ), specialist_followups={ "runtime": "CUDA fine on each GPU. NCCL cross-domain communication is the issue.", "dispatch": "Tensor parallel dispatch is correct. NCCL topology config is wrong.", "kernel": "Set NCCL_CROSS_NIC=1 and NCCL_NET_GDR_LEVEL=5 for cross-NVSwitch-domain communication.", "loader": "Weights sharded correctly. Communication config is the problem.", }, )) scenarios.append(Scenario( id="distributed_comm_02", root_cause="distributed_comm", correct_fix="fix_comm_config", incident_ticket=( "INCIDENT: Pipeline parallel Qwen3-235B on 4xMI300X has 50% lower throughput " "than expected. Bubble overhead is 60% instead of expected 15%. " "Pipeline stages are severely imbalanced." ), hardware="AMD MI300X", model_name="Qwen3-235B-A22B", backend="SGLang 0.5.x", initial_log=( "[SGLang] Pipeline parallel: 4 stages\n" "[SGLang] Stage 0 (layers 0-14): avg 45ms\n" "[SGLang] Stage 1 (layers 15-29): avg 42ms\n" "[SGLang] Stage 2 (layers 30-44): avg 180ms (MoE layers!)\n" "[SGLang] Stage 3 (layers 45-59): avg 175ms\n" "[SGLang] Pipeline bubble: 60% (stages 0-1 idle waiting for 2-3)" ), initial_snippet=( "# sglang_config.yaml\n" "pipeline_parallel_size: 4\n" "# Default: equal layer split (15 layers per stage)\n" "# Qwen3 MoE: layers 30-59 have MoE blocks (4x slower)\n" "# Uniform split creates severe load imbalance\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "ROCm runtime healthy on all 4 GPUs. No communication errors.", 0.75, False ), "dispatch": SpecialistOpinion( "Pipeline parallel stages have severe load imbalance. MoE layers in stages 2-3 " "are 4x slower than dense layers in stages 0-1. Need non-uniform stage partitioning.", 0.94, True ), "kernel": SpecialistOpinion( "MoE expert dispatch takes longer due to all-to-all communication within each " "pipeline stage. Consider tensor parallel for MoE layers instead.", 0.55, False ), "loader": SpecialistOpinion( "Pipeline split should put fewer MoE layers per stage. " "Recommended: 20 layers in stages 0-1, 10 layers in stages 2-3.", 0.90, True ), }, inspect_results=InspectResult( logs=( "[SGLang] Stage timing (ms): [45, 42, 180, 175]\n" "[SGLang] Slowest stage: 180ms (4x faster stages)\n" "[SGLang] Pipeline bubble: 135ms per micro-batch\n" "[SGLang] MoE layers: 30-59 (all in stages 2-3)\n" "[SGLang] Dense layers: 0-29 (all in stages 0-1)" ), config=( "pipeline_parallel_size: 4\n" "layers_per_stage: [15, 15, 15, 15]\n" "moe_layers: [30, 31, ..., 59]\n" "stage_0_1_type: dense\n" "stage_2_3_type: moe" ), snippet=( "# Uniform split: [0-14, 15-29, 30-44, 45-59]\n" "# Stages 0-1: dense only (fast)\n" "# Stages 2-3: MoE only (slow)\n" "# Fix: rebalance to [0-19, 20-39, 40-49, 50-59]\n" "# Or: [0-24, 25-44, 45-54, 55-59] for more even timing" ), metrics=( "pipeline_bubble_pct: 60\n" "expected_bubble_pct: 15\n" "throughput_tokens_per_sec: 850\n" "expected_throughput: 1700\n" "stage_imbalance_ratio: 4.3x" ), ), specialist_followups={ "runtime": "No communication errors. Throughput issue from pipeline imbalance.", "dispatch": "Rebalance pipeline stages: put more dense layers per stage, fewer MoE layers per stage.", "kernel": "MoE layers are just slower. The fix is pipeline partitioning, not kernel changes.", "loader": "Re-partition: stages 0-1 get 20 layers each, stages 2-3 get 10 layers each.", }, )) scenarios.append(Scenario( id="distributed_comm_03", root_cause="distributed_comm", correct_fix="fix_comm_config", incident_ticket=( "INCIDENT: vLLM tensor parallel on 2xRTX 5090 via PCIe has 90% communication " "overhead. All-reduce takes 85ms but compute only takes 9ms per layer. " "Expected NVLink but system has PCIe only." ), hardware="NVIDIA SM120 (GeForce RTX 5090)", model_name="Llama-3.3-70B-Instruct", backend="vLLM 0.8.x", initial_log=( "[vLLM] Tensor parallel size: 2\n" "[vLLM] NCCL init: using PCIe transport (no NVLink detected)\n" "[vLLM] All-reduce per layer: 85ms (PCIe 5.0 x16)\n" "[vLLM] Compute per layer: 9ms\n" "[vLLM] Communication overhead: 90.4%\n" "[vLLM] Total latency per token: 7520ms (expected: ~800ms)" ), initial_snippet=( "# system config\n" "# 2x RTX 5090 in PCIe slots (no NVLink bridge)\n" "# PCIe 5.0 x16: ~64 GB/s bidirectional\n" "# NVLink 5: ~900 GB/s (not available)\n" "tensor_parallel_size: 2\n" "# All-reduce over PCIe is bottleneck\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "CUDA runtime fine. PCIe link healthy but slow for TP all-reduce.", 0.80, True ), "dispatch": SpecialistOpinion( "Tensor parallel is wrong strategy for PCIe-only multi-GPU. " "Should use pipeline parallel instead — it only sends activations between stages " "(much less data than TP all-reduce). Or use single-GPU with quantization.", 0.94, True ), "kernel": SpecialistOpinion( "Maybe the NCCL kernel is using a suboptimal algorithm for PCIe.", 0.50, False ), "loader": SpecialistOpinion( "Model weights split correctly across 2 GPUs. Loading is fine.", 0.65, False ), }, inspect_results=InspectResult( logs=( "[NCCL] Transport: PCIe 5.0 x16\n" "[NCCL] Bandwidth: 32 GB/s per direction\n" "[NCCL] All-reduce data per layer: 2.7 GB\n" "[NCCL] All-reduce time: 2.7GB / 32GB/s = 84ms\n" "[NCCL] 80 layers * 84ms = 6720ms communication per token" ), config=( "transport: pcie_5_x16\n" "bandwidth_per_dir_gbs: 32\n" "allreduce_per_layer_gb: 2.7\n" "nvlink_available: false\n" "tensor_parallel_size: 2" ), snippet=( "# TP all-reduce sends hidden_size * dtype_bytes per layer\n" "# 8192 * 2 * batch * seq_len = ~2.7GB per layer\n" "# PCIe: 2.7GB / 32GB/s = 84ms per layer\n" "# Fix: switch to pipeline parallel (sends only activations)\n" "# Or: use single-GPU with INT4 quantization (fits in 32GB)" ), metrics=( "allreduce_ms_per_layer: 85\n" "compute_ms_per_layer: 9\n" "comm_overhead_pct: 90.4\n" "total_latency_ms: 7520\n" "expected_with_nvlink_ms: 800" ), ), specialist_followups={ "runtime": "PCIe link is healthy but bandwidth-limited for TP. Switch parallel strategy.", "dispatch": "Switch from tensor parallel to pipeline parallel, or use single-GPU with INT4 quantization.", "kernel": "NCCL algorithm is optimal for PCIe. The interconnect is just too slow for TP.", "loader": "Weights loaded fine. The parallel strategy needs to change.", }, )) scenarios.append(Scenario( id="distributed_comm_04", root_cause="distributed_comm", correct_fix="fix_comm_config", incident_ticket=( "INCIDENT: 4-node distributed vLLM serving DeepSeek-V3-671B has intermittent " "NCCL errors. Every ~100 requests, one node reports 'unhandled system error' " "in all-gather. Network MTU mismatch between nodes suspected." ), hardware="NVIDIA B200", model_name="DeepSeek-V3-671B", backend="vLLM 0.8.x", initial_log=( "[vLLM] Distributed: 4 nodes, 8 GPUs each (32 total)\n" "[NCCL] Init: IB transport, 4 HCAs per node\n" "[vLLM] Request 87: NCCL error on node 2\n" "[NCCL] node2: unhandled system error (NCCL_SYSTEM_ERROR)\n" "[NCCL] Suspected: IB message size exceeds MTU on switch" ), initial_snippet=( "# Network config\n" "# Nodes 0,1,3: MTU=4096 (IB)\n" "# Node 2: MTU=2048 (different switch port config)\n" "# NCCL_IB_GID_INDEX=3\n" "# No explicit NCCL_IB_TC or NCCL_IB_TIMEOUT set\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "CUDA runtime healthy on all nodes. NCCL error is network-level.", 0.82, True ), "dispatch": SpecialistOpinion( "The error seems random but always involves node 2. Check node 2's IB config.", 0.75, False ), "kernel": SpecialistOpinion( "NCCL uses IB RDMA with MTU-sized messages. Node 2 has MTU=2048 while others " "have 4096. When NCCL sends a 4096-byte message to node 2, the IB switch drops it. " "Fix: align MTU to 4096 on node 2's switch port.", 0.94, True ), "loader": SpecialistOpinion( "Model weights distributed correctly across 32 GPUs. Not a loading issue.", 0.65, False ), }, inspect_results=InspectResult( logs=( "[NCCL] Node 0 IB MTU: 4096\n" "[NCCL] Node 1 IB MTU: 4096\n" "[NCCL] Node 2 IB MTU: 2048 (MISMATCH)\n" "[NCCL] Node 3 IB MTU: 4096\n" "[NCCL] Messages >2048 to node 2: DROPPED\n" "[NCCL] Error frequency: ~1 per 100 requests (when large all-gather hits node 2)" ), config=( "nodes: 4\n" "gpus_per_node: 8\n" "ib_mtu_node0: 4096\n" "ib_mtu_node1: 4096\n" "ib_mtu_node2: 2048\n" "ib_mtu_node3: 4096\n" "nccl_ib_timeout: default" ), snippet=( "# Node 2 IB switch port configured with MTU=2048\n" "# Other nodes: MTU=4096\n" "# NCCL sends 4096-byte IB messages -> dropped at node 2's switch\n" "# Fix: reconfigure node 2's switch port to MTU=4096\n" "# Workaround: NCCL_IB_TC=128, NCCL_IB_TIMEOUT=22" ), metrics=( "nccl_errors: 47 in 1 hour\n" "errors_on_node2: 47\n" "errors_on_other_nodes: 0\n" "avg_requests_between_errors: 103\n" "ib_mtu_mismatch: true" ), ), specialist_followups={ "runtime": "CUDA runtime is fine. NCCL errors from IB MTU mismatch on node 2.", "dispatch": "Node 2 is the problem. IB config needs to match other nodes.", "kernel": "Set node 2 IB MTU to 4096 to match the cluster. Or set NCCL_IB_TC=128 as workaround.", "loader": "Weight distribution is correct. Network MTU mismatch is the root cause.", }, )) scenarios.append(Scenario( id="distributed_comm_05", root_cause="distributed_comm", correct_fix="fix_comm_config", incident_ticket=( "INCIDENT: Expert parallel DeepSeek-V3-671B on 8xMI355X has all-to-all " "communication deadlock. Token routing to remote experts hangs when " "more than 4 GPUs are involved. 4-GPU EP works fine." ), hardware="AMD MI355X", model_name="DeepSeek-V3-671B", backend="vLLM 0.8.x", initial_log=( "[vLLM] Expert parallel: 8 GPUs, 32 experts per GPU\n" "[RCCL] All-to-all init: 8 ranks\n" "[vLLM] MoE token routing: sending tokens to remote experts...\n" "[RCCL] HANG: all-to-all stuck at rank 6 (waiting for rank 2)\n" "[RCCL] Timeout after 600s. 4-GPU EP (ranks 0-3) works fine." ), initial_snippet=( "# RCCL config\n" "RCCL_ALLTOALL_KERNEL=1 # basic kernel\n" "# 8 MI355X connected via xGMI + Infinity Fabric\n" "# xGMI: 4-GPU pod (GPUs 0-3) and (GPUs 4-7)\n" "# Cross-pod: Infinity Fabric bridge\n" "# RCCL kernel 1 doesn't handle cross-pod routing\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "ROCm runtime healthy. RCCL all-to-all kernel selection is wrong for 8-GPU topology.", 0.85, True ), "dispatch": SpecialistOpinion( "Expert parallel dispatch routes tokens correctly. The communication primitive is the issue.", 0.72, False ), "kernel": SpecialistOpinion( "RCCL all-to-all kernel 1 assumes fully connected topology. " "8 MI355X has 2 xGMI pods bridged by Infinity Fabric. " "Need kernel 2 (hierarchical) for cross-pod communication.", 0.93, True ), "loader": SpecialistOpinion( "Expert weights distributed correctly. All-to-all comm is the bottleneck.", 0.68, False ), }, inspect_results=InspectResult( logs=( "[RCCL] Topology: 2 xGMI pods [0-3] [4-7]\n" "[RCCL] Intra-pod bandwidth: 896 GB/s\n" "[RCCL] Cross-pod bandwidth: 128 GB/s (Infinity Fabric)\n" "[RCCL] All-to-all kernel 1: flat ring (assumes uniform links)\n" "[RCCL] Ring 0->1->2->3->4 blocked at 3->4 (cross-pod)\n" "[RCCL] Deadlock: rank 6 waits for rank 2 via pod bridge" ), config=( "expert_parallel_size: 8\n" "rccl_alltoall_kernel: 1\n" "topology: 2_xgmi_pods\n" "intra_pod_bw_gbs: 896\n" "cross_pod_bw_gbs: 128" ), snippet=( "# RCCL kernel 1: flat ring all-to-all\n" "# Assumes all links have equal bandwidth\n" "# Cross-pod link (128 GB/s) vs intra-pod (896 GB/s)\n" "# Flat ring deadlocks when cross-pod link saturates\n" "# Fix: RCCL_ALLTOALL_KERNEL=2 (hierarchical)\n" "# Or: set RCCL_NCHANNELS=16 for cross-pod" ), metrics=( "deadlock_count: 12 in 1 hour\n" "4gpu_ep_deadlocks: 0\n" "8gpu_ep_deadlocks: 12\n" "cross_pod_saturation_pct: 100\n" "intra_pod_utilization_pct: 15" ), ), specialist_followups={ "runtime": "ROCm fine. RCCL kernel selection needs to account for 2-pod topology.", "dispatch": "Expert routing is correct. RCCL transport is the issue.", "kernel": "Set RCCL_ALLTOALL_KERNEL=2 for hierarchical all-to-all across xGMI pods.", "loader": "Expert weight distribution is correct. Fix RCCL kernel config.", }, )) scenarios.append(Scenario( id="distributed_comm_06", root_cause="distributed_comm", correct_fix="fix_comm_config", incident_ticket=( "INCIDENT: TensorRT-LLM multi-node inference on 2x DGX Spark (16 GPUs total) " "has 40% throughput drop compared to single-node 8-GPU. " "Inter-node all-reduce over RoCE is bottlenecked. TCP fallback detected." ), hardware="NVIDIA SM121 (DGX Spark)", model_name="Mistral-Large-2", backend="TensorRT-LLM 0.18", initial_log=( "[TensorRT-LLM] 2 nodes, 8 GPUs each, TP=16\n" "[NCCL] Transport: NET/Socket (TCP fallback!)\n" "[NCCL] WARNING: RDMA not available, using TCP\n" "[NCCL] Inter-node all-reduce: 340ms (expected: 12ms with RDMA)\n" "[TensorRT-LLM] Throughput: 420 tok/s (expected: 700 tok/s)" ), initial_snippet=( "# Network config\n" "# RoCE v2 NICs installed but not configured\n" "# NCCL_SOCKET_IFNAME=eth0 (TCP fallback)\n" "# RoCE NIC: mlx5_0 (not specified in NCCL config)\n" "# GDR (GPU Direct RDMA) not enabled\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "CUDA runtime fine. Network transport is the bottleneck.", 0.78, False ), "dispatch": SpecialistOpinion( "NCCL fell back to TCP sockets instead of RDMA over RoCE. " "Need to configure NCCL_SOCKET_IFNAME to point to the RoCE NIC " "and enable GPU Direct RDMA.", 0.94, True ), "kernel": SpecialistOpinion( "TensorRT-LLM kernels are fine. Inter-node communication is bottlenecked by TCP.", 0.82, True ), "loader": SpecialistOpinion( "Maybe the model is too large for 16-way tensor parallel. Try pipeline parallel.", 0.40, False ), }, inspect_results=InspectResult( logs=( "[NCCL] NCCL_SOCKET_IFNAME=eth0 (1Gbps TCP)\n" "[NCCL] mlx5_0: RoCE v2, 400Gbps (UNCONFIGURED)\n" "[NCCL] GPU Direct RDMA: disabled\n" "[NCCL] TCP all-reduce: 340ms per layer\n" "[NCCL] Expected RDMA all-reduce: 12ms per layer" ), config=( "nccl_socket_ifname: eth0\n" "eth0_speed: 1Gbps\n" "mlx5_0_speed: 400Gbps\n" "rdma_enabled: false\n" "gdr_enabled: false" ), snippet=( "# NCCL using TCP over 1Gbps eth0 instead of 400Gbps RoCE\n" "# RoCE NIC (mlx5_0) available but not configured\n" "# Fix: NCCL_SOCKET_IFNAME=mlx5_0\n" "# Fix: NCCL_NET_GDR_LEVEL=5 (enable GPU Direct RDMA)\n" "# Fix: NCCL_IB_DISABLE=0" ), metrics=( "throughput_tok_s: 420\n" "expected_throughput: 700\n" "tcp_allreduce_ms: 340\n" "rdma_allreduce_ms: 12\n" "inter_node_bandwidth_gbps: 1" ), ), specialist_followups={ "runtime": "Network stack works. Just using wrong NIC for NCCL.", "dispatch": "Set NCCL_SOCKET_IFNAME=mlx5_0 and enable GDR for 400Gbps RDMA transport.", "kernel": "Compute kernels are fine. Inter-node latency will drop 28x with RDMA.", "loader": "Model size is fine for TP=16. The network config is the problem.", }, )) # --- driver_compat scenarios --- scenarios.append(Scenario( id="driver_compat_01", root_cause="driver_compat", correct_fix="update_driver_config", incident_ticket=( "INCIDENT: DGX Spark (SM121) with CUDA 12.4 driver fails to run FlashInfer 0.4. " "Error: 'CUDA driver version is insufficient for CUDA runtime version'. " "FlashInfer compiled against CUDA 13.0 but driver only supports up to 12.4." ), hardware="NVIDIA SM121 (DGX Spark)", model_name="DeepSeek-V3-671B", backend="FlashInfer 0.4", initial_log=( "[FlashInfer] Loading CUDA kernels...\n" "[CUDA] Driver version: 12040 (12.4)\n" "[CUDA] Runtime version: 13000 (13.0)\n" "[FlashInfer] ERROR: CUDA driver version is insufficient for CUDA runtime version\n" "[FlashInfer] Required: CUDA driver >= 13.0, found: 12.4" ), initial_snippet=( "# System info\n" "nvidia-smi: Driver Version: 550.54.15 (CUDA 12.4)\n" "# FlashInfer 0.4 was compiled with CUDA 13.0 toolkit\n" "# SM121 GPUs require CUDA 13.0+ driver\n" "# Driver 550.x only supports up to CUDA 12.4\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "CUDA driver 550.x (12.4) cannot load CUDA 13.0 compiled kernels. " "Need to upgrade to driver 570+ which supports CUDA 13.0.", 0.95, True ), "dispatch": SpecialistOpinion( "FlashInfer dispatch cannot even start. This is a driver-level incompatibility.", 0.80, True ), "kernel": SpecialistOpinion( "The FlashInfer kernel PTX targets SM121 which needs CUDA 13.0 JIT compilation. " "Driver 550.x cannot JIT compile for SM121.", 0.85, True ), "loader": SpecialistOpinion( "Try downgrading FlashInfer to a version compiled with CUDA 12.4.", 0.40, False ), }, inspect_results=InspectResult( logs=( "[CUDA] Driver: 550.54.15 (supports up to CUDA 12.4)\n" "[CUDA] Runtime: 13.0 (required by FlashInfer 0.4)\n" "[CUDA] SM121 JIT compilation requires driver with CUDA 13.0 support\n" "[CUDA] cuModuleLoadData failed: CUDA_ERROR_NO_BINARY_FOR_GPU" ), config=( "driver_version: 550.54.15\n" "max_cuda_version: 12.4\n" "required_cuda_version: 13.0\n" "gpu_arch: sm_121\n" "flashinfer_cuda_version: 13.0" ), snippet=( "# Driver 550.x: CUDA 12.4 max\n" "# FlashInfer 0.4: compiled with CUDA 13.0\n" "# SM121 PTX needs CUDA 13.0 JIT\n" "# Fix: upgrade driver to 570+ (CUDA 13.0 support)" ), metrics=( "driver_cuda_max: 12.4\n" "required_cuda: 13.0\n" "kernel_load_failures: 1\n" "jit_compilation: failed" ), ), specialist_followups={ "runtime": "Upgrade NVIDIA driver to 570+ for CUDA 13.0 support.", "dispatch": "Can't dispatch until driver supports CUDA 13.0.", "kernel": "SM121 kernels need CUDA 13.0 JIT. Driver 550.x can't do it.", "loader": "Downgrading FlashInfer won't help — SM121 requires CUDA 13.0 regardless.", }, )) scenarios.append(Scenario( id="driver_compat_02", root_cause="driver_compat", correct_fix="update_driver_config", incident_ticket=( "INCIDENT: ROCm 6.3 driver on MI300X silently produces wrong FP8 results. " "Model outputs have subtle accuracy degradation (perplexity 14 vs expected 9). " "ROCm 6.3.0 has known bug in FP8 MFMA instruction for certain matrix sizes." ), hardware="AMD MI300X", model_name="DeepSeek-R1-Distill-70B", backend="vLLM 0.8.x", initial_log=( "[vLLM] ROCm version: 6.3.0\n" "[vLLM] FP8 inference: e4m3fnuz format\n" "[vLLM] Output quality: marginal (perplexity 14.2, threshold 10)\n" "[vLLM] BF16 reference: perplexity 8.9 (correct)\n" "[vLLM] FP8 accuracy gap: 5.3 perplexity points (abnormal)" ), initial_snippet=( "# ROCm 6.3.0 release notes (known issues)\n" "# BUG: FP8 MFMA instruction produces incorrect results\n" "# when M=16, N=16, K=32 and matrix A is transposed\n" "# Affects: MI300X, MI355X\n" "# Fixed in: ROCm 6.3.1\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "ROCm 6.3.0 has a known FP8 MFMA bug. The driver produces silently wrong results " "for specific matrix dimensions. Need ROCm 6.3.1 hotfix.", 0.95, True ), "dispatch": SpecialistOpinion( "FP8 backend selected correctly. The hardware instruction has a bug.", 0.70, False ), "kernel": SpecialistOpinion( "The FP8 MFMA kernel triggers a known ROCm 6.3.0 driver bug for M16N16K32 " "with transposed A. This isn't a software kernel issue — it's the hardware " "microcode. Update to ROCm 6.3.1.", 0.92, True ), "loader": SpecialistOpinion( "Weights loaded correctly. BF16 works fine, only FP8 is affected.", 0.68, False ), }, inspect_results=InspectResult( logs=( "[ROCm] Version: 6.3.0 (known FP8 MFMA bug)\n" "[FP8] MFMA M16N16K32 with transA: incorrect accumulation\n" "[FP8] Affected layers: 23 of 80 (those using transposed A)\n" "[FP8] BF16 MFMA: correct (no bug in BF16 path)\n" "[ROCm] Fix available: ROCm 6.3.1" ), config=( "rocm_version: 6.3.0\n" "fp8_mfma_bug: true\n" "affected_config: M16N16K32_transA\n" "fix_version: 6.3.1\n" "bf16_affected: false" ), snippet=( "# ROCm 6.3.0 FP8 MFMA bug:\n" "# mfma_f32_16x16x32_fp8 with transposed A matrix\n" "# Accumulator gets wrong partial sums\n" "# Silent data corruption (no error, just wrong results)\n" "# Fix: upgrade to ROCm 6.3.1 or use BF16 as workaround" ), metrics=( "fp8_perplexity: 14.2\n" "bf16_perplexity: 8.9\n" "accuracy_gap: 5.3\n" "affected_layers: 23\n" "total_layers: 80" ), ), specialist_followups={ "runtime": "Upgrade to ROCm 6.3.1 which fixes the FP8 MFMA accumulator bug.", "dispatch": "Dispatch is correct. The hardware instruction is buggy.", "kernel": "Not a kernel bug — it's ROCm 6.3.0 microcode. Update to 6.3.1.", "loader": "Weights are fine. Only FP8 MFMA is affected.", }, )) scenarios.append(Scenario( id="driver_compat_03", root_cause="driver_compat", correct_fix="update_driver_config", incident_ticket=( "INCIDENT: RTX 5090 (SM120) fails TensorRT-LLM engine build. Error: " "'No registered converter for SM120'. TensorRT-LLM 0.18 was built against " "CUDA 12.8 but SM120 needs CUDA 13.0 compiler support." ), hardware="NVIDIA SM120 (GeForce RTX 5090)", model_name="Llama-3.3-70B-Instruct", backend="TensorRT-LLM 0.18", initial_log=( "[TensorRT-LLM] Building engine for SM120...\n" "[TensorRT] CUDA toolkit: 12.8\n" "[TensorRT] ERROR: No code generator for sm_120\n" "[TensorRT] sm_120 requires CUDA toolkit >= 13.0\n" "[TensorRT] Available targets: sm_70, sm_75, sm_80, sm_86, sm_89, sm_90" ), initial_snippet=( "# TensorRT-LLM build environment\n" "cuda_toolkit: 12.8\n" "tensorrt_version: 10.4\n" "# SM120 code generation requires CUDA 13.0+\n" "# TensorRT-LLM 0.18 was compiled with CUDA 12.8\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "CUDA toolkit 12.8 cannot generate code for SM120. Need CUDA 13.0+ toolkit " "and matching TensorRT version.", 0.94, True ), "dispatch": SpecialistOpinion( "TensorRT engine can't be built. Need newer CUDA toolkit.", 0.82, True ), "kernel": SpecialistOpinion( "Try using compute_90 as a fallback target for SM120.", 0.45, False ), "loader": SpecialistOpinion( "The model format is fine. It's the build toolchain that's outdated.", 0.70, False ), }, inspect_results=InspectResult( logs=( "[TensorRT] CUDA toolkit 12.8: sm_120 not in target list\n" "[TensorRT] sm_120 added in CUDA 13.0\n" "[TensorRT] TensorRT 10.4 built with CUDA 12.8\n" "[TensorRT] Need TensorRT 10.6+ with CUDA 13.0" ), config=( "cuda_toolkit: 12.8\n" "tensorrt: 10.4\n" "sm120_support_min_cuda: 13.0\n" "sm120_support_min_trt: 10.6\n" "available_sm_targets: [70,75,80,86,89,90]" ), snippet=( "# CUDA 12.8 code generator has no SM120 target\n" "# SM120 introduced in CUDA 13.0 toolkit\n" "# TensorRT 10.6+ bundles CUDA 13.0 codegen\n" "# Fix: upgrade to TensorRT-LLM with TRT 10.6 / CUDA 13.0" ), metrics=( "engine_build_failures: 1\n" "sm120_codegen: unavailable\n" "cuda_toolkit: 12.8\n" "required_cuda_toolkit: 13.0" ), ), specialist_followups={ "runtime": "Upgrade CUDA toolkit to 13.0 and TensorRT to 10.6+ for SM120 support.", "dispatch": "Engine can't be built without SM120 codegen. Upgrade toolchain.", "kernel": "compute_90 fallback won't work — SM120 has different register file layout.", "loader": "Model is fine. Toolchain needs updating for SM120.", }, )) scenarios.append(Scenario( id="driver_compat_04", root_cause="driver_compat", correct_fix="update_driver_config", incident_ticket=( "INCIDENT: NVIDIA driver 570.86 on B200 causes random GPU resets during " "long inference sequences (>8K tokens). System log shows 'Xid 79: GPU has " "fallen off the bus'. Known driver bug in 570.86 for B200." ), hardware="NVIDIA B200", model_name="Llama-4-Maverick-17Bx128E", backend="vLLM 0.8.x", initial_log=( "[System] Xid 79: GPU 3 has fallen off the bus\n" "[vLLM] ERROR: CUDA error on GPU 3: device-side assert triggered\n" "[vLLM] Inference failed at token position 8247\n" "[System] nvidia-smi: GPU 3: ERR! (needs reset)\n" "[System] Driver: 570.86.01" ), initial_snippet=( "# NVIDIA driver 570.86 release notes:\n" "# Known issue: B200 GPUs may experience Xid 79 errors\n" "# during sustained high-power workloads\n" "# Affected: long-running inference >8K tokens\n" "# Fixed in: 570.100+\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "Driver 570.86 has a known bug for B200 that causes GPU bus resets " "during sustained high-power inference. Upgrade to 570.100+.", 0.95, True ), "dispatch": SpecialistOpinion( "Cannot fix at the application level. This is a driver bug.", 0.75, False ), "kernel": SpecialistOpinion( "Xid 79 is a hardware-level bus error. The GPU physically disconnects from PCIe. " "This is a driver power management bug, not a kernel issue.", 0.88, True ), "loader": SpecialistOpinion( "Try reducing GPU power limit to prevent thermal issues.", 0.35, False ), }, inspect_results=InspectResult( logs=( "[System] dmesg: NVRM: Xid 79 at PCI:0000:81:00.0\n" "[System] Driver: 570.86.01\n" "[System] GPU 3 power at crash: 680W (TDP: 700W)\n" "[System] Crash occurs at token position: 8000-10000\n" "[NVIDIA] Known bug: 570.86 B200 power sequencing error" ), config=( "driver_version: 570.86.01\n" "gpu: B200\n" "known_bug: xid_79_b200\n" "fix_version: 570.100\n" "crash_power_watts: 680\n" "tdp_watts: 700" ), snippet=( "# Driver 570.86 B200 power management bug:\n" "# Sustained near-TDP workloads trigger power sequencing error\n" "# GPU drops off PCIe bus (Xid 79)\n" "# Fix: upgrade to driver 570.100+\n" "# Workaround: nvidia-smi -pl 600 (reduce power limit)" ), metrics=( "xid_79_events: 7 in 24 hours\n" "avg_tokens_at_crash: 8743\n" "gpu_power_at_crash_w: 680\n" "driver_version: 570.86\n" "fix_version: 570.100" ), ), specialist_followups={ "runtime": "Upgrade driver to 570.100+ to fix the B200 power sequencing bug.", "dispatch": "Application-level fix won't help. Driver upgrade needed.", "kernel": "Xid 79 is driver-level. Upgrade to 570.100 or use power limit workaround.", "loader": "Reducing power limit is a workaround but the real fix is driver 570.100+.", }, )) scenarios.append(Scenario( id="driver_compat_05", root_cause="driver_compat", correct_fix="update_driver_config", incident_ticket=( "INCIDENT: MI355X with ROCm 6.2 driver cannot use FlashAttention-2 for MI355X. " "Error: 'Unsupported AMDGPU target: gfx950'. MI355X (gfx950) was added in " "ROCm 6.4. System running old ROCm from base OS install." ), hardware="AMD MI355X", model_name="Qwen3-235B-A22B", backend="vLLM 0.8.x", initial_log=( "[vLLM] GPU: AMD MI355X (gfx950)\n" "[ROCm] Version: 6.2.0\n" "[FlashAttention] Compiling for gfx950...\n" "[FlashAttention] ERROR: gfx950 not supported by ROCm 6.2 compiler\n" "[FlashAttention] Available targets: gfx900, gfx906, gfx908, gfx90a, gfx942" ), initial_snippet=( "# ROCm version check\n" "rocm_version: 6.2.0 # from base OS install\n" "# MI355X (gfx950) support added in ROCm 6.4\n" "# ROCm 6.2 compiler cannot generate gfx950 ISA\n" "# FlashAttention-2 kernel compilation fails\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "ROCm 6.2 does not support gfx950 (MI355X). Need ROCm 6.4+ for this GPU.", 0.95, True ), "dispatch": SpecialistOpinion( "FlashAttention cannot compile kernels for gfx950 on ROCm 6.2. " "This blocks all optimized attention paths.", 0.85, True ), "kernel": SpecialistOpinion( "Try compiling FlashAttention targeting gfx942 as a fallback.", 0.40, False ), "loader": SpecialistOpinion( "Model weights load fine. The kernel compilation is the issue.", 0.65, False ), }, inspect_results=InspectResult( logs=( "[ROCm] Installed: 6.2.0\n" "[ROCm] gfx950 support: NOT AVAILABLE (added in 6.4)\n" "[hipcc] --offload-arch=gfx950: error: unknown target\n" "[FlashAttention] Cannot JIT compile attention kernels\n" "[vLLM] Falling back to naive attention (10x slower)" ), config=( "rocm_version: 6.2.0\n" "gpu_target: gfx950\n" "gfx950_min_rocm: 6.4\n" "available_targets: [gfx900,gfx906,gfx908,gfx90a,gfx942]\n" "flashattention_status: compilation_failed" ), snippet=( "# ROCm 6.2 compiler targets: gfx900-gfx942\n" "# gfx950 (MI355X) added in ROCm 6.4\n" "# No backward compatibility — gfx950 ISA differs from gfx942\n" "# Fix: upgrade ROCm to 6.4+" ), metrics=( "kernel_compilation_failures: 1\n" "attention_backend: naive_fallback\n" "attention_slowdown: 10x\n" "rocm_version: 6.2.0\n" "required_rocm: 6.4" ), ), specialist_followups={ "runtime": "Upgrade ROCm to 6.4+ to get gfx950 compiler support.", "dispatch": "All optimized attention paths blocked. ROCm upgrade is required.", "kernel": "gfx942 fallback won't work — ISA is incompatible. Must use native gfx950 from ROCm 6.4.", "loader": "Model loads but runs on naive attention at 10x penalty. Fix is ROCm upgrade.", }, )) scenarios.append(Scenario( id="driver_compat_06", root_cause="driver_compat", correct_fix="update_driver_config", incident_ticket=( "INCIDENT: H100 with driver 535 LTS cannot use FP8 tensor cores. " "FP8 GEMM falls back to FP16 emulation causing 3x slowdown. " "Driver 535 technically supports H100 but has incomplete FP8 support." ), hardware="NVIDIA H100", model_name="Mistral-Large-2", backend="vLLM 0.8.x", initial_log=( "[vLLM] GPU: H100 SXM (SM90)\n" "[CUDA] Driver: 535.183.01 (LTS)\n" "[vLLM] FP8 GEMM: attempting to use tensor cores...\n" "[cuBLAS] WARNING: FP8 GEMM not supported by driver, falling back to FP16 emulation\n" "[vLLM] Token generation: 45 tok/s (expected: 135 tok/s with native FP8)" ), initial_snippet=( "# Driver 535 (LTS) limitations:\n" "# - H100 SM90 basic support: YES\n" "# - FP8 tensor core (e4m3fn GEMM): INCOMPLETE\n" "# - cuBLAS FP8 API: stub only (returns CUBLAS_STATUS_NOT_SUPPORTED)\n" "# - Full FP8 requires driver 545+\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "Driver 535 LTS has incomplete FP8 cuBLAS support. Upgrade to 545+ for " "full FP8 tensor core acceleration on H100.", 0.94, True ), "dispatch": SpecialistOpinion( "vLLM correctly attempts FP8 but cuBLAS returns NOT_SUPPORTED. " "Fallback to FP16 emulation is working but slow.", 0.80, True ), "kernel": SpecialistOpinion( "Maybe the FP8 weights are in the wrong format for this cuBLAS version.", 0.45, False ), "loader": SpecialistOpinion( "FP8 model loaded fine. The slowdown is from driver-level FP8 emulation.", 0.72, False ), }, inspect_results=InspectResult( logs=( "[cuBLAS] Version: 12.3 (driver 535)\n" "[cuBLAS] cublasLtMatmul with FP8: CUBLAS_STATUS_NOT_SUPPORTED\n" "[cuBLAS] Fallback: FP8 -> FP16 upcast -> FP16 GEMM -> FP8 downcast\n" "[cuBLAS] FP8 emulation overhead: 3.1x\n" "[cuBLAS] Native FP8 requires cuBLAS 12.5+ (driver 545+)" ), config=( "driver_version: 535.183.01\n" "cublas_version: 12.3\n" "fp8_native_support: false\n" "fp8_emulation: true\n" "min_driver_for_fp8: 545" ), snippet=( "# Driver 535 LTS: cuBLAS 12.3 stubs FP8 API\n" "# Returns NOT_SUPPORTED, triggers FP16 emulation path\n" "# FP8 -> FP16 upcast + FP16 GEMM + FP8 downcast = 3x slower\n" "# Fix: upgrade to driver 545+ for native FP8 cuBLAS" ), metrics=( "throughput_tok_s: 45\n" "expected_throughput: 135\n" "fp8_emulation_overhead: 3.1x\n" "cublas_fp8_status: NOT_SUPPORTED\n" "driver: 535.183.01" ), ), specialist_followups={ "runtime": "Upgrade from driver 535 LTS to 545+ for native FP8 cuBLAS support.", "dispatch": "FP8 dispatch falls back to emulation. Driver upgrade will enable native path.", "kernel": "FP8 weight format is correct. cuBLAS just can't use FP8 tensor cores with driver 535.", "loader": "Weights are fine. Driver needs upgrading for FP8 acceleration.", }, )) # --- runtime_loader scenarios 07-10 --- scenarios.append(Scenario( id="runtime_loader_07", root_cause="runtime_loader", correct_fix="fix_runtime_path", incident_ticket=( "INCIDENT: TensorRT-LLM engine build fails on H100 with 'libnccl.so.2: cannot open " "shared object file'. NCCL installed via pip but TensorRT-LLM searches system paths only. " "Single-GPU mode works, multi-GPU engine build crashes." ), hardware="NVIDIA H100", model_name="DeepSeek-V3-671B", backend="TensorRT-LLM 0.18", initial_log=( "[TensorRT-LLM] Building engine for TP=8...\n" "[TensorRT-LLM] Loading NCCL for multi-GPU build...\n" "[TensorRT-LLM] ERROR: libnccl.so.2: cannot open shared object file: No such file or directory\n" "[TensorRT-LLM] LD_LIBRARY_PATH=/usr/local/tensorrt/lib:/usr/local/cuda-13/lib64\n" "[TensorRT-LLM] Note: pip-installed NCCL at /usr/local/lib/python3.11/dist-packages/nvidia/nccl/lib/" ), initial_snippet=( "# Environment setup\n" "LD_LIBRARY_PATH=/usr/local/tensorrt/lib:/usr/local/cuda-13/lib64\n" "# NCCL installed via: pip install nvidia-nccl-cu13\n" "# pip puts libnccl.so.2 at:\n" "# /usr/local/lib/python3.11/dist-packages/nvidia/nccl/lib/\n" "# TensorRT-LLM only searches LD_LIBRARY_PATH and /usr/lib\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "NCCL is installed but its .so lives inside the pip package directory, not " "on LD_LIBRARY_PATH. Add the pip NCCL lib path to LD_LIBRARY_PATH.", 0.95, True ), "dispatch": SpecialistOpinion("Engine build never starts. Can't assess dispatch.", 0.45, False), "kernel": SpecialistOpinion( "Not a kernel issue. The multi-GPU build just can't locate the NCCL shared library.", 0.68, False ), "loader": SpecialistOpinion( "Dynamic linker can't find libnccl.so.2. The pip-installed path is non-standard and " "needs to be added to LD_LIBRARY_PATH or symlinked to a system lib dir.", 0.91, True ), }, inspect_results=InspectResult( logs=( "[System] find / -name 'libnccl.so.2':\n" " /usr/local/lib/python3.11/dist-packages/nvidia/nccl/lib/libnccl.so.2\n" "[System] ldconfig -p | grep nccl -> (empty)\n" "[System] LD_LIBRARY_PATH=/usr/local/tensorrt/lib:/usr/local/cuda-13/lib64" ), config=( "LD_LIBRARY_PATH=/usr/local/tensorrt/lib:/usr/local/cuda-13/lib64\n" "nccl_install_method: pip\n" "nccl_lib_path=/usr/local/lib/python3.11/dist-packages/nvidia/nccl/lib\n" "ldconfig_nccl: not_cached" ), snippet=( "# libnccl.so.2 exists but in pip package path\n" "# Not in LD_LIBRARY_PATH or ldconfig cache\n" "# Fix: export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:" "/usr/local/lib/python3.11/dist-packages/nvidia/nccl/lib\n" "# Or: ln -s .../nccl/lib/libnccl.so.2 /usr/local/cuda-13/lib64/" ), metrics=( "engine_build_attempts: 4\n" "engine_build_failures: 4\n" "single_gpu_build: success\n" "multi_gpu_build: failed (missing libnccl)" ), ), specialist_followups={ "runtime": "Add pip NCCL lib path to LD_LIBRARY_PATH. Pip installs NCCL to a non-standard location.", "dispatch": "Engine never built. No dispatch to analyze.", "kernel": "No kernel issue. Just a library path problem.", "loader": "Symlink libnccl.so.2 into /usr/local/cuda-13/lib64/ or extend LD_LIBRARY_PATH.", }, )) scenarios.append(Scenario( id="runtime_loader_08", root_cause="runtime_loader", correct_fix="fix_runtime_path", incident_ticket=( "INCIDENT: SGLang container on H100 cluster crashes with 'libcublas.so.12: version " "`libcublas.so.12.6` not found'. Host has CUDA 12.6 but container ships CUDA 12.2 " "cublas. Version symbol mismatch on bind-mounted library." ), hardware="NVIDIA H100", model_name="Qwen3-235B-A22B", backend="SGLang 0.5.x", initial_log=( "[SGLang] Starting model worker...\n" "[SGLang] Loading torch with CUDA support...\n" "[SGLang] ERROR: /usr/local/cuda/lib64/libcublas.so.12: version `libcublas.so.12.6` " "not found (required by /opt/sglang/lib/python3.11/site-packages/torch/lib/libtorch_cuda.so)\n" "[Container] libcublas.so.12 -> CUDA 12.2 (container-bundled)\n" "[Host] libcublas.so.12 -> CUDA 12.6" ), initial_snippet=( "# Dockerfile\n" "FROM nvidia/cuda:12.2.0-devel-ubuntu22.04\n" "# Container has CUDA 12.2 cublas\n" "# But PyTorch wheel built against CUDA 12.6 cublas symbols\n" "# nvidia-container-toolkit mounts host cublas over container's\n" "# ... except when NVIDIA_DRIVER_CAPABILITIES doesn't include 'compute'\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "Container's libcublas.so.12 is CUDA 12.2 but PyTorch needs 12.6 symbols. " "The nvidia-container-toolkit should mount the host's 12.6 cublas but " "NVIDIA_DRIVER_CAPABILITIES is set to 'utility' only, skipping compute libs.", 0.94, True ), "dispatch": SpecialistOpinion("Server crashes on import. No dispatch reached.", 0.50, False), "kernel": SpecialistOpinion( "Maybe rebuild PyTorch against CUDA 12.2 to match the container.", 0.40, False ), "loader": SpecialistOpinion( "Symbol version mismatch in libcublas. Container's 12.2 cublas lacks the " "12.6 version tag. Fix NVIDIA_DRIVER_CAPABILITIES to include 'compute' so " "host libs get mounted.", 0.92, True ), }, inspect_results=InspectResult( logs=( "[Container] strings /usr/local/cuda/lib64/libcublas.so.12 | grep 'cublas.so.12' ->\n" " libcublas.so.12.2\n" "[Container] NVIDIA_DRIVER_CAPABILITIES=utility\n" "[Host] /usr/lib/x86_64-linux-gnu/libcublas.so.12 -> 12.6.4\n" "[nvidia-ctk] compute libs NOT mounted (capabilities missing 'compute')" ), config=( "container_cuda: 12.2\n" "host_cuda: 12.6\n" "NVIDIA_DRIVER_CAPABILITIES=utility\n" "pytorch_cublas_requirement: libcublas.so.12.6\n" "container_cublas_provides: libcublas.so.12.2" ), snippet=( "# NVIDIA_DRIVER_CAPABILITIES=utility only mounts nvidia-smi, not CUDA libs\n" "# Need: NVIDIA_DRIVER_CAPABILITIES=compute,utility\n" "# This tells nvidia-container-toolkit to bind-mount host's CUDA compute libs\n" "# Fix: docker run -e NVIDIA_DRIVER_CAPABILITIES=compute,utility ..." ), metrics=( "container_start_failures: 1\n" "cublas_version_container: 12.2\n" "cublas_version_host: 12.6\n" "cublas_version_required: 12.6" ), ), specialist_followups={ "runtime": "Set NVIDIA_DRIVER_CAPABILITIES=compute,utility so host CUDA 12.6 libs get mounted into container.", "dispatch": "Can't diagnose — server fails on torch import.", "kernel": "Rebuilding PyTorch is wrong approach. Fix container runtime config to use host libs.", "loader": "Host cublas 12.6 will be mounted over container's 12.2 once NVIDIA_DRIVER_CAPABILITIES includes 'compute'.", }, )) scenarios.append(Scenario( id="runtime_loader_09", root_cause="runtime_loader", correct_fix="fix_runtime_path", incident_ticket=( "INCIDENT: vLLM on B200 node fails with 'libcudart.so.12: ELF load command address/offset " "not page-aligned'. Container image was built on x86_64 but B200 nodes use aarch64 (GH200). " "Wrong architecture .so files loaded." ), hardware="NVIDIA B200", model_name="Llama-3.3-70B-Instruct", backend="vLLM 0.8.x", initial_log=( "[vLLM] Initializing CUDA runtime...\n" "[vLLM] ERROR: /usr/local/cuda/lib64/libcudart.so.12: ELF load command " "address/offset not page-aligned\n" "[System] uname -m: aarch64\n" "[System] file libcudart.so.12: ELF 64-bit LSB shared object, x86-64\n" "[vLLM] FATAL: Architecture mismatch — x86_64 .so on aarch64 host" ), initial_snippet=( "# Dockerfile (built on x86_64 CI runner)\n" "FROM nvidia/cuda:12.8.0-runtime-ubuntu22.04\n" "# This pulls x86_64 image on x86_64 CI\n" "# Deployed to aarch64 B200/GH200 nodes\n" "# CUDA libs inside container are x86_64 ELF binaries\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "Container was built for x86_64 but deployed on aarch64. All .so files are wrong " "architecture. Need to rebuild the container image targeting linux/arm64.", 0.96, True ), "dispatch": SpecialistOpinion("Can't assess — CUDA fails to initialize.", 0.42, False), "kernel": SpecialistOpinion( "The 'not page-aligned' error sometimes means corrupt download. Try re-pulling image.", 0.35, False ), "loader": SpecialistOpinion( "ELF architecture mismatch. The dynamic loader is trying to load x86_64 shared " "objects on an aarch64 system. Rebuild with --platform linux/arm64.", 0.93, True ), }, inspect_results=InspectResult( logs=( "[System] uname -m: aarch64\n" "[System] file /usr/local/cuda/lib64/libcudart.so.12:\n" " ELF 64-bit LSB shared object, x86-64, version 1 (SYSV)\n" "[System] docker inspect image -> Architecture: amd64\n" "[System] host architecture: arm64 (GH200 Grace-Hopper)" ), config=( "host_arch: aarch64\n" "container_arch: x86_64\n" "image_platform: linux/amd64\n" "target_platform: linux/arm64\n" "cuda_libs_arch: x86-64" ), snippet=( "# Container built with: docker build . (on x86_64 CI)\n" "# Deployed to aarch64 GH200 nodes\n" "# Fix: docker build --platform linux/arm64 .\n" "# Or: use multi-arch base image and build on arm64 runner" ), metrics=( "container_start_failures: 1\n" "host_arch: aarch64\n" "image_arch: amd64\n" "elf_load_errors: 1" ), ), specialist_followups={ "runtime": "Rebuild container targeting linux/arm64 for GH200/B200 aarch64 nodes.", "dispatch": "Nothing runs — wrong architecture binary.", "kernel": "Not a corruption issue. The ELF binary is simply for the wrong CPU architecture.", "loader": "Dynamic loader rejects x86_64 .so on aarch64. Rebuild image for arm64 platform.", }, )) scenarios.append(Scenario( id="runtime_loader_10", root_cause="runtime_loader", correct_fix="fix_runtime_path", incident_ticket=( "INCIDENT: PyTorch on SM121 DGX Spark fails with 'undefined symbol: cublasLtMatmulAlgoGetIds' " "in libcublas. Multiple CUDA installations present; LD_LIBRARY_PATH ordering causes " "older libcublas to be loaded before the correct one." ), hardware="NVIDIA SM121 (DGX Spark)", model_name="DeepSeek-R1-Distill-70B", backend="SGLang 0.5.x", initial_log=( "[SGLang] Importing torch...\n" "[SGLang] ERROR: undefined symbol: cublasLtMatmulAlgoGetIds\n" "[SGLang] Traceback: libtorch_cuda.so -> libcublas.so.12\n" "[System] Multiple libcublas.so.12 found:\n" " /usr/local/cuda-12.2/lib64/libcublas.so.12 (v12.2.5)\n" " /usr/local/cuda-13/lib64/libcublas.so.12 (v12.6.4)\n" "[System] LD_LIBRARY_PATH=/usr/local/cuda-12.2/lib64:/usr/local/cuda-13/lib64" ), initial_snippet=( "# /etc/profile.d/cuda.sh\n" "export LD_LIBRARY_PATH=/usr/local/cuda-12.2/lib64:/usr/local/cuda-13/lib64\n" "# cuda-12.2 listed FIRST — its older libcublas.so.12 gets loaded\n" "# PyTorch needs cublasLtMatmulAlgoGetIds (added in cublas 12.4)\n" "# cuda-12.2's cublas lacks this symbol\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "LD_LIBRARY_PATH has cuda-12.2 before cuda-13. The linker loads the older " "libcublas.so.12 which lacks symbols needed by PyTorch. Swap the path order " "or remove cuda-12.2 from LD_LIBRARY_PATH.", 0.96, True ), "dispatch": SpecialistOpinion( "Server crashes on import. Cannot reach dispatch phase.", 0.48, False ), "kernel": SpecialistOpinion( "Maybe the SM121 needs a newer cuBLAS version. Try updating CUDA toolkit.", 0.50, False ), "loader": SpecialistOpinion( "Classic LD_LIBRARY_PATH ordering bug. Two CUDA installs both provide " "libcublas.so.12 but with different symbol versions. The older one (12.2) is " "found first and loaded. Fix: put cuda-13 first or remove cuda-12.2 from path.", 0.94, True ), }, inspect_results=InspectResult( logs=( "[System] ldd /opt/sglang/lib/python3.11/site-packages/torch/lib/libtorch_cuda.so:\n" " libcublas.so.12 => /usr/local/cuda-12.2/lib64/libcublas.so.12 (WRONG)\n" "[System] nm -D /usr/local/cuda-12.2/lib64/libcublas.so.12 | grep cublasLtMatmulAlgoGetIds -> (empty)\n" "[System] nm -D /usr/local/cuda-13/lib64/libcublas.so.12 | grep cublasLtMatmulAlgoGetIds -> FOUND" ), config=( "LD_LIBRARY_PATH=/usr/local/cuda-12.2/lib64:/usr/local/cuda-13/lib64\n" "cublas_loaded: /usr/local/cuda-12.2/lib64/libcublas.so.12 (v12.2.5)\n" "cublas_needed: libcublas.so.12 >= v12.4 (for cublasLtMatmulAlgoGetIds)\n" "cuda_installations: [12.2, 13]" ), snippet=( "# LD_LIBRARY_PATH lists cuda-12.2 before cuda-13\n" "# Dynamic linker picks first match: cuda-12.2's cublas (v12.2.5)\n" "# PyTorch needs cublasLtMatmulAlgoGetIds (cublas >= 12.4)\n" "# Fix: export LD_LIBRARY_PATH=/usr/local/cuda-13/lib64:$LD_LIBRARY_PATH\n" "# Or remove /usr/local/cuda-12.2/lib64 from path entirely" ), metrics=( "import_failures: 1\n" "cublas_version_loaded: 12.2.5\n" "cublas_version_required: >=12.4\n" "cuda_installations: 2" ), ), specialist_followups={ "runtime": "Fix LD_LIBRARY_PATH ordering: cuda-13 must come before cuda-12.2. Or purge cuda-12.2.", "dispatch": "Can't diagnose until import succeeds.", "kernel": "CUDA toolkit is new enough. The problem is which libcublas the linker resolves first.", "loader": "Put /usr/local/cuda-13/lib64 first in LD_LIBRARY_PATH so the newer cublas is found.", }, )) # --- distributed_comm scenarios 07-10 --- scenarios.append(Scenario( id="distributed_comm_07", root_cause="distributed_comm", correct_fix="fix_comm_config", incident_ticket=( "INCIDENT: vLLM tensor parallel on 4xH100 fails with 'NCCL error: unhandled " "cuda error, NCCL version 2.21.5'. P2P access between GPU 0 and GPU 2 fails. " "GPUs 0-1 and 2-3 are on separate PCIe switches. nvidia-smi topo shows no P2P." ), hardware="NVIDIA H100", model_name="Llama-4-Maverick-17Bx128E", backend="vLLM 0.8.x", initial_log=( "[vLLM] Tensor parallel size: 4\n" "[NCCL] Testing P2P access...\n" "[NCCL] GPU 0 <-> GPU 1: P2P OK (NVLink)\n" "[NCCL] GPU 2 <-> GPU 3: P2P OK (NVLink)\n" "[NCCL] GPU 0 <-> GPU 2: P2P FAIL (different PCIe switch, no NVLink)\n" "[NCCL] ERROR: unhandled cuda error on ncclCommInitRank\n" "[NCCL] Falling back to SHM transport... FAILED (NCCL_SHM_DISABLE=1)" ), initial_snippet=( "# Environment\n" "NCCL_P2P_LEVEL=NVL # Only use NVLink for P2P\n" "NCCL_SHM_DISABLE=1 # Shared memory disabled by admin\n" "NCCL_NET_DISABLE=1 # Network transport disabled\n" "# GPUs 0-1 on NVSwitch A, GPUs 2-3 on NVSwitch B\n" "# No NVLink between switch domains, no fallback transport enabled\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "CUDA runtime is fine on each GPU individually. NCCL can't establish communication " "between GPUs on different PCIe switches because all fallback transports are disabled.", 0.88, True ), "dispatch": SpecialistOpinion( "Try reducing tensor parallel to 2 and use pipeline parallel for the other dimension.", 0.55, False ), "kernel": SpecialistOpinion( "NCCL needs at least one working transport between all GPU pairs. NVLink only " "connects 0-1 and 2-3. With SHM and NET disabled, there's no path from 0 to 2. " "Fix: enable SHM or NET transport, or set NCCL_P2P_LEVEL=SYS to allow PCIe P2P.", 0.95, True ), "loader": SpecialistOpinion( "Weights shard correctly but NCCL can't communicate across the GPU pairs.", 0.62, False ), }, inspect_results=InspectResult( logs=( "[nvidia-smi topo -m]\n" " GPU0 GPU1 GPU2 GPU3\n" "GPU0 X NV12 SYS SYS\n" "GPU1 NV12 X SYS SYS\n" "GPU2 SYS SYS X NV12\n" "GPU3 SYS SYS NV12 X\n" "[NCCL] P2P level: NVL (NVLink only)\n" "[NCCL] SHM: disabled\n" "[NCCL] NET: disabled\n" "[NCCL] No transport available for GPU0 <-> GPU2" ), config=( "tensor_parallel_size: 4\n" "NCCL_P2P_LEVEL=NVL\n" "NCCL_SHM_DISABLE=1\n" "NCCL_NET_DISABLE=1\n" "nvlink_pairs: [[0,1],[2,3]]\n" "cross_switch_transport: none" ), snippet=( "# NVLink only connects GPU pairs (0,1) and (2,3)\n" "# NCCL_P2P_LEVEL=NVL restricts to NVLink transport\n" "# SHM and NET are both disabled\n" "# No transport path from GPU 0 to GPU 2\n" "# Fix: NCCL_P2P_LEVEL=SYS (allow PCIe P2P)\n" "# Or: NCCL_SHM_DISABLE=0 (enable shared memory fallback)" ), metrics=( "nccl_init_failures: 1\n" "p2p_nvlink_pairs: 2\n" "p2p_failed_pairs: 4\n" "available_transports: 0 (for cross-switch)" ), ), specialist_followups={ "runtime": "All individual GPUs work. NCCL transport config is too restrictive.", "dispatch": "TP=2 with PP=2 would work around it, but better to fix transport config.", "kernel": "Set NCCL_P2P_LEVEL=SYS or enable SHM transport. Cross-switch GPUs need a fallback path.", "loader": "Weight sharding is correct. Communication config blocks cross-switch traffic.", }, )) scenarios.append(Scenario( id="distributed_comm_08", root_cause="distributed_comm", correct_fix="fix_comm_config", incident_ticket=( "INCIDENT: SGLang multi-GPU serving of DeepSeek-V3-671B on 8xB200 hangs during " "rendezvous. Workers 0-6 ready but worker 7 never joins. Timeout after 1800s. " "All 8 GPUs visible via nvidia-smi. Single-GPU serving works." ), hardware="NVIDIA B200", model_name="DeepSeek-V3-671B", backend="SGLang 0.5.x", initial_log=( "[SGLang] Launching 8 workers for TP=8...\n" "[Worker 0-6] Rendezvous: connected to master at 127.0.0.1:29500\n" "[Worker 7] Rendezvous: connecting to master at 127.0.0.1:29500...\n" "[Worker 7] ERROR: Connection refused (127.0.0.1:29500)\n" "[Worker 7] Retrying in 5s... (attempt 1/360)\n" "[Master] WARNING: torchrun nproc=8 but only 7 workers joined\n" "[Master] TIMEOUT: rendezvous deadline exceeded (1800s)" ), initial_snippet=( "# launch.sh\n" "torchrun --nproc_per_node=8 \\\n" " --master_addr=127.0.0.1 \\\n" " --master_port=29500 \\\n" " -m sglang.launch_server \\\n" " --tp 8 --model DeepSeek-V3-671B\n" "# Worker 7 can't connect — port 29500 hits ulimit on file descriptors\n" "# System ulimit -n = 1024 (default), too low for 8 workers + model\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "CUDA runtime healthy on all GPUs. Worker 7 fails at TCP rendezvous, not CUDA init. " "The master process runs out of file descriptors before worker 7 can connect.", 0.90, True ), "dispatch": SpecialistOpinion( "Maybe the model is too large for 8-way TP. Try reducing worker count.", 0.38, False ), "kernel": SpecialistOpinion( "Not a GPU kernel issue. This is a process-level rendezvous failure. " "Check system limits — ulimit, max connections, etc.", 0.78, True ), "loader": SpecialistOpinion( "Workers 0-6 load model shards fine. Worker 7 never gets past rendezvous to load anything.", 0.60, False ), }, inspect_results=InspectResult( logs=( "[System] ulimit -n: 1024\n" "[System] lsof -p | wc -l: 1021\n" "[System] Worker 7 connect() -> errno 24 (EMFILE: too many open files)\n" "[System] Workers 0-6 each hold ~120 FDs (CUDA ctx + model + sockets)\n" "[System] Master holds ~150 FDs (7 worker connections + logging + CUDA)" ), config=( "ulimit_nofile: 1024\n" "master_fd_usage: 1021\n" "worker_fd_usage_each: ~120\n" "total_workers: 8\n" "master_port: 29500" ), snippet=( "# Master process FD usage: 7 workers * ~120 FDs + 150 base = ~990\n" "# Worker 7 connect needs FD 1022+ but ulimit is 1024\n" "# Master can't accept() — out of file descriptors\n" "# Fix: ulimit -n 65536 before launching\n" "# Or: add 'LimitNOFILE=65536' to systemd unit" ), metrics=( "workers_joined: 7\n" "workers_expected: 8\n" "rendezvous_timeout_s: 1800\n" "master_fd_count: 1021\n" "ulimit_nofile: 1024" ), ), specialist_followups={ "runtime": "Raise file descriptor limit: ulimit -n 65536. Master runs out of FDs before worker 7 connects.", "dispatch": "All 8 GPUs are needed for TP=8. The model requires this parallelism level.", "kernel": "System ulimit too low. Raise to 65536 and relaunch.", "loader": "Worker 7 can't even connect. Fix rendezvous FD limit first.", }, )) scenarios.append(Scenario( id="distributed_comm_09", root_cause="distributed_comm", correct_fix="fix_comm_config", incident_ticket=( "INCIDENT: TensorRT-LLM pipeline parallel on 2xSM121 DGX Spark (16 GPUs) has " "NaN outputs after stage boundary. Pipeline stage 0 (GPUs 0-7) produces valid " "activations but stage 1 (GPUs 8-15) receives corrupted tensors. " "Suspected NCCL send/recv data corruption over InfiniBand." ), hardware="NVIDIA SM121 (DGX Spark)", model_name="Mistral-Large-2", backend="TensorRT-LLM 0.18", initial_log=( "[TensorRT-LLM] Pipeline parallel: 2 stages, 8 GPUs each\n" "[TensorRT-LLM] Stage 0 output: tensor([-0.23, 1.45, 0.87, ...]) OK\n" "[TensorRT-LLM] NCCL send rank 7 -> rank 8 (cross-node)\n" "[TensorRT-LLM] Stage 1 input: tensor([nan, nan, inf, nan, ...]) CORRUPTED\n" "[NCCL] Transport: IB/RoCE (mlx5_0)\n" "[NCCL] WARNING: IB error count: 847 (CRC errors on link)" ), initial_snippet=( "# Network config\n" "NCCL_IB_HCA=mlx5_0 # Single HCA\n" "NCCL_IB_GID_INDEX=3 # RoCE v2\n" "NCCL_IB_DISABLE=0\n" "# IB link between nodes has high CRC error rate\n" "# PFC (Priority Flow Control) not enabled on switch\n" "# RoCE without PFC -> packet drops -> NCCL data corruption\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "CUDA runtime fine. Data corruption happens during NCCL cross-node transfer. " "The IB link has CRC errors suggesting a physical or config issue.", 0.86, True ), "dispatch": SpecialistOpinion( "Pipeline parallel dispatch is correct. Corruption is in the transport layer.", 0.72, False ), "kernel": SpecialistOpinion( "NCCL over RoCE without PFC causes silent data corruption under congestion. " "The switch is dropping packets and RoCE has no reliable retransmission without PFC. " "Enable PFC on the switch or use NCCL_IB_RETRY_CNT and NCCL_IB_TIMEOUT.", 0.94, True ), "loader": SpecialistOpinion( "Model weights load correctly on both stages. Corruption only on activation transfer.", 0.65, False ), }, inspect_results=InspectResult( logs=( "[NCCL] IB transport: RoCE v2 over mlx5_0\n" "[NCCL] Link CRC errors: 847 in 60s\n" "[NCCL] Packet drops: 312\n" "[Switch] PFC: DISABLED on ports 17-18 (inter-node trunk)\n" "[Switch] ECN: DISABLED\n" "[NCCL] NCCL_IB_RETRY_CNT=0 (no retries on failure)" ), config=( "nccl_transport: ib_roce_v2\n" "nccl_ib_hca: mlx5_0\n" "pfc_enabled: false\n" "ecn_enabled: false\n" "nccl_ib_retry_cnt: 0\n" "link_crc_errors: 847" ), snippet=( "# RoCE v2 requires PFC for lossless transport\n" "# Without PFC, congestion causes packet drops\n" "# Dropped packets -> corrupted NCCL data (NaN activations)\n" "# Fix 1: Enable PFC on switch ports 17-18\n" "# Fix 2: Set NCCL_IB_RETRY_CNT=7, NCCL_IB_TIMEOUT=22\n" "# Fix 3: Enable ECN for congestion signaling" ), metrics=( "nan_outputs: 100%_after_stage_boundary\n" "ib_crc_errors: 847\n" "packet_drops: 312\n" "pfc_enabled: false\n" "nccl_retries: 0" ), ), specialist_followups={ "runtime": "IB link is unreliable. CRC errors point to congestion-related packet drops without PFC.", "dispatch": "Pipeline routing is fine. The data gets corrupted in transit over the wire.", "kernel": "Enable PFC on the switch and set NCCL_IB_RETRY_CNT=7. RoCE needs lossless fabric.", "loader": "Weight loading fine. Activation transfer across nodes is corrupted by packet drops.", }, )) scenarios.append(Scenario( id="distributed_comm_10", root_cause="distributed_comm", correct_fix="fix_comm_config", incident_ticket=( "INCIDENT: vLLM expert parallel on 8xH100 (DGX H100) sees 70% throughput " "degradation for MoE all-to-all. NCCL chooses single-ring algorithm but " "8-GPU NVSwitch topology supports tree/NVLS. Manual NCCL_ALGO override needed. " "Profiler shows all-to-all using only 12% of available NVLink bandwidth." ), hardware="NVIDIA H100", model_name="DeepSeek-V3-671B", backend="vLLM 0.8.x", initial_log=( "[vLLM] Expert parallel: 8 GPUs, MoE all-to-all\n" "[NCCL] Algorithm: Ring (auto-selected)\n" "[NCCL] NVSwitch detected but NVLS (NVLink SHARP) not enabled\n" "[NCCL] All-to-all per MoE layer: 8.2ms (ring)\n" "[NCCL] Expected with NVLS: 2.4ms\n" "[vLLM] MoE throughput: 380 tok/s (expected: 1200 tok/s)\n" "[NCCL] NVLink bandwidth utilization: 12%" ), initial_snippet=( "# Environment\n" "NCCL_ALGO=Ring # Explicitly set to Ring (suboptimal for NVSwitch)\n" "NCCL_NVLS_ENABLE=0 # NVLS disabled\n" "# DGX H100 has full NVSwitch — supports Tree and NVLS algorithms\n" "# Ring algorithm doesn't exploit NVSwitch all-to-all hardware\n" ), specialist_opinions={ "runtime": SpecialistOpinion( "CUDA runtime and NVSwitch hardware are healthy. NCCL is using a suboptimal " "communication algorithm that doesn't leverage the NVSwitch topology.", 0.85, True ), "dispatch": SpecialistOpinion( "Expert dispatch routing is correct. The all-to-all primitive itself is slow.", 0.70, False ), "kernel": SpecialistOpinion( "NCCL Ring algorithm sends data through all 8 GPUs sequentially. On NVSwitch, " "the Tree or NVLS algorithm can do all-to-all in one hop via the switch. " "NCCL_ALGO=Ring is overriding the auto-selector. Remove it and set " "NCCL_NVLS_ENABLE=1 to enable NVLink SHARP.", 0.95, True ), "loader": SpecialistOpinion( "Expert weights sharded correctly. Communication algorithm is the bottleneck.", 0.58, False ), }, inspect_results=InspectResult( logs=( "[NCCL] NCCL_ALGO=Ring (user override)\n" "[NCCL] NCCL_NVLS_ENABLE=0 (disabled)\n" "[NCCL] NVSwitch: present, 8-way full connectivity\n" "[NCCL] Ring all-to-all: 7 hops per message (8 GPUs)\n" "[NCCL] NVLS all-to-all: 1 hop per message (via NVSwitch)\n" "[NCCL] Ring BW utilization: 12% of NVLink capacity\n" "[NCCL] NVLS BW utilization (projected): 85% of NVLink capacity" ), config=( "NCCL_ALGO=Ring\n" "NCCL_NVLS_ENABLE=0\n" "nvswitch_present: true\n" "gpu_count: 8\n" "nvlink_bw_per_gpu_gbs: 900\n" "ring_alltoall_ms: 8.2\n" "nvls_alltoall_ms_projected: 2.4" ), snippet=( "# NCCL_ALGO=Ring forces ring algorithm (7 hops for 8 GPUs)\n" "# NVSwitch allows 1-hop all-to-all via NVLS\n" "# Fix: unset NCCL_ALGO (let NCCL auto-select Tree/NVLS)\n" "# Fix: NCCL_NVLS_ENABLE=1\n" "# Expected speedup: 3.4x on all-to-all latency" ), metrics=( "alltoall_ms_ring: 8.2\n" "alltoall_ms_nvls_projected: 2.4\n" "nvlink_bw_utilization_pct: 12\n" "throughput_tok_s: 380\n" "expected_throughput_nvls: 1200" ), ), specialist_followups={ "runtime": "NVSwitch hardware works. NCCL just isn't using it optimally due to forced Ring algorithm.", "dispatch": "Expert routing is fine. The NCCL collective is the bottleneck.", "kernel": "Remove NCCL_ALGO=Ring override and set NCCL_NVLS_ENABLE=1 for NVSwitch-optimized all-to-all.", "loader": "Expert distribution is correct. Fix NCCL algorithm selection.", }, )) return scenarios # Build the full scenario pool _HANDCRAFTED_SCENARIOS = _make_scenarios() # --------------------------------------------------------------------------- # Load scraped scenarios from generated_scenarios_full.json # --------------------------------------------------------------------------- def load_scraped_scenarios(path: str) -> list[Scenario]: """Load scraped scenarios from a JSON file and convert to Scenario objects. Missing fields (inspect_results, specialist_followups) are synthesized from the available data so every Scenario is fully populated. """ with open(path, "r", encoding="utf-8") as f: raw = json.load(f) scenarios: list[Scenario] = [] for entry in raw: # --- specialist_opinions: dict-of-dicts -> dict-of-SpecialistOpinion --- specialist_opinions: dict[str, SpecialistOpinion] = {} for name, op_dict in entry.get("specialist_opinions", {}).items(): specialist_opinions[name] = SpecialistOpinion( opinion=op_dict["opinion"], confidence=float(op_dict["confidence"]), is_correct=bool(op_dict["is_correct"]), ) # --- inspect_results: synthesize from available data --- hardware = entry.get("hardware", "Unknown") backend = entry.get("backend", "Unknown") model_name = entry.get("model_name", "Unknown") initial_log = entry.get("initial_log", "") initial_snippet = entry.get("initial_snippet", "") config_str = ( f"hardware: {hardware}\n" f"backend: {backend}\n" f"model: {model_name}\n" f"root_cause_family: {entry.get('root_cause', 'unknown')}" ) metrics_str = ( "error_count: 1\n" "restart_attempts: 0\n" "gpu_utilization: N/A\n" "inference_latency: N/A" ) inspect_results = InspectResult( logs=initial_log if initial_log else "No detailed logs available.", config=config_str, snippet=initial_snippet if initial_snippet else "No code snippet available.", metrics=metrics_str, ) # --- specialist_followups: generate from opinion text --- specialist_followups: dict[str, str] = {} for name, op in specialist_opinions.items(): if op.is_correct: specialist_followups[name] = ( f"Confirmed: {op.opinion} " f"I stand by my earlier assessment with {op.confidence:.0%} confidence." ) else: specialist_followups[name] = ( f"On further review, my initial assessment may not be the primary issue. " f"Original observation: {op.opinion}" ) scenarios.append(Scenario( id=entry["id"], root_cause=entry["root_cause"], correct_fix=entry["correct_fix"], incident_ticket=entry.get("incident_ticket", ""), hardware=hardware, model_name=model_name, backend=backend, initial_log=initial_log, initial_snippet=initial_snippet, specialist_opinions=specialist_opinions, inspect_results=inspect_results, specialist_followups=specialist_followups, )) return scenarios # Load scraped scenarios (if the data file exists) _DATA_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data") _SCRAPED_PATH = os.path.join(_DATA_DIR, "generated_scenarios_full.json") if os.path.exists(_SCRAPED_PATH): _SCRAPED_SCENARIOS = load_scraped_scenarios(_SCRAPED_PATH) else: _SCRAPED_SCENARIOS = [] # Stratified 80/20 train/eval split for scraped scenarios (by root_cause) def _stratified_split(scenarios, train_frac=0.8, seed=42): """Split scenarios into train/eval, stratified by root_cause.""" from collections import defaultdict rng = random.Random(seed) groups = defaultdict(list) for s in scenarios: groups[s.root_cause].append(s) train, eval_ = [], [] for rc in sorted(groups): # sorted for determinism items = groups[rc] rng.shuffle(items) n_eval = max(1, round(len(items) * (1 - train_frac))) if len(items) >= 2 else 0 train.extend(items[n_eval:]) eval_.extend(items[:n_eval]) return train, eval_ SCRAPED_TRAIN_SCENARIOS, SCRAPED_EVAL_SCENARIOS = _stratified_split(_SCRAPED_SCENARIOS) # Combine hand-crafted and scraped SCENARIOS = _HANDCRAFTED_SCENARIOS + _SCRAPED_SCENARIOS # _01, _03, _04, _05, _07, _08, _09, _10 = train; _02, _06 = eval (hand-crafted) HANDCRAFTED_TRAIN = [s for s in _HANDCRAFTED_SCENARIOS if s.id.endswith(("_01", "_03", "_04", "_05", "_07", "_08", "_09", "_10"))] HANDCRAFTED_EVAL = [s for s in _HANDCRAFTED_SCENARIOS if s.id.endswith(("_02", "_06"))] TRAIN_SCENARIOS = HANDCRAFTED_TRAIN + SCRAPED_TRAIN_SCENARIOS EVAL_SCENARIOS = HANDCRAFTED_EVAL + SCRAPED_EVAL_SCENARIOS def get_scenario(scenario_id: str | None = None, split: str = "train") -> Scenario: """Get a scenario by ID, or random from the given split.""" if scenario_id: for s in SCENARIOS: if s.id == scenario_id: return s raise ValueError(f"Unknown scenario: {scenario_id}") pool = TRAIN_SCENARIOS if split == "train" else EVAL_SCENARIOS return random.choice(pool) def randomize_specialist_opinions( scenario: Scenario, ) -> dict[str, SpecialistOpinion]: """Return a new dict of specialist opinions with per-episode randomization. - Randomly pick 1-2 specialists to have their correctness swapped (a correct one becomes wrong, a wrong one becomes correct). - Add noise to confidence scores (multiply by uniform(0.85, 1.15), clamped to [0.3, 0.99]). - The original scenario is NOT mutated. """ names = list(scenario.specialist_opinions.keys()) correct_names = [n for n in names if scenario.specialist_opinions[n].is_correct] incorrect_names = [n for n in names if not scenario.specialist_opinions[n].is_correct] # Determine how many to swap (1 or 2), limited by pool sizes max_swaps = min(len(correct_names), len(incorrect_names)) if max_swaps == 0: num_swaps = 0 else: num_swaps = random.randint(1, min(2, max_swaps)) # Pick which specialists get swapped swap_correct = random.sample(correct_names, num_swaps) if num_swaps > 0 else [] swap_incorrect = random.sample(incorrect_names, num_swaps) if num_swaps > 0 else [] # Build swap mapping: correct[i] gets incorrect[i]'s opinion text, and vice versa swap_pairs: dict[str, str] = {} for c, ic in zip(swap_correct, swap_incorrect): swap_pairs[c] = ic swap_pairs[ic] = c new_opinions: dict[str, SpecialistOpinion] = {} for name in names: orig = scenario.specialist_opinions[name] if name in swap_pairs: # Swap: take the opinion text from the partner, flip is_correct partner = scenario.specialist_opinions[swap_pairs[name]] opinion_text = partner.opinion is_correct = not orig.is_correct else: opinion_text = orig.opinion is_correct = orig.is_correct # Add noise to confidence noisy_confidence = orig.confidence * random.uniform(0.85, 1.15) noisy_confidence = max(0.3, min(0.99, noisy_confidence)) new_opinions[name] = SpecialistOpinion( opinion=opinion_text, confidence=noisy_confidence, is_correct=is_correct, ) return new_opinions