| |
| |
| |
| |
| |
|
|
| import { generateText, streamText } from 'ai'; |
| import type { GenerateTextResult, StreamTextResult } from 'ai'; |
| import { createLogger } from '@/lib/logger'; |
| import { PROVIDERS } from './providers'; |
| import { thinkingContext } from './thinking-context'; |
| import { getModelMetadataKey } from './model-metadata'; |
| import type { ThinkingCapability, ThinkingConfig } from '@/lib/types/provider'; |
| import { |
| getThinkingMode, |
| pickThinkingBudget, |
| pickThinkingEffort, |
| pickThinkingLevel, |
| } from '@/lib/ai/thinking-config'; |
| const log = createLogger('LLM'); |
|
|
| |
| export type { ThinkingConfig } from '@/lib/types/provider'; |
|
|
| |
| type GenerateTextParams = Parameters<typeof generateText>[0]; |
| type StreamTextParams = Parameters<typeof streamText>[0]; |
|
|
| function _extractRequestInfo(params: GenerateTextParams | StreamTextParams) { |
| const tools = params.tools ? Object.keys(params.tools as Record<string, unknown>) : undefined; |
|
|
| const p = params as Record<string, unknown>; |
| return { |
| system: p.system as string | undefined, |
| prompt: p.prompt as string | undefined, |
| messages: p.messages as unknown[] | undefined, |
| tools, |
| maxOutputTokens: p.maxOutputTokens as number | undefined, |
| }; |
| } |
|
|
| function getModelId(params: GenerateTextParams | StreamTextParams): string { |
| const m = params.model; |
| if (typeof m === 'string') return m; |
| if (m && typeof m === 'object' && 'modelId' in m) return (m as { modelId: string }).modelId; |
| return 'unknown'; |
| } |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
|
|
| interface ModelThinkingInfo { |
| thinking?: ThinkingCapability; |
| } |
|
|
| |
| const MODEL_THINKING_MAP: Map<string, ModelThinkingInfo> = (() => { |
| const map = new Map<string, ModelThinkingInfo>(); |
| for (const provider of Object.values(PROVIDERS)) { |
| for (const model of provider.models) { |
| map.set(getModelMetadataKey(provider.id, model.id), { |
| thinking: model.capabilities?.thinking, |
| }); |
| } |
| } |
| return map; |
| })(); |
|
|
| |
| const UNIQUE_MODEL_THINKING_MAP: Map<string, ModelThinkingInfo> = (() => { |
| const counts = new Map<string, number>(); |
| for (const provider of Object.values(PROVIDERS)) { |
| for (const model of provider.models) { |
| counts.set(model.id, (counts.get(model.id) ?? 0) + 1); |
| } |
| } |
|
|
| const map = new Map<string, ModelThinkingInfo>(); |
| for (const provider of Object.values(PROVIDERS)) { |
| for (const model of provider.models) { |
| if (counts.get(model.id) === 1) { |
| map.set(model.id, { |
| thinking: model.capabilities?.thinking, |
| }); |
| } |
| } |
| } |
| return map; |
| })(); |
|
|
| |
| function getGlobalThinkingConfig(): ThinkingConfig | undefined { |
| if (import.meta.env?.VITE_LLM_THINKING_DISABLED === 'true') { |
| return { mode: 'disabled', enabled: false }; |
| } |
| return undefined; |
| } |
|
|
| type ProviderOptions = Record<string, Record<string, unknown>>; |
|
|
| function getAnthropicEffort( |
| thinking: ThinkingCapability, |
| config: ThinkingConfig, |
| ): 'low' | 'medium' | 'high' | 'xhigh' | 'max' | undefined { |
| const effort = pickThinkingEffort(thinking, config); |
| if (!effort || effort === 'none' || effort === 'minimal') return undefined; |
| return effort; |
| } |
|
|
| function getModelProviderId(params: GenerateTextParams | StreamTextParams): string | undefined { |
| const m = params.model; |
| if (!m || typeof m !== 'object' || !('provider' in m)) return undefined; |
| const provider = (m as { provider?: string }).provider; |
| if (!provider) return undefined; |
| if (provider in PROVIDERS) return provider; |
| const prefix = provider.split('.')[0]; |
| return prefix in PROVIDERS ? prefix : undefined; |
| } |
|
|
| |
| |
| |
| function buildThinkingProviderOptions( |
| providerId: string | undefined, |
| modelId: string, |
| config: ThinkingConfig, |
| ): ProviderOptions | undefined { |
| const info = providerId |
| ? MODEL_THINKING_MAP.get(getModelMetadataKey(providerId, modelId)) |
| : UNIQUE_MODEL_THINKING_MAP.get(modelId); |
| if (!info?.thinking) return undefined; |
| const thinking = info.thinking; |
| if (thinking.control === 'none') return undefined; |
|
|
| const mode = getThinkingMode(config); |
|
|
| switch (thinking.requestAdapter) { |
| case 'openai': { |
| const effort = pickThinkingEffort(thinking, config); |
| return effort ? { openai: { reasoningEffort: effort } } : undefined; |
| } |
|
|
| case 'anthropic': { |
| if (mode === 'disabled') return { anthropic: { thinking: { type: 'disabled' } } }; |
|
|
| if (thinking.control === 'toggle-budget' || thinking.control === 'budget-only') { |
| const budget = pickThinkingBudget(thinking, config); |
| return budget === undefined |
| ? undefined |
| : { anthropic: { thinking: { type: 'enabled', budgetTokens: budget } } }; |
| } |
|
|
| const effort = getAnthropicEffort(thinking, config); |
| if (!effort) return undefined; |
|
|
| if (thinking.anthropicThinking?.type === 'adaptive') { |
| return { |
| anthropic: { |
| thinking: { type: 'adaptive' }, |
| effort, |
| }, |
| }; |
| } |
|
|
| const manualEffort = effort === 'xhigh' ? 'max' : effort; |
| const budget = thinking.anthropicThinking?.budgetByEffort?.[manualEffort]; |
| if (!budget) return undefined; |
| return { |
| anthropic: { |
| thinking: { type: 'enabled', budgetTokens: budget }, |
| effort: manualEffort, |
| }, |
| }; |
| } |
|
|
| case 'google': { |
| if (thinking.control === 'level') { |
| const level = pickThinkingLevel(thinking, config); |
| return level ? { google: { thinkingConfig: { thinkingLevel: level } } } : undefined; |
| } |
|
|
| const budget = pickThinkingBudget(thinking, config); |
| if (budget === undefined) return undefined; |
| return { google: { thinkingConfig: { thinkingBudget: budget } } }; |
| } |
|
|
| default: |
| |
| return undefined; |
| } |
| } |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| |
| function injectProviderOptions<T extends GenerateTextParams | StreamTextParams>( |
| params: T, |
| thinking?: ThinkingConfig, |
| ): T { |
| if ((params as Record<string, unknown>).providerOptions) return params; |
|
|
| const modelId = getModelId(params); |
| const providerId = getModelProviderId(params); |
|
|
| if (thinking) { |
| const opts = buildThinkingProviderOptions(providerId, modelId, thinking); |
| if (opts) return { ...params, providerOptions: opts }; |
| } |
|
|
| return params; |
| } |
|
|
| |
| |
| |
| |
| export interface LLMRetryOptions { |
| |
| retries?: number; |
| |
| |
| validate?: (text: string) => boolean; |
| } |
|
|
| const DEFAULT_VALIDATE = (text: string) => text.trim().length > 0; |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| export async function callLLM<T extends GenerateTextParams>( |
| params: T, |
| source: string, |
| retryOptions?: LLMRetryOptions, |
| thinking?: ThinkingConfig, |
| |
| ): Promise<GenerateTextResult<any, any>> { |
| const maxAttempts = (retryOptions?.retries ?? 0) + 1; |
| const validate = retryOptions?.validate ?? (maxAttempts > 1 ? DEFAULT_VALIDATE : undefined); |
|
|
| |
| let lastResult: GenerateTextResult<any, any> | undefined; |
| let lastError: unknown; |
|
|
| for (let attempt = 1; attempt <= maxAttempts; attempt++) { |
| try { |
| |
| const effectiveThinking = thinking ?? getGlobalThinkingConfig(); |
| const injectedParams = injectProviderOptions(params, effectiveThinking); |
|
|
| |
| |
| |
| const result = await thinkingContext.run(effectiveThinking, () => |
| generateText(injectedParams), |
| ); |
|
|
| |
| if (validate && !validate(result.text)) { |
| log.warn( |
| `[${source}] Validation failed (attempt ${attempt}/${maxAttempts}), ${attempt < maxAttempts ? 'retrying...' : 'giving up'}`, |
| ); |
| lastResult = result; |
| continue; |
| } |
|
|
| return result; |
| } catch (error) { |
| lastError = error; |
|
|
| if (attempt < maxAttempts) { |
| log.warn(`[${source}] Call failed (attempt ${attempt}/${maxAttempts}), retrying...`, error); |
| continue; |
| } |
| } |
| } |
|
|
| |
| if (lastResult) return lastResult; |
| throw lastError; |
| } |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| |
| export function streamLLM<T extends StreamTextParams>( |
| params: T, |
| source: string, |
| thinking?: ThinkingConfig, |
| |
| ): StreamTextResult<any, any> { |
| |
| const effectiveThinking = thinking ?? getGlobalThinkingConfig(); |
| const injectedParams = injectProviderOptions(params, effectiveThinking); |
| const result = thinkingContext.run(effectiveThinking, () => streamText(injectedParams)); |
|
|
| return result; |
| } |
|
|