| import URLHandler from '../utils/URLHandler'; |
| import * as completionResultCache from '../utils/completionResultCache'; |
| import type { TokenWithOffset } from './generatedSchemas'; |
|
|
| |
| const COMPLETIONS_PATH = '/api/v1/completions'; |
| const COMPLETIONS_PROMPT_PATH = '/api/v1/completions/prompt'; |
| const COMPLETIONS_STOP_PATH = '/api/v1/completions/stop'; |
|
|
| |
| export type OpenAICompletionsRequest = { |
| model: string; |
| prompt: string; |
| max_tokens?: number; |
| temperature?: number; |
| top_p?: number; |
| stop?: string | string[]; |
| [key: string]: unknown; |
| }; |
|
|
| export type OpenAICompletionChoice = { |
| text?: string; |
| index?: number; |
| finish_reason?: string | null; |
| }; |
|
|
| |
| export type InfoRadarCompletionPayload = { |
| bpe_strings: TokenWithOffset[]; |
| }; |
|
|
| |
| export type OpenAICompletionsResponse = { |
| id: string; |
| object: 'text_completion'; |
| created: number; |
| model: string; |
| choices: OpenAICompletionChoice[]; |
| usage?: { |
| prompt_tokens?: number; |
| completion_tokens?: number; |
| total_tokens?: number; |
| }; |
| |
| info_radar?: InfoRadarCompletionPayload; |
| }; |
|
|
| |
| |
| |
| |
| export function postCompletionsStop(): void { |
| const url = URLHandler.basicURL() + COMPLETIONS_STOP_PATH; |
| void fetch(url, { |
| method: 'POST', |
| headers: { 'Content-Type': 'application/json; charset=UTF-8' }, |
| body: '{}' |
| }).catch(() => { |
| |
| }); |
| } |
|
|
| export type PostCompletionsPromptOptions = { |
| signal?: AbortSignal; |
| }; |
|
|
| |
| |
| |
| export async function postCompletionsPrompt( |
| body: { model: string; prompt: string; system?: string }, |
| options: PostCompletionsPromptOptions = {} |
| ): Promise<{ prompt_used: string }> { |
| const { signal } = options; |
| const url = URLHandler.basicURL() + COMPLETIONS_PROMPT_PATH; |
| const payload: { model: string; prompt: string; system?: string } = { |
| model: body.model, |
| prompt: body.prompt |
| }; |
| if (body.system !== undefined) { |
| payload.system = body.system; |
| } |
| const res = await fetch(url, { |
| method: 'POST', |
| headers: { 'Content-Type': 'application/json; charset=UTF-8' }, |
| body: JSON.stringify(payload), |
| signal |
| }); |
| const text = await res.text(); |
| let parsed: { success?: boolean; message?: string; prompt_used?: string }; |
| try { |
| parsed = JSON.parse(text) as typeof parsed; |
| } catch { |
| throw new Error(`POST ${COMPLETIONS_PROMPT_PATH} failed: ${res.status} ${text.slice(0, 500)}`); |
| } |
| if (!res.ok) { |
| const msg = |
| typeof parsed.message === 'string' ? parsed.message : text.slice(0, 500); |
| throw new Error(msg || `POST ${COMPLETIONS_PROMPT_PATH} failed: ${res.status}`); |
| } |
| if (typeof parsed.prompt_used !== 'string' || !parsed.prompt_used.length) { |
| throw new Error('completions/prompt response missing prompt_used'); |
| } |
| return { prompt_used: parsed.prompt_used }; |
| } |
|
|
| export type PostCompletionsOptions = { |
| signal?: AbortSignal; |
| |
| onDelta?: (text: string, streamEnd: boolean) => void; |
| |
| cacheKey?: completionResultCache.CompletionResultCacheKey; |
| |
| forceRefresh?: boolean; |
| }; |
|
|
| |
| |
| |
| |
| export async function postCompletions( |
| body: OpenAICompletionsRequest, |
| options: PostCompletionsOptions = {} |
| ): Promise<OpenAICompletionsResponse> { |
| const { signal, onDelta, cacheKey, forceRefresh } = options; |
| const modelName = body.model; |
|
|
| return new Promise((resolve, reject) => { |
| let settled = false; |
| let streamedText = ''; |
|
|
| const safeReject = (e: unknown) => { |
| if (!settled) { |
| settled = true; |
| reject(e); |
| } |
| }; |
|
|
| const safeResolve = (v: OpenAICompletionsResponse) => { |
| if (settled) return; |
| if (signal?.aborted) { |
| safeReject(new DOMException('Aborted', 'AbortError')); |
| return; |
| } |
| settled = true; |
| if (typeof v.choices?.[0]?.text === 'string' && cacheKey) { |
| void completionResultCache.save(cacheKey, v, 'complete'); |
| } |
| resolve(v); |
| }; |
|
|
| const rejectIfAborted = (): boolean => { |
| if (!signal?.aborted) return false; |
| if (cacheKey && streamedText.length > 0) { |
| void completionResultCache.save( |
| cacheKey, |
| { |
| id: `partial-${Date.now()}`, |
| object: 'text_completion', |
| created: Math.floor(Date.now() / 1000), |
| model: modelName, |
| choices: [{ text: streamedText, index: 0, finish_reason: 'abort' }], |
| }, |
| 'partial' |
| ); |
| } |
| safeReject(new DOMException('Aborted', 'AbortError')); |
| return true; |
| }; |
|
|
| if (cacheKey) { |
| void (async () => { |
| try { |
| if (forceRefresh) { |
| await completionResultCache.removeCachedEntryByContentKey( |
| completionResultCache.contentKeyForPrompt(cacheKey.prompt) |
| ); |
| } |
| const cached = forceRefresh ? undefined : await completionResultCache.get(cacheKey); |
| if (cached) { |
| await completionResultCache.touch(cacheKey); |
| queueMicrotask(() => { |
| if (settled) return; |
| if (rejectIfAborted()) return; |
| const text = cached.choices?.[0]?.text; |
| if (typeof text !== 'string') { |
| safeReject(new Error('completions cache: invalid choices[0].text')); |
| return; |
| } |
| onDelta?.(text, true); |
| if (settled) return; |
| if (signal?.aborted) { |
| safeReject(new DOMException('Aborted', 'AbortError')); |
| return; |
| } |
| settled = true; |
| resolve(cached); |
| }); |
| return; |
| } |
| } catch (e) { |
| console.warn('[completions] read cache failed:', e); |
| } |
| fetchRemote(); |
| })(); |
| return; |
| } |
|
|
| fetchRemote(); |
|
|
| function fetchRemote(): void { |
| fetch(URLHandler.basicURL() + COMPLETIONS_PATH, { |
| method: 'POST', |
| headers: { 'Content-Type': 'application/json; charset=UTF-8' }, |
| body: JSON.stringify(body), |
| signal |
| }) |
| .then((response) => { |
| if (!response.ok) { |
| return response.text().then((t) => { |
| throw new Error(`POST ${COMPLETIONS_PATH} failed: ${response.status} ${t.slice(0, 500)}`); |
| }); |
| } |
| const reader = response.body!.getReader(); |
| signal?.addEventListener('abort', () => reader.cancel(), { once: true }); |
|
|
| const decoder = new TextDecoder(); |
| let buffer = ''; |
|
|
| const processDataLine = (jsonStr: string) => { |
| if (settled) return; |
| if (rejectIfAborted()) return; |
| let parsed: { |
| type?: string; |
| text?: string; |
| stream_end?: boolean; |
| data?: OpenAICompletionsResponse; |
| message?: string; |
| }; |
| try { |
| parsed = JSON.parse(jsonStr) as typeof parsed; |
| } catch (e) { |
| safeReject( |
| new Error( |
| `SSE event JSON parse failed: ${ |
| e instanceof SyntaxError ? e.message : String(e) |
| }` |
| ) |
| ); |
| return; |
| } |
| if (parsed.type === 'delta') { |
| const delta = parsed.text ?? ''; |
| streamedText += delta; |
| onDelta?.(delta, Boolean(parsed.stream_end)); |
| } else if (parsed.type === 'result') { |
| const data = parsed.data; |
| if (data && typeof data === 'object' && 'choices' in data) { |
| safeResolve(data as OpenAICompletionsResponse); |
| } else { |
| safeReject(new Error('completions stream: invalid result payload')); |
| } |
| } else if (parsed.type === 'error') { |
| safeReject(new Error(parsed.message || 'completions stream failed')); |
| } |
| }; |
|
|
| const readChunk = (): Promise<void> => { |
| return reader.read().then(({ done, value }) => { |
| if (settled) return; |
| if (rejectIfAborted()) return; |
| if (done) { |
| if (buffer.trim()) { |
| const line = buffer; |
| if (line.startsWith('data: ')) processDataLine(line.slice(6)); |
| } |
| if (!settled) { |
| safeReject(new Error('completions stream ended without result')); |
| } |
| return; |
| } |
| buffer += decoder.decode(value, { stream: true }); |
| const lines = buffer.split('\n'); |
| buffer = lines.pop() || ''; |
| for (const line of lines) { |
| if (line.startsWith('data: ')) processDataLine(line.slice(6)); |
| } |
| return readChunk(); |
| }); |
| }; |
| return readChunk(); |
| }) |
| .catch((e) => { |
| if (!settled) safeReject(e); |
| }); |
| } |
| }); |
| } |
|
|