| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
|
|
| import type { Scene } from '@/lib/types/stage'; |
| import type { Action, SpeechAction, DiscussionAction } from '@/lib/types/action'; |
| import type { |
| EngineMode, |
| TopicState, |
| PlaybackEngineCallbacks, |
| PlaybackSnapshot, |
| TriggerEvent, |
| Effect, |
| } from './types'; |
| import type { AudioPlayer } from '@/lib/utils/audio-player'; |
| import { ActionEngine } from '@/lib/action/engine'; |
| import { useCanvasStore } from '@/lib/store/canvas'; |
| import { useSettingsStore } from '@/lib/store/settings'; |
| import { createLogger } from '@/lib/logger'; |
|
|
| const log = createLogger('PlaybackEngine'); |
|
|
| |
| |
| |
| |
| |
| const CJK_LANG_THRESHOLD = 0.3; |
|
|
| export class PlaybackEngine { |
| private scenes: Scene[] = []; |
| private sceneIndex: number = 0; |
| private actionIndex: number = 0; |
| private mode: EngineMode = 'idle'; |
| private consumedDiscussions: Set<string> = new Set(); |
|
|
| |
| private savedSceneIndex: number | null = null; |
| private savedActionIndex: number | null = null; |
|
|
| |
| private currentTopicState: TopicState | null = null; |
|
|
| |
| private audioPlayer: AudioPlayer; |
| private actionEngine: ActionEngine; |
| private callbacks: PlaybackEngineCallbacks; |
|
|
| |
| private sceneId: string | undefined; |
|
|
| |
| private currentTrigger: TriggerEvent | null = null; |
| private triggerDelayTimer: ReturnType<typeof setTimeout> | null = null; |
| |
| private speechTimer: ReturnType<typeof setTimeout> | null = null; |
| private speechTimerStart: number = 0; |
| |
| private browserTTSActive: boolean = false; |
| private browserTTSChunks: string[] = []; |
| private browserTTSChunkIndex: number = 0; |
| private browserTTSPausedChunks: string[] = []; |
| private speechTimerRemaining: number = 0; |
|
|
| constructor( |
| scenes: Scene[], |
| actionEngine: ActionEngine, |
| audioPlayer: AudioPlayer, |
| callbacks: PlaybackEngineCallbacks = {}, |
| ) { |
| this.scenes = scenes; |
| this.sceneId = scenes[0]?.id; |
| this.actionEngine = actionEngine; |
| this.audioPlayer = audioPlayer; |
| this.callbacks = callbacks; |
| } |
|
|
| |
|
|
| |
| getMode(): EngineMode { |
| return this.mode; |
| } |
|
|
| |
| getSnapshot(): PlaybackSnapshot { |
| return { |
| sceneIndex: this.sceneIndex, |
| actionIndex: this.actionIndex, |
| consumedDiscussions: [...this.consumedDiscussions], |
| sceneId: this.sceneId, |
| }; |
| } |
|
|
| |
| restoreFromSnapshot(snapshot: PlaybackSnapshot): void { |
| this.sceneIndex = snapshot.sceneIndex; |
| this.actionIndex = snapshot.actionIndex; |
| this.consumedDiscussions = new Set(snapshot.consumedDiscussions); |
| } |
|
|
| |
| start(): void { |
| if (this.mode !== 'idle') { |
| log.warn('Cannot start: not idle, current mode:', this.mode); |
| return; |
| } |
|
|
| this.sceneIndex = 0; |
| this.actionIndex = 0; |
| this.setMode('playing'); |
| this.processNext(); |
| } |
|
|
| |
| continuePlayback(): void { |
| if (this.mode !== 'idle') { |
| log.warn('Cannot continue: not idle, current mode:', this.mode); |
| return; |
| } |
| this.setMode('playing'); |
| this.processNext(); |
| } |
|
|
| |
| pause(): void { |
| if (this.mode === 'playing') { |
| |
| if (this.triggerDelayTimer) { |
| clearTimeout(this.triggerDelayTimer); |
| this.triggerDelayTimer = null; |
| } |
| if (this.speechTimer) { |
| |
| this.speechTimerRemaining = Math.max( |
| 0, |
| this.speechTimerRemaining - (Date.now() - this.speechTimerStart), |
| ); |
| clearTimeout(this.speechTimer); |
| this.speechTimer = null; |
| } |
| this.setMode('paused'); |
| |
| if (!this.currentTrigger) { |
| if (this.browserTTSActive) { |
| |
| |
| |
| this.browserTTSPausedChunks = this.browserTTSChunks.slice(this.browserTTSChunkIndex); |
| window.speechSynthesis?.cancel(); |
| |
| } else if (this.audioPlayer.isPlaying()) { |
| this.audioPlayer.pause(); |
| } |
| } |
| } else if (this.mode === 'live') { |
| this.setMode('paused'); |
| this.currentTopicState = 'pending'; |
| |
| } else { |
| log.warn('Cannot pause: mode is', this.mode); |
| } |
| } |
|
|
| |
| resume(): void { |
| if (this.mode !== 'paused') { |
| log.warn('Cannot resume: not paused, mode is', this.mode); |
| return; |
| } |
|
|
| if (this.currentTopicState === 'pending') { |
| |
| this.currentTopicState = 'active'; |
| this.setMode('live'); |
| } else if (this.currentTrigger) { |
| |
| this.setMode('playing'); |
| } else { |
| |
| this.setMode('playing'); |
| if (this.browserTTSPausedChunks.length > 0) { |
| |
| this.browserTTSActive = true; |
| this.browserTTSChunks = this.browserTTSPausedChunks; |
| this.browserTTSChunkIndex = 0; |
| this.browserTTSPausedChunks = []; |
| this.playBrowserTTSChunk(); |
| } else if (this.audioPlayer.hasActiveAudio()) { |
| |
| this.audioPlayer.resume(); |
| } else if (this.speechTimerRemaining > 0) { |
| |
| this.speechTimerStart = Date.now(); |
| this.speechTimer = setTimeout(() => { |
| this.speechTimer = null; |
| this.speechTimerRemaining = 0; |
| this.callbacks.onSpeechEnd?.(); |
| if (this.mode === 'playing') this.processNext(); |
| }, this.speechTimerRemaining); |
| } else { |
| |
| this.processNext(); |
| } |
| } |
| } |
|
|
| |
| stop(): void { |
| |
| |
| this.setMode('idle'); |
| this.audioPlayer.stop(); |
| this.cancelBrowserTTS(); |
| this.actionEngine.clearEffects(); |
| if (this.triggerDelayTimer) { |
| clearTimeout(this.triggerDelayTimer); |
| this.triggerDelayTimer = null; |
| } |
| if (this.speechTimer) { |
| clearTimeout(this.speechTimer); |
| this.speechTimer = null; |
| } |
| this.speechTimerRemaining = 0; |
| this.sceneIndex = 0; |
| this.actionIndex = 0; |
| this.savedSceneIndex = null; |
| this.savedActionIndex = null; |
| this.currentTopicState = null; |
| this.currentTrigger = null; |
| } |
|
|
| |
| confirmDiscussion(): void { |
| if (!this.currentTrigger) { |
| log.warn('confirmDiscussion called but no trigger'); |
| return; |
| } |
|
|
| |
| this.consumedDiscussions.add(this.currentTrigger.id); |
|
|
| |
| |
| |
| this.savedSceneIndex = this.sceneIndex; |
| this.savedActionIndex = this.actionIndex; |
|
|
| |
| this.currentTopicState = 'active'; |
| this.setMode('live'); |
|
|
| |
| this.callbacks.onProactiveHide?.(); |
| this.callbacks.onDiscussionConfirmed?.( |
| this.currentTrigger.question, |
| this.currentTrigger.prompt, |
| this.currentTrigger.agentId, |
| ); |
| this.currentTrigger = null; |
| } |
|
|
| |
| skipDiscussion(): void { |
| if (this.currentTrigger) { |
| this.consumedDiscussions.add(this.currentTrigger.id); |
| this.currentTrigger = null; |
| } |
| this.callbacks.onProactiveHide?.(); |
|
|
| if (this.mode === 'playing') { |
| this.processNext(); |
| } |
| } |
|
|
| |
| handleEndDiscussion(): void { |
| this.actionEngine.clearEffects(); |
| this.currentTopicState = 'closed'; |
|
|
| |
| useCanvasStore.getState().setWhiteboardOpen(false); |
|
|
| this.callbacks.onDiscussionEnd?.(); |
|
|
| |
| this.restoreSavedLectureState(); |
|
|
| this.setMode('idle'); |
| } |
|
|
| |
| |
| |
| |
| |
| handleDiscussionError(): void { |
| const hasSavedLectureState = this.savedSceneIndex !== null && this.savedActionIndex !== null; |
| const isLiveTopic = |
| this.mode === 'live' || (this.mode === 'paused' && this.currentTopicState === 'pending'); |
|
|
| if (!isLiveTopic && !hasSavedLectureState) { |
| return; |
| } |
|
|
| this.actionEngine.clearEffects(); |
| useCanvasStore.getState().setWhiteboardOpen(false); |
| this.currentTopicState = 'closed'; |
| this.currentTrigger = null; |
| this.restoreSavedLectureState(); |
| this.setMode('idle'); |
| } |
|
|
| |
| handleUserInterrupt(text: string): void { |
| if (this.mode === 'playing' || this.mode === 'paused') { |
| |
| |
| |
| |
| if (this.savedSceneIndex === null) { |
| this.savedSceneIndex = this.sceneIndex; |
| this.savedActionIndex = Math.max(0, this.actionIndex - 1); |
| } |
|
|
| |
| if (this.triggerDelayTimer) { |
| clearTimeout(this.triggerDelayTimer); |
| this.triggerDelayTimer = null; |
| } |
| } |
|
|
| |
| |
| |
| |
| this.currentTopicState = 'active'; |
| this.setMode('live'); |
| this.audioPlayer.stop(); |
| this.cancelBrowserTTS(); |
| this.callbacks.onUserInterrupt?.(text); |
| } |
|
|
| |
| isExhausted(): boolean { |
| let si = this.sceneIndex; |
| let ai = this.actionIndex; |
| while (si < this.scenes.length) { |
| const actions = this.scenes[si].actions || []; |
| while (ai < actions.length) { |
| const action = actions[ai]; |
| |
| if (action.type === 'discussion' && this.consumedDiscussions.has(action.id)) { |
| ai++; |
| continue; |
| } |
| return false; |
| } |
| si++; |
| ai = 0; |
| } |
| return true; |
| } |
|
|
| |
|
|
| private setMode(mode: EngineMode): void { |
| if (this.mode === mode) return; |
| this.mode = mode; |
| this.callbacks.onModeChange?.(mode); |
| } |
|
|
| private restoreSavedLectureState(): void { |
| if (this.savedSceneIndex !== null && this.savedActionIndex !== null) { |
| this.sceneIndex = this.savedSceneIndex; |
| this.actionIndex = this.savedActionIndex; |
| } |
| this.savedSceneIndex = null; |
| this.savedActionIndex = null; |
| } |
|
|
| |
| |
| |
| |
| private getCurrentAction(): { action: Action; sceneId: string } | null { |
| while (this.sceneIndex < this.scenes.length) { |
| const scene = this.scenes[this.sceneIndex]; |
| const actions = scene.actions || []; |
|
|
| if (this.actionIndex < actions.length) { |
| return { action: actions[this.actionIndex], sceneId: scene.id }; |
| } |
|
|
| |
| this.sceneIndex++; |
| this.actionIndex = 0; |
| } |
| return null; |
| } |
|
|
| |
| |
| |
| private async processNext(): Promise<void> { |
| if (this.mode !== 'playing') return; |
|
|
| |
| if (this.actionIndex === 0 && this.sceneIndex < this.scenes.length) { |
| const scene = this.scenes[this.sceneIndex]; |
| this.actionEngine.clearEffects(); |
| this.callbacks.onSceneChange?.(scene.id); |
| this.callbacks.onSpeakerChange?.('teacher'); |
| } |
|
|
| const current = this.getCurrentAction(); |
| if (!current) { |
| |
| this.actionEngine.clearEffects(); |
| this.setMode('idle'); |
| this.callbacks.onComplete?.(); |
| return; |
| } |
|
|
| const { action } = current; |
|
|
| |
| |
| |
| this.callbacks.onProgress?.(this.getSnapshot()); |
|
|
| this.actionIndex++; |
|
|
| switch (action.type) { |
| case 'speech': { |
| const speechAction = action as SpeechAction; |
| this.callbacks.onSpeechStart?.(speechAction.text); |
|
|
| |
| this.audioPlayer.onEnded(() => { |
| this.callbacks.onSpeechEnd?.(); |
| if (this.mode === 'playing') { |
| this.processNext(); |
| } |
| }); |
|
|
| |
| |
| |
| |
| const scheduleReadingTimer = () => { |
| const text = speechAction.text; |
| const cjkCount = ( |
| text.match(/[\u4e00-\u9fff\u3400-\u4dbf\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]/g) || [] |
| ).length; |
| const isCJK = cjkCount > text.length * 0.3; |
| const speed = this.callbacks.getPlaybackSpeed?.() ?? 1; |
| const rawMs = isCJK |
| ? Math.max(2000, text.length * 150) |
| : Math.max(2000, text.split(/\s+/).filter(Boolean).length * 240); |
| const readingMs = rawMs / speed; |
| this.speechTimerStart = Date.now(); |
| this.speechTimerRemaining = readingMs; |
| this.speechTimer = setTimeout(() => { |
| this.speechTimer = null; |
| this.speechTimerRemaining = 0; |
| this.callbacks.onSpeechEnd?.(); |
| if (this.mode === 'playing') this.processNext(); |
| }, readingMs); |
| }; |
|
|
| this.audioPlayer |
| .play(speechAction.audioId || '', speechAction.audioUrl) |
| .then((audioStarted) => { |
| if (!audioStarted) { |
| |
| const settings = useSettingsStore.getState(); |
| if ( |
| settings.ttsEnabled && |
| settings.ttsProviderId === 'browser-native-tts' && |
| typeof window !== 'undefined' && |
| window.speechSynthesis |
| ) { |
| this.playBrowserTTS(speechAction); |
| } else { |
| scheduleReadingTimer(); |
| } |
| } |
| }) |
| .catch((err) => { |
| log.error('TTS error:', err); |
| scheduleReadingTimer(); |
| }); |
| break; |
| } |
|
|
| case 'spotlight': |
| case 'laser': { |
| |
| this.actionEngine.execute(action); |
| this.callbacks.onEffectFire?.({ |
| kind: action.type, |
| targetId: action.elementId, |
| ...(action.type === 'spotlight' |
| ? { dimOpacity: action.dimOpacity } |
| : { color: action.color }), |
| } as Effect); |
| |
| |
| |
| queueMicrotask(() => this.processNext()); |
| break; |
| } |
|
|
| case 'discussion': { |
| const discussionAction = action as DiscussionAction; |
| |
| if (this.consumedDiscussions.has(discussionAction.id)) { |
| this.processNext(); |
| return; |
| } |
| |
| if ( |
| discussionAction.agentId && |
| this.callbacks.isAgentSelected && |
| !this.callbacks.isAgentSelected(discussionAction.agentId) |
| ) { |
| this.consumedDiscussions.add(discussionAction.id); |
| this.processNext(); |
| return; |
| } |
|
|
| |
| const trigger: TriggerEvent = { |
| id: discussionAction.id, |
| question: discussionAction.topic, |
| prompt: discussionAction.prompt, |
| agentId: discussionAction.agentId, |
| }; |
|
|
| this.triggerDelayTimer = setTimeout(() => { |
| this.triggerDelayTimer = null; |
| if (this.mode !== 'playing') return; |
| this.currentTrigger = trigger; |
| this.callbacks.onProactiveShow?.(trigger); |
| |
| }, 3000); |
| break; |
| } |
|
|
| case 'play_video': |
| case 'wb_open': |
| case 'wb_draw_text': |
| case 'wb_draw_shape': |
| case 'wb_draw_chart': |
| case 'wb_draw_latex': |
| case 'wb_draw_table': |
| case 'wb_clear': |
| case 'wb_delete': |
| case 'wb_close': |
| case 'widget_highlight': |
| case 'widget_setState': |
| case 'widget_annotation': |
| case 'widget_reveal': { |
| |
| await this.actionEngine.execute(action); |
| if (this.mode === 'playing') { |
| this.processNext(); |
| } |
| break; |
| } |
|
|
| default: |
| |
| this.processNext(); |
| break; |
| } |
| } |
|
|
| |
|
|
| |
| |
| |
| |
| |
| private splitIntoChunks(text: string): string[] { |
| |
| const chunks = text |
| .split(/(?<=[.!?γοΌοΌ\n])\s*/) |
| .map((s) => s.trim()) |
| .filter((s) => s.length > 0); |
| |
| return chunks.length > 0 ? chunks : [text]; |
| } |
|
|
| |
| |
| |
| |
| |
| private playBrowserTTS(speechAction: SpeechAction): void { |
| this.browserTTSChunks = this.splitIntoChunks(speechAction.text); |
| this.browserTTSChunkIndex = 0; |
| this.browserTTSPausedChunks = []; |
| this.browserTTSActive = true; |
| this.playBrowserTTSChunk(); |
| } |
|
|
| |
| private async playBrowserTTSChunk(): Promise<void> { |
| if (this.browserTTSChunkIndex >= this.browserTTSChunks.length) { |
| |
| this.browserTTSActive = false; |
| this.browserTTSChunks = []; |
| this.callbacks.onSpeechEnd?.(); |
| if (this.mode === 'playing') this.processNext(); |
| return; |
| } |
|
|
| const settings = useSettingsStore.getState(); |
| const chunkText = this.browserTTSChunks[this.browserTTSChunkIndex]; |
| const utterance = new SpeechSynthesisUtterance(chunkText); |
|
|
| |
| const speed = this.callbacks.getPlaybackSpeed?.() ?? 1; |
| utterance.rate = (settings.ttsSpeed ?? 1) * speed; |
| utterance.volume = settings.ttsMuted ? 0 : (settings.ttsVolume ?? 1); |
|
|
| |
| const voices = await this.ensureVoicesLoaded(); |
|
|
| |
| let voiceFound = false; |
| if (settings.ttsVoice && settings.ttsVoice !== 'default') { |
| const voice = voices.find((v) => v.voiceURI === settings.ttsVoice); |
| if (voice) { |
| utterance.voice = voice; |
| utterance.lang = voice.lang; |
| voiceFound = true; |
| } |
| } |
| if (!voiceFound) { |
| |
| |
| const cjkRatio = |
| chunkText.length > 0 |
| ? (chunkText.match(/[\u4e00-\u9fff\u3400-\u4dbf]/g) || []).length / chunkText.length |
| : 0; |
| utterance.lang = cjkRatio > CJK_LANG_THRESHOLD ? 'zh-CN' : 'en-US'; |
| } |
|
|
| utterance.onend = () => { |
| this.browserTTSChunkIndex++; |
| if (this.mode === 'playing') { |
| this.playBrowserTTSChunk(); |
| } |
| }; |
|
|
| utterance.onerror = (event) => { |
| |
| if (event.error !== 'canceled') { |
| log.warn('Browser TTS chunk error:', event.error); |
| |
| this.browserTTSChunkIndex++; |
| if (this.mode === 'playing') { |
| this.playBrowserTTSChunk(); |
| } |
| } |
| |
| }; |
|
|
| |
| |
| window.speechSynthesis.cancel(); |
| window.speechSynthesis.speak(utterance); |
| } |
|
|
| |
| |
| |
| |
| private cachedVoices: SpeechSynthesisVoice[] | null = null; |
| private async ensureVoicesLoaded(): Promise<SpeechSynthesisVoice[]> { |
| if (this.cachedVoices && this.cachedVoices.length > 0) { |
| return this.cachedVoices; |
| } |
|
|
| let voices = window.speechSynthesis.getVoices(); |
| if (voices.length > 0) { |
| this.cachedVoices = voices; |
| return voices; |
| } |
|
|
| |
| await new Promise<void>((resolve) => { |
| const onVoicesChanged = () => { |
| window.speechSynthesis.removeEventListener('voiceschanged', onVoicesChanged); |
| resolve(); |
| }; |
| window.speechSynthesis.addEventListener('voiceschanged', onVoicesChanged); |
| |
| setTimeout(() => { |
| window.speechSynthesis.removeEventListener('voiceschanged', onVoicesChanged); |
| resolve(); |
| }, 2000); |
| }); |
|
|
| voices = window.speechSynthesis.getVoices(); |
| this.cachedVoices = voices; |
| return voices; |
| } |
|
|
| |
| private cancelBrowserTTS(): void { |
| if (this.browserTTSActive) { |
| this.browserTTSActive = false; |
| this.browserTTSChunks = []; |
| this.browserTTSChunkIndex = 0; |
| this.browserTTSPausedChunks = []; |
| window.speechSynthesis?.cancel(); |
| } |
| } |
| } |
|
|