File size: 9,522 Bytes
f56a29b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 | /**
* Media Generation Orchestrator
*
* Dispatches media generation API calls for all mediaGenerations across outlines.
* Runs entirely on the frontend — calls /api/generate/image and /api/generate/video,
* fetches result blobs, stores in IndexedDB, and updates the Zustand store.
*/
import { useMediaGenerationStore } from '@/lib/store/media-generation';
import { useSettingsStore } from '@/lib/store/settings';
import { db, mediaFileKey } from '@/lib/utils/database';
import type { SceneOutline } from '@/lib/types/generation';
import type { MediaGenerationRequest } from '@/lib/media/types';
import { createLogger } from '@/lib/logger';
const log = createLogger('MediaOrchestrator');
/** Error with a structured errorCode from the API */
class MediaApiError extends Error {
errorCode?: string;
constructor(message: string, errorCode?: string) {
super(message);
this.errorCode = errorCode;
}
}
/**
* Launch media generation for all mediaGenerations declared in outlines.
* Runs in parallel with content/action generation — does not block.
*/
export async function generateMediaForOutlines(
outlines: SceneOutline[],
stageId: string,
abortSignal?: AbortSignal,
): Promise<void> {
const settings = useSettingsStore.getState();
const store = useMediaGenerationStore.getState();
// Collect all media requests
const allRequests: MediaGenerationRequest[] = [];
for (const outline of outlines) {
if (!outline.mediaGenerations) continue;
for (const mg of outline.mediaGenerations) {
// Filter by enabled flags
if (mg.type === 'image' && !settings.imageGenerationEnabled) continue;
if (mg.type === 'video' && !settings.videoGenerationEnabled) continue;
// Skip already completed or permanently failed (restored from DB)
const existing = store.getTask(mg.elementId);
if (existing?.status === 'done' || existing?.status === 'failed') continue;
allRequests.push(mg);
}
}
if (allRequests.length === 0) return;
// Enqueue all as pending
useMediaGenerationStore.getState().enqueueTasks(stageId, allRequests);
// Process requests serially — image/video APIs have limited concurrency
for (const req of allRequests) {
if (abortSignal?.aborted) break;
await generateSingleMedia(req, stageId, abortSignal);
}
}
/**
* Retry a single failed media task.
*/
export async function retryMediaTask(elementId: string): Promise<void> {
const store = useMediaGenerationStore.getState();
const task = store.getTask(elementId);
if (!task || task.status !== 'failed') return;
// Check if the corresponding generation type is still enabled in global settings
const settings = useSettingsStore.getState();
if (task.type === 'image' && !settings.imageGenerationEnabled) {
store.markFailed(elementId, 'Generation disabled', 'GENERATION_DISABLED');
return;
}
if (task.type === 'video' && !settings.videoGenerationEnabled) {
store.markFailed(elementId, 'Generation disabled', 'GENERATION_DISABLED');
return;
}
// Remove persisted failure record from DB so a fresh result can be written
const dbKey = mediaFileKey(task.stageId, elementId);
await db.mediaFiles.delete(dbKey).catch(() => {});
store.markPendingForRetry(elementId);
await generateSingleMedia(
{
type: task.type,
prompt: task.prompt,
elementId: task.elementId,
aspectRatio: task.params.aspectRatio as MediaGenerationRequest['aspectRatio'],
style: task.params.style,
},
task.stageId,
);
}
// ==================== Internal ====================
async function generateSingleMedia(
req: MediaGenerationRequest,
stageId: string,
abortSignal?: AbortSignal,
): Promise<void> {
const store = useMediaGenerationStore.getState();
store.markGenerating(req.elementId);
try {
let resultUrl: string;
let posterUrl: string | undefined;
let mimeType: string;
if (req.type === 'image') {
const result = await callImageApi(req, abortSignal);
resultUrl = result.url;
mimeType = 'image/png';
} else {
const result = await callVideoApi(req, abortSignal);
resultUrl = result.url;
posterUrl = result.poster;
mimeType = 'video/mp4';
}
if (abortSignal?.aborted) return;
// Fetch blob from URL
const blob = await fetchAsBlob(resultUrl);
const posterBlob = posterUrl ? await fetchAsBlob(posterUrl).catch(() => undefined) : undefined;
// Store in IndexedDB
await db.mediaFiles.put({
id: mediaFileKey(stageId, req.elementId),
stageId,
type: req.type,
blob,
mimeType,
size: blob.size,
poster: posterBlob,
prompt: req.prompt,
params: JSON.stringify({
aspectRatio: req.aspectRatio,
style: req.style,
}),
createdAt: Date.now(),
});
// Update store with object URL
const objectUrl = URL.createObjectURL(blob);
const posterObjectUrl = posterBlob ? URL.createObjectURL(posterBlob) : undefined;
useMediaGenerationStore.getState().markDone(req.elementId, objectUrl, posterObjectUrl);
} catch (err) {
if (abortSignal?.aborted) return;
const message = err instanceof Error ? err.message : String(err);
const errorCode = err instanceof MediaApiError ? err.errorCode : undefined;
log.error(`Failed ${req.elementId}:`, message);
useMediaGenerationStore.getState().markFailed(req.elementId, message, errorCode);
// Persist non-retryable failures to IndexedDB so they survive page refresh
if (errorCode) {
await db.mediaFiles
.put({
id: mediaFileKey(stageId, req.elementId),
stageId,
type: req.type,
blob: new Blob(), // empty placeholder
mimeType: req.type === 'image' ? 'image/png' : 'video/mp4',
size: 0,
prompt: req.prompt,
params: JSON.stringify({
aspectRatio: req.aspectRatio,
style: req.style,
}),
error: message,
errorCode,
createdAt: Date.now(),
})
.catch(() => {}); // best-effort
}
}
}
async function callImageApi(
req: MediaGenerationRequest,
abortSignal?: AbortSignal,
): Promise<{ url: string }> {
const settings = useSettingsStore.getState();
const providerConfig = settings.imageProvidersConfig?.[settings.imageProviderId];
const response = await fetch('/api/generate/image', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'x-image-provider': settings.imageProviderId || '',
'x-image-model': settings.imageModelId || '',
'x-api-key': providerConfig?.apiKey || '',
'x-base-url': providerConfig?.baseUrl || '',
},
body: JSON.stringify({
prompt: req.prompt,
aspectRatio: req.aspectRatio,
style: req.style,
}),
signal: abortSignal,
});
if (!response.ok) {
const data = await response.json().catch(() => ({}));
throw new MediaApiError(data.error || `Image API returned ${response.status}`, data.errorCode);
}
const data = await response.json();
if (!data.success)
throw new MediaApiError(data.error || 'Image generation failed', data.errorCode);
// Result may have url or base64
const url =
data.result?.url || (data.result?.base64 ? `data:image/png;base64,${data.result.base64}` : '');
if (!url) throw new Error('No image URL in response');
return { url };
}
async function callVideoApi(
req: MediaGenerationRequest,
abortSignal?: AbortSignal,
): Promise<{ url: string; poster?: string }> {
const settings = useSettingsStore.getState();
const providerConfig = settings.videoProvidersConfig?.[settings.videoProviderId];
const response = await fetch('/api/generate/video', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'x-video-provider': settings.videoProviderId || '',
'x-video-model': settings.videoModelId || '',
'x-api-key': providerConfig?.apiKey || '',
'x-base-url': providerConfig?.baseUrl || '',
},
body: JSON.stringify({
prompt: req.prompt,
aspectRatio: req.aspectRatio,
}),
signal: abortSignal,
});
if (!response.ok) {
const data = await response.json().catch(() => ({}));
throw new MediaApiError(data.error || `Video API returned ${response.status}`, data.errorCode);
}
const data = await response.json();
if (!data.success)
throw new MediaApiError(data.error || 'Video generation failed', data.errorCode);
const url = data.result?.url;
if (!url) throw new Error('No video URL in response');
return { url, poster: data.result?.poster };
}
async function fetchAsBlob(url: string): Promise<Blob> {
// For data URLs, convert directly
if (url.startsWith('data:')) {
const res = await fetch(url);
return res.blob();
}
// For remote URLs, proxy through our server to bypass CORS restrictions
if (url.startsWith('http://') || url.startsWith('https://')) {
const res = await fetch('/api/proxy-media', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ url }),
});
if (!res.ok) {
const data = await res.json().catch(() => ({}));
throw new Error(data.error || `Proxy fetch failed: ${res.status}`);
}
return res.blob();
}
// Relative URLs (shouldn't happen, but handle gracefully)
const res = await fetch(url);
if (!res.ok) throw new Error(`Failed to fetch blob: ${res.status}`);
return res.blob();
}
|