feat(canvas): add video-prompt node and enhance video generation support
- Introduced a new node type "video-prompt" for AI video generation, including its integration into the canvas command palette and node template picker. - Updated connection validation to allow connections from text nodes to video-prompt and from video-prompt to ai-video nodes. - Enhanced error handling and messaging for video generation failures, including specific cases for provider issues. - Added tests to validate new video-prompt functionality and connection policies. - Updated localization files to include new labels and prompts for video-prompt and ai-video nodes.
This commit is contained in:
@@ -2,8 +2,52 @@
|
||||
|
||||
import { v } from "convex/values";
|
||||
import { action } from "./_generated/server";
|
||||
import { shouldLogVideoPollResult, type VideoPollStatus } from "../lib/video-poll-logging";
|
||||
|
||||
const FREEPIK_BASE = "https://api.freepik.com";
|
||||
const FREEPIK_REQUEST_TIMEOUT_MS = 30_000;
|
||||
const FREEPIK_MAX_RETRIES = 2;
|
||||
|
||||
export type FreepikVideoTaskStatus =
|
||||
| "CREATED"
|
||||
| "IN_PROGRESS"
|
||||
| "COMPLETED"
|
||||
| "FAILED";
|
||||
|
||||
export interface FreepikVideoTaskStatusResponse {
|
||||
status: FreepikVideoTaskStatus;
|
||||
generated?: Array<{ url: string }>;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface FreepikMappedError {
|
||||
code: "model_unavailable" | "timeout" | "transient" | "unknown";
|
||||
message: string;
|
||||
retryable: boolean;
|
||||
}
|
||||
|
||||
export class FreepikApiError extends Error {
|
||||
readonly source = "freepik" as const;
|
||||
readonly status?: number;
|
||||
readonly code: FreepikMappedError["code"];
|
||||
readonly retryable: boolean;
|
||||
readonly body?: unknown;
|
||||
|
||||
constructor(args: {
|
||||
status?: number;
|
||||
code: FreepikMappedError["code"];
|
||||
message: string;
|
||||
retryable: boolean;
|
||||
body?: unknown;
|
||||
}) {
|
||||
super(args.message);
|
||||
this.name = "FreepikApiError";
|
||||
this.status = args.status;
|
||||
this.code = args.code;
|
||||
this.retryable = args.retryable;
|
||||
this.body = args.body;
|
||||
}
|
||||
}
|
||||
|
||||
type AssetType = "photo" | "vector" | "icon";
|
||||
|
||||
@@ -39,6 +83,495 @@ function parseSize(size?: string): { width?: number; height?: number } {
|
||||
return { width, height };
|
||||
}
|
||||
|
||||
function wait(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
setTimeout(resolve, ms);
|
||||
});
|
||||
}
|
||||
|
||||
function getFreepikApiKeyOrThrow(): string {
|
||||
const apiKey = process.env.FREEPIK_API_KEY;
|
||||
if (!apiKey) {
|
||||
throw new FreepikApiError({
|
||||
code: "model_unavailable",
|
||||
message: "FREEPIK_API_KEY not set",
|
||||
retryable: false,
|
||||
});
|
||||
}
|
||||
return apiKey;
|
||||
}
|
||||
|
||||
function normalizeFreepikEndpoint(path: string): string {
|
||||
if (path.startsWith("http://") || path.startsWith("https://")) {
|
||||
return path;
|
||||
}
|
||||
if (path.startsWith("/")) {
|
||||
return `${FREEPIK_BASE}${path}`;
|
||||
}
|
||||
return `${FREEPIK_BASE}/${path}`;
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return value !== null && typeof value === "object";
|
||||
}
|
||||
|
||||
function extractErrorDetail(body: unknown): string | undefined {
|
||||
if (typeof body === "string" && body.trim().length > 0) {
|
||||
return body.trim();
|
||||
}
|
||||
if (!isRecord(body)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const direct =
|
||||
typeof body.error === "string"
|
||||
? body.error
|
||||
: typeof body.message === "string"
|
||||
? body.message
|
||||
: undefined;
|
||||
if (direct && direct.trim().length > 0) {
|
||||
return direct.trim();
|
||||
}
|
||||
|
||||
const data = body.data;
|
||||
if (isRecord(data)) {
|
||||
const nested =
|
||||
typeof data.error === "string"
|
||||
? data.error
|
||||
: typeof data.message === "string"
|
||||
? data.message
|
||||
: undefined;
|
||||
if (nested && nested.trim().length > 0) {
|
||||
return nested.trim();
|
||||
}
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function mapFreepikError(status: number, body: unknown): FreepikMappedError {
|
||||
const detail = extractErrorDetail(body);
|
||||
|
||||
if (status === 401) {
|
||||
return {
|
||||
code: "model_unavailable",
|
||||
message: "Freepik API-Key ungueltig",
|
||||
retryable: false,
|
||||
};
|
||||
}
|
||||
|
||||
if (status === 400) {
|
||||
return {
|
||||
code: "unknown",
|
||||
message: detail ?? "Ungueltige Parameter fuer dieses Modell",
|
||||
retryable: false,
|
||||
};
|
||||
}
|
||||
|
||||
if (status === 404) {
|
||||
return {
|
||||
code: "transient",
|
||||
message: detail ?? "Freepik Task noch nicht verfuegbar",
|
||||
retryable: true,
|
||||
};
|
||||
}
|
||||
|
||||
if (status === 503) {
|
||||
return {
|
||||
code: "model_unavailable",
|
||||
message: "Freepik temporaer nicht verfuegbar",
|
||||
retryable: true,
|
||||
};
|
||||
}
|
||||
|
||||
if (status === 408 || status === 504) {
|
||||
return {
|
||||
code: "timeout",
|
||||
message: detail ?? "Freepik timeout",
|
||||
retryable: true,
|
||||
};
|
||||
}
|
||||
|
||||
if (status === 429) {
|
||||
return {
|
||||
code: "transient",
|
||||
message: detail ?? "Freepik Rate-Limit erreicht",
|
||||
retryable: true,
|
||||
};
|
||||
}
|
||||
|
||||
if (status >= 500) {
|
||||
return {
|
||||
code: "transient",
|
||||
message: detail ?? "Freepik Serverfehler",
|
||||
retryable: true,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
code: "unknown",
|
||||
message: detail ?? "Unbekannter Freepik-Fehler",
|
||||
retryable: false,
|
||||
};
|
||||
}
|
||||
|
||||
function isNetworkLikeError(error: unknown): boolean {
|
||||
if (!(error instanceof Error)) {
|
||||
return false;
|
||||
}
|
||||
const lower = error.message.toLowerCase();
|
||||
return (
|
||||
lower.includes("fetch failed") ||
|
||||
lower.includes("network") ||
|
||||
lower.includes("connection") ||
|
||||
lower.includes("econn")
|
||||
);
|
||||
}
|
||||
|
||||
async function parseResponseBody(response: Response): Promise<unknown> {
|
||||
const text = await response.text();
|
||||
if (!text) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
return text;
|
||||
}
|
||||
}
|
||||
|
||||
async function freepikJsonRequest<TResponse>(params: {
|
||||
path: string;
|
||||
method: "GET" | "POST";
|
||||
body?: string;
|
||||
useApiKey?: boolean;
|
||||
}): Promise<TResponse> {
|
||||
const apiKey = params.useApiKey === false ? null : getFreepikApiKeyOrThrow();
|
||||
const url = normalizeFreepikEndpoint(params.path);
|
||||
|
||||
for (let attempt = 0; attempt <= FREEPIK_MAX_RETRIES; attempt++) {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), FREEPIK_REQUEST_TIMEOUT_MS);
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: params.method,
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
...(apiKey ? { "x-freepik-api-key": apiKey } : {}),
|
||||
...(params.body ? { "Content-Type": "application/json" } : {}),
|
||||
},
|
||||
body: params.body,
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const responseBody = await parseResponseBody(response);
|
||||
const mapped = mapFreepikError(response.status, responseBody);
|
||||
const mappedError = new FreepikApiError({
|
||||
status: response.status,
|
||||
code: mapped.code,
|
||||
message: mapped.message,
|
||||
retryable: mapped.retryable,
|
||||
body: responseBody,
|
||||
});
|
||||
|
||||
if (mapped.retryable && attempt < FREEPIK_MAX_RETRIES) {
|
||||
await wait(Math.min(1200, 300 * (attempt + 1)));
|
||||
continue;
|
||||
}
|
||||
|
||||
throw mappedError;
|
||||
}
|
||||
|
||||
return (await response.json()) as TResponse;
|
||||
} catch (error) {
|
||||
const isTimeout = error instanceof Error && error.name === "AbortError";
|
||||
const retryable = isTimeout || isNetworkLikeError(error);
|
||||
|
||||
if (retryable && attempt < FREEPIK_MAX_RETRIES) {
|
||||
await wait(Math.min(1200, 300 * (attempt + 1)));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isTimeout) {
|
||||
throw new FreepikApiError({
|
||||
code: "timeout",
|
||||
message: "Freepik timeout",
|
||||
retryable: true,
|
||||
});
|
||||
}
|
||||
|
||||
if (isNetworkLikeError(error)) {
|
||||
throw new FreepikApiError({
|
||||
code: "transient",
|
||||
message: error instanceof Error ? error.message : "Netzwerkfehler bei Freepik",
|
||||
retryable: true,
|
||||
});
|
||||
}
|
||||
|
||||
throw error;
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
throw new FreepikApiError({
|
||||
code: "unknown",
|
||||
message: "Freepik request failed",
|
||||
retryable: false,
|
||||
});
|
||||
}
|
||||
|
||||
function buildTaskStatusPath(statusEndpointPath: string, taskId: string): string {
|
||||
const trimmedTaskId = taskId.trim();
|
||||
if (!trimmedTaskId) {
|
||||
throw new FreepikApiError({
|
||||
code: "unknown",
|
||||
message: "Missing Freepik task_id for status polling",
|
||||
retryable: false,
|
||||
});
|
||||
}
|
||||
|
||||
if (statusEndpointPath.includes("{task-id}")) {
|
||||
return statusEndpointPath.replaceAll("{task-id}", encodeURIComponent(trimmedTaskId));
|
||||
}
|
||||
|
||||
const suffix = statusEndpointPath.endsWith("/") ? "" : "/";
|
||||
return `${statusEndpointPath}${suffix}${encodeURIComponent(trimmedTaskId)}`;
|
||||
}
|
||||
|
||||
export async function createVideoTask(params: {
|
||||
endpoint: string;
|
||||
prompt: string;
|
||||
durationSeconds: 5 | 10;
|
||||
webhookUrl?: string;
|
||||
imageUrl?: string;
|
||||
}): Promise<{ task_id: string }> {
|
||||
const payload: Record<string, unknown> = {
|
||||
prompt: params.prompt,
|
||||
duration: params.durationSeconds,
|
||||
};
|
||||
if (params.webhookUrl) {
|
||||
payload.webhook_url = params.webhookUrl;
|
||||
}
|
||||
if (params.imageUrl) {
|
||||
payload.image_url = params.imageUrl;
|
||||
}
|
||||
|
||||
const result = await freepikJsonRequest<{ data?: { task_id?: string } }>({
|
||||
path: params.endpoint,
|
||||
method: "POST",
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
|
||||
console.info("[freepik.createVideoTask] response", {
|
||||
endpoint: params.endpoint,
|
||||
durationSeconds: params.durationSeconds,
|
||||
hasImageUrl: Boolean(params.imageUrl),
|
||||
promptLength: params.prompt.length,
|
||||
responseKeys: isRecord(result) ? Object.keys(result) : [],
|
||||
dataKeys: isRecord(result.data) ? Object.keys(result.data) : [],
|
||||
});
|
||||
|
||||
const taskId =
|
||||
typeof result.data?.task_id === "string"
|
||||
? result.data.task_id
|
||||
: typeof (result as { task_id?: unknown }).task_id === "string"
|
||||
? (result as { task_id: string }).task_id
|
||||
: undefined;
|
||||
if (typeof taskId !== "string" || taskId.trim().length === 0) {
|
||||
throw new FreepikApiError({
|
||||
code: "unknown",
|
||||
message: "Freepik response missing task_id",
|
||||
retryable: false,
|
||||
body: result,
|
||||
});
|
||||
}
|
||||
|
||||
return { task_id: taskId };
|
||||
}
|
||||
|
||||
export async function getVideoTaskStatus(params: {
|
||||
taskId: string;
|
||||
statusEndpointPath: string;
|
||||
attempt?: number;
|
||||
}): Promise<FreepikVideoTaskStatusResponse> {
|
||||
const statusPath = buildTaskStatusPath(params.statusEndpointPath, params.taskId);
|
||||
const result = await freepikJsonRequest<{
|
||||
data?: {
|
||||
status?: string;
|
||||
generated?: unknown;
|
||||
error?: string;
|
||||
message?: string;
|
||||
};
|
||||
status?: string;
|
||||
generated?: unknown;
|
||||
error?: string;
|
||||
message?: string;
|
||||
}>({
|
||||
path: statusPath,
|
||||
method: "GET",
|
||||
});
|
||||
|
||||
const statusRaw =
|
||||
typeof result.data?.status === "string"
|
||||
? result.data.status
|
||||
: typeof result.status === "string"
|
||||
? result.status
|
||||
: undefined;
|
||||
const status =
|
||||
statusRaw === "CREATED" ||
|
||||
statusRaw === "IN_PROGRESS" ||
|
||||
statusRaw === "COMPLETED" ||
|
||||
statusRaw === "FAILED"
|
||||
? statusRaw
|
||||
: null;
|
||||
|
||||
if (
|
||||
status &&
|
||||
shouldLogVideoPollResult(params.attempt ?? 1, status as VideoPollStatus)
|
||||
) {
|
||||
console.info("[freepik.getVideoTaskStatus] response", {
|
||||
taskId: params.taskId,
|
||||
statusPath,
|
||||
statusRaw: typeof statusRaw === "string" ? statusRaw : null,
|
||||
acceptedStatus: status,
|
||||
dataKeys: isRecord(result.data) ? Object.keys(result.data) : [],
|
||||
generatedCount: Array.isArray(result.data?.generated)
|
||||
? result.data.generated.length
|
||||
: Array.isArray(result.generated)
|
||||
? result.generated.length
|
||||
: 0,
|
||||
hasError:
|
||||
typeof result.data?.error === "string" || typeof result.error === "string",
|
||||
hasMessage:
|
||||
typeof result.data?.message === "string" || typeof result.message === "string",
|
||||
});
|
||||
}
|
||||
|
||||
if (!status) {
|
||||
console.warn("[freepik.getVideoTaskStatus] unexpected response", {
|
||||
taskId: params.taskId,
|
||||
statusPath,
|
||||
result,
|
||||
});
|
||||
throw new FreepikApiError({
|
||||
code: "unknown",
|
||||
message: "Freepik task status missing or invalid",
|
||||
retryable: false,
|
||||
body: result,
|
||||
});
|
||||
}
|
||||
|
||||
const generatedRaw = Array.isArray(result.data?.generated)
|
||||
? result.data.generated
|
||||
: Array.isArray(result.generated)
|
||||
? result.generated
|
||||
: undefined;
|
||||
|
||||
const generated = Array.isArray(generatedRaw)
|
||||
? generatedRaw
|
||||
.map((entry) => {
|
||||
const url =
|
||||
typeof entry === "string"
|
||||
? entry
|
||||
: isRecord(entry) && typeof entry.url === "string"
|
||||
? entry.url
|
||||
: undefined;
|
||||
if (!url) return null;
|
||||
return { url };
|
||||
})
|
||||
.filter((entry): entry is { url: string } => entry !== null)
|
||||
: undefined;
|
||||
|
||||
const error =
|
||||
typeof result.data?.error === "string"
|
||||
? result.data.error
|
||||
: typeof result.data?.message === "string"
|
||||
? result.data.message
|
||||
: typeof result.error === "string"
|
||||
? result.error
|
||||
: typeof result.message === "string"
|
||||
? result.message
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
status,
|
||||
...(generated && generated.length > 0 ? { generated } : {}),
|
||||
...(error ? { error } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export async function downloadVideoAsBlob(url: string): Promise<Blob> {
|
||||
for (let attempt = 0; attempt <= FREEPIK_MAX_RETRIES; attempt++) {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), FREEPIK_REQUEST_TIMEOUT_MS);
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: "GET",
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const body = await parseResponseBody(response);
|
||||
const mapped = mapFreepikError(response.status, body);
|
||||
const mappedError = new FreepikApiError({
|
||||
status: response.status,
|
||||
code: mapped.code,
|
||||
message: mapped.message,
|
||||
retryable: mapped.retryable,
|
||||
body,
|
||||
});
|
||||
|
||||
if (mapped.retryable && attempt < FREEPIK_MAX_RETRIES) {
|
||||
await wait(Math.min(1200, 300 * (attempt + 1)));
|
||||
continue;
|
||||
}
|
||||
|
||||
throw mappedError;
|
||||
}
|
||||
|
||||
return await response.blob();
|
||||
} catch (error) {
|
||||
const isTimeout = error instanceof Error && error.name === "AbortError";
|
||||
const retryable = isTimeout || isNetworkLikeError(error);
|
||||
|
||||
if (retryable && attempt < FREEPIK_MAX_RETRIES) {
|
||||
await wait(Math.min(1200, 300 * (attempt + 1)));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isTimeout) {
|
||||
throw new FreepikApiError({
|
||||
code: "timeout",
|
||||
message: "Freepik video download timeout",
|
||||
retryable: true,
|
||||
});
|
||||
}
|
||||
if (error instanceof FreepikApiError) {
|
||||
throw error;
|
||||
}
|
||||
if (isNetworkLikeError(error)) {
|
||||
throw new FreepikApiError({
|
||||
code: "transient",
|
||||
message: error instanceof Error ? error.message : "Netzwerkfehler beim Video-Download",
|
||||
retryable: true,
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
throw new FreepikApiError({
|
||||
code: "unknown",
|
||||
message: "Freepik video download failed",
|
||||
retryable: false,
|
||||
});
|
||||
}
|
||||
|
||||
export const search = action({
|
||||
args: {
|
||||
term: v.string(),
|
||||
|
||||
Reference in New Issue
Block a user