feat(canvas): add video-prompt node and enhance video generation support
- Introduced a new node type "video-prompt" for AI video generation, including its integration into the canvas command palette and node template picker. - Updated connection validation to allow connections from text nodes to video-prompt and from video-prompt to ai-video nodes. - Enhanced error handling and messaging for video generation failures, including specific cases for provider issues. - Added tests to validate new video-prompt functionality and connection policies. - Updated localization files to include new labels and prompts for video-prompt and ai-video nodes.
This commit is contained in:
611
convex/ai.ts
611
convex/ai.ts
@@ -13,8 +13,23 @@ import {
|
||||
} from "./openrouter";
|
||||
import type { Id } from "./_generated/dataModel";
|
||||
import { assertNodeBelongsToCanvasOrThrow } from "./ai_utils";
|
||||
import {
|
||||
createVideoTask,
|
||||
downloadVideoAsBlob,
|
||||
FreepikApiError,
|
||||
getVideoTaskStatus,
|
||||
} from "./freepik";
|
||||
import { getVideoModel, isVideoModelId } from "../lib/ai-video-models";
|
||||
import {
|
||||
shouldLogVideoPollAttempt,
|
||||
shouldLogVideoPollResult,
|
||||
type VideoPollStatus,
|
||||
} from "../lib/video-poll-logging";
|
||||
import { normalizePublicTier } from "../lib/tier-credits";
|
||||
|
||||
const MAX_IMAGE_RETRIES = 2;
|
||||
const MAX_VIDEO_POLL_ATTEMPTS = 30;
|
||||
const MAX_VIDEO_POLL_TOTAL_MS = 10 * 60 * 1000;
|
||||
|
||||
type ErrorCategory =
|
||||
| "credits"
|
||||
@@ -34,9 +49,36 @@ function getErrorCode(error: unknown): string | undefined {
|
||||
const data = error.data as ErrorData;
|
||||
return data?.code;
|
||||
}
|
||||
if (error instanceof FreepikApiError) {
|
||||
return error.code;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function getErrorSource(error: unknown): string | undefined {
|
||||
if (error instanceof FreepikApiError) {
|
||||
return error.source;
|
||||
}
|
||||
if (error && typeof error === "object") {
|
||||
const source = (error as { source?: unknown }).source;
|
||||
return typeof source === "string" ? source : undefined;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function getProviderStatus(error: unknown): number | null {
|
||||
if (error instanceof FreepikApiError) {
|
||||
return typeof error.status === "number" ? error.status : null;
|
||||
}
|
||||
if (error && typeof error === "object") {
|
||||
const status = (error as { status?: unknown }).status;
|
||||
if (typeof status === "number" && Number.isFinite(status)) {
|
||||
return status;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown): string {
|
||||
if (error instanceof Error) return error.message;
|
||||
return String(error ?? "Generation failed");
|
||||
@@ -54,9 +96,25 @@ function categorizeError(error: unknown): {
|
||||
retryable: boolean;
|
||||
} {
|
||||
const code = getErrorCode(error);
|
||||
const source = getErrorSource(error);
|
||||
const message = errorMessage(error);
|
||||
const lower = message.toLowerCase();
|
||||
const status = parseOpenRouterStatus(message);
|
||||
const status = getProviderStatus(error) ?? parseOpenRouterStatus(message);
|
||||
|
||||
if (source === "freepik") {
|
||||
if (code === "model_unavailable") {
|
||||
return {
|
||||
category: "provider",
|
||||
retryable: status === 503,
|
||||
};
|
||||
}
|
||||
if (code === "timeout") {
|
||||
return { category: "timeout", retryable: true };
|
||||
}
|
||||
if (code === "transient") {
|
||||
return { category: "transient", retryable: true };
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
code === "CREDITS_TEST_DISABLED" ||
|
||||
@@ -552,6 +610,7 @@ export const generateImage = action({
|
||||
model: modelId,
|
||||
nodeId: verifiedNodeId,
|
||||
canvasId: verifiedCanvasId,
|
||||
provider: "openrouter",
|
||||
})
|
||||
: null;
|
||||
|
||||
@@ -627,3 +686,553 @@ export const generateImage = action({
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
function isVideoModelAllowedForTier(modelTier: "free" | "starter" | "pro", userTier: "free" | "starter" | "pro" | "max") {
|
||||
const tierOrder = { free: 0, starter: 1, pro: 2, max: 3 } as const;
|
||||
return tierOrder[userTier] >= tierOrder[modelTier];
|
||||
}
|
||||
|
||||
export const setVideoTaskInfo = internalMutation({
|
||||
args: {
|
||||
nodeId: v.id("nodes"),
|
||||
taskId: v.string(),
|
||||
},
|
||||
handler: async (ctx, { nodeId, taskId }) => {
|
||||
const node = await ctx.db.get(nodeId);
|
||||
if (!node) {
|
||||
throw new Error("Node not found");
|
||||
}
|
||||
|
||||
const prev =
|
||||
node.data && typeof node.data === "object"
|
||||
? (node.data as Record<string, unknown>)
|
||||
: {};
|
||||
|
||||
await ctx.db.patch(nodeId, {
|
||||
data: {
|
||||
...prev,
|
||||
taskId,
|
||||
},
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
export const markVideoPollingRetry = internalMutation({
|
||||
args: {
|
||||
nodeId: v.id("nodes"),
|
||||
attempt: v.number(),
|
||||
maxAttempts: v.number(),
|
||||
failureMessage: v.string(),
|
||||
},
|
||||
handler: async (ctx, { nodeId, attempt, maxAttempts, failureMessage }) => {
|
||||
await ctx.db.patch(nodeId, {
|
||||
status: "executing",
|
||||
retryCount: attempt,
|
||||
statusMessage: `Retry ${attempt}/${maxAttempts} - ${failureMessage}`,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
export const finalizeVideoSuccess = internalMutation({
|
||||
args: {
|
||||
nodeId: v.id("nodes"),
|
||||
prompt: v.string(),
|
||||
modelId: v.string(),
|
||||
durationSeconds: v.union(v.literal(5), v.literal(10)),
|
||||
storageId: v.id("_storage"),
|
||||
retryCount: v.number(),
|
||||
creditCost: v.number(),
|
||||
},
|
||||
handler: async (
|
||||
ctx,
|
||||
{ nodeId, prompt, modelId, durationSeconds, storageId, retryCount, creditCost }
|
||||
) => {
|
||||
const model = getVideoModel(modelId);
|
||||
if (!model) {
|
||||
throw new Error(`Unknown video model: ${modelId}`);
|
||||
}
|
||||
|
||||
const existing = await ctx.db.get(nodeId);
|
||||
if (!existing) {
|
||||
throw new Error("Node not found");
|
||||
}
|
||||
|
||||
const prev =
|
||||
existing.data && typeof existing.data === "object"
|
||||
? (existing.data as Record<string, unknown>)
|
||||
: {};
|
||||
|
||||
await ctx.db.patch(nodeId, {
|
||||
status: "done",
|
||||
retryCount,
|
||||
statusMessage: undefined,
|
||||
data: {
|
||||
...prev,
|
||||
taskId: undefined,
|
||||
storageId,
|
||||
prompt,
|
||||
model: modelId,
|
||||
modelLabel: model.label,
|
||||
durationSeconds,
|
||||
generatedAt: Date.now(),
|
||||
creditCost,
|
||||
},
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
export const finalizeVideoFailure = internalMutation({
|
||||
args: {
|
||||
nodeId: v.id("nodes"),
|
||||
retryCount: v.number(),
|
||||
statusMessage: v.string(),
|
||||
},
|
||||
handler: async (ctx, { nodeId, retryCount, statusMessage }) => {
|
||||
const existing = await ctx.db.get(nodeId);
|
||||
if (!existing) {
|
||||
throw new Error("Node not found");
|
||||
}
|
||||
const prev =
|
||||
existing.data && typeof existing.data === "object"
|
||||
? (existing.data as Record<string, unknown>)
|
||||
: {};
|
||||
|
||||
await ctx.db.patch(nodeId, {
|
||||
status: "error",
|
||||
retryCount,
|
||||
statusMessage,
|
||||
data: {
|
||||
...prev,
|
||||
taskId: undefined,
|
||||
},
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
export const processVideoGeneration = internalAction({
|
||||
args: {
|
||||
outputNodeId: v.id("nodes"),
|
||||
prompt: v.string(),
|
||||
modelId: v.string(),
|
||||
durationSeconds: v.union(v.literal(5), v.literal(10)),
|
||||
creditCost: v.number(),
|
||||
reservationId: v.optional(v.id("creditTransactions")),
|
||||
shouldDecrementConcurrency: v.boolean(),
|
||||
userId: v.string(),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const model = getVideoModel(args.modelId);
|
||||
if (!model) {
|
||||
throw new Error(`Unknown video model: ${args.modelId}`);
|
||||
}
|
||||
|
||||
console.info("[processVideoGeneration] start", {
|
||||
outputNodeId: args.outputNodeId,
|
||||
modelId: args.modelId,
|
||||
endpoint: model.freepikEndpoint,
|
||||
durationSeconds: args.durationSeconds,
|
||||
promptLength: args.prompt.length,
|
||||
hasReservation: Boolean(args.reservationId),
|
||||
shouldDecrementConcurrency: args.shouldDecrementConcurrency,
|
||||
});
|
||||
|
||||
try {
|
||||
const { task_id } = await createVideoTask({
|
||||
endpoint: model.freepikEndpoint,
|
||||
prompt: args.prompt,
|
||||
durationSeconds: args.durationSeconds,
|
||||
});
|
||||
|
||||
console.info("[processVideoGeneration] task created", {
|
||||
outputNodeId: args.outputNodeId,
|
||||
taskId: task_id,
|
||||
modelId: args.modelId,
|
||||
});
|
||||
|
||||
await ctx.runMutation(internal.ai.setVideoTaskInfo, {
|
||||
nodeId: args.outputNodeId,
|
||||
taskId: task_id,
|
||||
});
|
||||
|
||||
await ctx.scheduler.runAfter(5000, internal.ai.pollVideoTask, {
|
||||
taskId: task_id,
|
||||
outputNodeId: args.outputNodeId,
|
||||
prompt: args.prompt,
|
||||
modelId: args.modelId,
|
||||
durationSeconds: args.durationSeconds,
|
||||
creditCost: args.creditCost,
|
||||
reservationId: args.reservationId,
|
||||
shouldDecrementConcurrency: args.shouldDecrementConcurrency,
|
||||
userId: args.userId,
|
||||
attempt: 1,
|
||||
startedAtMs: Date.now(),
|
||||
});
|
||||
} catch (error) {
|
||||
console.warn("[processVideoGeneration] failed before polling", {
|
||||
outputNodeId: args.outputNodeId,
|
||||
modelId: args.modelId,
|
||||
errorMessage: errorMessage(error),
|
||||
errorCode: getErrorCode(error) ?? null,
|
||||
source: getErrorSource(error) ?? null,
|
||||
providerStatus: getProviderStatus(error),
|
||||
freepikBody: error instanceof FreepikApiError ? error.body : undefined,
|
||||
});
|
||||
|
||||
if (args.reservationId) {
|
||||
try {
|
||||
await ctx.runMutation(internal.credits.releaseInternal, {
|
||||
transactionId: args.reservationId,
|
||||
});
|
||||
} catch {
|
||||
// Keep node failure updates best-effort even if release fails.
|
||||
}
|
||||
}
|
||||
|
||||
await ctx.runMutation(internal.ai.finalizeVideoFailure, {
|
||||
nodeId: args.outputNodeId,
|
||||
retryCount: 0,
|
||||
statusMessage: formatTerminalStatusMessage(error),
|
||||
});
|
||||
|
||||
if (args.shouldDecrementConcurrency) {
|
||||
await ctx.runMutation(internal.credits.decrementConcurrency, {
|
||||
userId: args.userId,
|
||||
});
|
||||
}
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
export const pollVideoTask = internalAction({
|
||||
args: {
|
||||
taskId: v.string(),
|
||||
outputNodeId: v.id("nodes"),
|
||||
prompt: v.string(),
|
||||
modelId: v.string(),
|
||||
durationSeconds: v.union(v.literal(5), v.literal(10)),
|
||||
creditCost: v.number(),
|
||||
reservationId: v.optional(v.id("creditTransactions")),
|
||||
shouldDecrementConcurrency: v.boolean(),
|
||||
userId: v.string(),
|
||||
attempt: v.number(),
|
||||
startedAtMs: v.number(),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const elapsedMs = Date.now() - args.startedAtMs;
|
||||
if (args.attempt > MAX_VIDEO_POLL_ATTEMPTS || elapsedMs > MAX_VIDEO_POLL_TOTAL_MS) {
|
||||
if (args.reservationId) {
|
||||
try {
|
||||
await ctx.runMutation(internal.credits.releaseInternal, {
|
||||
transactionId: args.reservationId,
|
||||
});
|
||||
} catch {
|
||||
// Keep node status updates best-effort.
|
||||
}
|
||||
}
|
||||
|
||||
await ctx.runMutation(internal.ai.finalizeVideoFailure, {
|
||||
nodeId: args.outputNodeId,
|
||||
retryCount: args.attempt,
|
||||
statusMessage: "Timeout: Video generation exceeded maximum polling time",
|
||||
});
|
||||
|
||||
if (args.shouldDecrementConcurrency) {
|
||||
await ctx.runMutation(internal.credits.decrementConcurrency, {
|
||||
userId: args.userId,
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
if (shouldLogVideoPollAttempt(args.attempt)) {
|
||||
console.info("[pollVideoTask] poll start", {
|
||||
outputNodeId: args.outputNodeId,
|
||||
taskId: args.taskId,
|
||||
attempt: args.attempt,
|
||||
elapsedMs,
|
||||
});
|
||||
}
|
||||
|
||||
const model = getVideoModel(args.modelId);
|
||||
if (!model) {
|
||||
throw new Error(`Unknown video model: ${args.modelId}`);
|
||||
}
|
||||
|
||||
const status = await getVideoTaskStatus({
|
||||
taskId: args.taskId,
|
||||
statusEndpointPath: model.statusEndpointPath,
|
||||
attempt: args.attempt,
|
||||
});
|
||||
|
||||
if (shouldLogVideoPollResult(args.attempt, status.status as VideoPollStatus)) {
|
||||
console.info("[pollVideoTask] poll result", {
|
||||
outputNodeId: args.outputNodeId,
|
||||
taskId: args.taskId,
|
||||
attempt: args.attempt,
|
||||
status: status.status,
|
||||
generatedCount: status.generated?.length ?? 0,
|
||||
hasError: Boolean(status.error),
|
||||
statusError: status.error ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
if (status.status === "FAILED") {
|
||||
if (args.reservationId) {
|
||||
try {
|
||||
await ctx.runMutation(internal.credits.releaseInternal, {
|
||||
transactionId: args.reservationId,
|
||||
});
|
||||
} catch {
|
||||
// Keep node status updates best-effort.
|
||||
}
|
||||
}
|
||||
|
||||
await ctx.runMutation(internal.ai.finalizeVideoFailure, {
|
||||
nodeId: args.outputNodeId,
|
||||
retryCount: args.attempt,
|
||||
statusMessage: status.error?.trim() || "Provider: Video generation failed",
|
||||
});
|
||||
|
||||
if (args.shouldDecrementConcurrency) {
|
||||
await ctx.runMutation(internal.credits.decrementConcurrency, {
|
||||
userId: args.userId,
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (status.status === "COMPLETED") {
|
||||
const generatedUrl = status.generated?.[0]?.url;
|
||||
if (!generatedUrl) {
|
||||
throw new Error("Freepik completed without generated video URL");
|
||||
}
|
||||
|
||||
const blob = await downloadVideoAsBlob(generatedUrl);
|
||||
const storageId = await ctx.storage.store(blob);
|
||||
|
||||
await ctx.runMutation(internal.ai.finalizeVideoSuccess, {
|
||||
nodeId: args.outputNodeId,
|
||||
prompt: args.prompt,
|
||||
modelId: args.modelId,
|
||||
durationSeconds: args.durationSeconds,
|
||||
storageId: storageId as Id<"_storage">,
|
||||
retryCount: args.attempt,
|
||||
creditCost: args.creditCost,
|
||||
});
|
||||
|
||||
if (args.reservationId) {
|
||||
await ctx.runMutation(internal.credits.commitInternal, {
|
||||
transactionId: args.reservationId,
|
||||
actualCost: args.creditCost,
|
||||
});
|
||||
}
|
||||
|
||||
if (args.shouldDecrementConcurrency) {
|
||||
await ctx.runMutation(internal.credits.decrementConcurrency, {
|
||||
userId: args.userId,
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn("[pollVideoTask] poll failed", {
|
||||
outputNodeId: args.outputNodeId,
|
||||
taskId: args.taskId,
|
||||
attempt: args.attempt,
|
||||
elapsedMs,
|
||||
errorMessage: errorMessage(error),
|
||||
errorCode: getErrorCode(error) ?? null,
|
||||
source: getErrorSource(error) ?? null,
|
||||
providerStatus: getProviderStatus(error),
|
||||
retryable: categorizeError(error).retryable,
|
||||
freepikBody: error instanceof FreepikApiError ? error.body : undefined,
|
||||
});
|
||||
|
||||
const { retryable } = categorizeError(error);
|
||||
if (retryable && args.attempt < MAX_VIDEO_POLL_ATTEMPTS) {
|
||||
await ctx.runMutation(internal.ai.markVideoPollingRetry, {
|
||||
nodeId: args.outputNodeId,
|
||||
attempt: args.attempt,
|
||||
maxAttempts: MAX_VIDEO_POLL_ATTEMPTS,
|
||||
failureMessage: errorMessage(error),
|
||||
});
|
||||
|
||||
const retryDelayMs =
|
||||
args.attempt <= 5 ? 5000 : args.attempt <= 15 ? 10000 : 20000;
|
||||
await ctx.scheduler.runAfter(retryDelayMs, internal.ai.pollVideoTask, {
|
||||
...args,
|
||||
attempt: args.attempt + 1,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (args.reservationId) {
|
||||
try {
|
||||
await ctx.runMutation(internal.credits.releaseInternal, {
|
||||
transactionId: args.reservationId,
|
||||
});
|
||||
} catch {
|
||||
// Keep node status updates best-effort.
|
||||
}
|
||||
}
|
||||
|
||||
await ctx.runMutation(internal.ai.finalizeVideoFailure, {
|
||||
nodeId: args.outputNodeId,
|
||||
retryCount: args.attempt,
|
||||
statusMessage: formatTerminalStatusMessage(error),
|
||||
});
|
||||
|
||||
if (args.shouldDecrementConcurrency) {
|
||||
await ctx.runMutation(internal.credits.decrementConcurrency, {
|
||||
userId: args.userId,
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const delayMs = args.attempt <= 5 ? 5000 : args.attempt <= 15 ? 10000 : 20000;
|
||||
await ctx.scheduler.runAfter(delayMs, internal.ai.pollVideoTask, {
|
||||
...args,
|
||||
attempt: args.attempt + 1,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
export const generateVideo = action({
|
||||
args: {
|
||||
canvasId: v.id("canvases"),
|
||||
sourceNodeId: v.id("nodes"),
|
||||
outputNodeId: v.id("nodes"),
|
||||
prompt: v.string(),
|
||||
modelId: v.string(),
|
||||
durationSeconds: v.union(v.literal(5), v.literal(10)),
|
||||
},
|
||||
handler: async (ctx, args): Promise<{ queued: true; outputNodeId: Id<"nodes"> }> => {
|
||||
const canvas = await ctx.runQuery(api.canvases.get, {
|
||||
canvasId: args.canvasId,
|
||||
});
|
||||
if (!canvas) {
|
||||
throw new Error("Canvas not found");
|
||||
}
|
||||
|
||||
const sourceNode = await ctx.runQuery(
|
||||
api.nodes.get as FunctionReference<"query", "public">,
|
||||
{
|
||||
nodeId: args.sourceNodeId,
|
||||
includeStorageUrl: false,
|
||||
}
|
||||
);
|
||||
if (!sourceNode) {
|
||||
throw new Error("Source node not found");
|
||||
}
|
||||
assertNodeBelongsToCanvasOrThrow(sourceNode, args.canvasId);
|
||||
|
||||
const outputNode = await ctx.runQuery(
|
||||
api.nodes.get as FunctionReference<"query", "public">,
|
||||
{
|
||||
nodeId: args.outputNodeId,
|
||||
includeStorageUrl: false,
|
||||
}
|
||||
);
|
||||
if (!outputNode) {
|
||||
throw new Error("Output node not found");
|
||||
}
|
||||
assertNodeBelongsToCanvasOrThrow(outputNode, args.canvasId);
|
||||
|
||||
if (outputNode.type !== "ai-video") {
|
||||
throw new Error("Output node must be ai-video");
|
||||
}
|
||||
|
||||
if (!isVideoModelId(args.modelId)) {
|
||||
throw new Error(`Unknown video model: ${args.modelId}`);
|
||||
}
|
||||
|
||||
const model = getVideoModel(args.modelId);
|
||||
if (!model) {
|
||||
throw new Error(`Unknown video model: ${args.modelId}`);
|
||||
}
|
||||
|
||||
const subscription = await ctx.runQuery(api.credits.getSubscription, {});
|
||||
const userTier = normalizePublicTier(subscription?.tier);
|
||||
if (!isVideoModelAllowedForTier(model.tier, userTier)) {
|
||||
throw new Error(`Model ${args.modelId} requires ${model.tier} tier`);
|
||||
}
|
||||
|
||||
const prompt = args.prompt.trim();
|
||||
if (!prompt) {
|
||||
throw new Error("Prompt is required");
|
||||
}
|
||||
|
||||
const userId = canvas.ownerId;
|
||||
const creditCost = model.creditCost[args.durationSeconds];
|
||||
const internalCreditsEnabled = process.env.INTERNAL_CREDITS_ENABLED === "true";
|
||||
|
||||
await ctx.runMutation(internal.credits.checkAbuseLimits, {});
|
||||
|
||||
let usageIncremented = false;
|
||||
const reservationId: Id<"creditTransactions"> | null = internalCreditsEnabled
|
||||
? await ctx.runMutation(api.credits.reserve, {
|
||||
estimatedCost: creditCost,
|
||||
description: `Videogenerierung - ${model.label} (${args.durationSeconds}s)`,
|
||||
model: args.modelId,
|
||||
nodeId: args.outputNodeId,
|
||||
canvasId: args.canvasId,
|
||||
provider: "freepik",
|
||||
videoMeta: {
|
||||
model: args.modelId,
|
||||
durationSeconds: args.durationSeconds,
|
||||
hasAudio: false,
|
||||
},
|
||||
})
|
||||
: null;
|
||||
|
||||
if (!internalCreditsEnabled) {
|
||||
await ctx.runMutation(internal.credits.incrementUsage, {});
|
||||
usageIncremented = true;
|
||||
}
|
||||
|
||||
try {
|
||||
await ctx.runMutation(internal.ai.markNodeExecuting, {
|
||||
nodeId: args.outputNodeId,
|
||||
});
|
||||
|
||||
await ctx.scheduler.runAfter(0, internal.ai.processVideoGeneration, {
|
||||
outputNodeId: args.outputNodeId,
|
||||
prompt,
|
||||
modelId: args.modelId,
|
||||
durationSeconds: args.durationSeconds,
|
||||
creditCost,
|
||||
reservationId: reservationId ?? undefined,
|
||||
shouldDecrementConcurrency: usageIncremented,
|
||||
userId,
|
||||
});
|
||||
|
||||
return { queued: true, outputNodeId: args.outputNodeId };
|
||||
} catch (error) {
|
||||
if (reservationId) {
|
||||
try {
|
||||
await ctx.runMutation(api.credits.release, {
|
||||
transactionId: reservationId,
|
||||
});
|
||||
} catch {
|
||||
// Prefer returning a clear node error over masking with cleanup failures.
|
||||
}
|
||||
}
|
||||
|
||||
await ctx.runMutation(internal.ai.finalizeVideoFailure, {
|
||||
nodeId: args.outputNodeId,
|
||||
retryCount: 0,
|
||||
statusMessage: formatTerminalStatusMessage(error),
|
||||
});
|
||||
|
||||
if (usageIncremented) {
|
||||
await ctx.runMutation(internal.credits.decrementConcurrency, {
|
||||
userId,
|
||||
});
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
@@ -420,6 +420,12 @@ export const reserve = mutation({
|
||||
nodeId: v.optional(v.id("nodes")),
|
||||
canvasId: v.optional(v.id("canvases")),
|
||||
model: v.optional(v.string()),
|
||||
provider: v.optional(v.union(v.literal("openrouter"), v.literal("freepik"))),
|
||||
videoMeta: v.optional(v.object({
|
||||
model: v.string(),
|
||||
durationSeconds: v.number(),
|
||||
hasAudio: v.boolean(),
|
||||
})),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const user = await requireAuth(ctx);
|
||||
@@ -502,6 +508,8 @@ export const reserve = mutation({
|
||||
nodeId: args.nodeId,
|
||||
canvasId: args.canvasId,
|
||||
model: args.model,
|
||||
provider: args.provider,
|
||||
videoMeta: args.videoMeta,
|
||||
});
|
||||
|
||||
return transactionId;
|
||||
|
||||
@@ -2,8 +2,52 @@
|
||||
|
||||
import { v } from "convex/values";
|
||||
import { action } from "./_generated/server";
|
||||
import { shouldLogVideoPollResult, type VideoPollStatus } from "../lib/video-poll-logging";
|
||||
|
||||
const FREEPIK_BASE = "https://api.freepik.com";
|
||||
const FREEPIK_REQUEST_TIMEOUT_MS = 30_000;
|
||||
const FREEPIK_MAX_RETRIES = 2;
|
||||
|
||||
export type FreepikVideoTaskStatus =
|
||||
| "CREATED"
|
||||
| "IN_PROGRESS"
|
||||
| "COMPLETED"
|
||||
| "FAILED";
|
||||
|
||||
export interface FreepikVideoTaskStatusResponse {
|
||||
status: FreepikVideoTaskStatus;
|
||||
generated?: Array<{ url: string }>;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface FreepikMappedError {
|
||||
code: "model_unavailable" | "timeout" | "transient" | "unknown";
|
||||
message: string;
|
||||
retryable: boolean;
|
||||
}
|
||||
|
||||
export class FreepikApiError extends Error {
|
||||
readonly source = "freepik" as const;
|
||||
readonly status?: number;
|
||||
readonly code: FreepikMappedError["code"];
|
||||
readonly retryable: boolean;
|
||||
readonly body?: unknown;
|
||||
|
||||
constructor(args: {
|
||||
status?: number;
|
||||
code: FreepikMappedError["code"];
|
||||
message: string;
|
||||
retryable: boolean;
|
||||
body?: unknown;
|
||||
}) {
|
||||
super(args.message);
|
||||
this.name = "FreepikApiError";
|
||||
this.status = args.status;
|
||||
this.code = args.code;
|
||||
this.retryable = args.retryable;
|
||||
this.body = args.body;
|
||||
}
|
||||
}
|
||||
|
||||
type AssetType = "photo" | "vector" | "icon";
|
||||
|
||||
@@ -39,6 +83,495 @@ function parseSize(size?: string): { width?: number; height?: number } {
|
||||
return { width, height };
|
||||
}
|
||||
|
||||
function wait(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
setTimeout(resolve, ms);
|
||||
});
|
||||
}
|
||||
|
||||
function getFreepikApiKeyOrThrow(): string {
|
||||
const apiKey = process.env.FREEPIK_API_KEY;
|
||||
if (!apiKey) {
|
||||
throw new FreepikApiError({
|
||||
code: "model_unavailable",
|
||||
message: "FREEPIK_API_KEY not set",
|
||||
retryable: false,
|
||||
});
|
||||
}
|
||||
return apiKey;
|
||||
}
|
||||
|
||||
function normalizeFreepikEndpoint(path: string): string {
|
||||
if (path.startsWith("http://") || path.startsWith("https://")) {
|
||||
return path;
|
||||
}
|
||||
if (path.startsWith("/")) {
|
||||
return `${FREEPIK_BASE}${path}`;
|
||||
}
|
||||
return `${FREEPIK_BASE}/${path}`;
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return value !== null && typeof value === "object";
|
||||
}
|
||||
|
||||
function extractErrorDetail(body: unknown): string | undefined {
|
||||
if (typeof body === "string" && body.trim().length > 0) {
|
||||
return body.trim();
|
||||
}
|
||||
if (!isRecord(body)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const direct =
|
||||
typeof body.error === "string"
|
||||
? body.error
|
||||
: typeof body.message === "string"
|
||||
? body.message
|
||||
: undefined;
|
||||
if (direct && direct.trim().length > 0) {
|
||||
return direct.trim();
|
||||
}
|
||||
|
||||
const data = body.data;
|
||||
if (isRecord(data)) {
|
||||
const nested =
|
||||
typeof data.error === "string"
|
||||
? data.error
|
||||
: typeof data.message === "string"
|
||||
? data.message
|
||||
: undefined;
|
||||
if (nested && nested.trim().length > 0) {
|
||||
return nested.trim();
|
||||
}
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function mapFreepikError(status: number, body: unknown): FreepikMappedError {
|
||||
const detail = extractErrorDetail(body);
|
||||
|
||||
if (status === 401) {
|
||||
return {
|
||||
code: "model_unavailable",
|
||||
message: "Freepik API-Key ungueltig",
|
||||
retryable: false,
|
||||
};
|
||||
}
|
||||
|
||||
if (status === 400) {
|
||||
return {
|
||||
code: "unknown",
|
||||
message: detail ?? "Ungueltige Parameter fuer dieses Modell",
|
||||
retryable: false,
|
||||
};
|
||||
}
|
||||
|
||||
if (status === 404) {
|
||||
return {
|
||||
code: "transient",
|
||||
message: detail ?? "Freepik Task noch nicht verfuegbar",
|
||||
retryable: true,
|
||||
};
|
||||
}
|
||||
|
||||
if (status === 503) {
|
||||
return {
|
||||
code: "model_unavailable",
|
||||
message: "Freepik temporaer nicht verfuegbar",
|
||||
retryable: true,
|
||||
};
|
||||
}
|
||||
|
||||
if (status === 408 || status === 504) {
|
||||
return {
|
||||
code: "timeout",
|
||||
message: detail ?? "Freepik timeout",
|
||||
retryable: true,
|
||||
};
|
||||
}
|
||||
|
||||
if (status === 429) {
|
||||
return {
|
||||
code: "transient",
|
||||
message: detail ?? "Freepik Rate-Limit erreicht",
|
||||
retryable: true,
|
||||
};
|
||||
}
|
||||
|
||||
if (status >= 500) {
|
||||
return {
|
||||
code: "transient",
|
||||
message: detail ?? "Freepik Serverfehler",
|
||||
retryable: true,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
code: "unknown",
|
||||
message: detail ?? "Unbekannter Freepik-Fehler",
|
||||
retryable: false,
|
||||
};
|
||||
}
|
||||
|
||||
function isNetworkLikeError(error: unknown): boolean {
|
||||
if (!(error instanceof Error)) {
|
||||
return false;
|
||||
}
|
||||
const lower = error.message.toLowerCase();
|
||||
return (
|
||||
lower.includes("fetch failed") ||
|
||||
lower.includes("network") ||
|
||||
lower.includes("connection") ||
|
||||
lower.includes("econn")
|
||||
);
|
||||
}
|
||||
|
||||
async function parseResponseBody(response: Response): Promise<unknown> {
|
||||
const text = await response.text();
|
||||
if (!text) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
return text;
|
||||
}
|
||||
}
|
||||
|
||||
async function freepikJsonRequest<TResponse>(params: {
|
||||
path: string;
|
||||
method: "GET" | "POST";
|
||||
body?: string;
|
||||
useApiKey?: boolean;
|
||||
}): Promise<TResponse> {
|
||||
const apiKey = params.useApiKey === false ? null : getFreepikApiKeyOrThrow();
|
||||
const url = normalizeFreepikEndpoint(params.path);
|
||||
|
||||
for (let attempt = 0; attempt <= FREEPIK_MAX_RETRIES; attempt++) {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), FREEPIK_REQUEST_TIMEOUT_MS);
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: params.method,
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
...(apiKey ? { "x-freepik-api-key": apiKey } : {}),
|
||||
...(params.body ? { "Content-Type": "application/json" } : {}),
|
||||
},
|
||||
body: params.body,
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const responseBody = await parseResponseBody(response);
|
||||
const mapped = mapFreepikError(response.status, responseBody);
|
||||
const mappedError = new FreepikApiError({
|
||||
status: response.status,
|
||||
code: mapped.code,
|
||||
message: mapped.message,
|
||||
retryable: mapped.retryable,
|
||||
body: responseBody,
|
||||
});
|
||||
|
||||
if (mapped.retryable && attempt < FREEPIK_MAX_RETRIES) {
|
||||
await wait(Math.min(1200, 300 * (attempt + 1)));
|
||||
continue;
|
||||
}
|
||||
|
||||
throw mappedError;
|
||||
}
|
||||
|
||||
return (await response.json()) as TResponse;
|
||||
} catch (error) {
|
||||
const isTimeout = error instanceof Error && error.name === "AbortError";
|
||||
const retryable = isTimeout || isNetworkLikeError(error);
|
||||
|
||||
if (retryable && attempt < FREEPIK_MAX_RETRIES) {
|
||||
await wait(Math.min(1200, 300 * (attempt + 1)));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isTimeout) {
|
||||
throw new FreepikApiError({
|
||||
code: "timeout",
|
||||
message: "Freepik timeout",
|
||||
retryable: true,
|
||||
});
|
||||
}
|
||||
|
||||
if (isNetworkLikeError(error)) {
|
||||
throw new FreepikApiError({
|
||||
code: "transient",
|
||||
message: error instanceof Error ? error.message : "Netzwerkfehler bei Freepik",
|
||||
retryable: true,
|
||||
});
|
||||
}
|
||||
|
||||
throw error;
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
throw new FreepikApiError({
|
||||
code: "unknown",
|
||||
message: "Freepik request failed",
|
||||
retryable: false,
|
||||
});
|
||||
}
|
||||
|
||||
function buildTaskStatusPath(statusEndpointPath: string, taskId: string): string {
|
||||
const trimmedTaskId = taskId.trim();
|
||||
if (!trimmedTaskId) {
|
||||
throw new FreepikApiError({
|
||||
code: "unknown",
|
||||
message: "Missing Freepik task_id for status polling",
|
||||
retryable: false,
|
||||
});
|
||||
}
|
||||
|
||||
if (statusEndpointPath.includes("{task-id}")) {
|
||||
return statusEndpointPath.replaceAll("{task-id}", encodeURIComponent(trimmedTaskId));
|
||||
}
|
||||
|
||||
const suffix = statusEndpointPath.endsWith("/") ? "" : "/";
|
||||
return `${statusEndpointPath}${suffix}${encodeURIComponent(trimmedTaskId)}`;
|
||||
}
|
||||
|
||||
export async function createVideoTask(params: {
|
||||
endpoint: string;
|
||||
prompt: string;
|
||||
durationSeconds: 5 | 10;
|
||||
webhookUrl?: string;
|
||||
imageUrl?: string;
|
||||
}): Promise<{ task_id: string }> {
|
||||
const payload: Record<string, unknown> = {
|
||||
prompt: params.prompt,
|
||||
duration: params.durationSeconds,
|
||||
};
|
||||
if (params.webhookUrl) {
|
||||
payload.webhook_url = params.webhookUrl;
|
||||
}
|
||||
if (params.imageUrl) {
|
||||
payload.image_url = params.imageUrl;
|
||||
}
|
||||
|
||||
const result = await freepikJsonRequest<{ data?: { task_id?: string } }>({
|
||||
path: params.endpoint,
|
||||
method: "POST",
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
|
||||
console.info("[freepik.createVideoTask] response", {
|
||||
endpoint: params.endpoint,
|
||||
durationSeconds: params.durationSeconds,
|
||||
hasImageUrl: Boolean(params.imageUrl),
|
||||
promptLength: params.prompt.length,
|
||||
responseKeys: isRecord(result) ? Object.keys(result) : [],
|
||||
dataKeys: isRecord(result.data) ? Object.keys(result.data) : [],
|
||||
});
|
||||
|
||||
const taskId =
|
||||
typeof result.data?.task_id === "string"
|
||||
? result.data.task_id
|
||||
: typeof (result as { task_id?: unknown }).task_id === "string"
|
||||
? (result as { task_id: string }).task_id
|
||||
: undefined;
|
||||
if (typeof taskId !== "string" || taskId.trim().length === 0) {
|
||||
throw new FreepikApiError({
|
||||
code: "unknown",
|
||||
message: "Freepik response missing task_id",
|
||||
retryable: false,
|
||||
body: result,
|
||||
});
|
||||
}
|
||||
|
||||
return { task_id: taskId };
|
||||
}
|
||||
|
||||
export async function getVideoTaskStatus(params: {
|
||||
taskId: string;
|
||||
statusEndpointPath: string;
|
||||
attempt?: number;
|
||||
}): Promise<FreepikVideoTaskStatusResponse> {
|
||||
const statusPath = buildTaskStatusPath(params.statusEndpointPath, params.taskId);
|
||||
const result = await freepikJsonRequest<{
|
||||
data?: {
|
||||
status?: string;
|
||||
generated?: unknown;
|
||||
error?: string;
|
||||
message?: string;
|
||||
};
|
||||
status?: string;
|
||||
generated?: unknown;
|
||||
error?: string;
|
||||
message?: string;
|
||||
}>({
|
||||
path: statusPath,
|
||||
method: "GET",
|
||||
});
|
||||
|
||||
const statusRaw =
|
||||
typeof result.data?.status === "string"
|
||||
? result.data.status
|
||||
: typeof result.status === "string"
|
||||
? result.status
|
||||
: undefined;
|
||||
const status =
|
||||
statusRaw === "CREATED" ||
|
||||
statusRaw === "IN_PROGRESS" ||
|
||||
statusRaw === "COMPLETED" ||
|
||||
statusRaw === "FAILED"
|
||||
? statusRaw
|
||||
: null;
|
||||
|
||||
if (
|
||||
status &&
|
||||
shouldLogVideoPollResult(params.attempt ?? 1, status as VideoPollStatus)
|
||||
) {
|
||||
console.info("[freepik.getVideoTaskStatus] response", {
|
||||
taskId: params.taskId,
|
||||
statusPath,
|
||||
statusRaw: typeof statusRaw === "string" ? statusRaw : null,
|
||||
acceptedStatus: status,
|
||||
dataKeys: isRecord(result.data) ? Object.keys(result.data) : [],
|
||||
generatedCount: Array.isArray(result.data?.generated)
|
||||
? result.data.generated.length
|
||||
: Array.isArray(result.generated)
|
||||
? result.generated.length
|
||||
: 0,
|
||||
hasError:
|
||||
typeof result.data?.error === "string" || typeof result.error === "string",
|
||||
hasMessage:
|
||||
typeof result.data?.message === "string" || typeof result.message === "string",
|
||||
});
|
||||
}
|
||||
|
||||
if (!status) {
|
||||
console.warn("[freepik.getVideoTaskStatus] unexpected response", {
|
||||
taskId: params.taskId,
|
||||
statusPath,
|
||||
result,
|
||||
});
|
||||
throw new FreepikApiError({
|
||||
code: "unknown",
|
||||
message: "Freepik task status missing or invalid",
|
||||
retryable: false,
|
||||
body: result,
|
||||
});
|
||||
}
|
||||
|
||||
const generatedRaw = Array.isArray(result.data?.generated)
|
||||
? result.data.generated
|
||||
: Array.isArray(result.generated)
|
||||
? result.generated
|
||||
: undefined;
|
||||
|
||||
const generated = Array.isArray(generatedRaw)
|
||||
? generatedRaw
|
||||
.map((entry) => {
|
||||
const url =
|
||||
typeof entry === "string"
|
||||
? entry
|
||||
: isRecord(entry) && typeof entry.url === "string"
|
||||
? entry.url
|
||||
: undefined;
|
||||
if (!url) return null;
|
||||
return { url };
|
||||
})
|
||||
.filter((entry): entry is { url: string } => entry !== null)
|
||||
: undefined;
|
||||
|
||||
const error =
|
||||
typeof result.data?.error === "string"
|
||||
? result.data.error
|
||||
: typeof result.data?.message === "string"
|
||||
? result.data.message
|
||||
: typeof result.error === "string"
|
||||
? result.error
|
||||
: typeof result.message === "string"
|
||||
? result.message
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
status,
|
||||
...(generated && generated.length > 0 ? { generated } : {}),
|
||||
...(error ? { error } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export async function downloadVideoAsBlob(url: string): Promise<Blob> {
|
||||
for (let attempt = 0; attempt <= FREEPIK_MAX_RETRIES; attempt++) {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), FREEPIK_REQUEST_TIMEOUT_MS);
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: "GET",
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const body = await parseResponseBody(response);
|
||||
const mapped = mapFreepikError(response.status, body);
|
||||
const mappedError = new FreepikApiError({
|
||||
status: response.status,
|
||||
code: mapped.code,
|
||||
message: mapped.message,
|
||||
retryable: mapped.retryable,
|
||||
body,
|
||||
});
|
||||
|
||||
if (mapped.retryable && attempt < FREEPIK_MAX_RETRIES) {
|
||||
await wait(Math.min(1200, 300 * (attempt + 1)));
|
||||
continue;
|
||||
}
|
||||
|
||||
throw mappedError;
|
||||
}
|
||||
|
||||
return await response.blob();
|
||||
} catch (error) {
|
||||
const isTimeout = error instanceof Error && error.name === "AbortError";
|
||||
const retryable = isTimeout || isNetworkLikeError(error);
|
||||
|
||||
if (retryable && attempt < FREEPIK_MAX_RETRIES) {
|
||||
await wait(Math.min(1200, 300 * (attempt + 1)));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isTimeout) {
|
||||
throw new FreepikApiError({
|
||||
code: "timeout",
|
||||
message: "Freepik video download timeout",
|
||||
retryable: true,
|
||||
});
|
||||
}
|
||||
if (error instanceof FreepikApiError) {
|
||||
throw error;
|
||||
}
|
||||
if (isNetworkLikeError(error)) {
|
||||
throw new FreepikApiError({
|
||||
code: "transient",
|
||||
message: error instanceof Error ? error.message : "Netzwerkfehler beim Video-Download",
|
||||
retryable: true,
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
throw new FreepikApiError({
|
||||
code: "unknown",
|
||||
message: "Freepik video download failed",
|
||||
retryable: false,
|
||||
});
|
||||
}
|
||||
|
||||
export const search = action({
|
||||
args: {
|
||||
term: v.string(),
|
||||
|
||||
@@ -222,6 +222,12 @@ export default defineSchema({
|
||||
canvasId: v.optional(v.id("canvases")), // Zugehöriger Canvas
|
||||
openRouterCost: v.optional(v.number()), // Tatsächliche API-Kosten (Cent)
|
||||
model: v.optional(v.string()), // OpenRouter Model ID
|
||||
provider: v.optional(v.union(v.literal("openrouter"), v.literal("freepik"))),
|
||||
videoMeta: v.optional(v.object({
|
||||
model: v.string(),
|
||||
durationSeconds: v.number(),
|
||||
hasAudio: v.boolean(),
|
||||
})),
|
||||
})
|
||||
.index("by_user", ["userId"])
|
||||
.index("by_user_type", ["userId", "type"])
|
||||
|
||||
Reference in New Issue
Block a user