feat(canvas): add video-prompt node and enhance video generation support

- Introduced a new node type "video-prompt" for AI video generation, including its integration into the canvas command palette and node template picker.
- Updated connection validation to allow connections from text nodes to video-prompt and from video-prompt to ai-video nodes.
- Enhanced error handling and messaging for video generation failures, including specific cases for provider issues.
- Added tests to validate new video-prompt functionality and connection policies.
- Updated localization files to include new labels and prompts for video-prompt and ai-video nodes.
This commit is contained in:
2026-04-07 08:50:59 +02:00
parent 456b910532
commit ed08b976f9
28 changed files with 2899 additions and 9 deletions

145
tests/ai-video-node.test.ts Normal file
View File

@@ -0,0 +1,145 @@
// @vitest-environment jsdom
import React from "react";
import { act } from "react";
import { createRoot, type Root } from "react-dom/client";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
generateVideo: vi.fn(async () => ({ queued: true, outputNodeId: "ai-video-1" })),
getEdges: vi.fn(() => [{ source: "video-prompt-1", target: "ai-video-1" }]),
getNode: vi.fn((id: string) => {
if (id === "video-prompt-1") {
return { id, type: "video-prompt", data: { canvasId: "canvas-1" } };
}
return null;
}),
toastPromise: vi.fn(async <T,>(promise: Promise<T>) => await promise),
toastWarning: vi.fn(),
}));
vi.mock("next-intl", () => ({
useTranslations: () => (key: string) => key,
}));
vi.mock("convex/react", () => ({
useAction: () => mocks.generateVideo,
}));
vi.mock("@/convex/_generated/api", () => ({
api: {
ai: {
generateVideo: "ai.generateVideo",
},
},
}));
vi.mock("@/components/canvas/canvas-sync-context", () => ({
useCanvasSync: () => ({
status: { isOffline: false, isSyncing: false, pendingCount: 0 },
}),
}));
vi.mock("@/lib/toast", () => ({
toast: {
promise: mocks.toastPromise,
warning: mocks.toastWarning,
},
}));
vi.mock("@/components/canvas/nodes/base-node-wrapper", () => ({
default: ({ children }: { children: React.ReactNode }) => React.createElement("div", null, children),
}));
vi.mock("@xyflow/react", () => ({
Handle: () => null,
Position: { Left: "left", Right: "right" },
useReactFlow: () => ({
getEdges: mocks.getEdges,
getNode: mocks.getNode,
}),
}));
import AiVideoNode from "@/components/canvas/nodes/ai-video-node";
(globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true;
describe("AiVideoNode", () => {
let container: HTMLDivElement | null = null;
let root: Root | null = null;
beforeEach(() => {
mocks.generateVideo.mockClear();
mocks.getEdges.mockClear();
mocks.getNode.mockClear();
mocks.toastPromise.mockClear();
mocks.toastWarning.mockClear();
});
afterEach(() => {
if (root) {
act(() => {
root?.unmount();
});
}
container?.remove();
container = null;
root = null;
});
it("retries generation from the connected video prompt node", async () => {
container = document.createElement("div");
document.body.appendChild(container);
root = createRoot(container);
await act(async () => {
root?.render(
React.createElement(AiVideoNode, {
id: "ai-video-1",
selected: false,
dragging: false,
draggable: true,
selectable: true,
deletable: true,
zIndex: 1,
isConnectable: true,
type: "ai-video",
data: {
prompt: "ein suesser Berner Sennenhund rennt ueber eine Wiese",
modelId: "wan-2-2-480p",
durationSeconds: 5,
canvasId: "canvas-1",
_status: "error",
_statusMessage: "Netzwerk: task not found yet",
} as Record<string, unknown>,
positionAbsoluteX: 0,
positionAbsoluteY: 0,
}),
);
});
const retryButton = Array.from(container.querySelectorAll("button")).find((element) =>
element.textContent?.includes("retryButton"),
);
if (!(retryButton instanceof HTMLButtonElement)) {
throw new Error("Retry button not found");
}
expect(retryButton.disabled).toBe(false);
await act(async () => {
retryButton.click();
});
expect(mocks.generateVideo).toHaveBeenCalledTimes(1);
expect(mocks.generateVideo).toHaveBeenCalledWith({
canvasId: "canvas-1",
sourceNodeId: "video-prompt-1",
outputNodeId: "ai-video-1",
prompt: "ein suesser Berner Sennenhund rennt ueber eine Wiese",
modelId: "wan-2-2-480p",
durationSeconds: 5,
});
});
});

View File

@@ -21,4 +21,60 @@ describe("canvas connection policy", () => {
getCanvasConnectionValidationMessage("compare-incoming-limit"),
).toBe("Compare-Nodes erlauben genau zwei eingehende Verbindungen.");
});
it("allows text to video-prompt", () => {
expect(
validateCanvasConnectionPolicy({
sourceType: "text",
targetType: "video-prompt",
targetIncomingCount: 0,
}),
).toBeNull();
});
it("allows video-prompt to ai-video", () => {
expect(
validateCanvasConnectionPolicy({
sourceType: "video-prompt",
targetType: "ai-video",
targetIncomingCount: 0,
}),
).toBeNull();
});
it("blocks direct video-prompt to image prompt flow", () => {
expect(
validateCanvasConnectionPolicy({
sourceType: "video-prompt",
targetType: "prompt",
targetIncomingCount: 0,
}),
).toBe("video-prompt-target-invalid");
});
it("blocks ai-video as adjustment source", () => {
expect(
validateCanvasConnectionPolicy({
sourceType: "ai-video",
targetType: "curves",
targetIncomingCount: 0,
}),
).toBe("adjustment-source-invalid");
});
it("blocks ai-video as render source", () => {
expect(
validateCanvasConnectionPolicy({
sourceType: "ai-video",
targetType: "render",
targetIncomingCount: 0,
}),
).toBe("render-source-invalid");
});
it("describes video-only ai-video input", () => {
expect(
getCanvasConnectionValidationMessage("ai-video-source-invalid"),
).toBe("KI-Video-Ausgabe akzeptiert nur Eingaben von KI-Video.");
});
});

View File

@@ -0,0 +1,219 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
createVideoTask,
downloadVideoAsBlob,
getVideoTaskStatus,
} from "@/convex/freepik";
type MockResponseInit = {
ok: boolean;
status: number;
statusText?: string;
json?: unknown;
text?: string;
blob?: Blob;
};
function createMockResponse(init: MockResponseInit): Response {
return {
ok: init.ok,
status: init.status,
statusText: init.statusText ?? "",
json: vi.fn(async () => init.json),
text: vi.fn(async () => init.text ?? JSON.stringify(init.json ?? {})),
blob: vi.fn(async () => init.blob ?? new Blob([])),
headers: new Headers(),
} as unknown as Response;
}
describe("freepik video client", () => {
const originalApiKey = process.env.FREEPIK_API_KEY;
const fetchMock = vi.fn<typeof fetch>();
beforeEach(() => {
process.env.FREEPIK_API_KEY = "test-key";
fetchMock.mockReset();
vi.stubGlobal("fetch", fetchMock);
});
afterEach(() => {
vi.unstubAllGlobals();
process.env.FREEPIK_API_KEY = originalApiKey;
});
it("creates a video task", async () => {
fetchMock.mockResolvedValueOnce(
createMockResponse({
ok: true,
status: 200,
json: { data: { task_id: "task_123" } },
}),
);
const result = await createVideoTask({
endpoint: "/v1/ai/video/seedance-1-5-pro-1080p",
prompt: "A cinematic city timelapse",
durationSeconds: 5,
});
expect(result.task_id).toBe("task_123");
expect(fetchMock).toHaveBeenCalledTimes(1);
});
it("accepts root-level task_id responses for current video endpoints", async () => {
fetchMock.mockResolvedValueOnce(
createMockResponse({
ok: true,
status: 200,
json: { task_id: "task_root", status: "CREATED" },
}),
);
const result = await createVideoTask({
endpoint: "/v1/ai/image-to-video/kling-v2-1-std",
prompt: "A cinematic city timelapse",
durationSeconds: 5,
});
expect(result.task_id).toBe("task_root");
});
it("reads completed video task status", async () => {
fetchMock.mockResolvedValueOnce(
createMockResponse({
ok: true,
status: 200,
json: {
data: {
status: "COMPLETED",
generated: [{ url: "https://cdn.example.com/video.mp4" }],
},
},
}),
);
const result = await getVideoTaskStatus({
taskId: "task_123",
statusEndpointPath: "/v1/ai/text-to-video/wan-2-5-t2v-720p/{task-id}",
});
expect(result.status).toBe("COMPLETED");
expect(result.generated?.[0]?.url).toBe("https://cdn.example.com/video.mp4");
expect(fetchMock).toHaveBeenCalledWith(
"https://api.freepik.com/v1/ai/text-to-video/wan-2-5-t2v-720p/task_123",
expect.any(Object),
);
});
it("downloads completed video as blob", async () => {
const blob = new Blob(["video-bytes"], { type: "video/mp4" });
fetchMock.mockResolvedValueOnce(
createMockResponse({
ok: true,
status: 200,
blob,
}),
);
const result = await downloadVideoAsBlob("https://cdn.example.com/video.mp4");
expect(result.type).toBe("video/mp4");
expect(fetchMock).toHaveBeenCalledTimes(1);
});
it("maps unauthorized responses to model_unavailable", async () => {
fetchMock.mockResolvedValueOnce(
createMockResponse({
ok: false,
status: 401,
statusText: "Unauthorized",
json: { message: "invalid api key" },
}),
);
await expect(
createVideoTask({
endpoint: "/v1/ai/video/seedance-1-5-pro-1080p",
prompt: "A cinematic city timelapse",
durationSeconds: 5,
}),
).rejects.toMatchObject({
source: "freepik",
code: "model_unavailable",
status: 401,
retryable: false,
});
expect(fetchMock).toHaveBeenCalledTimes(1);
});
it("retries once on 503 and succeeds", async () => {
fetchMock
.mockResolvedValueOnce(
createMockResponse({
ok: false,
status: 503,
statusText: "Service Unavailable",
json: { message: "temporary outage" },
}),
)
.mockResolvedValueOnce(
createMockResponse({
ok: true,
status: 200,
json: { data: { task_id: "task_after_retry" } },
}),
);
const result = await createVideoTask({
endpoint: "/v1/ai/video/seedance-1-5-pro-1080p",
prompt: "A cinematic city timelapse",
durationSeconds: 10,
});
expect(result.task_id).toBe("task_after_retry");
expect(fetchMock).toHaveBeenCalledTimes(2);
});
it("treats task 404 during polling as retryable transient provider state", async () => {
fetchMock
.mockResolvedValueOnce(
createMockResponse({
ok: false,
status: 404,
statusText: "Not Found",
json: { message: "Not found" },
}),
)
.mockResolvedValueOnce(
createMockResponse({
ok: false,
status: 404,
statusText: "Not Found",
json: { message: "Not found" },
}),
)
.mockResolvedValueOnce(
createMockResponse({
ok: false,
status: 404,
statusText: "Not Found",
json: { message: "Not found" },
}),
);
await expect(
getVideoTaskStatus({
taskId: "task_404",
statusEndpointPath: "/v1/ai/text-to-video/wan-2-5-t2v-720p/{task-id}",
}),
).rejects.toMatchObject({
source: "freepik",
code: "transient",
status: 404,
retryable: true,
});
expect(fetchMock).toHaveBeenCalledTimes(3);
});
});

View File

@@ -0,0 +1,73 @@
import { describe, expect, it } from "vitest";
import {
DEFAULT_VIDEO_MODEL_ID,
VIDEO_MODELS,
getAvailableVideoModels,
getVideoModel,
isVideoModelId,
} from "@/lib/ai-video-models";
describe("ai video models registry", () => {
it("contains all planned MVP models", () => {
expect(Object.keys(VIDEO_MODELS)).toHaveLength(5);
expect(Object.keys(VIDEO_MODELS)).toEqual([
"wan-2-2-480p",
"wan-2-2-720p",
"kling-std-2-1",
"seedance-pro-1080p",
"kling-pro-2-6",
]);
expect(isVideoModelId(DEFAULT_VIDEO_MODEL_ID)).toBe(true);
});
it("keeps credit costs consistent for 5s and 10s durations", () => {
for (const model of Object.values(VIDEO_MODELS)) {
expect(model.creditCost[5]).toBeGreaterThan(0);
expect(model.creditCost[10]).toBeGreaterThan(0);
expect(model.creditCost[10]).toBe(model.creditCost[5] * 2);
}
});
it("filters available models by tier", () => {
expect(getAvailableVideoModels("free").map((model) => model.id)).toEqual([
"wan-2-2-480p",
"wan-2-2-720p",
]);
expect(getAvailableVideoModels("starter").map((model) => model.id)).toEqual([
"wan-2-2-480p",
"wan-2-2-720p",
"kling-std-2-1",
"seedance-pro-1080p",
]);
expect(getAvailableVideoModels("pro").map((model) => model.id)).toEqual([
"wan-2-2-480p",
"wan-2-2-720p",
"kling-std-2-1",
"seedance-pro-1080p",
"kling-pro-2-6",
]);
});
it("supports lookup and model id guards", () => {
expect(isVideoModelId("wan-2-2-480p")).toBe(true);
expect(isVideoModelId("not-a-model")).toBe(false);
const validModel = getVideoModel("wan-2-2-720p");
expect(validModel?.label).toBe("WAN 2.2 720p");
expect(getVideoModel("not-a-model")).toBeUndefined();
});
it("stores model-specific status endpoints for polling", () => {
expect(getVideoModel("wan-2-2-480p")?.statusEndpointPath).toBe(
"/v1/ai/text-to-video/wan-2-5-t2v-720p/{task-id}",
);
expect(getVideoModel("seedance-pro-1080p")?.statusEndpointPath).toBe(
"/v1/ai/video/seedance-1-5-pro-1080p/{task-id}",
);
expect(getVideoModel("kling-std-2-1")?.statusEndpointPath).toBe(
"/v1/ai/image-to-video/kling-v2-1/{task-id}",
);
});
});

View File

@@ -0,0 +1,22 @@
import { describe, expect, it } from "vitest";
import {
shouldLogVideoPollAttempt,
shouldLogVideoPollResult,
} from "@/lib/video-poll-logging";
describe("video poll logging", () => {
it("logs only the first and every fifth in-progress attempt", () => {
expect(shouldLogVideoPollAttempt(1)).toBe(true);
expect(shouldLogVideoPollAttempt(2)).toBe(false);
expect(shouldLogVideoPollAttempt(5)).toBe(true);
expect(shouldLogVideoPollAttempt(6)).toBe(false);
});
it("always logs terminal poll results", () => {
expect(shouldLogVideoPollResult(2, "IN_PROGRESS")).toBe(false);
expect(shouldLogVideoPollResult(5, "IN_PROGRESS")).toBe(true);
expect(shouldLogVideoPollResult(17, "COMPLETED")).toBe(true);
expect(shouldLogVideoPollResult(3, "FAILED")).toBe(true);
});
});

View File

@@ -0,0 +1,235 @@
// @vitest-environment jsdom
import React from "react";
import { act } from "react";
import { createRoot, type Root } from "react-dom/client";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { Id } from "@/convex/_generated/dataModel";
const mocks = vi.hoisted(() => ({
edges: [] as Array<{ source: string; target: string }>,
nodes: [] as Array<{ id: string; type: string; data: Record<string, unknown> }>,
balance: { balance: 100, reserved: 0 } as { balance: number; reserved: number } | undefined,
queueNodeDataUpdate: vi.fn(async () => undefined),
createNodeConnectedFromSource: vi.fn(async () => "ai-video-node-1" as Id<"nodes">),
generateVideo: vi.fn(async () => ({ queued: true, outputNodeId: "ai-video-node-1" })),
getEdges: vi.fn(() => [] as Array<{ source: string; target: string }>),
getNode: vi.fn((id: string) =>
id === "video-prompt-1"
? { id, position: { x: 100, y: 50 }, measured: { width: 260, height: 220 } }
: null,
),
push: vi.fn(),
toastPromise: vi.fn(async <T,>(promise: Promise<T>) => await promise),
toastWarning: vi.fn(),
toastAction: vi.fn(),
toastError: vi.fn(),
}));
vi.mock("next-intl", () => ({
useTranslations: () => (key: string) => key,
}));
vi.mock("next/navigation", () => ({
useRouter: () => ({ push: mocks.push }),
}));
vi.mock("convex/react", () => ({
useAction: () => mocks.generateVideo,
}));
vi.mock("@/convex/_generated/api", () => ({
api: {
ai: {
generateVideo: "ai.generateVideo",
},
credits: {
getBalance: "credits.getBalance",
},
},
}));
vi.mock("@/hooks/use-auth-query", () => ({
useAuthQuery: () => mocks.balance,
}));
vi.mock("@/hooks/use-debounced-callback", () => ({
useDebouncedCallback: (callback: (...args: Array<unknown>) => void) => callback,
}));
vi.mock("@/components/canvas/canvas-sync-context", () => ({
useCanvasSync: () => ({
queueNodeDataUpdate: mocks.queueNodeDataUpdate,
status: { isOffline: false, isSyncing: false, pendingCount: 0 },
}),
}));
vi.mock("@/components/canvas/canvas-placement-context", () => ({
useCanvasPlacement: () => ({
createNodeConnectedFromSource: mocks.createNodeConnectedFromSource,
}),
}));
vi.mock("@/lib/toast", () => ({
toast: {
promise: mocks.toastPromise,
warning: mocks.toastWarning,
action: mocks.toastAction,
error: mocks.toastError,
},
}));
vi.mock("@/lib/ai-errors", () => ({
classifyError: (error: unknown) => ({
type: "generic",
rawMessage: error instanceof Error ? error.message : String(error),
}),
}));
vi.mock("@/components/ui/label", () => ({
Label: ({ children, htmlFor }: { children: React.ReactNode; htmlFor?: string }) =>
React.createElement("label", { htmlFor }, children),
}));
vi.mock("@/components/ui/select", () => ({
Select: ({ value, onValueChange, children }: {
value: string;
onValueChange: (value: string) => void;
children: React.ReactNode;
}) =>
React.createElement(
"select",
{
"aria-label": "video-model",
value,
onChange: (event: Event) => {
onValueChange((event.target as HTMLSelectElement).value);
},
},
children,
),
SelectTrigger: ({ children }: { children: React.ReactNode }) => children,
SelectValue: () => null,
SelectContent: ({ children }: { children: React.ReactNode }) => children,
SelectItem: ({ children, value }: { children: React.ReactNode; value: string }) =>
React.createElement("option", { value }, children),
}));
vi.mock("@/components/canvas/nodes/base-node-wrapper", () => ({
default: ({ children }: { children: React.ReactNode }) => React.createElement("div", null, children),
}));
vi.mock("@xyflow/react", () => ({
Handle: () => null,
Position: { Left: "left", Right: "right" },
useStore: (selector: (state: { edges: typeof mocks.edges; nodes: typeof mocks.nodes }) => unknown) =>
selector({ edges: mocks.edges, nodes: mocks.nodes }),
useReactFlow: () => ({
getEdges: mocks.getEdges,
getNode: mocks.getNode,
}),
}));
import VideoPromptNode from "@/components/canvas/nodes/video-prompt-node";
(globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true;
describe("VideoPromptNode", () => {
let container: HTMLDivElement | null = null;
let root: Root | null = null;
beforeEach(() => {
mocks.edges = [];
mocks.nodes = [];
mocks.balance = { balance: 100, reserved: 0 };
mocks.queueNodeDataUpdate.mockClear();
mocks.createNodeConnectedFromSource.mockClear();
mocks.generateVideo.mockClear();
mocks.getEdges.mockClear();
mocks.getNode.mockClear();
mocks.push.mockClear();
mocks.toastPromise.mockClear();
mocks.toastWarning.mockClear();
mocks.toastAction.mockClear();
mocks.toastError.mockClear();
});
afterEach(() => {
if (root) {
act(() => {
root?.unmount();
});
}
container?.remove();
container = null;
root = null;
});
it("creates an ai-video node and queues video generation when generate is clicked", async () => {
container = document.createElement("div");
document.body.appendChild(container);
root = createRoot(container);
await act(async () => {
root?.render(
React.createElement(VideoPromptNode, {
id: "video-prompt-1",
selected: false,
dragging: false,
draggable: true,
selectable: true,
deletable: true,
zIndex: 1,
isConnectable: true,
type: "video-prompt",
data: {
prompt: "ein suesser Berner Sennenhund rennt ueber eine Wiese",
modelId: "wan-2-2-480p",
durationSeconds: 5,
canvasId: "canvas-1",
},
positionAbsoluteX: 0,
positionAbsoluteY: 0,
}),
);
});
const button = Array.from(container.querySelectorAll("button")).find((element) =>
element.textContent?.includes("generateButton"),
);
if (!(button instanceof HTMLButtonElement)) {
throw new Error("Generate button not found");
}
await act(async () => {
button.click();
});
expect(mocks.createNodeConnectedFromSource).toHaveBeenCalledTimes(1);
expect(mocks.createNodeConnectedFromSource).toHaveBeenCalledWith(
expect.objectContaining({
type: "ai-video",
sourceNodeId: "video-prompt-1",
sourceHandle: "video-prompt-out",
targetHandle: "video-in",
data: expect.objectContaining({
prompt: "ein suesser Berner Sennenhund rennt ueber eine Wiese",
modelId: "wan-2-2-480p",
durationSeconds: 5,
}),
}),
);
expect(mocks.generateVideo).toHaveBeenCalledTimes(1);
expect(mocks.generateVideo).toHaveBeenCalledWith({
canvasId: "canvas-1",
sourceNodeId: "video-prompt-1",
outputNodeId: "ai-video-node-1",
prompt: "ein suesser Berner Sennenhund rennt ueber eine Wiese",
modelId: "wan-2-2-480p",
durationSeconds: 5,
});
});
});