From ed08b976f996ef8dddfcf0f8f990ad52e3b936b7 Mon Sep 17 00:00:00 2001 From: Matthias Meister Date: Tue, 7 Apr 2026 08:50:59 +0200 Subject: [PATCH] feat(canvas): add video-prompt node and enhance video generation support - Introduced a new node type "video-prompt" for AI video generation, including its integration into the canvas command palette and node template picker. - Updated connection validation to allow connections from text nodes to video-prompt and from video-prompt to ai-video nodes. - Enhanced error handling and messaging for video generation failures, including specific cases for provider issues. - Added tests to validate new video-prompt functionality and connection policies. - Updated localization files to include new labels and prompts for video-prompt and ai-video nodes. --- .../__tests__/use-canvas-connections.test.tsx | 58 ++ .../use-canvas-edge-insertions.test.tsx | 29 + components/canvas/canvas-command-palette.tsx | 1 + .../canvas/canvas-generation-failures.ts | 4 +- .../canvas/canvas-node-template-picker.tsx | 2 + components/canvas/canvas-sidebar.tsx | 1 + components/canvas/node-types.ts | 4 + components/canvas/nodes/ai-video-node.tsx | 251 +++++++ components/canvas/nodes/video-prompt-node.tsx | 418 ++++++++++++ convex/ai.ts | 611 +++++++++++++++++- convex/credits.ts | 8 + convex/freepik.ts | 533 +++++++++++++++ convex/schema.ts | 6 + lib/ai-video-models.ts | 109 ++++ lib/canvas-connection-policy.ts | 24 +- lib/canvas-node-catalog.ts | 8 +- lib/canvas-node-templates.ts | 12 + lib/canvas-node-types.ts | 2 + lib/canvas-utils.ts | 15 + lib/video-poll-logging.ts | 12 + messages/de.json | 25 + messages/en.json | 25 + tests/ai-video-node.test.ts | 145 +++++ tests/canvas-connection-policy.test.ts | 56 ++ tests/convex/freepik-video-client.test.ts | 219 +++++++ tests/lib/ai-video-models.test.ts | 73 +++ tests/lib/video-poll-logging.test.ts | 22 + tests/video-prompt-node.test.ts | 235 +++++++ 28 files changed, 2899 insertions(+), 9 deletions(-) create mode 100644 components/canvas/nodes/ai-video-node.tsx create mode 100644 components/canvas/nodes/video-prompt-node.tsx create mode 100644 lib/ai-video-models.ts create mode 100644 lib/video-poll-logging.ts create mode 100644 tests/ai-video-node.test.ts create mode 100644 tests/convex/freepik-video-client.test.ts create mode 100644 tests/lib/ai-video-models.test.ts create mode 100644 tests/lib/video-poll-logging.test.ts create mode 100644 tests/video-prompt-node.test.ts diff --git a/components/canvas/__tests__/use-canvas-connections.test.tsx b/components/canvas/__tests__/use-canvas-connections.test.tsx index c2f18f9..0b9a849 100644 --- a/components/canvas/__tests__/use-canvas-connections.test.tsx +++ b/components/canvas/__tests__/use-canvas-connections.test.tsx @@ -432,6 +432,64 @@ describe("useCanvasConnections", () => { expect(latestHandlersRef.current?.connectionDropMenu).toBeNull(); }); + it("rejects text to ai-video body drops", async () => { + const runCreateEdgeMutation = vi.fn(async () => undefined); + const showConnectionRejectedToast = vi.fn(); + + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); + + await act(async () => { + root?.render( + , + ); + }); + + await act(async () => { + latestHandlersRef.current?.onConnectStart?.( + {} as MouseEvent, + { + nodeId: "node-source", + handleId: null, + handleType: "source", + } as never, + ); + latestHandlersRef.current?.onConnectEnd( + { clientX: 400, clientY: 260 } as MouseEvent, + { + isValid: false, + from: { x: 0, y: 0 }, + fromNode: { id: "node-source", type: "text" }, + fromHandle: { id: null, type: "source" }, + fromPosition: null, + to: { x: 400, y: 260 }, + toHandle: null, + toNode: null, + toPosition: null, + pointer: null, + } as never, + ); + }); + + expect(runCreateEdgeMutation).not.toHaveBeenCalled(); + expect(showConnectionRejectedToast).toHaveBeenCalledWith("ai-video-source-invalid"); + expect(latestHandlersRef.current?.connectionDropMenu).toBeNull(); + }); + it("ignores onConnectEnd when no connect drag is active", async () => { const runCreateEdgeMutation = vi.fn(async () => undefined); const showConnectionRejectedToast = vi.fn(); diff --git a/components/canvas/__tests__/use-canvas-edge-insertions.test.tsx b/components/canvas/__tests__/use-canvas-edge-insertions.test.tsx index 3f87fc4..23de626 100644 --- a/components/canvas/__tests__/use-canvas-edge-insertions.test.tsx +++ b/components/canvas/__tests__/use-canvas-edge-insertions.test.tsx @@ -740,4 +740,33 @@ describe("useCanvasEdgeInsertions", () => { expect(templateTypes).not.toContain("text"); expect(templateTypes).not.toContain("ai-image"); }); + + it("offers video-prompt as valid split for text to ai-video", async () => { + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); + + await act(async () => { + root?.render( + , + ); + }); + + await act(async () => { + latestHandlersRef.current?.openEdgeInsertMenu({ edgeId: "edge-1", screenX: 20, screenY: 20 }); + }); + + const templateTypes = (latestHandlersRef.current?.edgeInsertTemplates ?? []).map( + (template) => template.type, + ); + + expect(templateTypes).toContain("video-prompt"); + expect(templateTypes).not.toContain("prompt"); + }); }); diff --git a/components/canvas/canvas-command-palette.tsx b/components/canvas/canvas-command-palette.tsx index 9acc49c..5b07aa7 100644 --- a/components/canvas/canvas-command-palette.tsx +++ b/components/canvas/canvas-command-palette.tsx @@ -55,6 +55,7 @@ const CATALOG_ICONS: Partial> = { image: Image, text: Type, prompt: Sparkles, + "video-prompt": Video, color: Palette, video: Video, asset: Package, diff --git a/components/canvas/canvas-generation-failures.ts b/components/canvas/canvas-generation-failures.ts index 650f1ac..0925ed9 100644 --- a/components/canvas/canvas-generation-failures.ts +++ b/components/canvas/canvas-generation-failures.ts @@ -26,7 +26,7 @@ export function useGenerationFailureWarnings( for (const node of convexNodes) { nextNodeStatusMap.set(node._id, node.status); - if (node.type !== "ai-image") { + if (node.type !== "ai-image" && node.type !== "ai-video") { continue; } @@ -61,7 +61,7 @@ export function useGenerationFailureWarnings( } if (recentFailures.length >= GENERATION_FAILURE_THRESHOLD) { - toast.warning(t('ai.openrouterIssuesTitle'), t('ai.openrouterIssuesDesc')); + toast.warning(t('ai.providerIssuesTitle'), t('ai.providerIssuesDesc')); recentGenerationFailureTimestampsRef.current = []; return; } diff --git a/components/canvas/canvas-node-template-picker.tsx b/components/canvas/canvas-node-template-picker.tsx index 0132bb3..251af14 100644 --- a/components/canvas/canvas-node-template-picker.tsx +++ b/components/canvas/canvas-node-template-picker.tsx @@ -27,6 +27,7 @@ const NODE_ICONS: Record = { image: Image, text: Type, prompt: Sparkles, + "video-prompt": Video, note: StickyNote, frame: Frame, compare: GitCompare, @@ -46,6 +47,7 @@ const NODE_SEARCH_KEYWORDS: Partial< image: ["image", "photo", "foto"], text: ["text", "typo"], prompt: ["prompt", "ai", "generate", "ki-bild", "ki", "bild"], + "video-prompt": ["video", "ai", "ki-video", "ki", "prompt"], note: ["note", "sticky", "notiz"], frame: ["frame", "artboard"], compare: ["compare", "before", "after", "vergleich"], diff --git a/components/canvas/canvas-sidebar.tsx b/components/canvas/canvas-sidebar.tsx index 442596a..dfde480 100644 --- a/components/canvas/canvas-sidebar.tsx +++ b/components/canvas/canvas-sidebar.tsx @@ -48,6 +48,7 @@ const CATALOG_ICONS: Partial> = { image: Image, text: Type, prompt: Sparkles, + "video-prompt": Video, color: Palette, video: Video, asset: Package, diff --git a/components/canvas/node-types.ts b/components/canvas/node-types.ts index 3812b53..09a72e1 100644 --- a/components/canvas/node-types.ts +++ b/components/canvas/node-types.ts @@ -1,7 +1,9 @@ import ImageNode from "./nodes/image-node"; import TextNode from "./nodes/text-node"; import PromptNode from "./nodes/prompt-node"; +import VideoPromptNode from "./nodes/video-prompt-node"; import AiImageNode from "./nodes/ai-image-node"; +import AiVideoNode from "./nodes/ai-video-node"; import GroupNode from "./nodes/group-node"; import FrameNode from "./nodes/frame-node"; import NoteNode from "./nodes/note-node"; @@ -25,7 +27,9 @@ export const nodeTypes = { image: ImageNode, text: TextNode, prompt: PromptNode, + "video-prompt": VideoPromptNode, "ai-image": AiImageNode, + "ai-video": AiVideoNode, group: GroupNode, frame: FrameNode, note: NoteNode, diff --git a/components/canvas/nodes/ai-video-node.tsx b/components/canvas/nodes/ai-video-node.tsx new file mode 100644 index 0000000..7aa6b22 --- /dev/null +++ b/components/canvas/nodes/ai-video-node.tsx @@ -0,0 +1,251 @@ +"use client"; + +import { useCallback, useState } from "react"; +import { useAction } from "convex/react"; +import type { FunctionReference } from "convex/server"; +import { useTranslations } from "next-intl"; +import { AlertCircle, Download, Loader2, RefreshCw, Video } from "lucide-react"; +import { Handle, Position, useReactFlow, type Node, type NodeProps } from "@xyflow/react"; + +import { api } from "@/convex/_generated/api"; +import type { Id } from "@/convex/_generated/dataModel"; +import { useCanvasSync } from "@/components/canvas/canvas-sync-context"; +import { classifyError } from "@/lib/ai-errors"; +import { getVideoModel, type VideoModelDurationSeconds } from "@/lib/ai-video-models"; +import { toast } from "@/lib/toast"; +import BaseNodeWrapper from "./base-node-wrapper"; + +type AiVideoNodeData = { + prompt?: string; + modelId?: string; + durationSeconds?: VideoModelDurationSeconds; + creditCost?: number; + canvasId?: string; + url?: string; + _status?: string; + _statusMessage?: string; +}; + +type NodeStatus = + | "idle" + | "analyzing" + | "clarifying" + | "executing" + | "done" + | "error"; + +export type AiVideoNodeType = Node; + +export default function AiVideoNode({ id, data, selected }: NodeProps) { + const t = useTranslations("aiVideoNode"); + const tToast = useTranslations("toasts"); + const nodeData = data as AiVideoNodeData; + const { getEdges, getNode } = useReactFlow(); + const { status: syncStatus } = useCanvasSync(); + const generateVideo = useAction( + (api as unknown as { + ai: { + generateVideo: FunctionReference< + "action", + "public", + { + canvasId: Id<"canvases">; + sourceNodeId: Id<"nodes">; + outputNodeId: Id<"nodes">; + prompt: string; + modelId: string; + durationSeconds: 5 | 10; + }, + { queued: true; outputNodeId: Id<"nodes"> } + >; + }; + }).ai.generateVideo, + ); + const status = (nodeData._status ?? "idle") as NodeStatus; + const [isRetrying, setIsRetrying] = useState(false); + const [localError, setLocalError] = useState(null); + const classifiedError = classifyError(nodeData._statusMessage ?? localError); + const isLoading = + status === "executing" || status === "analyzing" || status === "clarifying" || isRetrying; + + const modelLabel = + typeof nodeData.modelId === "string" + ? getVideoModel(nodeData.modelId)?.label ?? nodeData.modelId + : "-"; + + const handleRetry = useCallback(async () => { + if (isRetrying) return; + + if (syncStatus.isOffline) { + toast.warning( + "Offline aktuell nicht unterstuetzt", + "KI-Generierung benoetigt eine aktive Verbindung.", + ); + return; + } + + const prompt = nodeData.prompt?.trim(); + const modelId = nodeData.modelId; + const durationSeconds = nodeData.durationSeconds; + + if (!prompt || !modelId || !durationSeconds) { + setLocalError(t("errorFallback")); + return; + } + + const incomingEdge = getEdges().find((edge) => edge.target === id); + if (!incomingEdge) { + setLocalError(t("errorFallback")); + return; + } + + const sourceNode = getNode(incomingEdge.source); + if (!sourceNode || sourceNode.type !== "video-prompt") { + setLocalError(t("errorFallback")); + return; + } + + const sourceData = sourceNode.data as { canvasId?: string } | undefined; + const canvasId = (nodeData.canvasId ?? sourceData?.canvasId) as Id<"canvases"> | undefined; + if (!canvasId) { + setLocalError(t("errorFallback")); + return; + } + + setLocalError(null); + setIsRetrying(true); + + try { + await toast.promise( + generateVideo({ + canvasId, + sourceNodeId: incomingEdge.source as Id<"nodes">, + outputNodeId: id as Id<"nodes">, + prompt, + modelId, + durationSeconds, + }), + { + loading: tToast("ai.generating"), + success: tToast("ai.generationQueued"), + error: tToast("ai.generationFailed"), + }, + ); + } catch (error) { + const classified = classifyError(error); + setLocalError(classified.rawMessage ?? tToast("ai.generationFailed")); + } finally { + setIsRetrying(false); + } + }, [ + generateVideo, + getEdges, + getNode, + id, + isRetrying, + nodeData.canvasId, + nodeData.durationSeconds, + nodeData.modelId, + nodeData.prompt, + syncStatus.isOffline, + t, + tToast, + ]); + + return ( + + + +
+
+
+
+ +
+ {status === "idle" && !nodeData.url ? ( +
+ {t("idleHint")} +
+ ) : null} + + {isLoading ? ( +
+ +

{t("generating")}

+
+ ) : null} + + {status === "error" && !isLoading ? ( +
+ +

+ {classifiedError.rawMessage ?? t("errorFallback")} +

+ +
+ ) : null} + + {nodeData.url && !isLoading ? ( +
+ +
+

+ {t("modelMeta", { model: modelLabel })} +

+ {typeof nodeData.durationSeconds === "number" ? ( +

{t("durationMeta", { duration: nodeData.durationSeconds })}

+ ) : null} + {typeof nodeData.creditCost === "number" ? ( +

{t("creditMeta", { credits: nodeData.creditCost })}

+ ) : null} + {nodeData.prompt ?

{nodeData.prompt}

: null} + {nodeData.url ? ( + + + {t("downloadButton")} + + ) : null} +
+ + + + ); +} diff --git a/components/canvas/nodes/video-prompt-node.tsx b/components/canvas/nodes/video-prompt-node.tsx new file mode 100644 index 0000000..7bd7ee0 --- /dev/null +++ b/components/canvas/nodes/video-prompt-node.tsx @@ -0,0 +1,418 @@ +"use client"; + +import { useCallback, useMemo, useState } from "react"; +import { Handle, Position, useReactFlow, useStore, type Node, type NodeProps } from "@xyflow/react"; +import { useAction } from "convex/react"; +import type { FunctionReference } from "convex/server"; +import { useRouter } from "next/navigation"; +import { Coins, Loader2, Sparkles, Video } from "lucide-react"; +import { useTranslations } from "next-intl"; + +import { useDebouncedCallback } from "@/hooks/use-debounced-callback"; +import { + DEFAULT_VIDEO_MODEL_ID, + getAvailableVideoModels, + getVideoModel, + isVideoModelId, + type VideoModelDurationSeconds, + type VideoModelId, +} from "@/lib/ai-video-models"; +import type { Id } from "@/convex/_generated/dataModel"; +import { useCanvasPlacement } from "@/components/canvas/canvas-placement-context"; +import { useCanvasSync } from "@/components/canvas/canvas-sync-context"; +import { useAuthQuery } from "@/hooks/use-auth-query"; +import { api } from "@/convex/_generated/api"; +import { toast } from "@/lib/toast"; +import { classifyError } from "@/lib/ai-errors"; +import BaseNodeWrapper from "./base-node-wrapper"; +import { Label } from "@/components/ui/label"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; + +type VideoPromptNodeData = { + prompt?: string; + modelId?: string; + durationSeconds?: number; + hasAudio?: boolean; + canvasId?: string; + _status?: string; + _statusMessage?: string; +}; + +export type VideoPromptNodeType = Node; + +function normalizeDuration(value: number | undefined): VideoModelDurationSeconds { + return value === 10 ? 10 : 5; +} + +export default function VideoPromptNode({ + id, + data, + selected, +}: NodeProps) { + const t = useTranslations("videoPromptNode"); + const tToast = useTranslations("toasts"); + const nodeData = data as VideoPromptNodeData; + const router = useRouter(); + const { getNode } = useReactFlow(); + const { queueNodeDataUpdate, status } = useCanvasSync(); + const { createNodeConnectedFromSource } = useCanvasPlacement(); + const balance = useAuthQuery(api.credits.getBalance); + const edges = useStore((store) => store.edges); + const nodes = useStore((store) => store.nodes); + const generateVideo = useAction( + (api as unknown as { + ai: { + generateVideo: FunctionReference< + "action", + "public", + { + canvasId: Id<"canvases">; + sourceNodeId: Id<"nodes">; + outputNodeId: Id<"nodes">; + prompt: string; + modelId: string; + durationSeconds: 5 | 10; + }, + { queued: true; outputNodeId: Id<"nodes"> } + >; + }; + }).ai.generateVideo, + ); + + const [prompt, setPrompt] = useState(nodeData.prompt ?? ""); + const [modelId, setModelId] = useState( + isVideoModelId(nodeData.modelId ?? "") + ? (nodeData.modelId as VideoModelId) + : DEFAULT_VIDEO_MODEL_ID, + ); + const [durationSeconds, setDurationSeconds] = useState( + normalizeDuration(nodeData.durationSeconds), + ); + const [isGenerating, setIsGenerating] = useState(false); + const [error, setError] = useState(null); + + const inputMeta = useMemo(() => { + const incomingEdges = edges.filter((edge) => edge.target === id); + let textPrompt: string | undefined; + let hasTextInput = false; + + for (const edge of incomingEdges) { + const sourceNode = nodes.find((node) => node.id === edge.source); + if (sourceNode?.type !== "text") continue; + hasTextInput = true; + const sourceData = sourceNode.data as { content?: string }; + if (typeof sourceData.content === "string") { + textPrompt = sourceData.content; + break; + } + } + + return { + hasTextInput, + textPrompt: textPrompt ?? "", + }; + }, [edges, id, nodes]); + + const effectivePrompt = inputMeta.hasTextInput ? inputMeta.textPrompt : prompt; + const selectedModel = getVideoModel(modelId) ?? getVideoModel(DEFAULT_VIDEO_MODEL_ID); + const creditCost = selectedModel?.creditCost[durationSeconds] ?? 0; + const availableCredits = + balance !== undefined ? balance.balance - balance.reserved : null; + const hasEnoughCredits = + availableCredits === null ? true : availableCredits >= creditCost; + + const debouncedSave = useDebouncedCallback( + ( + nextPrompt: string, + nextModelId: VideoModelId, + nextDurationSeconds: VideoModelDurationSeconds, + ) => { + const raw = data as Record; + const { _status, _statusMessage, ...rest } = raw; + void _status; + void _statusMessage; + + void queueNodeDataUpdate({ + nodeId: id as Id<"nodes">, + data: { + ...rest, + prompt: nextPrompt, + modelId: nextModelId, + durationSeconds: nextDurationSeconds, + }, + }); + }, + 500, + ); + + const handlePromptChange = useCallback( + (event: React.ChangeEvent) => { + const value = event.target.value; + setPrompt(value); + debouncedSave(value, modelId, durationSeconds); + }, + [debouncedSave, durationSeconds, modelId], + ); + + const handleModelChange = useCallback( + (value: string) => { + if (!isVideoModelId(value)) return; + setModelId(value); + debouncedSave(prompt, value, durationSeconds); + }, + [debouncedSave, durationSeconds, prompt], + ); + + const handleDurationChange = useCallback( + (value: VideoModelDurationSeconds) => { + setDurationSeconds(value); + debouncedSave(prompt, modelId, value); + }, + [debouncedSave, modelId, prompt], + ); + + const generateDisabled = + !effectivePrompt.trim() || balance === undefined || !hasEnoughCredits || isGenerating; + + const handleGenerate = useCallback(async () => { + if (!effectivePrompt.trim() || isGenerating) return; + + if (status.isOffline) { + toast.warning( + "Offline aktuell nicht unterstuetzt", + "KI-Generierung benoetigt eine aktive Verbindung.", + ); + return; + } + + if (availableCredits !== null && !hasEnoughCredits) { + toast.action(tToast("ai.insufficientCreditsTitle"), { + description: tToast("ai.insufficientCreditsDesc", { + needed: creditCost, + available: availableCredits, + }), + label: tToast("billing.topUp"), + onClick: () => router.push("/settings/billing"), + type: "warning", + }); + return; + } + + setError(null); + setIsGenerating(true); + + try { + const canvasId = nodeData.canvasId as Id<"canvases">; + if (!canvasId) { + throw new Error("Canvas-ID fehlt in der Node"); + } + + const promptToUse = effectivePrompt.trim(); + if (!promptToUse) return; + + const currentNode = getNode(id); + const offsetX = (currentNode?.measured?.width ?? 260) + 32; + const position = { + x: (currentNode?.position?.x ?? 0) + offsetX, + y: currentNode?.position?.y ?? 0, + }; + + const clientRequestId = crypto.randomUUID(); + const outputNodeId = await createNodeConnectedFromSource({ + type: "ai-video", + position, + data: { + prompt: promptToUse, + modelId, + durationSeconds, + creditCost, + canvasId, + }, + clientRequestId, + sourceNodeId: id as Id<"nodes">, + sourceHandle: "video-prompt-out", + targetHandle: "video-in", + }); + + await toast.promise( + generateVideo({ + canvasId, + sourceNodeId: id as Id<"nodes">, + outputNodeId, + prompt: promptToUse, + modelId, + durationSeconds, + }), + { + loading: tToast("ai.generating"), + success: tToast("ai.generationQueued"), + error: tToast("ai.generationFailed"), + }, + ); + } catch (err) { + const classified = classifyError(err); + + if (classified.type === "dailyCap") { + toast.error( + tToast("billing.dailyLimitReachedTitle"), + "Morgen stehen wieder Generierungen zur Verfuegung.", + ); + } else if (classified.type === "concurrency") { + toast.warning( + tToast("ai.concurrentLimitReachedTitle"), + tToast("ai.concurrentLimitReachedDesc"), + ); + } else { + setError(classified.rawMessage || tToast("ai.generationFailed")); + } + } finally { + setIsGenerating(false); + } + }, [ + availableCredits, + createNodeConnectedFromSource, + creditCost, + durationSeconds, + effectivePrompt, + generateVideo, + getNode, + hasEnoughCredits, + id, + isGenerating, + modelId, + nodeData.canvasId, + router, + status.isOffline, + tToast, + ]); + + return ( + + + +
+
+
+ + {inputMeta.hasTextInput ? ( +
+

+ {t("promptFromTextNode")} +

+

+ {inputMeta.textPrompt.trim() || t("noPromptHint")} +

+
+ ) : ( +