feat: enhance AI image generation and prompt handling in canvas components
- Introduced shimmer animation for loading states in AI image nodes. - Updated prompt node to handle image generation with improved error handling and user feedback. - Refactored AI image node to manage generation status and display loading indicators. - Enhanced data handling in canvas components to include canvasId for better context management. - Improved status message handling in Convex mutations for clearer user feedback.
This commit is contained in:
@@ -127,4 +127,19 @@
|
|||||||
html {
|
html {
|
||||||
@apply font-sans;
|
@apply font-sans;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@keyframes shimmer {
|
||||||
|
0% {
|
||||||
|
transform: translateX(-100%);
|
||||||
|
}
|
||||||
|
100% {
|
||||||
|
transform: translateX(100%);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@layer utilities {
|
||||||
|
.animate-shimmer {
|
||||||
|
animation: shimmer 1.5s ease-in-out infinite;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -26,7 +26,7 @@ const nodeTemplates = [
|
|||||||
label: "Prompt",
|
label: "Prompt",
|
||||||
width: 320,
|
width: 320,
|
||||||
height: 140,
|
height: 140,
|
||||||
defaultData: { content: "", model: "" },
|
defaultData: { prompt: "", model: "" },
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
type: "note",
|
type: "note",
|
||||||
@@ -67,7 +67,7 @@ export default function CanvasToolbar({ canvasId }: CanvasToolbarProps) {
|
|||||||
positionY: 100 + offset,
|
positionY: 100 + offset,
|
||||||
width,
|
width,
|
||||||
height,
|
height,
|
||||||
data,
|
data: { ...data, canvasId },
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -192,7 +192,7 @@ function CanvasInner({ canvasId }: CanvasInnerProps) {
|
|||||||
positionY: position.y,
|
positionY: position.y,
|
||||||
width: defaults.width,
|
width: defaults.width,
|
||||||
height: defaults.height,
|
height: defaults.height,
|
||||||
data: defaults.data,
|
data: { ...defaults.data, canvasId },
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
[screenToFlowPosition, createNode, canvasId],
|
[screenToFlowPosition, createNode, canvasId],
|
||||||
|
|||||||
@@ -1,78 +1,209 @@
|
|||||||
"use client";
|
"use client";
|
||||||
|
|
||||||
import { Handle, Position, type NodeProps, type Node } from "@xyflow/react";
|
import { useCallback, useState } from "react";
|
||||||
|
import { Handle, Position, useReactFlow, type NodeProps, type Node } from "@xyflow/react";
|
||||||
|
import { useAction } from "convex/react";
|
||||||
|
import { api } from "@/convex/_generated/api";
|
||||||
|
import type { Id } from "@/convex/_generated/dataModel";
|
||||||
import BaseNodeWrapper from "./base-node-wrapper";
|
import BaseNodeWrapper from "./base-node-wrapper";
|
||||||
|
import { DEFAULT_MODEL_ID, getModel } from "@/lib/ai-models";
|
||||||
|
import {
|
||||||
|
Loader2,
|
||||||
|
AlertCircle,
|
||||||
|
RefreshCw,
|
||||||
|
ImageIcon,
|
||||||
|
} from "lucide-react";
|
||||||
|
|
||||||
type AiImageNodeData = {
|
type AiImageNodeData = {
|
||||||
|
storageId?: string;
|
||||||
url?: string;
|
url?: string;
|
||||||
prompt?: string;
|
prompt?: string;
|
||||||
model?: string;
|
model?: string;
|
||||||
|
modelTier?: string;
|
||||||
|
generatedAt?: number;
|
||||||
|
canvasId?: string;
|
||||||
_status?: string;
|
_status?: string;
|
||||||
_statusMessage?: string;
|
_statusMessage?: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
export type AiImageNode = Node<AiImageNodeData, "ai-image">;
|
export type AiImageNode = Node<AiImageNodeData, "ai-image">;
|
||||||
|
|
||||||
|
type NodeStatus =
|
||||||
|
| "idle"
|
||||||
|
| "analyzing"
|
||||||
|
| "clarifying"
|
||||||
|
| "executing"
|
||||||
|
| "done"
|
||||||
|
| "error";
|
||||||
|
|
||||||
export default function AiImageNode({
|
export default function AiImageNode({
|
||||||
|
id,
|
||||||
data,
|
data,
|
||||||
selected,
|
selected,
|
||||||
}: NodeProps<AiImageNode>) {
|
}: NodeProps<AiImageNode>) {
|
||||||
const status = data._status ?? "idle";
|
const nodeData = data as AiImageNodeData;
|
||||||
|
const { getEdges, getNode } = useReactFlow();
|
||||||
|
|
||||||
|
const [isGenerating, setIsGenerating] = useState(false);
|
||||||
|
const [localError, setLocalError] = useState<string | null>(null);
|
||||||
|
|
||||||
|
const generateImage = useAction(api.ai.generateImage);
|
||||||
|
|
||||||
|
const status = (nodeData._status ?? "idle") as NodeStatus;
|
||||||
|
const errorMessage = nodeData._statusMessage;
|
||||||
|
|
||||||
|
const isLoading =
|
||||||
|
status === "executing" ||
|
||||||
|
status === "analyzing" ||
|
||||||
|
status === "clarifying" ||
|
||||||
|
isGenerating;
|
||||||
|
|
||||||
|
const handleRegenerate = useCallback(async () => {
|
||||||
|
if (isLoading) return;
|
||||||
|
setLocalError(null);
|
||||||
|
setIsGenerating(true);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const canvasId = nodeData.canvasId as Id<"canvases">;
|
||||||
|
if (!canvasId) throw new Error("Missing canvasId");
|
||||||
|
|
||||||
|
const prompt = nodeData.prompt;
|
||||||
|
if (!prompt) throw new Error("No prompt — use Generate from a Prompt node");
|
||||||
|
|
||||||
|
const edges = getEdges();
|
||||||
|
const incomingEdges = edges.filter((e) => e.target === id);
|
||||||
|
let referenceStorageId: Id<"_storage"> | undefined;
|
||||||
|
for (const edge of incomingEdges) {
|
||||||
|
const src = getNode(edge.source);
|
||||||
|
if (src?.type === "image") {
|
||||||
|
const srcData = src.data as { storageId?: string };
|
||||||
|
if (srcData.storageId) {
|
||||||
|
referenceStorageId = srcData.storageId as Id<"_storage">;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await generateImage({
|
||||||
|
canvasId,
|
||||||
|
nodeId: id as Id<"nodes">,
|
||||||
|
prompt,
|
||||||
|
referenceStorageId,
|
||||||
|
model: nodeData.model ?? DEFAULT_MODEL_ID,
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
setLocalError(err instanceof Error ? err.message : "Generation failed");
|
||||||
|
} finally {
|
||||||
|
setIsGenerating(false);
|
||||||
|
}
|
||||||
|
}, [isLoading, nodeData, id, getEdges, getNode, generateImage]);
|
||||||
|
|
||||||
|
const modelName =
|
||||||
|
getModel(nodeData.model ?? DEFAULT_MODEL_ID)?.name ?? "AI";
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<BaseNodeWrapper
|
<BaseNodeWrapper selected={selected} className="w-[320px] overflow-hidden">
|
||||||
selected={selected}
|
<Handle
|
||||||
status={status}
|
type="target"
|
||||||
statusMessage={data._statusMessage}
|
position={Position.Left}
|
||||||
>
|
id="prompt-in"
|
||||||
<div className="p-2">
|
className="!h-3 !w-3 !bg-violet-500 !border-2 !border-background"
|
||||||
<div className="text-xs font-medium text-emerald-500 mb-1">
|
/>
|
||||||
🤖 KI-Bild
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{status === "executing" && (
|
<div className="border-b border-border px-3 py-2">
|
||||||
<div className="flex h-36 w-56 items-center justify-center rounded-lg bg-muted">
|
<div className="text-xs font-medium text-emerald-600 dark:text-emerald-400">
|
||||||
<div className="h-6 w-6 animate-spin rounded-full border-2 border-primary border-t-transparent" />
|
🖼️ AI Image
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="relative h-[320px] overflow-hidden bg-muted">
|
||||||
|
{status === "idle" && !nodeData.url && (
|
||||||
|
<div className="absolute inset-0 flex flex-col items-center justify-center gap-3 text-muted-foreground">
|
||||||
|
<ImageIcon className="h-10 w-10 opacity-30" />
|
||||||
|
<p className="px-6 text-center text-xs opacity-60">
|
||||||
|
Connect a Prompt node and click Generate
|
||||||
|
</p>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{status === "done" && data.url && (
|
{isLoading && (
|
||||||
|
<div className="absolute inset-0 z-10 flex flex-col items-center justify-center gap-3 bg-muted">
|
||||||
|
<div className="absolute inset-0 overflow-hidden">
|
||||||
|
<div className="animate-shimmer absolute inset-0 bg-gradient-to-r from-transparent via-white/10 to-transparent" />
|
||||||
|
</div>
|
||||||
|
<Loader2 className="relative z-10 h-8 w-8 animate-spin text-violet-500" />
|
||||||
|
<p className="relative z-10 text-xs text-muted-foreground">
|
||||||
|
{status === "analyzing" && "Analyzing…"}
|
||||||
|
{status === "clarifying" && "Clarifying…"}
|
||||||
|
{(status === "executing" || isGenerating) && "Generating…"}
|
||||||
|
</p>
|
||||||
|
<p className="relative z-10 text-[10px] text-muted-foreground/60">
|
||||||
|
{modelName}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{status === "error" && !isLoading && (
|
||||||
|
<div className="absolute inset-0 z-10 flex flex-col items-center justify-center gap-3 bg-muted">
|
||||||
|
<AlertCircle className="h-8 w-8 text-destructive" />
|
||||||
|
<p className="px-4 text-center text-xs font-medium text-destructive">
|
||||||
|
Generation failed
|
||||||
|
</p>
|
||||||
|
<p className="px-6 text-center text-[10px] text-muted-foreground">
|
||||||
|
{errorMessage ?? localError ?? "Unknown error"} — Credits not
|
||||||
|
charged
|
||||||
|
</p>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={() => void handleRegenerate()}
|
||||||
|
className="nodrag mt-1 flex items-center gap-1.5 rounded-md border border-border bg-background px-3 py-1.5 text-xs font-medium transition-colors hover:bg-accent"
|
||||||
|
>
|
||||||
|
<RefreshCw className="h-3 w-3" />
|
||||||
|
Try again
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{nodeData.url && !isLoading && (
|
||||||
|
// eslint-disable-next-line @next/next/no-img-element
|
||||||
<img
|
<img
|
||||||
src={data.url}
|
src={nodeData.url}
|
||||||
alt={data.prompt ?? "KI-generiertes Bild"}
|
alt={nodeData.prompt ?? "AI generated image"}
|
||||||
className="rounded-lg object-cover max-w-[260px]"
|
className="absolute inset-0 h-full w-full object-contain"
|
||||||
draggable={false}
|
draggable={false}
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{status === "error" && (
|
{status === "done" && nodeData.url && !isLoading && (
|
||||||
<div className="flex h-36 w-56 items-center justify-center rounded-lg bg-red-50 dark:bg-red-950/20 text-sm text-red-600">
|
<div className="absolute inset-0 z-20 flex items-end justify-end p-2 opacity-0 transition-opacity hover:opacity-100">
|
||||||
{data._statusMessage ?? "Fehler bei der Generierung"}
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={() => void handleRegenerate()}
|
||||||
|
className="nodrag flex items-center gap-1.5 rounded-md border border-border bg-background/90 px-2.5 py-1.5 text-xs font-medium backdrop-blur-sm transition-colors hover:bg-background"
|
||||||
|
>
|
||||||
|
<RefreshCw className="h-3 w-3" />
|
||||||
|
Regenerate
|
||||||
|
</button>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{status === "idle" && (
|
|
||||||
<div className="flex h-36 w-56 items-center justify-center rounded-lg border-2 border-dashed text-sm text-muted-foreground">
|
|
||||||
Prompt verbinden
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
|
|
||||||
{data.prompt && status === "done" && (
|
|
||||||
<p className="mt-1 text-xs text-muted-foreground truncate max-w-[260px]">
|
|
||||||
{data.prompt}
|
|
||||||
</p>
|
|
||||||
)}
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<Handle
|
{nodeData.prompt && (
|
||||||
type="target"
|
<div className="border-t border-border px-3 py-2">
|
||||||
position={Position.Left}
|
<p className="line-clamp-2 text-[10px] text-muted-foreground">
|
||||||
className="!h-3 !w-3 !bg-emerald-500 !border-2 !border-background"
|
{nodeData.prompt}
|
||||||
/>
|
</p>
|
||||||
|
<p className="mt-0.5 text-[10px] text-muted-foreground/60">
|
||||||
|
{modelName}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
<Handle
|
<Handle
|
||||||
type="source"
|
type="source"
|
||||||
position={Position.Right}
|
position={Position.Right}
|
||||||
className="!h-3 !w-3 !bg-primary !border-2 !border-background"
|
id="image-out"
|
||||||
|
className="!h-3 !w-3 !bg-violet-500 !border-2 !border-background"
|
||||||
/>
|
/>
|
||||||
</BaseNodeWrapper>
|
</BaseNodeWrapper>
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -1,16 +1,19 @@
|
|||||||
"use client";
|
"use client";
|
||||||
|
|
||||||
import { useState, useCallback, useEffect } from "react";
|
import { useCallback, useEffect, useRef, useState } from "react";
|
||||||
import { Handle, Position, type NodeProps, type Node } from "@xyflow/react";
|
import { Handle, Position, useReactFlow, type NodeProps, type Node } from "@xyflow/react";
|
||||||
import { useMutation } from "convex/react";
|
import { useMutation, useAction } from "convex/react";
|
||||||
import { api } from "@/convex/_generated/api";
|
import { api } from "@/convex/_generated/api";
|
||||||
import type { Id } from "@/convex/_generated/dataModel";
|
import type { Id } from "@/convex/_generated/dataModel";
|
||||||
import { useDebouncedCallback } from "@/hooks/use-debounced-callback";
|
|
||||||
import BaseNodeWrapper from "./base-node-wrapper";
|
import BaseNodeWrapper from "./base-node-wrapper";
|
||||||
|
import { useDebouncedCallback } from "@/hooks/use-debounced-callback";
|
||||||
|
import { DEFAULT_MODEL_ID } from "@/lib/ai-models";
|
||||||
|
import { Sparkles, Loader2 } from "lucide-react";
|
||||||
|
|
||||||
type PromptNodeData = {
|
type PromptNodeData = {
|
||||||
prompt?: string;
|
prompt?: string;
|
||||||
model?: string;
|
model?: string;
|
||||||
|
canvasId?: string;
|
||||||
_status?: string;
|
_status?: string;
|
||||||
_statusMessage?: string;
|
_statusMessage?: string;
|
||||||
};
|
};
|
||||||
@@ -22,82 +25,166 @@ export default function PromptNode({
|
|||||||
data,
|
data,
|
||||||
selected,
|
selected,
|
||||||
}: NodeProps<PromptNode>) {
|
}: NodeProps<PromptNode>) {
|
||||||
const updateData = useMutation(api.nodes.updateData);
|
const nodeData = data as PromptNodeData;
|
||||||
const [prompt, setPrompt] = useState(data.prompt ?? "");
|
const { getEdges, getNode } = useReactFlow();
|
||||||
const [isEditing, setIsEditing] = useState(false);
|
|
||||||
|
const [prompt, setPrompt] = useState(nodeData.prompt ?? "");
|
||||||
|
const [isGenerating, setIsGenerating] = useState(false);
|
||||||
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!isEditing) {
|
setPrompt(nodeData.prompt ?? "");
|
||||||
setPrompt(data.prompt ?? "");
|
}, [nodeData.prompt]);
|
||||||
}
|
|
||||||
}, [data.prompt, isEditing]);
|
|
||||||
|
|
||||||
const savePrompt = useDebouncedCallback(
|
const dataRef = useRef(data);
|
||||||
(newPrompt: string) => {
|
dataRef.current = data;
|
||||||
updateData({
|
|
||||||
nodeId: id as Id<"nodes">,
|
const updateData = useMutation(api.nodes.updateData);
|
||||||
|
const createNode = useMutation(api.nodes.create);
|
||||||
|
const generateImage = useAction(api.ai.generateImage);
|
||||||
|
|
||||||
|
const debouncedSave = useDebouncedCallback((value: string) => {
|
||||||
|
const raw = dataRef.current as Record<string, unknown>;
|
||||||
|
const { _status, _statusMessage, ...rest } = raw;
|
||||||
|
void _status;
|
||||||
|
void _statusMessage;
|
||||||
|
updateData({
|
||||||
|
nodeId: id as Id<"nodes">,
|
||||||
|
data: { ...rest, prompt: value },
|
||||||
|
});
|
||||||
|
}, 500);
|
||||||
|
|
||||||
|
const handlePromptChange = useCallback(
|
||||||
|
(e: React.ChangeEvent<HTMLTextAreaElement>) => {
|
||||||
|
const value = e.target.value;
|
||||||
|
setPrompt(value);
|
||||||
|
debouncedSave(value);
|
||||||
|
},
|
||||||
|
[debouncedSave]
|
||||||
|
);
|
||||||
|
|
||||||
|
const handleGenerate = useCallback(async () => {
|
||||||
|
if (!prompt.trim() || isGenerating) return;
|
||||||
|
setError(null);
|
||||||
|
setIsGenerating(true);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const canvasId = nodeData.canvasId as Id<"canvases">;
|
||||||
|
if (!canvasId) throw new Error("Missing canvasId on node");
|
||||||
|
|
||||||
|
const edges = getEdges();
|
||||||
|
const incomingEdges = edges.filter((e) => e.target === id);
|
||||||
|
let referenceStorageId: Id<"_storage"> | undefined;
|
||||||
|
|
||||||
|
for (const edge of incomingEdges) {
|
||||||
|
const sourceNode = getNode(edge.source);
|
||||||
|
if (sourceNode?.type === "image") {
|
||||||
|
const srcData = sourceNode.data as { storageId?: string };
|
||||||
|
if (srcData.storageId) {
|
||||||
|
referenceStorageId = srcData.storageId as Id<"_storage">;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const currentNode = getNode(id);
|
||||||
|
const offsetX = (currentNode?.measured?.width ?? 280) + 32;
|
||||||
|
const posX = (currentNode?.position?.x ?? 0) + offsetX;
|
||||||
|
const posY = currentNode?.position?.y ?? 0;
|
||||||
|
|
||||||
|
const aiNodeId = await createNode({
|
||||||
|
canvasId,
|
||||||
|
type: "ai-image",
|
||||||
|
positionX: posX,
|
||||||
|
positionY: posY,
|
||||||
|
width: 320,
|
||||||
|
height: 320,
|
||||||
data: {
|
data: {
|
||||||
...data,
|
prompt,
|
||||||
prompt: newPrompt,
|
model: DEFAULT_MODEL_ID,
|
||||||
_status: undefined,
|
modelTier: "standard",
|
||||||
_statusMessage: undefined,
|
canvasId,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
},
|
|
||||||
500,
|
|
||||||
);
|
|
||||||
|
|
||||||
const handleChange = useCallback(
|
await generateImage({
|
||||||
(e: React.ChangeEvent<HTMLTextAreaElement>) => {
|
canvasId,
|
||||||
const newPrompt = e.target.value;
|
nodeId: aiNodeId,
|
||||||
setPrompt(newPrompt);
|
prompt,
|
||||||
savePrompt(newPrompt);
|
referenceStorageId,
|
||||||
},
|
model: DEFAULT_MODEL_ID,
|
||||||
[savePrompt],
|
});
|
||||||
);
|
} catch (err) {
|
||||||
|
setError(err instanceof Error ? err.message : "Generation failed");
|
||||||
|
} finally {
|
||||||
|
setIsGenerating(false);
|
||||||
|
}
|
||||||
|
}, [
|
||||||
|
prompt,
|
||||||
|
isGenerating,
|
||||||
|
nodeData.canvasId,
|
||||||
|
id,
|
||||||
|
getEdges,
|
||||||
|
getNode,
|
||||||
|
createNode,
|
||||||
|
generateImage,
|
||||||
|
]);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<BaseNodeWrapper
|
<BaseNodeWrapper
|
||||||
selected={selected}
|
selected={selected}
|
||||||
status={data._status}
|
status={nodeData._status}
|
||||||
className="border-purple-500/30"
|
statusMessage={nodeData._statusMessage}
|
||||||
|
className="min-w-[240px] border-violet-500/30"
|
||||||
>
|
>
|
||||||
<div className="w-72 p-3">
|
<Handle
|
||||||
<div className="text-xs font-medium text-purple-500 mb-1">
|
type="target"
|
||||||
|
position={Position.Left}
|
||||||
|
id="image-in"
|
||||||
|
className="!h-3 !w-3 !bg-violet-500 !border-2 !border-background"
|
||||||
|
/>
|
||||||
|
|
||||||
|
<div className="flex flex-col gap-2 p-3">
|
||||||
|
<div className="text-xs font-medium text-violet-600 dark:text-violet-400">
|
||||||
✨ Prompt
|
✨ Prompt
|
||||||
</div>
|
</div>
|
||||||
{isEditing ? (
|
<textarea
|
||||||
<textarea
|
value={prompt}
|
||||||
value={prompt}
|
onChange={handlePromptChange}
|
||||||
onChange={handleChange}
|
placeholder="Describe what you want to generate…"
|
||||||
onBlur={() => setIsEditing(false)}
|
rows={4}
|
||||||
autoFocus
|
className="nodrag nowheel w-full resize-none rounded-md border border-border bg-background px-3 py-2 text-sm placeholder:text-muted-foreground focus:outline-none focus:ring-1 focus:ring-violet-500"
|
||||||
className="nodrag nowheel w-full resize-none rounded-md border-0 bg-transparent p-0 text-sm outline-none focus:ring-0 min-h-[3rem]"
|
/>
|
||||||
placeholder="Prompt eingeben…"
|
|
||||||
rows={4}
|
{error && (
|
||||||
/>
|
<p className="text-xs text-destructive">{error}</p>
|
||||||
) : (
|
|
||||||
<div
|
|
||||||
onDoubleClick={() => setIsEditing(true)}
|
|
||||||
className="min-h-[2rem] cursor-text text-sm whitespace-pre-wrap"
|
|
||||||
>
|
|
||||||
{prompt || (
|
|
||||||
<span className="text-muted-foreground">
|
|
||||||
Doppelklick zum Bearbeiten
|
|
||||||
</span>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
{data.model && (
|
|
||||||
<div className="mt-2 text-xs text-muted-foreground">
|
|
||||||
Modell: {data.model}
|
|
||||||
</div>
|
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={() => void handleGenerate()}
|
||||||
|
disabled={!prompt.trim() || isGenerating}
|
||||||
|
className="nodrag flex items-center justify-center gap-2 rounded-md bg-violet-600 px-3 py-2 text-sm font-medium text-white transition-colors hover:bg-violet-700 disabled:cursor-not-allowed disabled:opacity-50"
|
||||||
|
>
|
||||||
|
{isGenerating ? (
|
||||||
|
<>
|
||||||
|
<Loader2 className="h-4 w-4 animate-spin" />
|
||||||
|
Generating…
|
||||||
|
</>
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
|
<Sparkles className="h-4 w-4" />
|
||||||
|
Generate Image
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</button>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<Handle
|
<Handle
|
||||||
type="source"
|
type="source"
|
||||||
position={Position.Right}
|
position={Position.Right}
|
||||||
className="!h-3 !w-3 !bg-purple-500 !border-2 !border-background"
|
id="prompt-out"
|
||||||
|
className="!h-3 !w-3 !bg-violet-500 !border-2 !border-background"
|
||||||
/>
|
/>
|
||||||
</BaseNodeWrapper>
|
</BaseNodeWrapper>
|
||||||
);
|
);
|
||||||
|
|||||||
4
convex/_generated/api.d.ts
vendored
4
convex/_generated/api.d.ts
vendored
@@ -8,6 +8,7 @@
|
|||||||
* @module
|
* @module
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
import type * as ai from "../ai.js";
|
||||||
import type * as auth from "../auth.js";
|
import type * as auth from "../auth.js";
|
||||||
import type * as canvases from "../canvases.js";
|
import type * as canvases from "../canvases.js";
|
||||||
import type * as credits from "../credits.js";
|
import type * as credits from "../credits.js";
|
||||||
@@ -15,6 +16,7 @@ import type * as edges from "../edges.js";
|
|||||||
import type * as helpers from "../helpers.js";
|
import type * as helpers from "../helpers.js";
|
||||||
import type * as http from "../http.js";
|
import type * as http from "../http.js";
|
||||||
import type * as nodes from "../nodes.js";
|
import type * as nodes from "../nodes.js";
|
||||||
|
import type * as openrouter from "../openrouter.js";
|
||||||
import type * as storage from "../storage.js";
|
import type * as storage from "../storage.js";
|
||||||
|
|
||||||
import type {
|
import type {
|
||||||
@@ -24,6 +26,7 @@ import type {
|
|||||||
} from "convex/server";
|
} from "convex/server";
|
||||||
|
|
||||||
declare const fullApi: ApiFromModules<{
|
declare const fullApi: ApiFromModules<{
|
||||||
|
ai: typeof ai;
|
||||||
auth: typeof auth;
|
auth: typeof auth;
|
||||||
canvases: typeof canvases;
|
canvases: typeof canvases;
|
||||||
credits: typeof credits;
|
credits: typeof credits;
|
||||||
@@ -31,6 +34,7 @@ declare const fullApi: ApiFromModules<{
|
|||||||
helpers: typeof helpers;
|
helpers: typeof helpers;
|
||||||
http: typeof http;
|
http: typeof http;
|
||||||
nodes: typeof nodes;
|
nodes: typeof nodes;
|
||||||
|
openrouter: typeof openrouter;
|
||||||
storage: typeof storage;
|
storage: typeof storage;
|
||||||
}>;
|
}>;
|
||||||
|
|
||||||
|
|||||||
109
convex/ai.ts
Normal file
109
convex/ai.ts
Normal file
@@ -0,0 +1,109 @@
|
|||||||
|
import { v } from "convex/values";
|
||||||
|
import { action } from "./_generated/server";
|
||||||
|
import { api } from "./_generated/api";
|
||||||
|
import {
|
||||||
|
generateImageViaOpenRouter,
|
||||||
|
DEFAULT_IMAGE_MODEL,
|
||||||
|
IMAGE_MODELS,
|
||||||
|
} from "./openrouter";
|
||||||
|
|
||||||
|
export const generateImage = action({
|
||||||
|
args: {
|
||||||
|
canvasId: v.id("canvases"),
|
||||||
|
nodeId: v.id("nodes"),
|
||||||
|
prompt: v.string(),
|
||||||
|
referenceStorageId: v.optional(v.id("_storage")),
|
||||||
|
model: v.optional(v.string()),
|
||||||
|
},
|
||||||
|
handler: async (ctx, args) => {
|
||||||
|
const apiKey = process.env.OPENROUTER_API_KEY;
|
||||||
|
if (!apiKey) {
|
||||||
|
throw new Error("OPENROUTER_API_KEY is not set");
|
||||||
|
}
|
||||||
|
|
||||||
|
const modelId = args.model ?? DEFAULT_IMAGE_MODEL;
|
||||||
|
const modelConfig = IMAGE_MODELS[modelId];
|
||||||
|
if (!modelConfig) {
|
||||||
|
throw new Error(`Unknown model: ${modelId}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!(await ctx.runQuery(api.auth.getCurrentUser, {}))) {
|
||||||
|
throw new Error("User not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
const reservationId = await ctx.runMutation(api.credits.reserve, {
|
||||||
|
estimatedCost: modelConfig.estimatedCostPerImage,
|
||||||
|
description: `Bildgenerierung — ${modelConfig.name}`,
|
||||||
|
model: modelId,
|
||||||
|
nodeId: args.nodeId,
|
||||||
|
canvasId: args.canvasId,
|
||||||
|
});
|
||||||
|
|
||||||
|
await ctx.runMutation(api.nodes.updateStatus, {
|
||||||
|
nodeId: args.nodeId,
|
||||||
|
status: "executing",
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
let referenceImageUrl: string | undefined;
|
||||||
|
if (args.referenceStorageId) {
|
||||||
|
referenceImageUrl =
|
||||||
|
(await ctx.storage.getUrl(args.referenceStorageId)) ?? undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await generateImageViaOpenRouter(apiKey, {
|
||||||
|
prompt: args.prompt,
|
||||||
|
referenceImageUrl,
|
||||||
|
model: modelId,
|
||||||
|
});
|
||||||
|
|
||||||
|
const binaryString = atob(result.imageBase64);
|
||||||
|
const bytes = new Uint8Array(binaryString.length);
|
||||||
|
for (let i = 0; i < binaryString.length; i++) {
|
||||||
|
bytes[i] = binaryString.charCodeAt(i);
|
||||||
|
}
|
||||||
|
|
||||||
|
const blob = new Blob([bytes], { type: result.mimeType });
|
||||||
|
const storageId = await ctx.storage.store(blob);
|
||||||
|
|
||||||
|
const existing = await ctx.runQuery(api.nodes.get, { nodeId: args.nodeId });
|
||||||
|
if (!existing) throw new Error("Node not found");
|
||||||
|
const prev = (existing.data ?? {}) as Record<string, unknown>;
|
||||||
|
|
||||||
|
await ctx.runMutation(api.nodes.updateData, {
|
||||||
|
nodeId: args.nodeId,
|
||||||
|
data: {
|
||||||
|
...prev,
|
||||||
|
storageId,
|
||||||
|
prompt: args.prompt,
|
||||||
|
model: modelId,
|
||||||
|
modelTier: modelConfig.tier,
|
||||||
|
generatedAt: Date.now(),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
await ctx.runMutation(api.nodes.updateStatus, {
|
||||||
|
nodeId: args.nodeId,
|
||||||
|
status: "done",
|
||||||
|
});
|
||||||
|
|
||||||
|
await ctx.runMutation(api.credits.commit, {
|
||||||
|
transactionId: reservationId,
|
||||||
|
actualCost: modelConfig.estimatedCostPerImage,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
await ctx.runMutation(api.credits.release, {
|
||||||
|
transactionId: reservationId,
|
||||||
|
});
|
||||||
|
|
||||||
|
await ctx.runMutation(api.nodes.updateStatus, {
|
||||||
|
nodeId: args.nodeId,
|
||||||
|
status: "error",
|
||||||
|
statusMessage:
|
||||||
|
error instanceof Error ? error.message : "Generation failed",
|
||||||
|
});
|
||||||
|
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
@@ -270,7 +270,15 @@ export const updateStatus = mutation({
|
|||||||
if (!node) throw new Error("Node not found");
|
if (!node) throw new Error("Node not found");
|
||||||
|
|
||||||
await getCanvasOrThrow(ctx, node.canvasId, user.userId);
|
await getCanvasOrThrow(ctx, node.canvasId, user.userId);
|
||||||
await ctx.db.patch(nodeId, { status, statusMessage });
|
const patch: { status: typeof status; statusMessage?: string } = {
|
||||||
|
status,
|
||||||
|
};
|
||||||
|
if (statusMessage !== undefined) {
|
||||||
|
patch.statusMessage = statusMessage;
|
||||||
|
} else if (status === "done" || status === "executing" || status === "idle") {
|
||||||
|
patch.statusMessage = undefined;
|
||||||
|
}
|
||||||
|
await ctx.db.patch(nodeId, patch);
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
112
convex/openrouter.ts
Normal file
112
convex/openrouter.ts
Normal file
@@ -0,0 +1,112 @@
|
|||||||
|
export const OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1";
|
||||||
|
|
||||||
|
export interface OpenRouterModel {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
tier: "budget" | "standard" | "premium";
|
||||||
|
estimatedCostPerImage: number; // in Euro-Cent (for credit reservation)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Phase 1: Gemini 2.5 Flash Image only.
|
||||||
|
// Add more models here in Phase 2 when the model selector UI is built.
|
||||||
|
export const IMAGE_MODELS: Record<string, OpenRouterModel> = {
|
||||||
|
"google/gemini-2.5-flash-image": {
|
||||||
|
id: "google/gemini-2.5-flash-image",
|
||||||
|
name: "Gemini 2.5 Flash",
|
||||||
|
tier: "standard",
|
||||||
|
estimatedCostPerImage: 4, // ~€0.04 in Euro-Cent
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
export const DEFAULT_IMAGE_MODEL = "google/gemini-2.5-flash-image";
|
||||||
|
|
||||||
|
export interface GenerateImageParams {
|
||||||
|
prompt: string;
|
||||||
|
referenceImageUrl?: string; // optional image-to-image input
|
||||||
|
model?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface OpenRouterImageResponse {
|
||||||
|
imageBase64: string; // base64-encoded PNG/JPEG
|
||||||
|
mimeType: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calls the OpenRouter API to generate an image.
|
||||||
|
* Uses the chat/completions endpoint with a vision-capable model that returns
|
||||||
|
* an inline image in the response (base64).
|
||||||
|
*
|
||||||
|
* Must be called from a Convex Action (has access to fetch + env vars).
|
||||||
|
*/
|
||||||
|
export async function generateImageViaOpenRouter(
|
||||||
|
apiKey: string,
|
||||||
|
params: GenerateImageParams
|
||||||
|
): Promise<OpenRouterImageResponse> {
|
||||||
|
const modelId = params.model ?? DEFAULT_IMAGE_MODEL;
|
||||||
|
|
||||||
|
// Build message content — text prompt, optionally with a reference image
|
||||||
|
const userContent: object[] = [];
|
||||||
|
|
||||||
|
if (params.referenceImageUrl) {
|
||||||
|
userContent.push({
|
||||||
|
type: "image_url",
|
||||||
|
image_url: { url: params.referenceImageUrl },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
userContent.push({
|
||||||
|
type: "text",
|
||||||
|
text: params.prompt,
|
||||||
|
});
|
||||||
|
|
||||||
|
const body = {
|
||||||
|
model: modelId,
|
||||||
|
modalities: ["image", "text"],
|
||||||
|
messages: [
|
||||||
|
{
|
||||||
|
role: "user",
|
||||||
|
content: userContent,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
const response = await fetch(`${OPENROUTER_BASE_URL}/chat/completions`, {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
Authorization: `Bearer ${apiKey}`,
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"HTTP-Referer": "https://app.lemonspace.io",
|
||||||
|
"X-Title": "LemonSpace",
|
||||||
|
},
|
||||||
|
body: JSON.stringify(body),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
const errorText = await response.text();
|
||||||
|
throw new Error(`OpenRouter API error ${response.status}: ${errorText}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const data = await response.json();
|
||||||
|
|
||||||
|
// OpenRouter returns generated images in message.images (separate from content)
|
||||||
|
const images = data?.choices?.[0]?.message?.images;
|
||||||
|
|
||||||
|
if (!images || images.length === 0) {
|
||||||
|
throw new Error("No image found in OpenRouter response");
|
||||||
|
}
|
||||||
|
|
||||||
|
const imageUrl = images[0]?.image_url?.url;
|
||||||
|
if (!imageUrl) {
|
||||||
|
throw new Error("Image block missing image_url.url");
|
||||||
|
}
|
||||||
|
|
||||||
|
// The URL is a data URI: "data:image/png;base64,<data>"
|
||||||
|
const dataUri: string = imageUrl;
|
||||||
|
const [meta, base64Data] = dataUri.split(",");
|
||||||
|
const mimeType = meta.replace("data:", "").replace(";base64", "");
|
||||||
|
|
||||||
|
return {
|
||||||
|
imageBase64: base64Data,
|
||||||
|
mimeType: mimeType || "image/png",
|
||||||
|
};
|
||||||
|
}
|
||||||
45
lib/ai-models.ts
Normal file
45
lib/ai-models.ts
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
// Client-side model definitions for the UI.
|
||||||
|
// Mirrors the backend config in convex/openrouter.ts — keep in sync.
|
||||||
|
|
||||||
|
export interface AiModel {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
tier: "budget" | "standard" | "premium";
|
||||||
|
description: string;
|
||||||
|
estimatedCost: string; // human-readable, e.g. "~€0.04"
|
||||||
|
minTier: "free" | "starter" | "pro" | "business"; // minimum subscription tier
|
||||||
|
}
|
||||||
|
|
||||||
|
export const IMAGE_MODELS: AiModel[] = [
|
||||||
|
{
|
||||||
|
id: "google/gemini-2.5-flash-image",
|
||||||
|
name: "Gemini 2.5 Flash",
|
||||||
|
tier: "standard",
|
||||||
|
description: "Fast, high-quality generation",
|
||||||
|
estimatedCost: "~€0.04",
|
||||||
|
minTier: "free",
|
||||||
|
},
|
||||||
|
// Phase 2 — uncomment when model selector UI is ready:
|
||||||
|
// {
|
||||||
|
// id: "black-forest-labs/flux.2-klein-4b",
|
||||||
|
// name: "FLUX.2 Klein",
|
||||||
|
// tier: "budget",
|
||||||
|
// description: "Photorealism, fastest Flux",
|
||||||
|
// estimatedCost: "~€0.02",
|
||||||
|
// minTier: "free",
|
||||||
|
// },
|
||||||
|
// {
|
||||||
|
// id: "openai/gpt-5-image",
|
||||||
|
// name: "GPT-5 Image",
|
||||||
|
// tier: "premium",
|
||||||
|
// description: "Best instruction following, text in image",
|
||||||
|
// estimatedCost: "~€0.15",
|
||||||
|
// minTier: "starter",
|
||||||
|
// },
|
||||||
|
];
|
||||||
|
|
||||||
|
export const DEFAULT_MODEL_ID = "google/gemini-2.5-flash-image";
|
||||||
|
|
||||||
|
export function getModel(id: string): AiModel | undefined {
|
||||||
|
return IMAGE_MODELS.find((m) => m.id === id);
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user