feat: enhance AI image generation and prompt handling in canvas components

- Introduced shimmer animation for loading states in AI image nodes.
- Updated prompt node to handle image generation with improved error handling and user feedback.
- Refactored AI image node to manage generation status and display loading indicators.
- Enhanced data handling in canvas components to include canvasId for better context management.
- Improved status message handling in Convex mutations for clearer user feedback.
This commit is contained in:
Matthias
2026-03-25 18:18:55 +01:00
parent 2f4d8a7172
commit 8d6ce275f8
10 changed files with 615 additions and 104 deletions

View File

@@ -8,6 +8,7 @@
* @module
*/
import type * as ai from "../ai.js";
import type * as auth from "../auth.js";
import type * as canvases from "../canvases.js";
import type * as credits from "../credits.js";
@@ -15,6 +16,7 @@ import type * as edges from "../edges.js";
import type * as helpers from "../helpers.js";
import type * as http from "../http.js";
import type * as nodes from "../nodes.js";
import type * as openrouter from "../openrouter.js";
import type * as storage from "../storage.js";
import type {
@@ -24,6 +26,7 @@ import type {
} from "convex/server";
declare const fullApi: ApiFromModules<{
ai: typeof ai;
auth: typeof auth;
canvases: typeof canvases;
credits: typeof credits;
@@ -31,6 +34,7 @@ declare const fullApi: ApiFromModules<{
helpers: typeof helpers;
http: typeof http;
nodes: typeof nodes;
openrouter: typeof openrouter;
storage: typeof storage;
}>;

109
convex/ai.ts Normal file
View File

@@ -0,0 +1,109 @@
import { v } from "convex/values";
import { action } from "./_generated/server";
import { api } from "./_generated/api";
import {
generateImageViaOpenRouter,
DEFAULT_IMAGE_MODEL,
IMAGE_MODELS,
} from "./openrouter";
export const generateImage = action({
args: {
canvasId: v.id("canvases"),
nodeId: v.id("nodes"),
prompt: v.string(),
referenceStorageId: v.optional(v.id("_storage")),
model: v.optional(v.string()),
},
handler: async (ctx, args) => {
const apiKey = process.env.OPENROUTER_API_KEY;
if (!apiKey) {
throw new Error("OPENROUTER_API_KEY is not set");
}
const modelId = args.model ?? DEFAULT_IMAGE_MODEL;
const modelConfig = IMAGE_MODELS[modelId];
if (!modelConfig) {
throw new Error(`Unknown model: ${modelId}`);
}
if (!(await ctx.runQuery(api.auth.getCurrentUser, {}))) {
throw new Error("User not found");
}
const reservationId = await ctx.runMutation(api.credits.reserve, {
estimatedCost: modelConfig.estimatedCostPerImage,
description: `Bildgenerierung — ${modelConfig.name}`,
model: modelId,
nodeId: args.nodeId,
canvasId: args.canvasId,
});
await ctx.runMutation(api.nodes.updateStatus, {
nodeId: args.nodeId,
status: "executing",
});
try {
let referenceImageUrl: string | undefined;
if (args.referenceStorageId) {
referenceImageUrl =
(await ctx.storage.getUrl(args.referenceStorageId)) ?? undefined;
}
const result = await generateImageViaOpenRouter(apiKey, {
prompt: args.prompt,
referenceImageUrl,
model: modelId,
});
const binaryString = atob(result.imageBase64);
const bytes = new Uint8Array(binaryString.length);
for (let i = 0; i < binaryString.length; i++) {
bytes[i] = binaryString.charCodeAt(i);
}
const blob = new Blob([bytes], { type: result.mimeType });
const storageId = await ctx.storage.store(blob);
const existing = await ctx.runQuery(api.nodes.get, { nodeId: args.nodeId });
if (!existing) throw new Error("Node not found");
const prev = (existing.data ?? {}) as Record<string, unknown>;
await ctx.runMutation(api.nodes.updateData, {
nodeId: args.nodeId,
data: {
...prev,
storageId,
prompt: args.prompt,
model: modelId,
modelTier: modelConfig.tier,
generatedAt: Date.now(),
},
});
await ctx.runMutation(api.nodes.updateStatus, {
nodeId: args.nodeId,
status: "done",
});
await ctx.runMutation(api.credits.commit, {
transactionId: reservationId,
actualCost: modelConfig.estimatedCostPerImage,
});
} catch (error) {
await ctx.runMutation(api.credits.release, {
transactionId: reservationId,
});
await ctx.runMutation(api.nodes.updateStatus, {
nodeId: args.nodeId,
status: "error",
statusMessage:
error instanceof Error ? error.message : "Generation failed",
});
throw error;
}
},
});

View File

@@ -270,7 +270,15 @@ export const updateStatus = mutation({
if (!node) throw new Error("Node not found");
await getCanvasOrThrow(ctx, node.canvasId, user.userId);
await ctx.db.patch(nodeId, { status, statusMessage });
const patch: { status: typeof status; statusMessage?: string } = {
status,
};
if (statusMessage !== undefined) {
patch.statusMessage = statusMessage;
} else if (status === "done" || status === "executing" || status === "idle") {
patch.statusMessage = undefined;
}
await ctx.db.patch(nodeId, patch);
},
});

112
convex/openrouter.ts Normal file
View File

@@ -0,0 +1,112 @@
export const OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1";
export interface OpenRouterModel {
id: string;
name: string;
tier: "budget" | "standard" | "premium";
estimatedCostPerImage: number; // in Euro-Cent (for credit reservation)
}
// Phase 1: Gemini 2.5 Flash Image only.
// Add more models here in Phase 2 when the model selector UI is built.
export const IMAGE_MODELS: Record<string, OpenRouterModel> = {
"google/gemini-2.5-flash-image": {
id: "google/gemini-2.5-flash-image",
name: "Gemini 2.5 Flash",
tier: "standard",
estimatedCostPerImage: 4, // ~€0.04 in Euro-Cent
},
};
export const DEFAULT_IMAGE_MODEL = "google/gemini-2.5-flash-image";
export interface GenerateImageParams {
prompt: string;
referenceImageUrl?: string; // optional image-to-image input
model?: string;
}
export interface OpenRouterImageResponse {
imageBase64: string; // base64-encoded PNG/JPEG
mimeType: string;
}
/**
* Calls the OpenRouter API to generate an image.
* Uses the chat/completions endpoint with a vision-capable model that returns
* an inline image in the response (base64).
*
* Must be called from a Convex Action (has access to fetch + env vars).
*/
export async function generateImageViaOpenRouter(
apiKey: string,
params: GenerateImageParams
): Promise<OpenRouterImageResponse> {
const modelId = params.model ?? DEFAULT_IMAGE_MODEL;
// Build message content — text prompt, optionally with a reference image
const userContent: object[] = [];
if (params.referenceImageUrl) {
userContent.push({
type: "image_url",
image_url: { url: params.referenceImageUrl },
});
}
userContent.push({
type: "text",
text: params.prompt,
});
const body = {
model: modelId,
modalities: ["image", "text"],
messages: [
{
role: "user",
content: userContent,
},
],
};
const response = await fetch(`${OPENROUTER_BASE_URL}/chat/completions`, {
method: "POST",
headers: {
Authorization: `Bearer ${apiKey}`,
"Content-Type": "application/json",
"HTTP-Referer": "https://app.lemonspace.io",
"X-Title": "LemonSpace",
},
body: JSON.stringify(body),
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`OpenRouter API error ${response.status}: ${errorText}`);
}
const data = await response.json();
// OpenRouter returns generated images in message.images (separate from content)
const images = data?.choices?.[0]?.message?.images;
if (!images || images.length === 0) {
throw new Error("No image found in OpenRouter response");
}
const imageUrl = images[0]?.image_url?.url;
if (!imageUrl) {
throw new Error("Image block missing image_url.url");
}
// The URL is a data URI: "data:image/png;base64,<data>"
const dataUri: string = imageUrl;
const [meta, base64Data] = dataUri.split(",");
const mimeType = meta.replace("data:", "").replace(";base64", "");
return {
imageBase64: base64Data,
mimeType: mimeType || "image/png",
};
}