feat: enhance AI image generation and prompt handling in canvas components
- Introduced shimmer animation for loading states in AI image nodes. - Updated prompt node to handle image generation with improved error handling and user feedback. - Refactored AI image node to manage generation status and display loading indicators. - Enhanced data handling in canvas components to include canvasId for better context management. - Improved status message handling in Convex mutations for clearer user feedback.
This commit is contained in:
4
convex/_generated/api.d.ts
vendored
4
convex/_generated/api.d.ts
vendored
@@ -8,6 +8,7 @@
|
||||
* @module
|
||||
*/
|
||||
|
||||
import type * as ai from "../ai.js";
|
||||
import type * as auth from "../auth.js";
|
||||
import type * as canvases from "../canvases.js";
|
||||
import type * as credits from "../credits.js";
|
||||
@@ -15,6 +16,7 @@ import type * as edges from "../edges.js";
|
||||
import type * as helpers from "../helpers.js";
|
||||
import type * as http from "../http.js";
|
||||
import type * as nodes from "../nodes.js";
|
||||
import type * as openrouter from "../openrouter.js";
|
||||
import type * as storage from "../storage.js";
|
||||
|
||||
import type {
|
||||
@@ -24,6 +26,7 @@ import type {
|
||||
} from "convex/server";
|
||||
|
||||
declare const fullApi: ApiFromModules<{
|
||||
ai: typeof ai;
|
||||
auth: typeof auth;
|
||||
canvases: typeof canvases;
|
||||
credits: typeof credits;
|
||||
@@ -31,6 +34,7 @@ declare const fullApi: ApiFromModules<{
|
||||
helpers: typeof helpers;
|
||||
http: typeof http;
|
||||
nodes: typeof nodes;
|
||||
openrouter: typeof openrouter;
|
||||
storage: typeof storage;
|
||||
}>;
|
||||
|
||||
|
||||
109
convex/ai.ts
Normal file
109
convex/ai.ts
Normal file
@@ -0,0 +1,109 @@
|
||||
import { v } from "convex/values";
|
||||
import { action } from "./_generated/server";
|
||||
import { api } from "./_generated/api";
|
||||
import {
|
||||
generateImageViaOpenRouter,
|
||||
DEFAULT_IMAGE_MODEL,
|
||||
IMAGE_MODELS,
|
||||
} from "./openrouter";
|
||||
|
||||
export const generateImage = action({
|
||||
args: {
|
||||
canvasId: v.id("canvases"),
|
||||
nodeId: v.id("nodes"),
|
||||
prompt: v.string(),
|
||||
referenceStorageId: v.optional(v.id("_storage")),
|
||||
model: v.optional(v.string()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const apiKey = process.env.OPENROUTER_API_KEY;
|
||||
if (!apiKey) {
|
||||
throw new Error("OPENROUTER_API_KEY is not set");
|
||||
}
|
||||
|
||||
const modelId = args.model ?? DEFAULT_IMAGE_MODEL;
|
||||
const modelConfig = IMAGE_MODELS[modelId];
|
||||
if (!modelConfig) {
|
||||
throw new Error(`Unknown model: ${modelId}`);
|
||||
}
|
||||
|
||||
if (!(await ctx.runQuery(api.auth.getCurrentUser, {}))) {
|
||||
throw new Error("User not found");
|
||||
}
|
||||
|
||||
const reservationId = await ctx.runMutation(api.credits.reserve, {
|
||||
estimatedCost: modelConfig.estimatedCostPerImage,
|
||||
description: `Bildgenerierung — ${modelConfig.name}`,
|
||||
model: modelId,
|
||||
nodeId: args.nodeId,
|
||||
canvasId: args.canvasId,
|
||||
});
|
||||
|
||||
await ctx.runMutation(api.nodes.updateStatus, {
|
||||
nodeId: args.nodeId,
|
||||
status: "executing",
|
||||
});
|
||||
|
||||
try {
|
||||
let referenceImageUrl: string | undefined;
|
||||
if (args.referenceStorageId) {
|
||||
referenceImageUrl =
|
||||
(await ctx.storage.getUrl(args.referenceStorageId)) ?? undefined;
|
||||
}
|
||||
|
||||
const result = await generateImageViaOpenRouter(apiKey, {
|
||||
prompt: args.prompt,
|
||||
referenceImageUrl,
|
||||
model: modelId,
|
||||
});
|
||||
|
||||
const binaryString = atob(result.imageBase64);
|
||||
const bytes = new Uint8Array(binaryString.length);
|
||||
for (let i = 0; i < binaryString.length; i++) {
|
||||
bytes[i] = binaryString.charCodeAt(i);
|
||||
}
|
||||
|
||||
const blob = new Blob([bytes], { type: result.mimeType });
|
||||
const storageId = await ctx.storage.store(blob);
|
||||
|
||||
const existing = await ctx.runQuery(api.nodes.get, { nodeId: args.nodeId });
|
||||
if (!existing) throw new Error("Node not found");
|
||||
const prev = (existing.data ?? {}) as Record<string, unknown>;
|
||||
|
||||
await ctx.runMutation(api.nodes.updateData, {
|
||||
nodeId: args.nodeId,
|
||||
data: {
|
||||
...prev,
|
||||
storageId,
|
||||
prompt: args.prompt,
|
||||
model: modelId,
|
||||
modelTier: modelConfig.tier,
|
||||
generatedAt: Date.now(),
|
||||
},
|
||||
});
|
||||
|
||||
await ctx.runMutation(api.nodes.updateStatus, {
|
||||
nodeId: args.nodeId,
|
||||
status: "done",
|
||||
});
|
||||
|
||||
await ctx.runMutation(api.credits.commit, {
|
||||
transactionId: reservationId,
|
||||
actualCost: modelConfig.estimatedCostPerImage,
|
||||
});
|
||||
} catch (error) {
|
||||
await ctx.runMutation(api.credits.release, {
|
||||
transactionId: reservationId,
|
||||
});
|
||||
|
||||
await ctx.runMutation(api.nodes.updateStatus, {
|
||||
nodeId: args.nodeId,
|
||||
status: "error",
|
||||
statusMessage:
|
||||
error instanceof Error ? error.message : "Generation failed",
|
||||
});
|
||||
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
});
|
||||
@@ -270,7 +270,15 @@ export const updateStatus = mutation({
|
||||
if (!node) throw new Error("Node not found");
|
||||
|
||||
await getCanvasOrThrow(ctx, node.canvasId, user.userId);
|
||||
await ctx.db.patch(nodeId, { status, statusMessage });
|
||||
const patch: { status: typeof status; statusMessage?: string } = {
|
||||
status,
|
||||
};
|
||||
if (statusMessage !== undefined) {
|
||||
patch.statusMessage = statusMessage;
|
||||
} else if (status === "done" || status === "executing" || status === "idle") {
|
||||
patch.statusMessage = undefined;
|
||||
}
|
||||
await ctx.db.patch(nodeId, patch);
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
112
convex/openrouter.ts
Normal file
112
convex/openrouter.ts
Normal file
@@ -0,0 +1,112 @@
|
||||
export const OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1";
|
||||
|
||||
export interface OpenRouterModel {
|
||||
id: string;
|
||||
name: string;
|
||||
tier: "budget" | "standard" | "premium";
|
||||
estimatedCostPerImage: number; // in Euro-Cent (for credit reservation)
|
||||
}
|
||||
|
||||
// Phase 1: Gemini 2.5 Flash Image only.
|
||||
// Add more models here in Phase 2 when the model selector UI is built.
|
||||
export const IMAGE_MODELS: Record<string, OpenRouterModel> = {
|
||||
"google/gemini-2.5-flash-image": {
|
||||
id: "google/gemini-2.5-flash-image",
|
||||
name: "Gemini 2.5 Flash",
|
||||
tier: "standard",
|
||||
estimatedCostPerImage: 4, // ~€0.04 in Euro-Cent
|
||||
},
|
||||
};
|
||||
|
||||
export const DEFAULT_IMAGE_MODEL = "google/gemini-2.5-flash-image";
|
||||
|
||||
export interface GenerateImageParams {
|
||||
prompt: string;
|
||||
referenceImageUrl?: string; // optional image-to-image input
|
||||
model?: string;
|
||||
}
|
||||
|
||||
export interface OpenRouterImageResponse {
|
||||
imageBase64: string; // base64-encoded PNG/JPEG
|
||||
mimeType: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calls the OpenRouter API to generate an image.
|
||||
* Uses the chat/completions endpoint with a vision-capable model that returns
|
||||
* an inline image in the response (base64).
|
||||
*
|
||||
* Must be called from a Convex Action (has access to fetch + env vars).
|
||||
*/
|
||||
export async function generateImageViaOpenRouter(
|
||||
apiKey: string,
|
||||
params: GenerateImageParams
|
||||
): Promise<OpenRouterImageResponse> {
|
||||
const modelId = params.model ?? DEFAULT_IMAGE_MODEL;
|
||||
|
||||
// Build message content — text prompt, optionally with a reference image
|
||||
const userContent: object[] = [];
|
||||
|
||||
if (params.referenceImageUrl) {
|
||||
userContent.push({
|
||||
type: "image_url",
|
||||
image_url: { url: params.referenceImageUrl },
|
||||
});
|
||||
}
|
||||
|
||||
userContent.push({
|
||||
type: "text",
|
||||
text: params.prompt,
|
||||
});
|
||||
|
||||
const body = {
|
||||
model: modelId,
|
||||
modalities: ["image", "text"],
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: userContent,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const response = await fetch(`${OPENROUTER_BASE_URL}/chat/completions`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
"HTTP-Referer": "https://app.lemonspace.io",
|
||||
"X-Title": "LemonSpace",
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`OpenRouter API error ${response.status}: ${errorText}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
// OpenRouter returns generated images in message.images (separate from content)
|
||||
const images = data?.choices?.[0]?.message?.images;
|
||||
|
||||
if (!images || images.length === 0) {
|
||||
throw new Error("No image found in OpenRouter response");
|
||||
}
|
||||
|
||||
const imageUrl = images[0]?.image_url?.url;
|
||||
if (!imageUrl) {
|
||||
throw new Error("Image block missing image_url.url");
|
||||
}
|
||||
|
||||
// The URL is a data URI: "data:image/png;base64,<data>"
|
||||
const dataUri: string = imageUrl;
|
||||
const [meta, base64Data] = dataUri.split(",");
|
||||
const mimeType = meta.replace("data:", "").replace(";base64", "");
|
||||
|
||||
return {
|
||||
imageBase64: base64Data,
|
||||
mimeType: mimeType || "image/png",
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user