matthias/lemonspace_app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'feat/mixer-overlay-resize-render-bake'

Browse Source
This commit is contained in:
Matthias Meister
2026-04-15 08:46:42 +02:00
parent 9a346554ec f1c61fd14e
commit 2679a0cc4e
30 changed files with 6361 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

203
lib/canvas-mixer-preview.ts
View File

@@ -19,8 +19,14 @@ export type MixerPreviewState = {
overlayUrl?: string;
blendMode: MixerBlendMode;
opacity: number;
offsetX: number;
offsetY: number;
overlayX: number;
overlayY: number;
overlayWidth: number;
overlayHeight: number;
cropLeft: number;
cropTop: number;
cropRight: number;
cropBottom: number;
error?: MixerPreviewError;
};
@@ -35,9 +41,18 @@ const DEFAULT_BLEND_MODE: MixerBlendMode = "normal";
const DEFAULT_OPACITY = 100;
const MIN_OPACITY = 0;
const MAX_OPACITY = 100;
const DEFAULT_OFFSET = 0;
const MIN_OFFSET = -2048;
const MAX_OFFSET = 2048;
const DEFAULT_OVERLAY_X = 0;
const DEFAULT_OVERLAY_Y = 0;
const DEFAULT_OVERLAY_WIDTH = 1;
const DEFAULT_OVERLAY_HEIGHT = 1;
const DEFAULT_CROP_LEFT = 0;
const DEFAULT_CROP_TOP = 0;
const DEFAULT_CROP_RIGHT = 0;
const DEFAULT_CROP_BOTTOM = 0;
const MIN_OVERLAY_POSITION = 0;
const MAX_OVERLAY_POSITION = 1;
const MIN_OVERLAY_SIZE = 0.1;
const MAX_OVERLAY_SIZE = 1;
function clamp(value: number, min: number, max: number): number {
return Math.max(min, Math.min(max, value));
@@ -65,18 +80,165 @@ function normalizeOpacity(value: unknown): number {
return clamp(parsed, MIN_OPACITY, MAX_OPACITY);
}
function normalizeOffset(value: unknown): number {
function normalizeOverlayNumber(value: unknown, fallback: number): number {
const parsed = parseNumeric(value);
if (parsed === null) {
return DEFAULT_OFFSET;
return fallback;
}
return clamp(parsed, MIN_OFFSET, MAX_OFFSET);
return parsed;
}
function normalizeUnitRect(args: {
x: unknown;
y: unknown;
width: unknown;
height: unknown;
defaults: { x: number; y: number; width: number; height: number };
}): { x: number; y: number; width: number; height: number } {
const x = clamp(
normalizeOverlayNumber(args.x, args.defaults.x),
MIN_OVERLAY_POSITION,
MAX_OVERLAY_POSITION - MIN_OVERLAY_SIZE,
);
const y = clamp(
normalizeOverlayNumber(args.y, args.defaults.y),
MIN_OVERLAY_POSITION,
MAX_OVERLAY_POSITION - MIN_OVERLAY_SIZE,
);
const width = clamp(
normalizeOverlayNumber(args.width, args.defaults.width),
MIN_OVERLAY_SIZE,
Math.min(MAX_OVERLAY_SIZE, MAX_OVERLAY_POSITION - x),
);
const height = clamp(
normalizeOverlayNumber(args.height, args.defaults.height),
MIN_OVERLAY_SIZE,
Math.min(MAX_OVERLAY_SIZE, MAX_OVERLAY_POSITION - y),
);
return { x, y, width, height };
}
function normalizeOverlayRect(record: Record<string, unknown>): Pick<
MixerPreviewState,
"overlayX" | "overlayY" | "overlayWidth" | "overlayHeight"
> {
const hasLegacyOffset = record.offsetX !== undefined || record.offsetY !== undefined;
const hasOverlayRectField =
record.overlayX !== undefined ||
record.overlayY !== undefined ||
record.overlayWidth !== undefined ||
record.overlayHeight !== undefined;
if (hasLegacyOffset && !hasOverlayRectField) {
return {
overlayX: DEFAULT_OVERLAY_X,
overlayY: DEFAULT_OVERLAY_Y,
overlayWidth: DEFAULT_OVERLAY_WIDTH,
overlayHeight: DEFAULT_OVERLAY_HEIGHT,
};
}
const normalized = normalizeUnitRect({
x: record.overlayX,
y: record.overlayY,
width: record.overlayWidth,
height: record.overlayHeight,
defaults: {
x: DEFAULT_OVERLAY_X,
y: DEFAULT_OVERLAY_Y,
width: DEFAULT_OVERLAY_WIDTH,
height: DEFAULT_OVERLAY_HEIGHT,
},
});
return {
overlayX: normalized.x,
overlayY: normalized.y,
overlayWidth: normalized.width,
overlayHeight: normalized.height,
};
}
function normalizeCropEdges(record: Record<string, unknown>): Pick<
MixerPreviewState,
"cropLeft" | "cropTop" | "cropRight" | "cropBottom"
> {
const hasCropField =
record.cropLeft !== undefined ||
record.cropTop !== undefined ||
record.cropRight !== undefined ||
record.cropBottom !== undefined;
const hasLegacyContentRectField =
record.contentX !== undefined ||
record.contentY !== undefined ||
record.contentWidth !== undefined ||
record.contentHeight !== undefined;
if (!hasCropField && hasLegacyContentRectField) {
const legacyRect = normalizeUnitRect({
x: record.contentX,
y: record.contentY,
width: record.contentWidth,
height: record.contentHeight,
defaults: {
x: 0,
y: 0,
width: 1,
height: 1,
},
});
return {
cropLeft: legacyRect.x,
cropTop: legacyRect.y,
cropRight: 1 - (legacyRect.x + legacyRect.width),
cropBottom: 1 - (legacyRect.y + legacyRect.height),
};
}
const cropLeft = clamp(
normalizeOverlayNumber(record.cropLeft, DEFAULT_CROP_LEFT),
0,
1 - MIN_OVERLAY_SIZE,
);
const cropTop = clamp(
normalizeOverlayNumber(record.cropTop, DEFAULT_CROP_TOP),
0,
1 - MIN_OVERLAY_SIZE,
);
const cropRight = clamp(
normalizeOverlayNumber(record.cropRight, DEFAULT_CROP_RIGHT),
0,
1 - cropLeft - MIN_OVERLAY_SIZE,
);
const cropBottom = clamp(
normalizeOverlayNumber(record.cropBottom, DEFAULT_CROP_BOTTOM),
0,
1 - cropTop - MIN_OVERLAY_SIZE,
);
return {
cropLeft,
cropTop,
cropRight,
cropBottom,
};
}
export function normalizeMixerPreviewData(data: unknown): Pick<
MixerPreviewState,
"blendMode" | "opacity" | "offsetX" | "offsetY"
| "blendMode"
| "opacity"
| "overlayX"
| "overlayY"
| "overlayWidth"
| "overlayHeight"
| "cropLeft"
| "cropTop"
| "cropRight"
| "cropBottom"
> {
const record = (data ?? {}) as Record<string, unknown>;
const blendMode = MIXER_BLEND_MODES.has(record.blendMode as MixerBlendMode)
@@ -86,8 +248,8 @@ export function normalizeMixerPreviewData(data: unknown): Pick<
return {
blendMode,
opacity: normalizeOpacity(record.opacity),
offsetX: normalizeOffset(record.offsetX),
offsetY: normalizeOffset(record.offsetY),
...normalizeOverlayRect(record),
...normalizeCropEdges(record),
};
}
@@ -119,6 +281,17 @@ function resolveSourceUrlFromNode(args: {
}
if (args.sourceNode.type === "render") {
const preview = resolveRenderPreviewInputFromGraph({
nodeId: args.sourceNode.id,
graph: args.graph,
});
if (preview.sourceComposition) {
return undefined;
}
if (preview.sourceUrl) {
return preview.sourceUrl;
}
const renderData = (args.sourceNode.data ?? {}) as Record<string, unknown>;
const renderOutputUrl =
typeof renderData.lastUploadUrl === "string" && renderData.lastUploadUrl.length > 0
@@ -133,11 +306,7 @@ function resolveSourceUrlFromNode(args: {
return directRenderUrl;
}
const preview = resolveRenderPreviewInputFromGraph({
nodeId: args.sourceNode.id,
graph: args.graph,
});
return preview.sourceUrl ?? undefined;
return undefined;
}
return resolveNodeImageUrl(args.sourceNode.data) ?? undefined;
@@ -172,6 +341,8 @@ export function resolveMixerPreviewFromGraph(args: {
if (base.duplicate || overlay.duplicate) {
return {
status: "error",
baseUrl: undefined,
overlayUrl: undefined,
...normalized,
error: "duplicate-handle-edge",
};

10
lib/canvas-node-templates.ts
View File

@@ -51,8 +51,14 @@ export const CANVAS_NODE_TEMPLATES = [
defaultData: {
blendMode: "normal",
opacity: 100,
offsetX: 0,
offsetY: 0,
overlayX: 0,
overlayY: 0,
overlayWidth: 1,
overlayHeight: 1,
cropLeft: 0,
cropTop: 0,
cropRight: 0,
cropBottom: 0,
},
},
{

355
lib/canvas-render-preview.ts
View File

@@ -15,10 +15,29 @@ export type RenderPreviewGraphEdge = {
};
export type RenderPreviewInput = {
sourceUrl: string;
sourceUrl: string | null;
sourceComposition?: RenderPreviewSourceComposition;
steps: PipelineStep[];
};
export type MixerBlendMode = "normal" | "multiply" | "screen" | "overlay";
export type RenderPreviewSourceComposition = {
kind: "mixer";
baseUrl: string;
overlayUrl: string;
blendMode: MixerBlendMode;
opacity: number;
overlayX: number;
overlayY: number;
overlayWidth: number;
overlayHeight: number;
cropLeft: number;
cropTop: number;
cropRight: number;
cropBottom: number;
};
export type CanvasGraphNodeLike = {
id: string;
type: string;
@@ -38,6 +57,8 @@ export type CanvasGraphSnapshot = {
incomingEdgesByTarget: ReadonlyMap<string, readonly CanvasGraphEdgeLike[]>;
};
type RenderPreviewResolvedInput = RenderPreviewInput;
export type CanvasGraphNodeDataOverrides = ReadonlyMap<string, unknown>;
export function shouldFastPathPreviewPipeline(
@@ -129,6 +150,188 @@ export const RENDER_PREVIEW_PIPELINE_TYPES = new Set([
"detail-adjust",
]);
const MIXER_SOURCE_NODE_TYPES = new Set(["image", "asset", "ai-image", "render"]);
const MIXER_BLEND_MODES = new Set<MixerBlendMode>([
"normal",
"multiply",
"screen",
"overlay",
]);
const DEFAULT_BLEND_MODE: MixerBlendMode = "normal";
const DEFAULT_OPACITY = 100;
const MIN_OPACITY = 0;
const MAX_OPACITY = 100;
const DEFAULT_OVERLAY_X = 0;
const DEFAULT_OVERLAY_Y = 0;
const DEFAULT_OVERLAY_WIDTH = 1;
const DEFAULT_OVERLAY_HEIGHT = 1;
const DEFAULT_CROP_LEFT = 0;
const DEFAULT_CROP_TOP = 0;
const DEFAULT_CROP_RIGHT = 0;
const DEFAULT_CROP_BOTTOM = 0;
const MIN_OVERLAY_POSITION = 0;
const MAX_OVERLAY_POSITION = 1;
const MIN_OVERLAY_SIZE = 0.1;
const MAX_OVERLAY_SIZE = 1;
function clamp(value: number, min: number, max: number): number {
return Math.max(min, Math.min(max, value));
}
function parseNumeric(value: unknown): number | null {
if (typeof value === "number") {
return Number.isFinite(value) ? value : null;
}
if (typeof value === "string") {
const parsed = Number(value);
return Number.isFinite(parsed) ? parsed : null;
}
return null;
}
function normalizeOpacity(value: unknown): number {
const parsed = parseNumeric(value);
if (parsed === null) {
return DEFAULT_OPACITY;
}
return clamp(parsed, MIN_OPACITY, MAX_OPACITY);
}
function normalizeOverlayNumber(value: unknown, fallback: number): number {
const parsed = parseNumeric(value);
if (parsed === null) {
return fallback;
}
return parsed;
}
function normalizeMixerCompositionRect(data: Record<string, unknown>): Pick<
RenderPreviewSourceComposition,
"overlayX" | "overlayY" | "overlayWidth" | "overlayHeight"
> {
const hasLegacyOffset = data.offsetX !== undefined || data.offsetY !== undefined;
const hasOverlayRectField =
data.overlayX !== undefined ||
data.overlayY !== undefined ||
data.overlayWidth !== undefined ||
data.overlayHeight !== undefined;
if (hasLegacyOffset && !hasOverlayRectField) {
return {
overlayX: DEFAULT_OVERLAY_X,
overlayY: DEFAULT_OVERLAY_Y,
overlayWidth: DEFAULT_OVERLAY_WIDTH,
overlayHeight: DEFAULT_OVERLAY_HEIGHT,
};
}
const overlayX = clamp(
normalizeOverlayNumber(data.overlayX, DEFAULT_OVERLAY_X),
MIN_OVERLAY_POSITION,
MAX_OVERLAY_POSITION - MIN_OVERLAY_SIZE,
);
const overlayY = clamp(
normalizeOverlayNumber(data.overlayY, DEFAULT_OVERLAY_Y),
MIN_OVERLAY_POSITION,
MAX_OVERLAY_POSITION - MIN_OVERLAY_SIZE,
);
const overlayWidth = clamp(
normalizeOverlayNumber(data.overlayWidth, DEFAULT_OVERLAY_WIDTH),
MIN_OVERLAY_SIZE,
Math.min(MAX_OVERLAY_SIZE, MAX_OVERLAY_POSITION - overlayX),
);
const overlayHeight = clamp(
normalizeOverlayNumber(data.overlayHeight, DEFAULT_OVERLAY_HEIGHT),
MIN_OVERLAY_SIZE,
Math.min(MAX_OVERLAY_SIZE, MAX_OVERLAY_POSITION - overlayY),
);
return {
overlayX,
overlayY,
overlayWidth,
overlayHeight,
};
}
function normalizeMixerCompositionCropEdges(data: Record<string, unknown>): Pick<
RenderPreviewSourceComposition,
"cropLeft" | "cropTop" | "cropRight" | "cropBottom"
> {
const hasCropField =
data.cropLeft !== undefined ||
data.cropTop !== undefined ||
data.cropRight !== undefined ||
data.cropBottom !== undefined;
const hasLegacyContentRectField =
data.contentX !== undefined ||
data.contentY !== undefined ||
data.contentWidth !== undefined ||
data.contentHeight !== undefined;
if (!hasCropField && hasLegacyContentRectField) {
const contentX = clamp(
normalizeOverlayNumber(data.contentX, 0),
MIN_OVERLAY_POSITION,
MAX_OVERLAY_POSITION - MIN_OVERLAY_SIZE,
);
const contentY = clamp(
normalizeOverlayNumber(data.contentY, 0),
MIN_OVERLAY_POSITION,
MAX_OVERLAY_POSITION - MIN_OVERLAY_SIZE,
);
const contentWidth = clamp(
normalizeOverlayNumber(data.contentWidth, 1),
MIN_OVERLAY_SIZE,
Math.min(MAX_OVERLAY_SIZE, MAX_OVERLAY_POSITION - contentX),
);
const contentHeight = clamp(
normalizeOverlayNumber(data.contentHeight, 1),
MIN_OVERLAY_SIZE,
Math.min(MAX_OVERLAY_SIZE, MAX_OVERLAY_POSITION - contentY),
);
return {
cropLeft: contentX,
cropTop: contentY,
cropRight: 1 - (contentX + contentWidth),
cropBottom: 1 - (contentY + contentHeight),
};
}
const cropLeft = clamp(
normalizeOverlayNumber(data.cropLeft, DEFAULT_CROP_LEFT),
0,
1 - MIN_OVERLAY_SIZE,
);
const cropTop = clamp(
normalizeOverlayNumber(data.cropTop, DEFAULT_CROP_TOP),
0,
1 - MIN_OVERLAY_SIZE,
);
const cropRight = clamp(
normalizeOverlayNumber(data.cropRight, DEFAULT_CROP_RIGHT),
0,
1 - cropLeft - MIN_OVERLAY_SIZE,
);
const cropBottom = clamp(
normalizeOverlayNumber(data.cropBottom, DEFAULT_CROP_BOTTOM),
0,
1 - cropTop - MIN_OVERLAY_SIZE,
);
return {
cropLeft,
cropTop,
cropRight,
cropBottom,
};
}
export function resolveRenderFingerprint(data: unknown): {
resolution: RenderResolutionOption;
customWidth?: number;
@@ -163,15 +366,19 @@ export function resolveRenderFingerprint(data: unknown): {
export function resolveRenderPipelineHash(args: {
sourceUrl: string | null;
sourceComposition?: RenderPreviewSourceComposition;
steps: PipelineStep[];
data: unknown;
}): string | null {
if (!args.sourceUrl) {
if (!args.sourceUrl && !args.sourceComposition) {
return null;
}
return hashPipeline(
{ sourceUrl: args.sourceUrl, render: resolveRenderFingerprint(args.data) },
{
source: args.sourceComposition ?? args.sourceUrl,
render: resolveRenderFingerprint(args.data),
},
args.steps,
);
}
@@ -212,6 +419,119 @@ function resolveSourceNodeUrl(node: CanvasGraphNodeLike): string | null {
return resolveNodeImageUrl(node.data);
}
function resolveRenderOutputUrl(node: CanvasGraphNodeLike): string | null {
const data = (node.data ?? {}) as Record<string, unknown>;
const lastUploadUrl =
typeof data.lastUploadUrl === "string" && data.lastUploadUrl.length > 0
? data.lastUploadUrl
: null;
if (lastUploadUrl) {
return lastUploadUrl;
}
return resolveNodeImageUrl(node.data);
}
function resolveMixerHandleEdge(args: {
incomingEdges: readonly CanvasGraphEdgeLike[];
handle: "base" | "overlay";
}): CanvasGraphEdgeLike | null {
const filtered = args.incomingEdges.filter((edge) => {
if (args.handle === "base") {
return edge.targetHandle === "base" || edge.targetHandle == null || edge.targetHandle === "";
}
return edge.targetHandle === "overlay";
});
if (filtered.length !== 1) {
return null;
}
return filtered[0] ?? null;
}
function resolveMixerSourceUrlFromNode(args: {
node: CanvasGraphNodeLike;
graph: CanvasGraphSnapshot;
}): string | null {
if (!MIXER_SOURCE_NODE_TYPES.has(args.node.type)) {
return null;
}
if (args.node.type === "render") {
const preview = resolveRenderPreviewInputFromGraph({
nodeId: args.node.id,
graph: args.graph,
});
if (preview.sourceComposition) {
return null;
}
if (preview.sourceUrl) {
return preview.sourceUrl;
}
const directRenderUrl = resolveRenderOutputUrl(args.node);
if (directRenderUrl) {
return directRenderUrl;
}
return null;
}
return resolveNodeImageUrl(args.node.data);
}
function resolveMixerSourceUrlFromEdge(args: {
edge: CanvasGraphEdgeLike | null;
graph: CanvasGraphSnapshot;
}): string | null {
if (!args.edge) {
return null;
}
const sourceNode = args.graph.nodesById.get(args.edge.source);
if (!sourceNode) {
return null;
}
return resolveMixerSourceUrlFromNode({
node: sourceNode,
graph: args.graph,
});
}
function resolveRenderMixerCompositionFromGraph(args: {
node: CanvasGraphNodeLike;
graph: CanvasGraphSnapshot;
}): RenderPreviewSourceComposition | null {
const incomingEdges = args.graph.incomingEdgesByTarget.get(args.node.id) ?? [];
const baseEdge = resolveMixerHandleEdge({ incomingEdges, handle: "base" });
const overlayEdge = resolveMixerHandleEdge({ incomingEdges, handle: "overlay" });
const baseUrl = resolveMixerSourceUrlFromEdge({ edge: baseEdge, graph: args.graph });
const overlayUrl = resolveMixerSourceUrlFromEdge({ edge: overlayEdge, graph: args.graph });
if (!baseUrl || !overlayUrl) {
return null;
}
const data = (args.node.data ?? {}) as Record<string, unknown>;
const blendMode = MIXER_BLEND_MODES.has(data.blendMode as MixerBlendMode)
? (data.blendMode as MixerBlendMode)
: DEFAULT_BLEND_MODE;
return {
kind: "mixer",
baseUrl,
overlayUrl,
blendMode,
opacity: normalizeOpacity(data.opacity),
...normalizeMixerCompositionRect(data),
...normalizeMixerCompositionCropEdges(data),
};
}
export function buildGraphSnapshot(
nodes: readonly CanvasGraphNodeLike[],
edges: readonly CanvasGraphEdgeLike[],
@@ -384,7 +704,32 @@ export function findSourceNodeFromGraph(
export function resolveRenderPreviewInputFromGraph(args: {
nodeId: string;
graph: CanvasGraphSnapshot;
}): { sourceUrl: string | null; steps: PipelineStep[] } {
}): RenderPreviewResolvedInput {
const renderIncoming = getSortedIncomingEdge(
args.graph.incomingEdgesByTarget.get(args.nodeId),
);
const renderInputNode = renderIncoming
? args.graph.nodesById.get(renderIncoming.source)
: null;
if (renderInputNode?.type === "mixer") {
const sourceComposition = resolveRenderMixerCompositionFromGraph({
node: renderInputNode,
graph: args.graph,
});
const steps = collectPipelineFromGraph(args.graph, {
nodeId: args.nodeId,
isPipelineNode: (node) => RENDER_PREVIEW_PIPELINE_TYPES.has(node.type ?? ""),
});
return {
sourceUrl: null,
sourceComposition: sourceComposition ?? undefined,
steps,
};
}
const sourceUrl = getSourceImageFromGraph(args.graph, {
nodeId: args.nodeId,
isSourceNode: (node) => SOURCE_NODE_TYPES.has(node.type ?? ""),
@@ -406,7 +751,7 @@ export function resolveRenderPreviewInput(args: {
nodeId: string;
nodes: readonly RenderPreviewGraphNode[];
edges: readonly RenderPreviewGraphEdge[];
}): { sourceUrl: string | null; steps: PipelineStep[] } {
}): RenderPreviewResolvedInput {
return resolveRenderPreviewInputFromGraph({
nodeId: args.nodeId,
graph: buildGraphSnapshot(args.nodes, args.edges),

10
lib/canvas-utils.ts
View File

@@ -437,8 +437,14 @@ export const NODE_DEFAULTS: Record<
data: {
blendMode: "normal",
opacity: 100,
offsetX: 0,
offsetY: 0,
overlayX: 0,
overlayY: 0,
overlayWidth: 1,
overlayHeight: 1,
cropLeft: 0,
cropTop: 0,
cropRight: 0,
cropBottom: 0,
},
},
"agent-output": {

8
lib/image-pipeline/bridge.ts
View File

@@ -10,7 +10,7 @@ import {
applyGeometryStepsToSource,
partitionPipelineSteps,
} from "@/lib/image-pipeline/geometry-transform";
import { loadSourceBitmap } from "@/lib/image-pipeline/source-loader";
import { loadRenderSourceBitmap } from "@/lib/image-pipeline/source-loader";
type SupportedCanvas = HTMLCanvasElement | OffscreenCanvas;
type SupportedContext = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
@@ -99,7 +99,11 @@ function resolveMimeType(format: RenderFormat): string {
export async function renderFull(options: RenderFullOptions): Promise<RenderFullResult> {
const { signal } = options;
const bitmap = await loadSourceBitmap(options.sourceUrl, { signal });
const bitmap = await loadRenderSourceBitmap({
sourceUrl: options.sourceUrl,
sourceComposition: options.sourceComposition,
signal,
});
const { geometrySteps, tonalSteps } = partitionPipelineSteps(options.steps);
const geometryResult = applyGeometryStepsToSource({
source: bitmap,

13
lib/image-pipeline/image-pipeline.worker.ts
View File

@@ -2,21 +2,26 @@ import { renderFull } from "@/lib/image-pipeline/bridge";
import { renderPreview } from "@/lib/image-pipeline/preview-renderer";
import type { PipelineStep } from "@/lib/image-pipeline/contracts";
import type { HistogramData } from "@/lib/image-pipeline/histogram";
import type { RenderFullOptions, RenderFullResult } from "@/lib/image-pipeline/render-types";
import type {
RenderFullOptions,
RenderFullResult,
RenderSourceComposition,
} from "@/lib/image-pipeline/render-types";
import {
IMAGE_PIPELINE_BACKEND_FLAG_KEYS,
type BackendFeatureFlags,
} from "@/lib/image-pipeline/backend/feature-flags";
type PreviewWorkerPayload = {
sourceUrl: string;
sourceUrl?: string;
sourceComposition?: RenderSourceComposition;
steps: readonly PipelineStep[];
previewWidth: number;
includeHistogram?: boolean;
featureFlags?: BackendFeatureFlags;
};
type FullWorkerPayload = RenderFullOptions & {
type FullWorkerPayload = Omit<RenderFullOptions, "signal"> & {
featureFlags?: BackendFeatureFlags;
};
@@ -112,6 +117,7 @@ async function handlePreviewRequest(requestId: number, payload: PreviewWorkerPay
applyWorkerFeatureFlags(payload.featureFlags);
const result = await renderPreview({
sourceUrl: payload.sourceUrl,
sourceComposition: payload.sourceComposition,
steps: payload.steps,
previewWidth: payload.previewWidth,
includeHistogram: payload.includeHistogram,
@@ -161,6 +167,7 @@ async function handleFullRequest(requestId: number, payload: FullWorkerPayload):
applyWorkerFeatureFlags(payload.featureFlags);
const result = await renderFull({
sourceUrl: payload.sourceUrl,
sourceComposition: payload.sourceComposition,
steps: payload.steps,
render: payload.render,
signal: controller.signal,

10
lib/image-pipeline/preview-renderer.ts
View File

@@ -8,7 +8,8 @@ import {
applyGeometryStepsToSource,
partitionPipelineSteps,
} from "@/lib/image-pipeline/geometry-transform";
import { loadSourceBitmap } from "@/lib/image-pipeline/source-loader";
import { loadRenderSourceBitmap } from "@/lib/image-pipeline/source-loader";
import type { RenderSourceComposition } from "@/lib/image-pipeline/render-types";
export type PreviewRenderResult = {
width: number;
@@ -64,13 +65,16 @@ async function yieldToMainOrWorkerLoop(): Promise<void> {
}
export async function renderPreview(options: {
sourceUrl: string;
sourceUrl?: string;
sourceComposition?: RenderSourceComposition;
steps: readonly PipelineStep[];
previewWidth: number;
includeHistogram?: boolean;
signal?: AbortSignal;
}): Promise<PreviewRenderResult> {
const bitmap = await loadSourceBitmap(options.sourceUrl, {
const bitmap = await loadRenderSourceBitmap({
sourceUrl: options.sourceUrl,
sourceComposition: options.sourceComposition,
signal: options.signal,
});
const { geometrySteps, tonalSteps } = partitionPipelineSteps(options.steps);

19
lib/image-pipeline/render-types.ts
View File

@@ -24,6 +24,22 @@ export type RenderSizeLimits = {
maxPixels?: number;
};
export type RenderSourceComposition = {
kind: "mixer";
baseUrl: string;
overlayUrl: string;
blendMode: "normal" | "multiply" | "screen" | "overlay";
opacity: number;
overlayX: number;
overlayY: number;
overlayWidth: number;
overlayHeight: number;
cropLeft: number;
cropTop: number;
cropRight: number;
cropBottom: number;
};
export type ResolvedRenderSize = {
width: number;
height: number;
@@ -32,7 +48,8 @@ export type ResolvedRenderSize = {
};
export type RenderFullOptions = {
sourceUrl: string;
sourceUrl?: string;
sourceComposition?: RenderSourceComposition;
steps: readonly PipelineStep[];
render: RenderOptions;
limits?: RenderSizeLimits;

225
lib/image-pipeline/source-loader.ts
View File

@@ -1,3 +1,6 @@
import type { RenderSourceComposition } from "@/lib/image-pipeline/render-types";
import { computeVisibleMixerContentRect } from "@/lib/mixer-crop-layout";
export const SOURCE_BITMAP_CACHE_MAX_ENTRIES = 32;
type CacheEntry = {
@@ -12,6 +15,12 @@ type LoadSourceBitmapOptions = {
signal?: AbortSignal;
};
type LoadRenderSourceBitmapOptions = {
sourceUrl?: string;
sourceComposition?: RenderSourceComposition;
signal?: AbortSignal;
};
function throwIfAborted(signal: AbortSignal | undefined): void {
if (signal?.aborted) {
throw new DOMException("The operation was aborted.", "AbortError");
@@ -215,3 +224,219 @@ export async function loadSourceBitmap(
const promise = getOrCreateSourceBitmapPromise(sourceUrl);
return await awaitWithLocalAbort(promise, options.signal);
}
function createWorkingCanvas(width: number, height: number):
| HTMLCanvasElement
| OffscreenCanvas {
if (typeof document !== "undefined") {
const canvas = document.createElement("canvas");
canvas.width = width;
canvas.height = height;
return canvas;
}
if (typeof OffscreenCanvas !== "undefined") {
return new OffscreenCanvas(width, height);
}
throw new Error("Canvas rendering is not available in this environment.");
}
function mixerBlendModeToCompositeOperation(
blendMode: RenderSourceComposition["blendMode"],
): GlobalCompositeOperation {
if (blendMode === "normal") {
return "source-over";
}
return blendMode;
}
function normalizeCompositionOpacity(value: number): number {
if (!Number.isFinite(value)) {
return 1;
}
return Math.max(0, Math.min(100, value)) / 100;
}
function normalizeRatio(value: number, fallback: number): number {
if (!Number.isFinite(value)) {
return fallback;
}
return value;
}
function normalizeMixerRect(source: RenderSourceComposition): {
x: number;
y: number;
width: number;
height: number;
} {
const overlayX = Math.max(0, Math.min(0.9, normalizeRatio(source.overlayX, 0)));
const overlayY = Math.max(0, Math.min(0.9, normalizeRatio(source.overlayY, 0)));
const overlayWidth = Math.max(
0.1,
Math.min(1, normalizeRatio(source.overlayWidth, 1), 1 - overlayX),
);
const overlayHeight = Math.max(
0.1,
Math.min(1, normalizeRatio(source.overlayHeight, 1), 1 - overlayY),
);
return {
x: overlayX,
y: overlayY,
width: overlayWidth,
height: overlayHeight,
};
}
function normalizeMixerCropEdges(source: RenderSourceComposition): {
left: number;
top: number;
right: number;
bottom: number;
} {
const legacySource = source as RenderSourceComposition & {
contentX?: number;
contentY?: number;
contentWidth?: number;
contentHeight?: number;
};
const hasLegacyContentRect =
legacySource.contentX !== undefined ||
legacySource.contentY !== undefined ||
legacySource.contentWidth !== undefined ||
legacySource.contentHeight !== undefined;
if (hasLegacyContentRect) {
const contentX = Math.max(
0,
Math.min(0.9, normalizeRatio(legacySource.contentX ?? Number.NaN, 0)),
);
const contentY = Math.max(
0,
Math.min(0.9, normalizeRatio(legacySource.contentY ?? Number.NaN, 0)),
);
const contentWidth = Math.max(
0.1,
Math.min(1, normalizeRatio(legacySource.contentWidth ?? Number.NaN, 1), 1 - contentX),
);
const contentHeight = Math.max(
0.1,
Math.min(1, normalizeRatio(legacySource.contentHeight ?? Number.NaN, 1), 1 - contentY),
);
return {
left: contentX,
top: contentY,
right: 1 - (contentX + contentWidth),
bottom: 1 - (contentY + contentHeight),
};
}
const cropLeft = Math.max(0, Math.min(0.9, normalizeRatio(source.cropLeft, 0)));
const cropTop = Math.max(0, Math.min(0.9, normalizeRatio(source.cropTop, 0)));
const cropRight = Math.max(0, Math.min(1 - cropLeft - 0.1, normalizeRatio(source.cropRight, 0)));
const cropBottom = Math.max(
0,
Math.min(1 - cropTop - 0.1, normalizeRatio(source.cropBottom, 0)),
);
return {
left: cropLeft,
top: cropTop,
right: cropRight,
bottom: cropBottom,
};
}
async function loadMixerCompositionBitmap(
sourceComposition: RenderSourceComposition,
signal?: AbortSignal,
): Promise<ImageBitmap> {
const [baseBitmap, overlayBitmap] = await Promise.all([
loadSourceBitmap(sourceComposition.baseUrl, { signal }),
loadSourceBitmap(sourceComposition.overlayUrl, { signal }),
]);
throwIfAborted(signal);
const canvas = createWorkingCanvas(baseBitmap.width, baseBitmap.height);
const context = canvas.getContext("2d", { willReadFrequently: true });
if (!context) {
throw new Error("Render composition could not create a 2D context.");
}
context.clearRect(0, 0, baseBitmap.width, baseBitmap.height);
context.drawImage(baseBitmap, 0, 0, baseBitmap.width, baseBitmap.height);
const rect = normalizeMixerRect(sourceComposition);
const frameX = rect.x * baseBitmap.width;
const frameY = rect.y * baseBitmap.height;
const frameWidth = rect.width * baseBitmap.width;
const frameHeight = rect.height * baseBitmap.height;
const cropEdges = normalizeMixerCropEdges(sourceComposition);
const sourceX = cropEdges.left * overlayBitmap.width;
const sourceY = cropEdges.top * overlayBitmap.height;
const sourceWidth = (1 - cropEdges.left - cropEdges.right) * overlayBitmap.width;
const sourceHeight = (1 - cropEdges.top - cropEdges.bottom) * overlayBitmap.height;
const visibleRect = computeVisibleMixerContentRect({
frameAspectRatio: frameHeight > 0 ? frameWidth / frameHeight : 1,
sourceWidth: overlayBitmap.width,
sourceHeight: overlayBitmap.height,
cropLeft: cropEdges.left,
cropTop: cropEdges.top,
cropRight: cropEdges.right,
cropBottom: cropEdges.bottom,
});
const destX = frameX + (visibleRect?.x ?? 0) * frameWidth;
const destY = frameY + (visibleRect?.y ?? 0) * frameHeight;
const destWidth = (visibleRect?.width ?? 1) * frameWidth;
const destHeight = (visibleRect?.height ?? 1) * frameHeight;
context.globalCompositeOperation = mixerBlendModeToCompositeOperation(
sourceComposition.blendMode,
);
context.globalAlpha = normalizeCompositionOpacity(sourceComposition.opacity);
context.save();
context.beginPath();
context.rect(frameX, frameY, frameWidth, frameHeight);
context.clip();
context.drawImage(
overlayBitmap,
sourceX,
sourceY,
sourceWidth,
sourceHeight,
destX,
destY,
destWidth,
destHeight,
);
context.restore();
context.globalCompositeOperation = "source-over";
context.globalAlpha = 1;
return await createImageBitmap(canvas);
}
export async function loadRenderSourceBitmap(
options: LoadRenderSourceBitmapOptions,
): Promise<ImageBitmap> {
if (options.sourceComposition) {
if (options.sourceComposition.kind !== "mixer") {
throw new Error(`Unsupported source composition '${options.sourceComposition.kind}'.`);
}
return await loadMixerCompositionBitmap(options.sourceComposition, options.signal);
}
if (!options.sourceUrl) {
throw new Error("Render source is required.");
}
return await loadSourceBitmap(options.sourceUrl, { signal: options.signal });
}

37
lib/image-pipeline/worker-client.ts
View File

@@ -5,7 +5,11 @@ import {
} from "@/lib/image-pipeline/preview-renderer";
import { hashPipeline, type PipelineStep } from "@/lib/image-pipeline/contracts";
import type { HistogramData } from "@/lib/image-pipeline/histogram";
import type { RenderFullOptions, RenderFullResult } from "@/lib/image-pipeline/render-types";
import type {
RenderFullOptions,
RenderFullResult,
RenderSourceComposition,
} from "@/lib/image-pipeline/render-types";
import {
getBackendFeatureFlags,
type BackendFeatureFlags,
@@ -20,14 +24,15 @@ export type BackendDiagnosticsMetadata = {
};
type PreviewWorkerPayload = {
sourceUrl: string;
sourceUrl?: string;
sourceComposition?: RenderSourceComposition;
steps: readonly PipelineStep[];
previewWidth: number;
includeHistogram?: boolean;
featureFlags?: BackendFeatureFlags;
};
type FullWorkerPayload = RenderFullOptions & {
type FullWorkerPayload = Omit<RenderFullOptions, "signal"> & {
featureFlags?: BackendFeatureFlags;
};
@@ -318,19 +323,20 @@ function runWorkerRequest<TResponse extends PreviewRenderResult | RenderFullResu
worker.postMessage({
kind: "full",
requestId,
payload: args.payload as RenderFullOptions,
payload: args.payload as FullWorkerPayload,
} satisfies WorkerRequestMessage);
});
}
function getPreviewRequestKey(options: {
sourceUrl: string;
sourceUrl?: string;
sourceComposition?: RenderSourceComposition;
steps: readonly PipelineStep[];
previewWidth: number;
includeHistogram?: boolean;
}): string {
return [
hashPipeline(options.sourceUrl, options.steps),
hashPipeline(options.sourceComposition ?? options.sourceUrl ?? null, options.steps),
options.previewWidth,
options.includeHistogram === true ? 1 : 0,
].join(":");
@@ -341,7 +347,8 @@ function getWorkerFeatureFlagsSnapshot(): BackendFeatureFlags {
}
async function runPreviewRequest(options: {
sourceUrl: string;
sourceUrl?: string;
sourceComposition?: RenderSourceComposition;
steps: readonly PipelineStep[];
previewWidth: number;
includeHistogram?: boolean;
@@ -352,6 +359,7 @@ async function runPreviewRequest(options: {
kind: "preview",
payload: {
sourceUrl: options.sourceUrl,
sourceComposition: options.sourceComposition,
steps: options.steps,
previewWidth: options.previewWidth,
includeHistogram: options.includeHistogram,
@@ -367,6 +375,7 @@ async function runPreviewRequest(options: {
if (!shouldFallbackToMainThread(error)) {
logWorkerClientDebug("preview request failed without fallback", {
sourceUrl: options.sourceUrl,
sourceComposition: options.sourceComposition,
previewWidth: options.previewWidth,
includeHistogram: options.includeHistogram,
diagnostics: getLastBackendDiagnostics(),
@@ -377,6 +386,7 @@ async function runPreviewRequest(options: {
logWorkerClientDebug("preview request falling back to main-thread", {
sourceUrl: options.sourceUrl,
sourceComposition: options.sourceComposition,
previewWidth: options.previewWidth,
includeHistogram: options.includeHistogram,
error,
@@ -387,7 +397,8 @@ async function runPreviewRequest(options: {
}
function getOrCreateSharedPreviewRequest(options: {
sourceUrl: string;
sourceUrl?: string;
sourceComposition?: RenderSourceComposition;
steps: readonly PipelineStep[];
previewWidth: number;
includeHistogram?: boolean;
@@ -419,7 +430,8 @@ function getOrCreateSharedPreviewRequest(options: {
}
export async function renderPreviewWithWorkerFallback(options: {
sourceUrl: string;
sourceUrl?: string;
sourceComposition?: RenderSourceComposition;
steps: readonly PipelineStep[];
previewWidth: number;
includeHistogram?: boolean;
@@ -431,6 +443,7 @@ export async function renderPreviewWithWorkerFallback(options: {
const sharedRequest = getOrCreateSharedPreviewRequest({
sourceUrl: options.sourceUrl,
sourceComposition: options.sourceComposition,
steps: options.steps,
previewWidth: options.previewWidth,
includeHistogram: options.includeHistogram,
@@ -488,14 +501,16 @@ export async function renderPreviewWithWorkerFallback(options: {
export async function renderFullWithWorkerFallback(
options: RenderFullOptions,
): Promise<RenderFullResult> {
const { signal, ...serializableOptions } = options;
try {
return await runWorkerRequest<RenderFullResult>({
kind: "full",
payload: {
...options,
...serializableOptions,
featureFlags: getWorkerFeatureFlagsSnapshot(),
},
signal: options.signal,
signal,
});
} catch (error: unknown) {
if (isAbortError(error)) {

219
lib/mixer-crop-layout.ts Normal file
View File

@@ -0,0 +1,219 @@
const MIN_CROP_REMAINING_SIZE = 0.1;
type MixerSurfaceFit = "contain" | "cover";
function formatPercent(value: number): string {
const normalized = Math.abs(value) < 1e-10 ? 0 : value;
return `${normalized}%`;
}
function computeFittedRect(args: {
sourceWidth: number;
sourceHeight: number;
boundsX: number;
boundsY: number;
boundsWidth: number;
boundsHeight: number;
fit?: MixerSurfaceFit;
}): { x: number; y: number; width: number; height: number } {
const {
sourceWidth,
sourceHeight,
boundsX,
boundsY,
boundsWidth,
boundsHeight,
fit = "contain",
} = args;
if (sourceWidth <= 0 || sourceHeight <= 0 || boundsWidth <= 0 || boundsHeight <= 0) {
return {
x: boundsX,
y: boundsY,
width: boundsWidth,
height: boundsHeight,
};
}
const scale =
fit === "cover"
? Math.max(boundsWidth / sourceWidth, boundsHeight / sourceHeight)
: Math.min(boundsWidth / sourceWidth, boundsHeight / sourceHeight);
if (!Number.isFinite(scale) || scale <= 0) {
return {
x: boundsX,
y: boundsY,
width: boundsWidth,
height: boundsHeight,
};
}
const width = sourceWidth * scale;
const height = sourceHeight * scale;
return {
x: boundsX + (boundsWidth - width) / 2,
y: boundsY + (boundsHeight - height) / 2,
width,
height,
};
}
export function computeMixerFrameRectInSurface(args: {
surfaceWidth: number;
surfaceHeight: number;
baseWidth: number;
baseHeight: number;
overlayX: number;
overlayY: number;
overlayWidth: number;
overlayHeight: number;
fit?: MixerSurfaceFit;
}): { x: number; y: number; width: number; height: number } | null {
if (args.baseWidth <= 0 || args.baseHeight <= 0 || args.surfaceWidth <= 0 || args.surfaceHeight <= 0) {
return null;
}
const baseRect = computeFittedRect({
sourceWidth: args.baseWidth,
sourceHeight: args.baseHeight,
boundsX: 0,
boundsY: 0,
boundsWidth: args.surfaceWidth,
boundsHeight: args.surfaceHeight,
fit: args.fit,
});
return {
x: (baseRect.x + args.overlayX * baseRect.width) / args.surfaceWidth,
y: (baseRect.y + args.overlayY * baseRect.height) / args.surfaceHeight,
width: (args.overlayWidth * baseRect.width) / args.surfaceWidth,
height: (args.overlayHeight * baseRect.height) / args.surfaceHeight,
};
}
export function computeVisibleMixerContentRect(args: {
frameAspectRatio: number;
sourceWidth: number;
sourceHeight: number;
cropLeft: number;
cropTop: number;
cropRight: number;
cropBottom: number;
}): { x: number; y: number; width: number; height: number } | null {
if (args.sourceWidth <= 0 || args.sourceHeight <= 0) {
return null;
}
const cropWidth = Math.max(1 - args.cropLeft - args.cropRight, MIN_CROP_REMAINING_SIZE);
const cropHeight = Math.max(1 - args.cropTop - args.cropBottom, MIN_CROP_REMAINING_SIZE);
const frameAspectRatio = args.frameAspectRatio > 0 ? args.frameAspectRatio : 1;
const rect = computeFittedRect({
sourceWidth: args.sourceWidth * cropWidth,
sourceHeight: args.sourceHeight * cropHeight,
boundsX: 0,
boundsY: 0,
boundsWidth: frameAspectRatio,
boundsHeight: 1,
});
return {
x: rect.x / frameAspectRatio,
y: rect.y,
width: rect.width / frameAspectRatio,
height: rect.height,
};
}
export function computeMixerCropImageStyle(args: {
frameAspectRatio: number;
sourceWidth: number;
sourceHeight: number;
cropLeft: number;
cropTop: number;
cropRight: number;
cropBottom: number;
}) {
const safeWidth = Math.max(1 - args.cropLeft - args.cropRight, MIN_CROP_REMAINING_SIZE);
const safeHeight = Math.max(1 - args.cropTop - args.cropBottom, MIN_CROP_REMAINING_SIZE);
const visibleRect = computeVisibleMixerContentRect(args);
if (!visibleRect) {
return {
left: formatPercent((-args.cropLeft / safeWidth) * 100),
top: formatPercent((-args.cropTop / safeHeight) * 100),
width: formatPercent((1 / safeWidth) * 100),
height: formatPercent((1 / safeHeight) * 100),
} as const;
}
const imageWidth = visibleRect.width / safeWidth;
const imageHeight = visibleRect.height / safeHeight;
return {
left: formatPercent((visibleRect.x - (args.cropLeft / safeWidth) * visibleRect.width) * 100),
top: formatPercent((visibleRect.y - (args.cropTop / safeHeight) * visibleRect.height) * 100),
width: formatPercent(imageWidth * 100),
height: formatPercent(imageHeight * 100),
} as const;
}
export function computeMixerCompareOverlayImageStyle(args: {
surfaceWidth: number;
surfaceHeight: number;
baseWidth: number;
baseHeight: number;
overlayX: number;
overlayY: number;
overlayWidth: number;
overlayHeight: number;
sourceWidth: number;
sourceHeight: number;
cropLeft: number;
cropTop: number;
cropRight: number;
cropBottom: number;
}) {
const frameRect = computeMixerFrameRectInSurface({
surfaceWidth: args.surfaceWidth,
surfaceHeight: args.surfaceHeight,
baseWidth: args.baseWidth,
baseHeight: args.baseHeight,
overlayX: args.overlayX,
overlayY: args.overlayY,
overlayWidth: args.overlayWidth,
overlayHeight: args.overlayHeight,
});
const frameAspectRatio =
frameRect && frameRect.width > 0 && frameRect.height > 0
? (frameRect.width * args.surfaceWidth) / (frameRect.height * args.surfaceHeight)
: args.overlayWidth > 0 && args.overlayHeight > 0
? args.overlayWidth / args.overlayHeight
: 1;
return computeMixerCropImageStyle({
frameAspectRatio,
sourceWidth: args.sourceWidth,
sourceHeight: args.sourceHeight,
cropLeft: args.cropLeft,
cropTop: args.cropTop,
cropRight: args.cropRight,
cropBottom: args.cropBottom,
});
}
export function isMixerCropImageReady(args: {
currentOverlayUrl: string | null | undefined;
loadedOverlayUrl: string | null;
sourceWidth: number;
sourceHeight: number;
}): boolean {
return Boolean(
args.currentOverlayUrl &&
args.loadedOverlayUrl === args.currentOverlayUrl &&
args.sourceWidth > 0 &&
args.sourceHeight > 0,
);
}