matthias/lemonspace_app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(image-pipeline): add backend capability and fallback diagnostics

Browse Source
This commit is contained in:
Matthias
2026-04-04 21:17:32 +02:00
parent a6bec59866
commit 8fb5482550
4 changed files with 535 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
lib/image-pipeline/backend/backend-router.ts
View File

@@ -2,6 +2,7 @@ import { applyPipelineStep, applyPipelineSteps } from "@/lib/image-pipeline/rend
import {
CPU_BACKEND_ID,
type BackendExecutionOptions,
type BackendHint,
type BackendRouter,
type FullBackendRequest,
@@ -9,6 +10,20 @@ import {
type PreviewBackendRequest,
} from "@/lib/image-pipeline/backend/backend-types";
type BackendFallbackReason = "unsupported_api" | "flag_disabled" | "runtime_error";
type BackendFallbackEvent = {
reason: BackendFallbackReason;
requestedBackend: string;
fallbackBackend: string;
error?: Error;
};
type BackendAvailability = {
supported?: boolean;
enabled?: boolean;
};
const cpuBackend: ImagePipelineBackend = {
id: CPU_BACKEND_ID,
runPreviewStep(request) {
@@ -43,6 +58,8 @@ function normalizeBackendHint(value: BackendHint): string | null {
export function createBackendRouter(options?: {
backends?: readonly ImagePipelineBackend[];
defaultBackendId?: string;
backendAvailability?: Readonly<Record<string, BackendAvailability>>;
onFallback?: (event: BackendFallbackEvent) => void;
}): BackendRouter {
const configuredBackends = options?.backends?.length ? [...options.backends] : [cpuBackend];
const byId = new Map(configuredBackends.map((backend) => [backend.id.toLowerCase(), backend]));
@@ -51,23 +68,124 @@ export function createBackendRouter(options?: {
byId.get(CPU_BACKEND_ID) ??
configuredBackends[0] ??
cpuBackend;
const normalizedDefaultId = defaultBackend.id.toLowerCase();
function readAvailability(backendId: string): BackendAvailability | undefined {
return options?.backendAvailability?.[backendId.toLowerCase()];
}
function emitFallback(event: BackendFallbackEvent): void {
options?.onFallback?.(event);
}
function resolveBackendWithFallbackReason(backendHint: BackendHint): {
backend: ImagePipelineBackend;
fallbackReason: BackendFallbackReason | null;
requestedBackend: string | null;
} {
const normalizedHint = normalizeBackendHint(backendHint);
if (!normalizedHint) {
return {
backend: defaultBackend,
fallbackReason: null,
requestedBackend: null,
};
}
const hintedBackend = byId.get(normalizedHint);
if (!hintedBackend) {
return {
backend: defaultBackend,
fallbackReason: "unsupported_api",
requestedBackend: normalizedHint,
};
}
const availability = readAvailability(normalizedHint);
if (availability?.enabled === false) {
return {
backend: defaultBackend,
fallbackReason: "flag_disabled",
requestedBackend: normalizedHint,
};
}
if (availability?.supported === false) {
return {
backend: defaultBackend,
fallbackReason: "unsupported_api",
requestedBackend: normalizedHint,
};
}
return {
backend: hintedBackend,
fallbackReason: null,
requestedBackend: normalizedHint,
};
}
function runWithRuntimeFallback(args: {
backendHint: BackendHint;
runBackend: (backend: ImagePipelineBackend) => void;
executionOptions?: BackendExecutionOptions;
}): void {
const selection = resolveBackendWithFallbackReason(args.backendHint);
if (selection.fallbackReason && selection.requestedBackend) {
emitFallback({
reason: selection.fallbackReason,
requestedBackend: selection.requestedBackend,
fallbackBackend: defaultBackend.id,
});
}
try {
args.runBackend(selection.backend);
return;
} catch (error: unknown) {
const shouldAbort = args.executionOptions?.shouldAbort;
if (shouldAbort?.()) {
throw error;
}
if (selection.backend.id.toLowerCase() === normalizedDefaultId) {
throw error;
}
const normalizedError =
error instanceof Error ? error : new Error("Image pipeline backend execution failed.");
emitFallback({
reason: "runtime_error",
requestedBackend: selection.backend.id.toLowerCase(),
fallbackBackend: defaultBackend.id,
error: normalizedError,
});
args.runBackend(defaultBackend);
}
}
return {
resolveBackend(backendHint) {
const normalizedHint = normalizeBackendHint(backendHint);
if (!normalizedHint) {
return defaultBackend;
}
return byId.get(normalizedHint) ?? defaultBackend;
return resolveBackendWithFallbackReason(backendHint).backend;
},
runPreviewStep(request) {
const backend = this.resolveBackend(request.backendHint);
backend.runPreviewStep(request);
runWithRuntimeFallback({
backendHint: request.backendHint,
executionOptions: request.executionOptions,
runBackend: (backend) => {
backend.runPreviewStep(request);
},
});
},
runFullPipeline(request) {
const backend = this.resolveBackend(request.backendHint);
backend.runFullPipeline(request);
runWithRuntimeFallback({
backendHint: request.backendHint,
executionOptions: request.executionOptions,
runBackend: (backend) => {
backend.runFullPipeline(request);
},
});
},
};
}

93
lib/image-pipeline/backend/capabilities.ts Normal file
View File

@@ -0,0 +1,93 @@
export type BackendCapabilities = {
webgl: boolean;
wasmSimd: boolean;
offscreenCanvas: boolean;
};
type CapabilityProbes = {
probeWebgl: () => boolean;
probeWasmSimd: () => boolean;
probeOffscreenCanvas: () => boolean;
};
const WASM_SIMD_PROBE_MODULE = new Uint8Array([
0x00,
0x61,
0x73,
0x6d,
0x01,
0x00,
0x00,
0x00,
0x01,
0x05,
0x01,
0x60,
0x00,
0x01,
0x7b,
0x03,
0x02,
0x01,
0x00,
0x0a,
0x0a,
0x01,
0x08,
0x00,
0x41,
0x00,
0xfd,
0x0f,
0x0b,
]);
function probeOffscreenCanvasAvailability(): boolean {
return typeof OffscreenCanvas !== "undefined";
}
function probeWebglAvailability(): boolean {
try {
if (typeof document !== "undefined") {
const canvas = document.createElement("canvas");
const context = canvas.getContext("webgl2") ?? canvas.getContext("webgl");
if (context) {
return true;
}
}
if (typeof OffscreenCanvas !== "undefined") {
const offscreenCanvas = new OffscreenCanvas(1, 1);
const context = offscreenCanvas.getContext("webgl2") ?? offscreenCanvas.getContext("webgl");
return Boolean(context);
}
return false;
} catch {
return false;
}
}
function probeWasmSimdAvailability(): boolean {
if (typeof WebAssembly === "undefined" || typeof WebAssembly.validate !== "function") {
return false;
}
try {
return WebAssembly.validate(WASM_SIMD_PROBE_MODULE);
} catch {
return false;
}
}
export function detectBackendCapabilities(probes?: Partial<CapabilityProbes>): BackendCapabilities {
const probeWebgl = probes?.probeWebgl ?? probeWebglAvailability;
const probeWasmSimd = probes?.probeWasmSimd ?? probeWasmSimdAvailability;
const probeOffscreenCanvas = probes?.probeOffscreenCanvas ?? probeOffscreenCanvasAvailability;
return {
webgl: probeWebgl(),
wasmSimd: probeWasmSimd(),
offscreenCanvas: probeOffscreenCanvas(),
};
}

29
lib/image-pipeline/worker-client.ts
View File

@@ -9,6 +9,12 @@ import type { RenderFullOptions, RenderFullResult } from "@/lib/image-pipeline/r
export type { PreviewRenderResult };
export type BackendDiagnosticsMetadata = {
backendId?: string;
fallbackReason?: string;
details?: Record<string, unknown>;
};
type PreviewWorkerPayload = {
sourceUrl: string;
steps: readonly PipelineStep[];
@@ -37,6 +43,7 @@ type WorkerResultPreviewPayload = {
height: number;
histogram: HistogramData;
pixels: ArrayBuffer;
diagnostics?: BackendDiagnosticsMetadata;
};
type WorkerResponseMessage =
@@ -48,7 +55,9 @@ type WorkerResponseMessage =
| {
kind: "full-result";
requestId: number;
payload: RenderFullResult;
payload: RenderFullResult & {
diagnostics?: BackendDiagnosticsMetadata;
};
}
| {
kind: "error";
@@ -56,6 +65,7 @@ type WorkerResponseMessage =
payload: {
name: string;
message: string;
diagnostics?: BackendDiagnosticsMetadata;
};
};
@@ -77,6 +87,7 @@ let workerInitError: Error | null = null;
let requestIdCounter = 0;
const pendingRequests = new Map<number, PendingRequest>();
const inFlightPreviewRequests = new Map<string, SharedPreviewRequest>();
let lastBackendDiagnostics: BackendDiagnosticsMetadata | null = null;
type SharedPreviewRequest = {
promise: Promise<PreviewRenderResult>;
@@ -126,6 +137,18 @@ function shouldFallbackToMainThread(error: unknown): error is WorkerUnavailableE
return error instanceof WorkerUnavailableError;
}
function updateLastBackendDiagnostics(metadata: BackendDiagnosticsMetadata | undefined): void {
if (!metadata) {
return;
}
lastBackendDiagnostics = metadata;
}
export function getLastBackendDiagnostics(): BackendDiagnosticsMetadata | null {
return lastBackendDiagnostics;
}
function getWorker(): Worker {
if (typeof window === "undefined" || typeof Worker === "undefined") {
throw new WorkerUnavailableError("Worker API is not available.");
@@ -154,6 +177,7 @@ function getWorker(): Worker {
pendingRequests.delete(message.requestId);
if (message.kind === "error") {
updateLastBackendDiagnostics(message.payload.diagnostics);
const workerError = new Error(message.payload.message);
workerError.name = message.payload.name;
pending.reject(workerError);
@@ -161,6 +185,7 @@ function getWorker(): Worker {
}
if (pending.kind === "preview" && message.kind === "preview-result") {
updateLastBackendDiagnostics(message.payload.diagnostics);
const pixels = new Uint8ClampedArray(message.payload.pixels);
pending.resolve({
width: message.payload.width,
@@ -172,6 +197,7 @@ function getWorker(): Worker {
}
if (pending.kind === "full" && message.kind === "full-result") {
updateLastBackendDiagnostics(message.payload.diagnostics);
pending.resolve(message.payload);
return;
}
@@ -206,6 +232,7 @@ function runWorkerRequest<TResponse extends PreviewRenderResult | RenderFullResu
}
const worker = getWorker();
lastBackendDiagnostics = null;
const requestId = nextRequestId();
return new Promise<TResponse>((resolve, reject) => {

286
tests/image-pipeline/backend-capabilities.test.ts Normal file
View File

@@ -0,0 +1,286 @@
// @vitest-environment jsdom
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { ImagePipelineBackend } from "@/lib/image-pipeline/backend/backend-types";
import { detectBackendCapabilities } from "@/lib/image-pipeline/backend/capabilities";
import { createBackendRouter } from "@/lib/image-pipeline/backend/backend-router";
const previewRendererMocks = vi.hoisted(() => ({
renderPreview: vi.fn(),
}));
const bridgeMocks = vi.hoisted(() => ({
renderFull: vi.fn(),
}));
vi.mock("@/lib/image-pipeline/preview-renderer", () => ({
renderPreview: previewRendererMocks.renderPreview,
}));
vi.mock("@/lib/image-pipeline/bridge", () => ({
renderFull: bridgeMocks.renderFull,
}));
describe("detectBackendCapabilities", () => {
it("reports webgl, wasmSimd and offscreenCanvas independently", () => {
expect(
detectBackendCapabilities({
probeWebgl: () => true,
probeWasmSimd: () => false,
probeOffscreenCanvas: () => true,
}),
).toEqual({
webgl: true,
wasmSimd: false,
offscreenCanvas: true,
});
expect(
detectBackendCapabilities({
probeWebgl: () => false,
probeWasmSimd: () => true,
probeOffscreenCanvas: () => false,
}),
).toEqual({
webgl: false,
wasmSimd: true,
offscreenCanvas: false,
});
});
});
describe("backend router fallback reasons", () => {
function createPreviewStep() {
return {
nodeId: "n1",
type: "color-adjust",
params: {
hsl: {
hue: 0,
saturation: 0,
luminance: 0,
},
temperature: 0,
tint: 0,
vibrance: 0,
},
} as const;
}
function createTestBackends(args: {
webglPreview: ImagePipelineBackend["runPreviewStep"];
cpuPreview?: ImagePipelineBackend["runPreviewStep"];
}): readonly ImagePipelineBackend[] {
return [
{
id: "cpu",
runPreviewStep: args.cpuPreview ?? vi.fn(),
runFullPipeline: vi.fn(),
},
{
id: "webgl",
runPreviewStep: args.webglPreview,
runFullPipeline: vi.fn(),
},
];
}
it("emits unsupported_api when backend is unavailable at runtime", () => {
const reasons: string[] = [];
const cpuPreview = vi.fn();
const router = createBackendRouter({
backends: createTestBackends({
cpuPreview,
webglPreview: vi.fn(),
}),
backendAvailability: {
webgl: {
supported: false,
enabled: true,
},
},
onFallback: (event) => {
reasons.push(event.reason);
},
});
router.runPreviewStep({
pixels: new Uint8ClampedArray(4),
step: createPreviewStep(),
width: 1,
height: 1,
backendHint: "webgl",
});
expect(cpuPreview).toHaveBeenCalledTimes(1);
expect(reasons).toEqual(["unsupported_api"]);
});
it("emits flag_disabled when backend is disabled by flags", () => {
const reasons: string[] = [];
const cpuPreview = vi.fn();
const router = createBackendRouter({
backends: createTestBackends({
cpuPreview,
webglPreview: vi.fn(),
}),
backendAvailability: {
webgl: {
supported: true,
enabled: false,
},
},
onFallback: (event) => {
reasons.push(event.reason);
},
});
router.runPreviewStep({
pixels: new Uint8ClampedArray(4),
step: createPreviewStep(),
width: 1,
height: 1,
backendHint: "webgl",
});
expect(cpuPreview).toHaveBeenCalledTimes(1);
expect(reasons).toEqual(["flag_disabled"]);
});
it("emits runtime_error when backend execution throws", () => {
const reasons: string[] = [];
const cpuPreview = vi.fn();
const router = createBackendRouter({
backends: createTestBackends({
cpuPreview,
webglPreview: () => {
throw new Error("WebGL kernel failed");
},
}),
backendAvailability: {
webgl: {
supported: true,
enabled: true,
},
},
onFallback: (event) => {
reasons.push(event.reason);
},
});
router.runPreviewStep({
pixels: new Uint8ClampedArray(4),
step: createPreviewStep(),
width: 1,
height: 1,
backendHint: "webgl",
});
expect(cpuPreview).toHaveBeenCalledTimes(1);
expect(reasons).toEqual(["runtime_error"]);
});
});
describe("worker-client backend diagnostics metadata", () => {
type WorkerMessage =
| {
kind: "preview" | "full";
requestId: number;
}
| {
kind: "cancel";
requestId: number;
};
class FakeWorker {
static behavior: (worker: FakeWorker, message: WorkerMessage) => void = () => {};
onmessage: ((event: MessageEvent) => void) | null = null;
onerror: (() => void) | null = null;
onmessageerror: (() => void) | null = null;
postMessage(message: WorkerMessage): void {
FakeWorker.behavior(this, message);
}
terminate(): void {
// no-op for test worker
}
}
beforeEach(() => {
vi.resetModules();
vi.unstubAllGlobals();
vi.stubGlobal(
"ImageData",
class ImageData {
data: Uint8ClampedArray;
width: number;
height: number;
constructor(data: Uint8ClampedArray, width: number, height: number) {
this.data = data;
this.width = width;
this.height = height;
}
},
);
vi.stubGlobal("Worker", FakeWorker as unknown as typeof Worker);
});
it("captures diagnostics metadata while keeping preview return contract", async () => {
FakeWorker.behavior = (worker, message) => {
if (message.kind !== "preview") {
return;
}
queueMicrotask(() => {
worker.onmessage?.({
data: {
kind: "preview-result",
requestId: message.requestId,
payload: {
width: 2,
height: 2,
histogram: {
red: new Uint32Array(256),
green: new Uint32Array(256),
blue: new Uint32Array(256),
luminance: new Uint32Array(256),
},
pixels: new Uint8ClampedArray(16).buffer,
diagnostics: {
backendId: "cpu",
fallbackReason: "unsupported_api",
details: {
requestedBackend: "webgl",
},
},
},
},
} as MessageEvent);
});
};
const workerClient = await import("@/lib/image-pipeline/worker-client");
const result = await workerClient.renderPreviewWithWorkerFallback({
sourceUrl: "https://cdn.example.com/source.png",
steps: [],
previewWidth: 128,
includeHistogram: true,
});
expect(result.width).toBe(2);
expect(result.height).toBe(2);
expect((result as { diagnostics?: unknown }).diagnostics).toBeUndefined();
expect(workerClient.getLastBackendDiagnostics()).toEqual({
backendId: "cpu",
fallbackReason: "unsupported_api",
details: {
requestedBackend: "webgl",
},
});
});
});