diff --git a/lib/image-pipeline/backend/backend-router.ts b/lib/image-pipeline/backend/backend-router.ts index 7428459..aee5df4 100644 --- a/lib/image-pipeline/backend/backend-router.ts +++ b/lib/image-pipeline/backend/backend-router.ts @@ -13,6 +13,11 @@ import { getBackendFeatureFlags, type BackendFeatureFlags, } from "@/lib/image-pipeline/backend/feature-flags"; +import { detectBackendCapabilities } from "@/lib/image-pipeline/backend/capabilities"; +import { + createWebglPreviewBackend, + isWebglPreviewPipelineSupported, +} from "@/lib/image-pipeline/backend/webgl/webgl-backend"; type BackendFallbackReason = "unsupported_api" | "flag_disabled" | "runtime_error"; @@ -103,12 +108,15 @@ export function createBackendRouter(options?: { return cpuFallbackBackend; } + const availability = readAvailability(configuredDefaultBackend.id); + if (availability?.enabled === false || availability?.supported === false) { + return cpuFallbackBackend; + } + return configuredDefaultBackend; } const defaultBackend = resolveDefaultBackend(); - const normalizedDefaultId = defaultBackend.id.toLowerCase(); - function readAvailability(backendId: string): BackendAvailability | undefined { return options?.backendAvailability?.[backendId.toLowerCase()]; } @@ -196,7 +204,7 @@ export function createBackendRouter(options?: { throw error; } - if (selection.backend.id.toLowerCase() === normalizedDefaultId) { + if (selection.backend.id.toLowerCase() === cpuFallbackBackend.id.toLowerCase()) { throw error; } @@ -205,10 +213,10 @@ export function createBackendRouter(options?: { emitFallback({ reason: "runtime_error", requestedBackend: selection.backend.id.toLowerCase(), - fallbackBackend: defaultBackend.id, + fallbackBackend: cpuFallbackBackend.id, error: normalizedError, }); - args.runBackend(defaultBackend); + args.runBackend(cpuFallbackBackend); } } @@ -237,10 +245,31 @@ export function createBackendRouter(options?: { }; } +const rolloutFeatureFlags = getBackendFeatureFlags(); +const rolloutCapabilities = detectBackendCapabilities(); +const rolloutWebglAvailable = rolloutCapabilities.webgl; +const rolloutWebglEnabled = rolloutFeatureFlags.webglEnabled && !rolloutFeatureFlags.forceCpu; + const rolloutRouter = createBackendRouter({ - featureFlags: getBackendFeatureFlags(), + backends: [cpuBackend, createWebglPreviewBackend()], + defaultBackendId: "webgl", + backendAvailability: { + webgl: { + supported: rolloutWebglAvailable, + enabled: rolloutWebglEnabled, + }, + }, + featureFlags: rolloutFeatureFlags, }); +export function getPreviewBackendHintForSteps(steps: readonly PreviewBackendRequest["step"][]): BackendHint { + if (!rolloutWebglEnabled || !rolloutWebglAvailable) { + return CPU_BACKEND_ID; + } + + return isWebglPreviewPipelineSupported(steps) ? "webgl" : CPU_BACKEND_ID; +} + export function runPreviewStepWithBackendRouter(request: PreviewBackendRequest): void { rolloutRouter.runPreviewStep(request); } diff --git a/lib/image-pipeline/backend/webgl/shaders/color-adjust.frag.glsl b/lib/image-pipeline/backend/webgl/shaders/color-adjust.frag.glsl new file mode 100644 index 0000000..20ac75e --- /dev/null +++ b/lib/image-pipeline/backend/webgl/shaders/color-adjust.frag.glsl @@ -0,0 +1,12 @@ +#version 100 +precision mediump float; + +varying vec2 vUv; +uniform sampler2D uSource; +uniform vec3 uColorShift; + +void main() { + vec4 color = texture2D(uSource, vUv); + color.rgb = clamp(color.rgb + uColorShift, 0.0, 1.0); + gl_FragColor = color; +} diff --git a/lib/image-pipeline/backend/webgl/shaders/curves.frag.glsl b/lib/image-pipeline/backend/webgl/shaders/curves.frag.glsl new file mode 100644 index 0000000..431555d --- /dev/null +++ b/lib/image-pipeline/backend/webgl/shaders/curves.frag.glsl @@ -0,0 +1,12 @@ +#version 100 +precision mediump float; + +varying vec2 vUv; +uniform sampler2D uSource; +uniform float uGamma; + +void main() { + vec4 color = texture2D(uSource, vUv); + color.rgb = pow(max(color.rgb, vec3(0.0)), vec3(max(uGamma, 0.001))); + gl_FragColor = color; +} diff --git a/lib/image-pipeline/backend/webgl/webgl-backend.ts b/lib/image-pipeline/backend/webgl/webgl-backend.ts new file mode 100644 index 0000000..53e0d5d --- /dev/null +++ b/lib/image-pipeline/backend/webgl/webgl-backend.ts @@ -0,0 +1,191 @@ +import { applyPipelineStep, applyPipelineSteps } from "@/lib/image-pipeline/render-core"; +import type { + BackendPipelineRequest, + BackendStepRequest, + ImagePipelineBackend, +} from "@/lib/image-pipeline/backend/backend-types"; +import type { PipelineStep } from "@/lib/image-pipeline/contracts"; + +const CURVES_FRAGMENT_SHADER_SOURCE = `#version 100 +precision mediump float; + +varying vec2 vUv; +uniform sampler2D uSource; +uniform float uGamma; + +void main() { + vec4 color = texture2D(uSource, vUv); + color.rgb = pow(max(color.rgb, vec3(0.0)), vec3(max(uGamma, 0.001))); + gl_FragColor = color; +} +`; + +const COLOR_ADJUST_FRAGMENT_SHADER_SOURCE = `#version 100 +precision mediump float; + +varying vec2 vUv; +uniform sampler2D uSource; +uniform vec3 uColorShift; + +void main() { + vec4 color = texture2D(uSource, vUv); + color.rgb = clamp(color.rgb + uColorShift, 0.0, 1.0); + gl_FragColor = color; +} +`; + +const VERTEX_SHADER_SOURCE = `#version 100 +attribute vec2 aPosition; +varying vec2 vUv; + +void main() { + vUv = (aPosition + 1.0) * 0.5; + gl_Position = vec4(aPosition, 0.0, 1.0); +} +`; + +type SupportedPreviewStepType = "curves" | "color-adjust"; + +const SUPPORTED_PREVIEW_STEP_TYPES = new Set([ + "curves", + "color-adjust", +]); + +function assertSupportedStep(step: PipelineStep): void { + if (SUPPORTED_PREVIEW_STEP_TYPES.has(step.type as SupportedPreviewStepType)) { + return; + } + + throw new Error(`WebGL backend does not support step type '${step.type}'.`); +} + +function createGlContext(): WebGLRenderingContext | WebGL2RenderingContext { + if (typeof document !== "undefined") { + const canvas = document.createElement("canvas"); + return ( + canvas.getContext("webgl2") ?? + canvas.getContext("webgl") ?? + (() => { + throw new Error("WebGL context is unavailable."); + })() + ); + } + + if (typeof OffscreenCanvas !== "undefined") { + const canvas = new OffscreenCanvas(1, 1); + return ( + canvas.getContext("webgl2") ?? + canvas.getContext("webgl") ?? + (() => { + throw new Error("WebGL context is unavailable."); + })() + ); + } + + throw new Error("WebGL context is unavailable."); +} + +function compileShader( + gl: WebGLRenderingContext | WebGL2RenderingContext, + source: string, + shaderType: number, +): WebGLShader { + const shader = gl.createShader(shaderType); + if (!shader) { + throw new Error("WebGL shader allocation failed."); + } + + gl.shaderSource(shader, source); + gl.compileShader(shader); + + if (gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { + return shader; + } + + const info = gl.getShaderInfoLog(shader) ?? "Unknown shader compile error."; + gl.deleteShader(shader); + throw new Error(`WebGL shader compile failed: ${info}`); +} + +function compileProgram( + gl: WebGLRenderingContext | WebGL2RenderingContext, + fragmentShaderSource: string, +): void { + const vertexShader = compileShader(gl, VERTEX_SHADER_SOURCE, gl.VERTEX_SHADER); + const fragmentShader = compileShader(gl, fragmentShaderSource, gl.FRAGMENT_SHADER); + const program = gl.createProgram(); + + if (!program) { + gl.deleteShader(vertexShader); + gl.deleteShader(fragmentShader); + throw new Error("WebGL program allocation failed."); + } + + gl.attachShader(program, vertexShader); + gl.attachShader(program, fragmentShader); + gl.linkProgram(program); + + gl.deleteShader(vertexShader); + gl.deleteShader(fragmentShader); + + if (gl.getProgramParameter(program, gl.LINK_STATUS)) { + gl.deleteProgram(program); + return; + } + + const info = gl.getProgramInfoLog(program) ?? "Unknown program link error."; + gl.deleteProgram(program); + throw new Error(`WebGL program link failed: ${info}`); +} + +export function isWebglPreviewStepSupported(step: PipelineStep): boolean { + return SUPPORTED_PREVIEW_STEP_TYPES.has(step.type as SupportedPreviewStepType); +} + +export function isWebglPreviewPipelineSupported(steps: readonly PipelineStep[]): boolean { + return steps.every((step) => isWebglPreviewStepSupported(step)); +} + +export function createWebglPreviewBackend(): ImagePipelineBackend { + let initialized = false; + + function ensureInitialized(): void { + if (initialized) { + return; + } + + const gl = createGlContext(); + compileProgram(gl, CURVES_FRAGMENT_SHADER_SOURCE); + compileProgram(gl, COLOR_ADJUST_FRAGMENT_SHADER_SOURCE); + initialized = true; + } + + return { + id: "webgl", + runPreviewStep(request: BackendStepRequest): void { + assertSupportedStep(request.step); + ensureInitialized(); + applyPipelineStep( + request.pixels, + request.step, + request.width, + request.height, + request.executionOptions, + ); + }, + runFullPipeline(request: BackendPipelineRequest): void { + if (!isWebglPreviewPipelineSupported(request.steps)) { + throw new Error("WebGL backend does not support all pipeline steps."); + } + + ensureInitialized(); + applyPipelineSteps( + request.pixels, + request.steps, + request.width, + request.height, + request.executionOptions, + ); + }, + }; +} diff --git a/lib/image-pipeline/preview-renderer.ts b/lib/image-pipeline/preview-renderer.ts index 7756c09..c0aeba6 100644 --- a/lib/image-pipeline/preview-renderer.ts +++ b/lib/image-pipeline/preview-renderer.ts @@ -1,5 +1,8 @@ import type { PipelineStep } from "@/lib/image-pipeline/contracts"; -import { runPreviewStepWithBackendRouter } from "@/lib/image-pipeline/backend/backend-router"; +import { + getPreviewBackendHintForSteps, + runPreviewStepWithBackendRouter, +} from "@/lib/image-pipeline/backend/backend-router"; import { computeHistogram, emptyHistogram, type HistogramData } from "@/lib/image-pipeline/histogram"; import { loadSourceBitmap } from "@/lib/image-pipeline/source-loader"; @@ -75,6 +78,7 @@ export async function renderPreview(options: { context.drawImage(bitmap, 0, 0, width, height); const imageData = context.getImageData(0, 0, width, height); + const backendHint = getPreviewBackendHintForSteps(options.steps); for (let index = 0; index < options.steps.length; index += 1) { runPreviewStepWithBackendRouter({ @@ -82,6 +86,7 @@ export async function renderPreview(options: { step: options.steps[index]!, width, height, + backendHint, executionOptions: { shouldAbort: () => Boolean(options.signal?.aborted), }, diff --git a/tests/image-pipeline/webgl-backend-poc.test.ts b/tests/image-pipeline/webgl-backend-poc.test.ts new file mode 100644 index 0000000..dcdf86e --- /dev/null +++ b/tests/image-pipeline/webgl-backend-poc.test.ts @@ -0,0 +1,280 @@ +// @vitest-environment jsdom + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { PipelineStep } from "@/lib/image-pipeline/contracts"; +import type { ImagePipelineBackend } from "@/lib/image-pipeline/backend/backend-types"; + +function createCurvesStep(): PipelineStep { + return { + nodeId: "curves-1", + type: "curves", + params: { + channelMode: "master", + levels: { + blackPoint: 0, + whitePoint: 255, + gamma: 1, + }, + points: { + rgb: [ + { x: 0, y: 0 }, + { x: 255, y: 255 }, + ], + red: [ + { x: 0, y: 0 }, + { x: 255, y: 255 }, + ], + green: [ + { x: 0, y: 0 }, + { x: 255, y: 255 }, + ], + blue: [ + { x: 0, y: 0 }, + { x: 255, y: 255 }, + ], + }, + }, + }; +} + +function createColorAdjustStep(): PipelineStep { + return { + nodeId: "color-1", + type: "color-adjust", + params: { + hsl: { + hue: 0, + saturation: 0, + luminance: 0, + }, + temperature: 0, + tint: 0, + vibrance: 0, + }, + }; +} + +function createUnsupportedStep(): PipelineStep { + return { + nodeId: "light-1", + type: "light-adjust", + params: { + exposure: 0, + }, + }; +} + +function createCpuAndWebglBackends(args: { + webglPreview?: ImagePipelineBackend["runPreviewStep"]; + cpuPreview?: ImagePipelineBackend["runPreviewStep"]; +}): readonly ImagePipelineBackend[] { + return [ + { + id: "cpu", + runPreviewStep: args.cpuPreview ?? vi.fn(), + runFullPipeline: vi.fn(), + }, + { + id: "webgl", + runPreviewStep: args.webglPreview ?? vi.fn(), + runFullPipeline: vi.fn(), + }, + ]; +} + +describe("webgl backend poc", () => { + beforeEach(() => { + vi.resetModules(); + vi.resetAllMocks(); + }); + + afterEach(() => { + vi.unmock("@/lib/image-pipeline/backend/capabilities"); + vi.unmock("@/lib/image-pipeline/backend/feature-flags"); + vi.unmock("@/lib/image-pipeline/backend/webgl/webgl-backend"); + vi.unmock("@/lib/image-pipeline/backend/backend-router"); + vi.unmock("@/lib/image-pipeline/source-loader"); + }); + + it("selects webgl for preview when webgl is available and enabled", async () => { + const webglPreview = vi.fn(); + + vi.doMock("@/lib/image-pipeline/backend/feature-flags", async () => { + const actual = await vi.importActual( + "@/lib/image-pipeline/backend/feature-flags", + ); + return { + ...actual, + getBackendFeatureFlags: () => ({ + forceCpu: false, + webglEnabled: true, + wasmEnabled: false, + }), + }; + }); + + vi.doMock("@/lib/image-pipeline/backend/capabilities", async () => { + const actual = await vi.importActual( + "@/lib/image-pipeline/backend/capabilities", + ); + return { + ...actual, + detectBackendCapabilities: () => ({ + webgl: true, + wasmSimd: false, + offscreenCanvas: true, + }), + }; + }); + + vi.doMock("@/lib/image-pipeline/backend/webgl/webgl-backend", () => ({ + createWebglPreviewBackend: () => ({ + id: "webgl", + runPreviewStep: webglPreview, + runFullPipeline: vi.fn(), + }), + })); + + const { runPreviewStepWithBackendRouter } = await import("@/lib/image-pipeline/backend/backend-router"); + + runPreviewStepWithBackendRouter({ + pixels: new Uint8ClampedArray(4), + step: createCurvesStep(), + width: 1, + height: 1, + }); + + expect(webglPreview).toHaveBeenCalledTimes(1); + }); + + it("uses cpu for every step in a mixed pipeline request", async () => { + vi.doMock("@/lib/image-pipeline/backend/feature-flags", async () => { + const actual = await vi.importActual( + "@/lib/image-pipeline/backend/feature-flags", + ); + return { + ...actual, + getBackendFeatureFlags: () => ({ + forceCpu: false, + webglEnabled: true, + wasmEnabled: false, + }), + }; + }); + + vi.doMock("@/lib/image-pipeline/backend/capabilities", async () => { + const actual = await vi.importActual( + "@/lib/image-pipeline/backend/capabilities", + ); + return { + ...actual, + detectBackendCapabilities: () => ({ + webgl: true, + wasmSimd: false, + offscreenCanvas: true, + }), + }; + }); + + vi.doMock("@/lib/image-pipeline/backend/webgl/webgl-backend", async () => { + const actual = await vi.importActual( + "@/lib/image-pipeline/backend/webgl/webgl-backend", + ); + return { + ...actual, + }; + }); + + const backendRouterModule = await import("@/lib/image-pipeline/backend/backend-router"); + const runPreviewStepWithBackendRouter = vi + .spyOn(backendRouterModule, "runPreviewStepWithBackendRouter") + .mockImplementation(() => {}); + + vi.doMock("@/lib/image-pipeline/source-loader", () => ({ + loadSourceBitmap: vi.fn().mockResolvedValue({ width: 2, height: 2 }), + })); + + vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({ + drawImage: vi.fn(), + getImageData: vi.fn(() => ({ + data: new Uint8ClampedArray(16), + })), + } as unknown as CanvasRenderingContext2D); + + vi.stubGlobal("requestAnimationFrame", ((callback: FrameRequestCallback) => { + callback(0); + return 1; + }) as typeof requestAnimationFrame); + + const { renderPreview } = await import("@/lib/image-pipeline/preview-renderer"); + + await renderPreview({ + sourceUrl: "https://cdn.example.com/source.png", + steps: [createColorAdjustStep(), createUnsupportedStep()], + previewWidth: 2, + includeHistogram: false, + }); + + expect(runPreviewStepWithBackendRouter).toHaveBeenCalledTimes(2); + for (const call of runPreviewStepWithBackendRouter.mock.calls) { + expect(call[0]).toMatchObject({ + backendHint: "cpu", + }); + } + }); + + it("downgrades compile/link failures to cpu with runtime_error reason", async () => { + const { createBackendRouter } = await import("@/lib/image-pipeline/backend/backend-router"); + const cpuPreview = vi.fn(); + const fallbackEvents: Array<{ + reason: string; + requestedBackend: string; + fallbackBackend: string; + }> = []; + + const router = createBackendRouter({ + backends: createCpuAndWebglBackends({ + cpuPreview, + webglPreview: () => { + throw new Error("WebGL shader compile failed"); + }, + }), + defaultBackendId: "webgl", + backendAvailability: { + webgl: { + supported: true, + enabled: true, + }, + }, + featureFlags: { + forceCpu: false, + webglEnabled: true, + wasmEnabled: false, + }, + onFallback: (event) => { + fallbackEvents.push({ + reason: event.reason, + requestedBackend: event.requestedBackend, + fallbackBackend: event.fallbackBackend, + }); + }, + }); + + router.runPreviewStep({ + pixels: new Uint8ClampedArray(4), + step: createCurvesStep(), + width: 1, + height: 1, + }); + + expect(cpuPreview).toHaveBeenCalledTimes(1); + expect(fallbackEvents).toEqual([ + { + reason: "runtime_error", + requestedBackend: "webgl", + fallbackBackend: "cpu", + }, + ]); + }); +});