feat(image-pipeline): add webgl preview backend poc

lib/image-pipeline/backend/backend-router.ts

@@ -13,6 +13,11 @@ import {
   getBackendFeatureFlags,
   type BackendFeatureFlags,
 } from "@/lib/image-pipeline/backend/feature-flags";
+import { detectBackendCapabilities } from "@/lib/image-pipeline/backend/capabilities";
+import {
+  createWebglPreviewBackend,
+  isWebglPreviewPipelineSupported,
+} from "@/lib/image-pipeline/backend/webgl/webgl-backend";
 
 type BackendFallbackReason = "unsupported_api" | "flag_disabled" | "runtime_error";
 
@@ -103,12 +108,15 @@ export function createBackendRouter(options?: {
       return cpuFallbackBackend;
     }
 
+    const availability = readAvailability(configuredDefaultBackend.id);
+    if (availability?.enabled === false || availability?.supported === false) {
+      return cpuFallbackBackend;
+    }
+
     return configuredDefaultBackend;
   }
 
   const defaultBackend = resolveDefaultBackend();
-  const normalizedDefaultId = defaultBackend.id.toLowerCase();
-
   function readAvailability(backendId: string): BackendAvailability | undefined {
     return options?.backendAvailability?.[backendId.toLowerCase()];
   }
@@ -196,7 +204,7 @@ export function createBackendRouter(options?: {
         throw error;
       }
 
-      if (selection.backend.id.toLowerCase() === normalizedDefaultId) {
+      if (selection.backend.id.toLowerCase() === cpuFallbackBackend.id.toLowerCase()) {
         throw error;
       }
 
@@ -205,10 +213,10 @@ export function createBackendRouter(options?: {
       emitFallback({
         reason: "runtime_error",
         requestedBackend: selection.backend.id.toLowerCase(),
-        fallbackBackend: defaultBackend.id,
+        fallbackBackend: cpuFallbackBackend.id,
         error: normalizedError,
       });
-      args.runBackend(defaultBackend);
+      args.runBackend(cpuFallbackBackend);
     }
   }
 
@@ -237,10 +245,31 @@ export function createBackendRouter(options?: {
   };
 }
 
+const rolloutFeatureFlags = getBackendFeatureFlags();
+const rolloutCapabilities = detectBackendCapabilities();
+const rolloutWebglAvailable = rolloutCapabilities.webgl;
+const rolloutWebglEnabled = rolloutFeatureFlags.webglEnabled && !rolloutFeatureFlags.forceCpu;
+
 const rolloutRouter = createBackendRouter({
-  featureFlags: getBackendFeatureFlags(),
+  backends: [cpuBackend, createWebglPreviewBackend()],
+  defaultBackendId: "webgl",
+  backendAvailability: {
+    webgl: {
+      supported: rolloutWebglAvailable,
+      enabled: rolloutWebglEnabled,
+    },
+  },
+  featureFlags: rolloutFeatureFlags,
 });
 
+export function getPreviewBackendHintForSteps(steps: readonly PreviewBackendRequest["step"][]): BackendHint {
+  if (!rolloutWebglEnabled || !rolloutWebglAvailable) {
+    return CPU_BACKEND_ID;
+  }
+
+  return isWebglPreviewPipelineSupported(steps) ? "webgl" : CPU_BACKEND_ID;
+}
+
 export function runPreviewStepWithBackendRouter(request: PreviewBackendRequest): void {
   rolloutRouter.runPreviewStep(request);
 }

lib/image-pipeline/backend/webgl/shaders/color-adjust.frag.glsl

@@ -0,0 +1,12 @@
+#version 100
+precision mediump float;
+
+varying vec2 vUv;
+uniform sampler2D uSource;
+uniform vec3 uColorShift;
+
+void main() {
+  vec4 color = texture2D(uSource, vUv);
+  color.rgb = clamp(color.rgb + uColorShift, 0.0, 1.0);
+  gl_FragColor = color;
+}

lib/image-pipeline/backend/webgl/shaders/curves.frag.glsl

@@ -0,0 +1,12 @@
+#version 100
+precision mediump float;
+
+varying vec2 vUv;
+uniform sampler2D uSource;
+uniform float uGamma;
+
+void main() {
+  vec4 color = texture2D(uSource, vUv);
+  color.rgb = pow(max(color.rgb, vec3(0.0)), vec3(max(uGamma, 0.001)));
+  gl_FragColor = color;
+}

lib/image-pipeline/backend/webgl/webgl-backend.ts

@@ -0,0 +1,191 @@
+import { applyPipelineStep, applyPipelineSteps } from "@/lib/image-pipeline/render-core";
+import type {
+  BackendPipelineRequest,
+  BackendStepRequest,
+  ImagePipelineBackend,
+} from "@/lib/image-pipeline/backend/backend-types";
+import type { PipelineStep } from "@/lib/image-pipeline/contracts";
+
+const CURVES_FRAGMENT_SHADER_SOURCE = `#version 100
+precision mediump float;
+
+varying vec2 vUv;
+uniform sampler2D uSource;
+uniform float uGamma;
+
+void main() {
+  vec4 color = texture2D(uSource, vUv);
+  color.rgb = pow(max(color.rgb, vec3(0.0)), vec3(max(uGamma, 0.001)));
+  gl_FragColor = color;
+}
+`;
+
+const COLOR_ADJUST_FRAGMENT_SHADER_SOURCE = `#version 100
+precision mediump float;
+
+varying vec2 vUv;
+uniform sampler2D uSource;
+uniform vec3 uColorShift;
+
+void main() {
+  vec4 color = texture2D(uSource, vUv);
+  color.rgb = clamp(color.rgb + uColorShift, 0.0, 1.0);
+  gl_FragColor = color;
+}
+`;
+
+const VERTEX_SHADER_SOURCE = `#version 100
+attribute vec2 aPosition;
+varying vec2 vUv;
+
+void main() {
+  vUv = (aPosition + 1.0) * 0.5;
+  gl_Position = vec4(aPosition, 0.0, 1.0);
+}
+`;
+
+type SupportedPreviewStepType = "curves" | "color-adjust";
+
+const SUPPORTED_PREVIEW_STEP_TYPES = new Set<SupportedPreviewStepType>([
+  "curves",
+  "color-adjust",
+]);
+
+function assertSupportedStep(step: PipelineStep): void {
+  if (SUPPORTED_PREVIEW_STEP_TYPES.has(step.type as SupportedPreviewStepType)) {
+    return;
+  }
+
+  throw new Error(`WebGL backend does not support step type '${step.type}'.`);
+}
+
+function createGlContext(): WebGLRenderingContext | WebGL2RenderingContext {
+  if (typeof document !== "undefined") {
+    const canvas = document.createElement("canvas");
+    return (
+      canvas.getContext("webgl2") ??
+      canvas.getContext("webgl") ??
+      (() => {
+        throw new Error("WebGL context is unavailable.");
+      })()
+    );
+  }
+
+  if (typeof OffscreenCanvas !== "undefined") {
+    const canvas = new OffscreenCanvas(1, 1);
+    return (
+      canvas.getContext("webgl2") ??
+      canvas.getContext("webgl") ??
+      (() => {
+        throw new Error("WebGL context is unavailable.");
+      })()
+    );
+  }
+
+  throw new Error("WebGL context is unavailable.");
+}
+
+function compileShader(
+  gl: WebGLRenderingContext | WebGL2RenderingContext,
+  source: string,
+  shaderType: number,
+): WebGLShader {
+  const shader = gl.createShader(shaderType);
+  if (!shader) {
+    throw new Error("WebGL shader allocation failed.");
+  }
+
+  gl.shaderSource(shader, source);
+  gl.compileShader(shader);
+
+  if (gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
+    return shader;
+  }
+
+  const info = gl.getShaderInfoLog(shader) ?? "Unknown shader compile error.";
+  gl.deleteShader(shader);
+  throw new Error(`WebGL shader compile failed: ${info}`);
+}
+
+function compileProgram(
+  gl: WebGLRenderingContext | WebGL2RenderingContext,
+  fragmentShaderSource: string,
+): void {
+  const vertexShader = compileShader(gl, VERTEX_SHADER_SOURCE, gl.VERTEX_SHADER);
+  const fragmentShader = compileShader(gl, fragmentShaderSource, gl.FRAGMENT_SHADER);
+  const program = gl.createProgram();
+
+  if (!program) {
+    gl.deleteShader(vertexShader);
+    gl.deleteShader(fragmentShader);
+    throw new Error("WebGL program allocation failed.");
+  }
+
+  gl.attachShader(program, vertexShader);
+  gl.attachShader(program, fragmentShader);
+  gl.linkProgram(program);
+
+  gl.deleteShader(vertexShader);
+  gl.deleteShader(fragmentShader);
+
+  if (gl.getProgramParameter(program, gl.LINK_STATUS)) {
+    gl.deleteProgram(program);
+    return;
+  }
+
+  const info = gl.getProgramInfoLog(program) ?? "Unknown program link error.";
+  gl.deleteProgram(program);
+  throw new Error(`WebGL program link failed: ${info}`);
+}
+
+export function isWebglPreviewStepSupported(step: PipelineStep): boolean {
+  return SUPPORTED_PREVIEW_STEP_TYPES.has(step.type as SupportedPreviewStepType);
+}
+
+export function isWebglPreviewPipelineSupported(steps: readonly PipelineStep[]): boolean {
+  return steps.every((step) => isWebglPreviewStepSupported(step));
+}
+
+export function createWebglPreviewBackend(): ImagePipelineBackend {
+  let initialized = false;
+
+  function ensureInitialized(): void {
+    if (initialized) {
+      return;
+    }
+
+    const gl = createGlContext();
+    compileProgram(gl, CURVES_FRAGMENT_SHADER_SOURCE);
+    compileProgram(gl, COLOR_ADJUST_FRAGMENT_SHADER_SOURCE);
+    initialized = true;
+  }
+
+  return {
+    id: "webgl",
+    runPreviewStep(request: BackendStepRequest): void {
+      assertSupportedStep(request.step);
+      ensureInitialized();
+      applyPipelineStep(
+        request.pixels,
+        request.step,
+        request.width,
+        request.height,
+        request.executionOptions,
+      );
+    },
+    runFullPipeline(request: BackendPipelineRequest): void {
+      if (!isWebglPreviewPipelineSupported(request.steps)) {
+        throw new Error("WebGL backend does not support all pipeline steps.");
+      }
+
+      ensureInitialized();
+      applyPipelineSteps(
+        request.pixels,
+        request.steps,
+        request.width,
+        request.height,
+        request.executionOptions,
+      );
+    },
+  };
+}

lib/image-pipeline/preview-renderer.ts

@@ -1,5 +1,8 @@
 import type { PipelineStep } from "@/lib/image-pipeline/contracts";
-import { runPreviewStepWithBackendRouter } from "@/lib/image-pipeline/backend/backend-router";
+import {
+  getPreviewBackendHintForSteps,
+  runPreviewStepWithBackendRouter,
+} from "@/lib/image-pipeline/backend/backend-router";
 import { computeHistogram, emptyHistogram, type HistogramData } from "@/lib/image-pipeline/histogram";
 import { loadSourceBitmap } from "@/lib/image-pipeline/source-loader";
 
@@ -75,6 +78,7 @@ export async function renderPreview(options: {
 
   context.drawImage(bitmap, 0, 0, width, height);
   const imageData = context.getImageData(0, 0, width, height);
+  const backendHint = getPreviewBackendHintForSteps(options.steps);
 
   for (let index = 0; index < options.steps.length; index += 1) {
     runPreviewStepWithBackendRouter({
@@ -82,6 +86,7 @@ export async function renderPreview(options: {
       step: options.steps[index]!,
       width,
       height,
+      backendHint,
       executionOptions: {
         shouldAbort: () => Boolean(options.signal?.aborted),
       },

tests/image-pipeline/webgl-backend-poc.test.ts

@@ -0,0 +1,280 @@
+// @vitest-environment jsdom
+
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+import type { PipelineStep } from "@/lib/image-pipeline/contracts";
+import type { ImagePipelineBackend } from "@/lib/image-pipeline/backend/backend-types";
+
+function createCurvesStep(): PipelineStep {
+  return {
+    nodeId: "curves-1",
+    type: "curves",
+    params: {
+      channelMode: "master",
+      levels: {
+        blackPoint: 0,
+        whitePoint: 255,
+        gamma: 1,
+      },
+      points: {
+        rgb: [
+          { x: 0, y: 0 },
+          { x: 255, y: 255 },
+        ],
+        red: [
+          { x: 0, y: 0 },
+          { x: 255, y: 255 },
+        ],
+        green: [
+          { x: 0, y: 0 },
+          { x: 255, y: 255 },
+        ],
+        blue: [
+          { x: 0, y: 0 },
+          { x: 255, y: 255 },
+        ],
+      },
+    },
+  };
+}
+
+function createColorAdjustStep(): PipelineStep {
+  return {
+    nodeId: "color-1",
+    type: "color-adjust",
+    params: {
+      hsl: {
+        hue: 0,
+        saturation: 0,
+        luminance: 0,
+      },
+      temperature: 0,
+      tint: 0,
+      vibrance: 0,
+    },
+  };
+}
+
+function createUnsupportedStep(): PipelineStep {
+  return {
+    nodeId: "light-1",
+    type: "light-adjust",
+    params: {
+      exposure: 0,
+    },
+  };
+}
+
+function createCpuAndWebglBackends(args: {
+  webglPreview?: ImagePipelineBackend["runPreviewStep"];
+  cpuPreview?: ImagePipelineBackend["runPreviewStep"];
+}): readonly ImagePipelineBackend[] {
+  return [
+    {
+      id: "cpu",
+      runPreviewStep: args.cpuPreview ?? vi.fn(),
+      runFullPipeline: vi.fn(),
+    },
+    {
+      id: "webgl",
+      runPreviewStep: args.webglPreview ?? vi.fn(),
+      runFullPipeline: vi.fn(),
+    },
+  ];
+}
+
+describe("webgl backend poc", () => {
+  beforeEach(() => {
+    vi.resetModules();
+    vi.resetAllMocks();
+  });
+
+  afterEach(() => {
+    vi.unmock("@/lib/image-pipeline/backend/capabilities");
+    vi.unmock("@/lib/image-pipeline/backend/feature-flags");
+    vi.unmock("@/lib/image-pipeline/backend/webgl/webgl-backend");
+    vi.unmock("@/lib/image-pipeline/backend/backend-router");
+    vi.unmock("@/lib/image-pipeline/source-loader");
+  });
+
+  it("selects webgl for preview when webgl is available and enabled", async () => {
+    const webglPreview = vi.fn();
+
+    vi.doMock("@/lib/image-pipeline/backend/feature-flags", async () => {
+      const actual = await vi.importActual<typeof import("@/lib/image-pipeline/backend/feature-flags")>(
+        "@/lib/image-pipeline/backend/feature-flags",
+      );
+      return {
+        ...actual,
+        getBackendFeatureFlags: () => ({
+          forceCpu: false,
+          webglEnabled: true,
+          wasmEnabled: false,
+        }),
+      };
+    });
+
+    vi.doMock("@/lib/image-pipeline/backend/capabilities", async () => {
+      const actual = await vi.importActual<typeof import("@/lib/image-pipeline/backend/capabilities")>(
+        "@/lib/image-pipeline/backend/capabilities",
+      );
+      return {
+        ...actual,
+        detectBackendCapabilities: () => ({
+          webgl: true,
+          wasmSimd: false,
+          offscreenCanvas: true,
+        }),
+      };
+    });
+
+    vi.doMock("@/lib/image-pipeline/backend/webgl/webgl-backend", () => ({
+      createWebglPreviewBackend: () => ({
+        id: "webgl",
+        runPreviewStep: webglPreview,
+        runFullPipeline: vi.fn(),
+      }),
+    }));
+
+    const { runPreviewStepWithBackendRouter } = await import("@/lib/image-pipeline/backend/backend-router");
+
+    runPreviewStepWithBackendRouter({
+      pixels: new Uint8ClampedArray(4),
+      step: createCurvesStep(),
+      width: 1,
+      height: 1,
+    });
+
+    expect(webglPreview).toHaveBeenCalledTimes(1);
+  });
+
+  it("uses cpu for every step in a mixed pipeline request", async () => {
+    vi.doMock("@/lib/image-pipeline/backend/feature-flags", async () => {
+      const actual = await vi.importActual<typeof import("@/lib/image-pipeline/backend/feature-flags")>(
+        "@/lib/image-pipeline/backend/feature-flags",
+      );
+      return {
+        ...actual,
+        getBackendFeatureFlags: () => ({
+          forceCpu: false,
+          webglEnabled: true,
+          wasmEnabled: false,
+        }),
+      };
+    });
+
+    vi.doMock("@/lib/image-pipeline/backend/capabilities", async () => {
+      const actual = await vi.importActual<typeof import("@/lib/image-pipeline/backend/capabilities")>(
+        "@/lib/image-pipeline/backend/capabilities",
+      );
+      return {
+        ...actual,
+        detectBackendCapabilities: () => ({
+          webgl: true,
+          wasmSimd: false,
+          offscreenCanvas: true,
+        }),
+      };
+    });
+
+    vi.doMock("@/lib/image-pipeline/backend/webgl/webgl-backend", async () => {
+      const actual = await vi.importActual<typeof import("@/lib/image-pipeline/backend/webgl/webgl-backend")>(
+        "@/lib/image-pipeline/backend/webgl/webgl-backend",
+      );
+      return {
+        ...actual,
+      };
+    });
+
+    const backendRouterModule = await import("@/lib/image-pipeline/backend/backend-router");
+    const runPreviewStepWithBackendRouter = vi
+      .spyOn(backendRouterModule, "runPreviewStepWithBackendRouter")
+      .mockImplementation(() => {});
+
+    vi.doMock("@/lib/image-pipeline/source-loader", () => ({
+      loadSourceBitmap: vi.fn().mockResolvedValue({ width: 2, height: 2 }),
+    }));
+
+    vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({
+      drawImage: vi.fn(),
+      getImageData: vi.fn(() => ({
+        data: new Uint8ClampedArray(16),
+      })),
+    } as unknown as CanvasRenderingContext2D);
+
+    vi.stubGlobal("requestAnimationFrame", ((callback: FrameRequestCallback) => {
+      callback(0);
+      return 1;
+    }) as typeof requestAnimationFrame);
+
+    const { renderPreview } = await import("@/lib/image-pipeline/preview-renderer");
+
+    await renderPreview({
+      sourceUrl: "https://cdn.example.com/source.png",
+      steps: [createColorAdjustStep(), createUnsupportedStep()],
+      previewWidth: 2,
+      includeHistogram: false,
+    });
+
+    expect(runPreviewStepWithBackendRouter).toHaveBeenCalledTimes(2);
+    for (const call of runPreviewStepWithBackendRouter.mock.calls) {
+      expect(call[0]).toMatchObject({
+        backendHint: "cpu",
+      });
+    }
+  });
+
+  it("downgrades compile/link failures to cpu with runtime_error reason", async () => {
+    const { createBackendRouter } = await import("@/lib/image-pipeline/backend/backend-router");
+    const cpuPreview = vi.fn();
+    const fallbackEvents: Array<{
+      reason: string;
+      requestedBackend: string;
+      fallbackBackend: string;
+    }> = [];
+
+    const router = createBackendRouter({
+      backends: createCpuAndWebglBackends({
+        cpuPreview,
+        webglPreview: () => {
+          throw new Error("WebGL shader compile failed");
+        },
+      }),
+      defaultBackendId: "webgl",
+      backendAvailability: {
+        webgl: {
+          supported: true,
+          enabled: true,
+        },
+      },
+      featureFlags: {
+        forceCpu: false,
+        webglEnabled: true,
+        wasmEnabled: false,
+      },
+      onFallback: (event) => {
+        fallbackEvents.push({
+          reason: event.reason,
+          requestedBackend: event.requestedBackend,
+          fallbackBackend: event.fallbackBackend,
+        });
+      },
+    });
+
+    router.runPreviewStep({
+      pixels: new Uint8ClampedArray(4),
+      step: createCurvesStep(),
+      width: 1,
+      height: 1,
+    });
+
+    expect(cpuPreview).toHaveBeenCalledTimes(1);
+    expect(fallbackEvents).toEqual([
+      {
+        reason: "runtime_error",
+        requestedBackend: "webgl",
+        fallbackBackend: "cpu",
+      },
+    ]);
+  });
+});