pitchfast/tests/external-audit-pipeline-source.test.ts

import assert from "node:assert/strict";
import { existsSync, readFileSync } from "node:fs";
import path from "node:path";
import test from "node:test";

const actionPath = path.join(process.cwd(), "convex", "auditGenerationAction.ts");
const actionSource = existsSync(actionPath) ? readFileSync(actionPath, "utf8") : "";
const generationPath = path.join(process.cwd(), "convex", "auditGeneration.ts");
const generationSource = existsSync(generationPath)
  ? readFileSync(generationPath, "utf8")
  : "";

function extractFunctionSource(functionName: string) {
  const declarationPattern = new RegExp(
    `(?:async\\s+)?function\\s+${functionName}\\s*\\([\\s\\S]*?\\n\\)\\s*(?::\\s*[^\\{]+)?\\{`,
  );
  const match = declarationPattern.exec(actionSource);
  assert.notEqual(match, null, `Expected function ${functionName}.`);

  const openBraceIndex = match!.index + match![0].lastIndexOf("{");
  let depth = 0;
  let end = -1;

  for (let index = openBraceIndex; index < actionSource.length; index += 1) {
    const char = actionSource[index];
    if (char === "{") {
      depth += 1;
    } else if (char === "}") {
      depth -= 1;
      if (depth === 0) {
        end = index;
        break;
      }
    }
  }

  assert.notEqual(end, -1, `Expected balanced braces for ${functionName}.`);
  return actionSource.slice(match!.index, end + 1);
}

test("audit generation action orchestrates external capture helpers when legacy crawl artifacts are absent", () => {
  assert.match(
    actionSource,
    /buildScreenshotOneRequests[\s\S]*buildJinaReaderAuditInput[\s\S]*estimateExternalAuditCostUsd|estimateExternalAuditCostUsd[\s\S]*buildScreenshotOneRequests[\s\S]*buildJinaReaderAuditInput/,
    "Action should import and use the approved external audit service helpers.",
  );
  assert.match(
    actionSource,
    /SCREENSHOTONE_API_KEY/,
    "ScreenshotOne capture should be guarded by the managed SCREENSHOTONE_API_KEY env key.",
  );
  assert.match(
    actionSource,
    /JINA_API_KEY/,
    "Jina capture should be compatible with the optional managed JINA_API_KEY env key.",
  );
  assert.match(
    actionSource,
    /evidence\.screenshots\.length\s*===\s*0[\s\S]*(started\.lead\.websiteUrl|started\.lead\.websiteDomain)/,
    "External capture should be prepared from the started lead URL/domain when legacy screenshots are missing.",
  );
});

test("audit generation action records provider usage events for capture and OpenRouter generation", () => {
  assert.match(
    actionSource,
    /internal\.usageEvents\.recordUsageEvent/,
    "Action should record usage through internal.usageEvents.recordUsageEvent.",
  );

  for (const provider of ["screenshotone", "jina", "openrouter"]) {
    assert.match(
      actionSource,
      new RegExp(`provider:\\s*["']${provider}["']`),
      `Action should record ${provider} usage.`,
    );
  }

  assert.match(
    actionSource,
    /provider:\s*["']openrouter["'][\s\S]*operation:\s*["']audit_generation["']/,
    "OpenRouter usage should be recorded as audit_generation.",
  );
  assert.match(
    actionSource,
    /provider:\s*["']screenshotone["'][\s\S]*operation:\s*["']audit_capture["']/,
    "ScreenshotOne usage should be recorded as audit_capture.",
  );
  assert.match(
    actionSource,
    /provider:\s*["']jina["'][\s\S]*operation:\s*["']audit_capture["']/,
    "Jina usage should be recorded as audit_capture.",
  );
});

test("Jina markdown joins the evidence prompt without requiring Playwright crawl pages", () => {
  assert.match(
    actionSource,
    /jina(?:Reader)?AuditInput[\s\S]*markdown/,
    "Action should keep Jina reader markdown as an audit evidence input.",
  );
  assert.match(
    actionSource,
    /buildAuditEvidenceInput\(\{[\s\S]*externalMarkdown|externalMarkdown[\s\S]*buildAuditEvidenceInput\(\{/,
    "Action should pass external markdown into the evidence builder.",
  );
  assert.match(
    generationSource,
    /externalMarkdown/,
    "Audit generation evidence types should expose external markdown for prompts.",
  );
});

test("external capture fetches use timeout, abort signal, and bounded response readers", () => {
  for (const constantName of [
    "EXTERNAL_CAPTURE_TIMEOUT_MS",
    "MAX_SCREENSHOT_BYTES",
    "MAX_JINA_MARKDOWN_BYTES",
    "MAX_JINA_MARKDOWN_CHARS",
  ]) {
    assert.match(
      actionSource,
      new RegExp(`const\\s+${constantName}\\s*=`),
      `Action should define ${constantName}.`,
    );
  }

  assert.match(
    actionSource,
    /AbortController/,
    "External fetches should use AbortController for per-request timeouts.",
  );
  assert.match(
    actionSource,
    /fetch\([\s\S]*signal:/,
    "External fetches should pass an AbortSignal.",
  );
  assert.doesNotMatch(
    actionSource,
    /response\.blob\(\)/,
    "ScreenshotOne capture should not call unbounded response.blob().",
  );
  assert.doesNotMatch(
    actionSource,
    /response\.text\(\)/,
    "Jina capture should not call unbounded response.text().",
  );
});

test("audit generation action sanitizes raw errors before run events and run failure summaries", () => {
  assert.match(
    actionSource,
    /function messageFromError[\s\S]*sanitizeSecretCandidates/,
    "messageFromError should sanitize/redact before returning error text.",
  );

  for (const secretName of ["SCREENSHOTONE_API_KEY", "JINA_API_KEY"]) {
    assert.match(
      actionSource,
      new RegExp(`["']${secretName}["']`),
      `Secret sanitizer should know ${secretName}.`,
    );
  }

  assert.doesNotMatch(
    actionSource,
    /value:\s*messageFromError\(error\)/,
    "Run event details should not receive raw messageFromError calls inline.",
  );
  assert.doesNotMatch(
    actionSource,
    /errorSummary\s*=\s*messageFromError\(error\)/,
    "Failure summaries should not assign unsanitized raw errors inline.",
  );
});

test("german-copy OpenRouter usage event aggregates all six generation calls", () => {
  assert.match(
    actionSource,
    /aggregateOpenRouterUsage/,
    "Action should expose an aggregation helper for stage-level OpenRouter usage.",
  );
  assert.match(
    actionSource,
    /aggregateOpenRouterUsage\(\[[\s\S]*publicSummaryResult\.usage[\s\S]*germanBodyResult\.usage[\s\S]*germanSubjectResult\.usage[\s\S]*germanEmailResult\.usage[\s\S]*germanCallScriptResult\.usage[\s\S]*germanFollowUpResult\.usage[\s\S]*\]\)/,
    "German-copy usage should aggregate public summary, body, subject, email, call script, and follow-up calls.",
  );
});

test("usage event recording is best-effort and cannot fail audit generation", () => {
  const usageRecorder = extractFunctionSource("recordAuditUsageEvent");

  assert.match(
    usageRecorder,
    /try\s*\{[\s\S]*await ctx\.runMutation\(internal\.usageEvents\.recordUsageEvent/,
    "Usage recorder should isolate recordUsageEvent in a try block.",
  );
  assert.match(
    usageRecorder,
    /catch\s*\(error\)\s*\{[\s\S]*messageFromError\(error\)[\s\S]*level:\s*["']warning["']/,
    "Usage recorder should sanitize/log failures as warnings.",
  );
  assert.match(
    usageRecorder,
    /catch\s*\(error\)\s*\{[\s\S]*try\s*\{[\s\S]*appendRunEvent[\s\S]*\}\s*catch/,
    "Warning logging for usage failures should also be best-effort.",
  );
});

test("external capture timeout covers body streaming and cancels readers", () => {
  const fetcher = extractFunctionSource("fetchExternalCapture");
  const reader = extractFunctionSource("readLimitedResponseBytes");

  assert.match(
    fetcher,
    /return\s*\{[\s\S]*response[\s\S]*abortController:\s*controller[\s\S]*timeout[\s\S]*\}/,
    "fetchExternalCapture should return the active deadline context for body reads.",
  );
  assert.doesNotMatch(
    fetcher,
    /finally\s*\{[\s\S]*clearTimeout\(timeout\)/,
    "fetchExternalCapture should not clear the timeout before body streaming completes.",
  );
  assert.match(
    reader,
    /signal\??:\s*AbortSignal/,
    "Bounded response reader should accept an AbortSignal.",
  );
  assert.match(
    reader,
    /signal\?\.addEventListener\(\s*["']abort["'][\s\S]*reader\.cancel/,
    "Bounded response reader should cancel the reader on timeout/abort.",
  );
  assert.match(
    reader,
    /totalBytes\s*>\s*maxBytes[\s\S]*await reader\.cancel\(/,
    "Bounded response reader should cancel the stream when the byte cap is exceeded.",
  );
  assert.match(
    actionSource,
    /readLimitedResponseBytes\([\s\S]*MAX_SCREENSHOT_BYTES[\s\S]*abortController\.signal/,
    "Screenshot body reads should use the active timeout signal.",
  );
  assert.match(
    actionSource,
    /readLimitedMarkdown\([\s\S]*abortController\.signal/,
    "Jina markdown body reads should use the active timeout signal.",
  );
  assert.match(
    actionSource,
    /finally\s*\{[\s\S]*clearExternalCaptureTimeout/,
    "Capture loops should clear the external timeout after fetch and body streaming finish.",
  );
});

test("external capture request builders are provider-level best-effort", () => {
  const capture = extractFunctionSource("captureExternalAuditArtifacts");

  assert.match(
    capture,
    /if\s*\(args\.needsScreenshots\)[\s\S]*try\s*\{[\s\S]*buildScreenshotOneRequests/,
    "ScreenshotOne request construction should be inside a provider-level try block.",
  );
  assert.match(
    capture,
    /buildScreenshotOneRequests[\s\S]*catch\s*\(error\)[\s\S]*messageFromError\(error\)[\s\S]*level:\s*["']warning["']/,
    "ScreenshotOne request construction failures should degrade to sanitized warnings.",
  );
  assert.match(
    capture,
    /if\s*\(args\.needsMarkdown\)[\s\S]*try\s*\{[\s\S]*buildJinaReaderAuditInput/,
    "Jina reader input construction should be inside a provider-level try block.",
  );
  assert.match(
    capture,
    /buildJinaReaderAuditInput[\s\S]*catch\s*\(error\)[\s\S]*messageFromError\(error\)[\s\S]*level:\s*["']warning["']/,
    "Jina reader input construction failures should degrade to sanitized warnings.",
  );
});

test("ScreenshotOne missing-key skip emits best-effort warning only when screenshots are needed", () => {
  const capture = extractFunctionSource("captureExternalAuditArtifacts");
  const needsScreenshotsIndex = capture.indexOf("if (args.needsScreenshots)");
  const needsMarkdownIndex = capture.indexOf("if (args.needsMarkdown)");
  const missingKeyWarningIndex = capture.indexOf(
    "ScreenshotOne ist nicht konfiguriert; Screenshot-Erfassung wurde übersprungen.",
  );

  assert.notEqual(
    needsScreenshotsIndex,
    -1,
    "External capture should branch on needsScreenshots.",
  );
  assert.notEqual(
    needsMarkdownIndex,
    -1,
    "External capture should keep the later needsMarkdown branch.",
  );
  assert.notEqual(
    missingKeyWarningIndex,
    -1,
    "Missing ScreenshotOne config should emit a clear warning message.",
  );
  assert.equal(
    missingKeyWarningIndex > needsScreenshotsIndex &&
      missingKeyWarningIndex < needsMarkdownIndex,
    true,
    "Missing-key warning should live inside the needsScreenshots branch, so legacy screenshots do not warn.",
  );
  assert.match(
    capture,
    /if\s*\(!screenshotOneApiKey\)\s*\{[\s\S]*try\s*\{[\s\S]*await appendRunEvent\(ctx,\s*\{[\s\S]*level:\s*["']warning["'][\s\S]*ScreenshotOne ist nicht konfiguriert; Screenshot-Erfassung wurde übersprungen\.[\s\S]*\}\s*\);[\s\S]*\}\s*catch\s*\{[\s\S]*\}/,
    "Missing-key warning logging should be best-effort and unable to fail the audit run.",
  );
});

test("external capture non-OK responses cancel bodies before continuing", () => {
  const capture = extractFunctionSource("captureExternalAuditArtifacts");
  const nonOkCancelCount = [
    ...capture.matchAll(
      /if\s*\(!response\.ok\)\s*\{[\s\S]*?await cancelExternalResponseBody\(response\);[\s\S]*?continue;/g,
    ),
  ].length;

  assert.match(
    actionSource,
    /async function cancelExternalResponseBody/,
    "Action should centralize best-effort body cancellation for non-OK responses.",
  );
  assert.equal(
    nonOkCancelCount,
    2,
    "Both ScreenshotOne and Jina non-OK branches should cancel bodies before continue.",
  );
});