webdev-pipeline/tests/audit-generation-action-source.test.ts

import assert from "node:assert/strict";
import { existsSync, readFileSync } from "node:fs";
import path from "node:path";
import test from "node:test";

const actionPath = path.join(process.cwd(), "convex", "auditGenerationAction.ts");
const actionSource = existsSync(actionPath) ? readFileSync(actionPath, "utf8") : "";
const generationSourcePath = path.join(process.cwd(), "convex", "auditGeneration.ts");
const generationSource = existsSync(generationSourcePath)
  ? readFileSync(generationSourcePath, "utf8")
  : "";

function hasPattern(source: string, pattern: RegExp) {
  return pattern.test(source);
}

function hasExportedInternalAction(exportName: string) {
  const pattern = new RegExp(
    `export const ${exportName}\\s*=\\s*internalAction\\s*\\(`,
  );
  return hasPattern(actionSource, pattern);
}

function hasStageCall(schema: string) {
  const escaped = schema.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
  return hasPattern(
    actionSource,
    new RegExp(
      `generateObject\\([\\s\\S]*schema:\\s*${escaped}[\\s\\S]*\\)`,
      "m",
    ),
  );
}

function extractFunctionSource(functionName: string) {
  const marker = `function ${functionName}`;
  const asyncMarker = `async function ${functionName}`;
  const declarationIndex = actionSource.indexOf(marker) === -1
    ? actionSource.indexOf(asyncMarker)
    : actionSource.indexOf(marker);
  assert.notEqual(
    declarationIndex,
    -1,
    `Expected function ${functionName} to exist.`,
  );

  const openBraceIndex = actionSource.indexOf("{", declarationIndex);
  let depth = 0;
  let end = -1;

  for (let index = openBraceIndex; index < actionSource.length; index += 1) {
    const char = actionSource[index];
    if (char === "{") {
      depth += 1;
    } else if (char === "}") {
      depth -= 1;
      if (depth === 0) {
        end = index;
        break;
      }
    }
  }

  assert.notEqual(end, -1, `Expected balanced braces for ${functionName}.`);
  return actionSource.slice(declarationIndex, end + 1);
}

test("auditGenerationAction module exists and is a Node action file", () => {
  assert.equal(existsSync(actionPath), true, "auditGenerationAction.ts should exist");
  assert.equal(
    hasPattern(actionSource, /^"use node";/m),
    true,
    "auditGenerationAction.ts should start with \"use node\"",
  );
});

test("auditGenerationAction exports processAuditGeneration with runId validator", () => {
  assert.equal(
    hasExportedInternalAction("processAuditGeneration"),
    true,
    "processAuditGeneration should be an internalAction",
  );

  assert.equal(
    hasPattern(
      actionSource,
      /processAuditGeneration\s*=\s*internalAction\(\s*{\s*args:\s*{\s*runId:\s*v\.id\(\s*["']agentRuns["']\s*\)\s*,?\s*}/,
    ),
    true,
    "processAuditGeneration should validate runId: v.id(\"agentRuns\")",
  );
});

test("action starts, queries evidence, and runs stage pipeline", () => {
  assert.equal(
    hasPattern(
      actionSource,
      /internal\.auditGeneration\.startAuditGenerationRun/,
    ),
    true,
    "Action should start the run via internal.auditGeneration.startAuditGenerationRun",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /internal\.auditGeneration\.getAuditGenerationEvidence/,
    ),
    true,
    "Action should load evidence via internal.auditGeneration.getAuditGenerationEvidence",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /internal\.auditGeneration\.persistAuditGenerationResult/,
    ),
    true,
    "Action should persist each stage result",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /internal\.auditGeneration\.finishAuditGenerationRun/,
    ),
    true,
    "Action should finish run via internal.auditGeneration.finishAuditGenerationRun",
  );
});

test("action includes all required audit stages", () => {
  for (const stage of [
    "classification",
    "multimodalAudit",
    "germanCopy",
    "qualityReview",
  ]) {
    const token = new RegExp(`stage:\\s*["']${stage}["']`);
    assert.equal(
      hasPattern(actionSource, token),
      true,
      `Action should reference ${stage} stage`,
    );
  }
});

test("action handles post-start failure paths in action-level catch", () => {
  assert.equal(
    hasPattern(
      actionSource,
      /try\s*{[\s\S]*internal\.auditGeneration\.getAuditGenerationEvidence[\s\S]*const provider = createOpenRouterProvider\(\)/,
    ),
    true,
    "Action should include evidence query and provider init inside catch-covered flow.",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /catch\s*\(error\)\s*{[\s\S]*appendRunEvent[\s\S]*finishAuditGenerationRun[\s\S]*"failed"/,
    ),
    true,
    "Action-level error handler should emit run events.",
  );
});

test("action calls generateObject with required schemas", () => {
  const requiredSchemas = [
    "auditClassificationSchema",
    "auditSummarySchema",
    "publicAuditTextSchema",
    "emailDraftSchema",
    "emailSubjectSchema",
    "callScriptSchema",
    "followUpDraftSchema",
    "qualityReviewSchema",
  ];

  for (const requiredSchema of requiredSchemas) {
    assert.equal(
      hasStageCall(requiredSchema),
      true,
      `Action should call generateObject with schema ${requiredSchema}`,
    );
  }
});

test("action loads v3 skill registry from v2 source for evidence input", () => {
  assert.equal(
    hasPattern(actionSource, /import\s*{[\s\S]*loadSkillsRegistry[\s\S]*}\s*from\s*["']\.\.\/lib\/skills-registry["']/),
    true,
    "Action should import loadSkillsRegistry from the shared registry parser.",
  );
  assert.equal(
    hasPattern(actionSource, /loadSkillsRegistry\(\s*(?:join\()?[\s\S]*v2_elemente[\s\S]*skills\.md[\s\S]*\)/),
    true,
    "Action should load the v3 registry from v2_elemente/skills.md.",
  );
  assert.equal(
    hasPattern(actionSource, /skillRegistry:\s*\[\s*\]/),
    false,
    "Action should not pass an always-empty skillRegistry to buildAuditEvidenceInput.",
  );
});

test("registry load warning logging is isolated from fallback return", () => {
  const loadRegistrySource = extractFunctionSource("loadAuditSkillRegistry");

  assert.equal(
    hasPattern(
      loadRegistrySource,
      /catch\s*\(error\)\s*{[\s\S]*try\s*{[\s\S]*appendRunEvent[\s\S]*}\s*catch\s*{[\s\S]*}\s*return\s*\[\s*\]/,
    ),
    true,
    "Registry load fallback should return [] even when warning event logging fails.",
  );
});

test("persistAuditStage omits undefined fields from Convex mutation args", () => {
  const persistSource = extractFunctionSource("persistAuditStage");
  const mutationPayloadSource = persistSource.slice(
    persistSource.indexOf("await ctx.runMutation"),
  );

  assert.doesNotMatch(
    actionSource,
    /persistAuditStage\(\s*{(?:(?!\n\s*}\s*\);)[\s\S])*(?:parsedJson|rawResponse|usage|finishReason|errorSummary):\s*undefined/,
    "Call sites should not pass explicit undefined stage payload fields.",
  );
  assert.doesNotMatch(
    persistSource,
    /usage:\s*usage\s*\?\s*toPersistedUsage\(usage\)\s*:\s*undefined/,
    "persistAuditStage should not emit usage: undefined.",
  );

  for (const field of [
    "systemPrompt",
    "rawResponse",
    "parsedJson",
    "finishReason",
    "errorSummary",
  ]) {
    assert.doesNotMatch(
      mutationPayloadSource,
      new RegExp(`\\n\\s*${field},`),
      `persistAuditStage should conditionally spread ${field}.`,
    );
  }
});

test("OpenRouter usage payloads omit undefined token fields", () => {
  const recordUsageSource = extractFunctionSource("recordOpenRouterUsage");

  assert.match(
    actionSource,
    /function toPersistedUsage[\s\S]*usage\.inputTokens\s*!==\s*undefined[\s\S]*promptTokens:\s*usage\.inputTokens/,
    "toPersistedUsage should omit promptTokens when inputTokens is undefined.",
  );
  assert.doesNotMatch(
    recordUsageSource,
    /tokens:\s*{[\s\S]*inputTokens:\s*args\.usage\.inputTokens/,
    "recordOpenRouterUsage should not build token payloads with undefined properties.",
  );
});

test("appendRunEvent omits undefined details from Convex mutation args", () => {
  assert.doesNotMatch(
    actionSource,
      /ctx\.runMutation\(internal\.runs\.appendEventInternal,\s*{[\s\S]*\n\s*details:\s*args\.details,\n/,
    "appendRunEvent should conditionally include details only when defined.",
  );
});

test("success finishAuditGenerationRun omits undefined errorSummary", () => {
  assert.doesNotMatch(
    actionSource,
    /finishAuditGenerationRun,\s*{[\s\S]*status:\s*["']succeeded["'][\s\S]*errorSummary:\s*qualityPassed\s*\?\s*undefined/,
    "Succeeded finishAuditGenerationRun payload should not send errorSummary: undefined.",
  );
});

test("quality review stage does not pass explicit undefined optional fields", () => {
  assert.doesNotMatch(
    actionSource,
    /persistAuditStage\(\s*{[\s\S]*stage:\s*["']qualityReview["'][\s\S]*errorSummary:\s*qualityPassed\s*\?\s*undefined/,
    "Quality persistAuditStage callsite should conditionally include errorSummary.",
  );
});

test("persistAuditStage callsites conditionally include optional auditId", () => {
  assert.doesNotMatch(
    actionSource,
    /await\s+persistAuditStage\(\s*{(?:(?!\n\s*}\s*\);)[\s\S])*\n\s*auditId,\n/,
    "persistAuditStage callsites should spread auditId only when defined.",
  );
});

test("audit generation helper callsites conditionally include optional auditId", () => {
  assert.doesNotMatch(
    actionSource,
    /(?:recordOpenRouterUsage|captureExternalAuditArtifacts)\(\s*ctx,\s*{(?:(?!\n\s*}\s*\);)[\s\S])*\n\s*auditId,\n/,
    "Helper callsites should spread auditId only when defined.",
  );
  assert.doesNotMatch(
    actionSource,
    /recordAuditUsageEvent\(\s*ctx,\s*{(?:(?!\n\s*}\s*\);)[\s\S])*\n\s*auditId:\s*args\.auditId,\n/,
    "recordAuditUsageEvent callsites should spread args.auditId only when defined.",
  );
});

test("persistAuditStage callsites avoid nested maybe-undefined usage objects", () => {
  assert.doesNotMatch(
    actionSource,
    /persistAuditStage\(\s*{(?:(?!\n\s*}\s*\);)[\s\S])*usage:\s*{[\s\S]*?(?:inputTokens|outputTokens|totalTokens|cacheReadTokens):/,
    "persistAuditStage callsites should use a usage helper or conditional spreads, not inline maybe-undefined usage objects.",
  );
});

test("classification stage uses v3 audit classification schema", () => {
  assert.equal(
    hasPattern(actionSource, /auditClassificationSchema/),
    true,
    "Action should reference the v3 auditClassificationSchema.",
  );
  assert.equal(
    hasStageCall("auditClassificationSchema"),
    true,
    "Classification generateObject call should validate v3 finding payloads.",
  );
  assert.equal(
    hasStageCall("internalFindingsSchema"),
    false,
    "Classification should no longer validate against legacy-only internalFindingsSchema.",
  );
});

test("action uses multimodal file parts with mediaType image/* when screenshots are available", () => {
  assert.equal(
    hasPattern(
      actionSource,
      /type:\s*["']file["'][\s\S]*mediaType:\s*(?:getValidMediaType|["']image\/)/,
    ),
    true,
    "Multimodal call should include AI file parts with image mediaType",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /ctx\.storage\.(get|getUrl)\(/,
    ),
    true,
    "Multimodal call should try to fetch screenshots from Convex storage",
  );
});

test("action handles missing screenshots with warning event fallback", () => {
  assert.equal(
    hasPattern(actionSource, /level:\s*["']warning["'][\s\S]*Screenshot|Vorschaubild/),
    true,
    "Action should append warning event when multimodal screenshot input is unavailable",
  );
  assert.equal(
    hasPattern(actionSource, /messages:\s*\[[\s\S]*type:\s*["']text["'][\s\S]*\]/),
    true,
    "Action should fall back to text-only multimodal calls when required parts are missing",
  );
});

test("action runs german copy guard and blocks outreach-ready on validation failure", () => {
  assert.equal(
    hasPattern(actionSource, /validateCustomerFacingCopy/),
    true,
    "Action should run German copy validation",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /qualityPassed\s*=\s*guardResult\.passed/,
    ),
    true,
    "Only deterministic German copy guard failures should hard-block the audit run.",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /qualityPassed\s*=\s*qualityResult\.object\.isValid\s*&&\s*guardResult\.passed/,
    ),
    false,
    "Subjective model QA warnings should not be combined with guardResult for terminal failure.",
  );
  assert.equal(
    hasPattern(actionSource, /internal\.leads\.reviewUpdateInternal/),
    true,
    "Action should patch lead via internal.leads.reviewUpdateInternal",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /isTerminalLeadContactStatus/,
    ),
    true,
    "Action should set contactStatus to outreach_ready only when terminal guard allows it.",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /do_not_contact|contacted|replied/i,
    ),
    true,
    "Action should explicitly guard against terminal lead statuses before outreach-ready.",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /Lead-Status wurde nicht auf outreach_ready gesetzt/,
    ),
    true,
    "Action should emit warning event when outreach-ready cannot be set.",
  );
});

test("action persists audit and outreach outputs before finishing succeeded run", () => {
  assert.equal(
    hasPattern(
      actionSource,
      /internal\.audits\.upsertFromAuditGeneration/,
    ),
    true,
    "Action should persist audit output via internal.audits.upsertFromAuditGeneration",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /internal\.outreach\.upsertFromAuditGeneration/,
    ),
    true,
    "Action should persist outreach output via internal.outreach.upsertFromAuditGeneration",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /internal\.audits\.upsertFromAuditGeneration[\s\S]*internal\.outreach\.upsertFromAuditGeneration[\s\S]*internal\.auditGeneration\.finishAuditGenerationRun[\s\S]*status:\s*["']succeeded["']/,
    ),
    true,
    "Action should finish success after persisted outputs",
  );
});

test("action uses model profiles for generation parameters", () => {
  assert.equal(
    hasPattern(actionSource, /resolveModelProfile\("classification"\)/),
    true,
    "classification generation should use resolveModelProfile.",
  );
  assert.equal(
    hasPattern(actionSource, /resolveModelProfile\("multimodalAudit"\)/),
    true,
    "multimodal generation should use resolveModelProfile.",
  );
  assert.equal(
    hasPattern(actionSource, /resolveModelProfile\("germanCopy"\)/),
    true,
    "german copy generation should use resolveModelProfile.",
  );
  assert.equal(
    hasPattern(actionSource, /resolveModelProfile\("qualityReview"\)/),
    true,
    "quality review generation should use resolveModelProfile.",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /temperature:\s*classificationProfile\.temperature[\s\S]*maxOutputTokens:\s*classificationProfile\.maxTokens/,
    ),
    true,
    "classification stage should use profile temperature/maxTokens.",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /temperature:\s*germanCopyProfile\.temperature[\s\S]*maxOutputTokens:\s*germanCopyProfile\.maxTokens/,
    ),
    true,
    "german copy stages should use profile temperature/maxTokens.",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /temperature:\s*qualityReviewProfile\.temperature[\s\S]*maxOutputTokens:\s*qualityReviewProfile\.maxTokens/,
    ),
    true,
    "quality review stage should use profile temperature/maxTokens.",
  );
});

test("action sanitization masks env-backed secrets", () => {
  assert.equal(
    hasPattern(
      actionSource,
      /sanitizeSecretCandidates\([\s\S]*process\.env/,
    ),
    true,
    "sanitize logic should include env-backed secret masking.",
  );
  assert.equal(
    hasPattern(actionSource, /OPENROUTER_API_KEY/),
    true,
    "sanitizer should include OPENROUTER_API_KEY in secret hints.",
  );
});

test("auditGeneration scheduler reference in queueLeadAuditGeneration is typed", () => {
  assert.equal(
    hasPattern(
      generationSource,
      /internal\.auditGenerationAction\.processAuditGeneration/,
    ),
    true,
    "queueLeadAuditGeneration should reference internal.auditGenerationAction.processAuditGeneration",
  );
  assert.equal(
    hasPattern(
      generationSource,
      /internal as any/,
    ),
    false,
    "No temporary internal cast should remain for the processAuditGeneration schedule",
  );
});