pitchfast/tests/audit-generation-action-source.test.ts

import assert from "node:assert/strict";
import { existsSync, readFileSync } from "node:fs";
import path from "node:path";
import test from "node:test";

const actionPath = path.join(process.cwd(), "convex", "auditGenerationAction.ts");
const actionSource = existsSync(actionPath) ? readFileSync(actionPath, "utf8") : "";
const toneGuidelinesPath = path.join(
  process.cwd(),
  "lib",
  "ai",
  "customer-tone-guidelines.ts",
);
const toneGuidelinesSource = existsSync(toneGuidelinesPath)
  ? readFileSync(toneGuidelinesPath, "utf8")
  : "";
const generationSourcePath = path.join(process.cwd(), "convex", "auditGeneration.ts");
const generationSource = existsSync(generationSourcePath)
  ? readFileSync(generationSourcePath, "utf8")
  : "";

function hasPattern(source: string, pattern: RegExp) {
  return pattern.test(source);
}

function hasExportedInternalAction(exportName: string) {
  const pattern = new RegExp(
    `export const ${exportName}\\s*=\\s*internalAction\\s*\\(`,
  );
  return hasPattern(actionSource, pattern);
}

function hasStageCall(schema: string) {
  const escaped = schema.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
  return hasPattern(
    actionSource,
    new RegExp(
      `generateObject\\([\\s\\S]*schema:\\s*${escaped}[\\s\\S]*\\)`,
      "m",
    ),
  );
}

function extractFunctionSource(functionName: string) {
  const marker = `function ${functionName}`;
  const asyncMarker = `async function ${functionName}`;
  const declarationIndex = actionSource.indexOf(marker) === -1
    ? actionSource.indexOf(asyncMarker)
    : actionSource.indexOf(marker);
  assert.notEqual(
    declarationIndex,
    -1,
    `Expected function ${functionName} to exist.`,
  );

  const openBraceIndex = actionSource.indexOf("{", declarationIndex);
  let depth = 0;
  let end = -1;

  for (let index = openBraceIndex; index < actionSource.length; index += 1) {
    const char = actionSource[index];
    if (char === "{") {
      depth += 1;
    } else if (char === "}") {
      depth -= 1;
      if (depth === 0) {
        end = index;
        break;
      }
    }
  }

  assert.notEqual(end, -1, `Expected balanced braces for ${functionName}.`);
  return actionSource.slice(declarationIndex, end + 1);
}

test("auditGenerationAction module exists and is a Node action file", () => {
  assert.equal(existsSync(actionPath), true, "auditGenerationAction.ts should exist");
  assert.equal(
    hasPattern(actionSource, /^"use node";/m),
    true,
    "auditGenerationAction.ts should start with \"use node\"",
  );
});

test("auditGenerationAction exports processAuditGeneration with runId validator", () => {
  assert.equal(
    hasExportedInternalAction("processAuditGeneration"),
    true,
    "processAuditGeneration should be an internalAction",
  );

  assert.equal(
    hasPattern(
      actionSource,
      /processAuditGeneration\s*=\s*internalAction\(\s*{\s*args:\s*{[\s\S]*?runId:\s*v\.id\(\s*["']agentRuns["']\s*\)[\s\S]*?rootRunId:\s*v\.optional\(v\.id\(\s*["']agentRuns["']\s*\)\)/,
    ),
    true,
    "processAuditGeneration should validate runId and optional rootRunId as agentRuns IDs",
  );
});

test("action starts, queries evidence, and runs stage pipeline", () => {
  assert.equal(
    hasPattern(
      actionSource,
      /internal\.auditGeneration\.startAuditGenerationRun/,
    ),
    true,
    "Action should start the run via internal.auditGeneration.startAuditGenerationRun",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /internal\.auditGeneration\.getAuditGenerationEvidence/,
    ),
    true,
    "Action should load evidence via internal.auditGeneration.getAuditGenerationEvidence",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /internal\.auditGeneration\.persistAuditGenerationResult/,
    ),
    true,
    "Action should persist each stage result",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /internal\.auditGeneration\.finishAuditGenerationRun/,
    ),
    true,
    "Action should finish run via internal.auditGeneration.finishAuditGenerationRun",
  );
});

test("action includes all required audit stages", () => {
  for (const stage of [
    "classification",
    "localSeoSpecialist",
    "conversionUxSpecialist",
    "visualTrustSpecialist",
    "critiqueSpecialist",
    "performanceAccessibilitySpecialist",
    "evidenceVerifier",
    "multimodalAudit",
    "germanCopy",
    "qualityReview",
  ]) {
    const token = new RegExp(`stage:\\s*["']${stage}["']`);
    assert.equal(
      hasPattern(actionSource, token),
      true,
      `Action should reference ${stage} stage`,
    );
  }
});

test("specialist fan-out runs after evidence input and before German copy", () => {
  const evidenceInputIndex = actionSource.indexOf("const evidenceInput = buildAuditEvidenceInput");
  const fanOutIndex = actionSource.indexOf("Promise.all(\n        specialistStageConfigs.map");
  const verifierIndex = actionSource.indexOf('currentStep = "evidenceVerifier"');
  const germanCopyIndex = actionSource.indexOf('currentStep = "germanCopy"');

  assert.notEqual(evidenceInputIndex, -1, "Action should build evidence input.");
  assert.notEqual(germanCopyIndex, -1, "Action should still run German copy.");
  assert.notEqual(fanOutIndex, -1, "Action should fan out specialist stage configs.");
  assert.notEqual(verifierIndex, -1, "Action should run the evidence verifier.");
  assert.equal(
    fanOutIndex > evidenceInputIndex && fanOutIndex < germanCopyIndex,
    true,
    "Specialist fan-out should run after evidence input and before German copy.",
  );
  assert.equal(
    verifierIndex > fanOutIndex && verifierIndex < germanCopyIndex,
    true,
    "Evidence verifier should run after specialist fan-out and before German copy.",
  );
});

test("specialist stages use specialist schemas and verified findings feed German copy", () => {
  assert.equal(
    hasStageCall("auditSpecialistResultSchema"),
    true,
    "Specialist stages should call generateObject with auditSpecialistResultSchema.",
  );
  assert.equal(
    hasStageCall("auditEvidenceVerificationSchema"),
    true,
    "Verifier stage should call generateObject with auditEvidenceVerificationSchema.",
  );
  assert.match(
    actionSource,
    /(?:const|let)\s+verifiedFindings\s*[:=]/,
    "Action should derive verifiedFindings before synthesis.",
  );
  assert.match(
    actionSource,
    /verifiedResult?\.?object|verifiedFindingIds/,
    "Verifier output should use compact finding IDs instead of echoing full findings.",
  );
  assert.match(
    actionSource,
    /verifiedFindingIds\.has\(candidate\.findingId\)/,
    "Action should map verifier-approved IDs back to original specialist findings.",
  );
  assert.match(
    actionSource,
    /buildGermanCopyPrompt\(\s*verifiedFindingsText/,
    "German copy should be generated from verified findings text.",
  );
  assert.doesNotMatch(
    actionSource,
    /buildGermanCopyPrompt\(\s*classificationSummary\s*,/,
    "German copy should no longer use raw classification summary as its primary finding input.",
  );
});

test("critique specialist translates impeccable critique guidance into the audit fan-out", () => {
  assert.match(
    actionSource,
    /stage:\s*["']critiqueSpecialist["']/,
    "Action should include a dedicated critique specialist stage.",
  );
  assert.match(
    actionSource,
    /impeccable-critique/,
    "Critique specialist should anchor findings to the impeccable critique skill id.",
  );
  assert.match(
    actionSource,
    /kognitive Last|Nielsen|AI-Slop|Informationsarchitektur/,
    "Critique specialist should include critique guidance beyond generic visual trust.",
  );
});

test("German copy prompt uses first-contact email tone guidelines without a new AI stage", () => {
  const buildPromptSource = extractFunctionSource("buildGermanCopyPrompt");

  assert.doesNotMatch(
    buildPromptSource,
    /Ich-Ich Kontext/,
    "German copy prompt should not force formulaic Ich-Ich copy.",
  );
  assert.match(
    actionSource,
    /buildCustomerTonePromptSection/,
    "German copy prompt should inject shared customer tone guidelines.",
  );
  assert.match(
    buildPromptSource,
    /evidence:\s*AuditEvidence/,
    "German copy prompt should accept explicit evidence context.",
  );
  assert.match(
    actionSource,
    /buildGermanCopyPrompt\([\s\S]*verifiedFindingsText[\s\S]*multimodalSummary[\s\S]*evidenceInput[\s\S]*\)/,
    "German copy prompt should receive the explicit evidence context at the callsite.",
  );
  assert.match(
    toneGuidelinesSource,
    /kollegial direkt/,
    "Tone guidelines should lock the selected sender posture.",
  );
  assert.match(
    toneGuidelinesSource,
    /maximal zwei verifizierte Befunde|max\. zwei verifizierte Befunde/,
    "Tone guidelines should keep outreach emails to at most two verified findings.",
  );
  assert.match(
    toneGuidelinesSource,
    /kein Mini-Audit/,
    "Tone guidelines should explicitly forbid mini-audit emails.",
  );
  assert.doesNotMatch(
    actionSource,
    /tone(?:Review|Rewrite|Specialist)|emailToneSpecialist|copyToneSpecialist/,
    "Tone work should not add another model-backed generation stage.",
  );
});

test("quality review can rewrite copy once without making copy feedback a hard failure", () => {
  const qualityPromptSource = extractFunctionSource("buildQualityReviewPrompt");

  assert.doesNotMatch(
    actionSource,
    /qualityPassed\s*=\s*qualityResult\.object\.isValid\s*&&\s*guardResult\.passed/,
    "Copy quality feedback should not be a hard AND-gate with the deterministic German copy guard.",
  );
  assert.doesNotMatch(
    actionSource,
    /qualityPassed\s*=\s*guardResult\.passed\s*;/,
    "The deterministic German copy guard should not be the quality pass condition.",
  );
  assert.match(
    actionSource,
    /rewriteRequired[\s\S]*revisedCopy[\s\S]*applyRevisedCopy/,
    "Quality review should be able to request one revised copy and apply it before persistence.",
  );
  assert.match(
    actionSource,
    /copyReviewAttempts\s*<\s*2/,
    "Quality review should run at most the initial review plus one rewrite review.",
  );
  assert.match(
    actionSource,
    /message:\s*["']Copy-Review hat korrigiert\.["']/,
    "A successful rewrite should be visible as a warning event.",
  );
  assert.match(
    actionSource,
    /message:\s*["']Copy-Review mit Hinweisen abgeschlossen\.["']/,
    "Remaining copy feedback should be stored as warning telemetry.",
  );
  assert.match(
    qualityPromptSource,
    /echte Erstmail von Matthias/,
    "Quality review should apply the selected first-contact email rubric.",
  );
  assert.match(
    qualityPromptSource,
    /KI-Verkaufstext/,
    "Quality review should reject AI-like sales copy.",
  );
  assert.match(
    qualityPromptSource,
    /verified findings|verifizierte Befunde/i,
    "Quality review should keep concrete claims tied to verified findings.",
  );
  assert.match(
    qualityPromptSource,
    /revisedCopy|rewriteRequired/,
    "Quality review prompt should ask for revised copy when rewrite is needed.",
  );
});

test("action handles post-start failure paths in action-level catch", () => {
  assert.equal(
    hasPattern(
      actionSource,
      /try\s*{[\s\S]*internal\.auditGeneration\.getAuditGenerationEvidence[\s\S]*const provider = createOpenRouterProvider\(\)/,
    ),
    true,
    "Action should include evidence query and provider init inside catch-covered flow.",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /catch\s*\(error\)\s*{[\s\S]*appendRunEvent[\s\S]*finishAuditGenerationRun[\s\S]*"failed"/,
    ),
    true,
    "Action-level error handler should emit run events.",
  );
});

test("action calls generateObject with required schemas", () => {
  const requiredSchemas = [
    "auditClassificationSchema",
    "auditSummarySchema",
    "publicAuditTextSchema",
    "emailDraftSchema",
    "emailSubjectSchema",
    "callScriptSchema",
    "followUpDraftSchema",
    "qualityReviewSchema",
  ];

  for (const requiredSchema of requiredSchemas) {
    assert.equal(
      hasStageCall(requiredSchema),
      true,
      `Action should call generateObject with schema ${requiredSchema}`,
    );
  }
});

test("action loads v3 skill registry from bundled MVP source for evidence input", () => {
  assert.equal(
    hasPattern(actionSource, /import\s*{[\s\S]*loadLocalAuditSkillRegistry[\s\S]*}\s*from\s*["']\.\.\/lib\/ai\/local-audit-skill-registry["']/),
    true,
    "Action should import the bundled MVP skill registry loader.",
  );
  assert.equal(
    hasPattern(actionSource, /loadLocalAuditSkillRegistry\(\s*\)/),
    true,
    "Action should load the v3 registry from a bundled MVP module.",
  );
  assert.doesNotMatch(
    actionSource,
    /v2_elemente|process\.cwd\(\)|loadSkillsRegistry\(|node:path/,
    "Action should not read v2 reference files or filesystem paths at runtime.",
  );
  assert.equal(
    hasPattern(actionSource, /skillRegistry:\s*\[\s*\]/),
    false,
    "Action should not pass an always-empty skillRegistry to buildAuditEvidenceInput.",
  );
});

test("registry load warning logging is isolated from fallback return", () => {
  const loadRegistrySource = extractFunctionSource("loadAuditSkillRegistry");

  assert.equal(
    hasPattern(
      loadRegistrySource,
      /catch\s*\(error\)\s*{[\s\S]*try\s*{[\s\S]*appendRunEvent[\s\S]*}\s*catch\s*{[\s\S]*}\s*return\s*\[\s*\]/,
    ),
    true,
    "Registry load fallback should return [] even when warning event logging fails.",
  );
});

test("persistAuditStage omits undefined fields from Convex mutation args", () => {
  const persistSource = extractFunctionSource("persistAuditStage");
  const mutationPayloadSource = persistSource.slice(
    persistSource.indexOf("await ctx.runMutation"),
  );

  assert.doesNotMatch(
    actionSource,
    /persistAuditStage\(\s*{(?:(?!\n\s*}\s*\);)[\s\S])*(?:parsedJson|rawResponse|usage|finishReason|errorSummary):\s*undefined/,
    "Call sites should not pass explicit undefined stage payload fields.",
  );
  assert.doesNotMatch(
    persistSource,
    /usage:\s*usage\s*\?\s*toPersistedUsage\(usage\)\s*:\s*undefined/,
    "persistAuditStage should not emit usage: undefined.",
  );

  for (const field of [
    "systemPrompt",
    "rawResponse",
    "parsedJson",
    "finishReason",
    "errorSummary",
  ]) {
    assert.doesNotMatch(
      mutationPayloadSource,
      new RegExp(`\\n\\s*${field},`),
      `persistAuditStage should conditionally spread ${field}.`,
    );
  }
});

test("OpenRouter usage payloads omit undefined token fields", () => {
  const recordUsageSource = extractFunctionSource("recordOpenRouterUsage");

  assert.match(
    actionSource,
    /function toPersistedUsage[\s\S]*usage\.inputTokens\s*!==\s*undefined[\s\S]*promptTokens:\s*usage\.inputTokens/,
    "toPersistedUsage should omit promptTokens when inputTokens is undefined.",
  );
  assert.doesNotMatch(
    recordUsageSource,
    /tokens:\s*{[\s\S]*inputTokens:\s*args\.usage\.inputTokens/,
    "recordOpenRouterUsage should not build token payloads with undefined properties.",
  );
});

test("appendRunEvent omits undefined details from Convex mutation args", () => {
  assert.doesNotMatch(
    actionSource,
      /ctx\.runMutation\(internal\.runs\.appendEventInternal,\s*{[\s\S]*\n\s*details:\s*args\.details,\n/,
    "appendRunEvent should conditionally include details only when defined.",
  );
});

test("success finishAuditGenerationRun omits undefined errorSummary", () => {
  assert.doesNotMatch(
    actionSource,
    /finishAuditGenerationRun,\s*{[\s\S]*status:\s*["']succeeded["'][\s\S]*errorSummary:\s*qualityPassed\s*\?\s*undefined/,
    "Succeeded finishAuditGenerationRun payload should not send errorSummary: undefined.",
  );
});

test("quality review stage does not pass explicit undefined optional fields", () => {
  assert.doesNotMatch(
    actionSource,
    /persistAuditStage\(\s*{[\s\S]*stage:\s*["']qualityReview["'][\s\S]*errorSummary:\s*qualityPassed\s*\?\s*undefined/,
    "Quality persistAuditStage callsite should conditionally include errorSummary.",
  );
});

test("persistAuditStage callsites conditionally include optional auditId", () => {
  assert.doesNotMatch(
    actionSource,
    /await\s+persistAuditStage\(\s*{(?:(?!\n\s*}\s*\);)[\s\S])*\n\s*auditId,\n/,
    "persistAuditStage callsites should spread auditId only when defined.",
  );
});

test("audit generation helper callsites conditionally include optional auditId", () => {
  assert.doesNotMatch(
    actionSource,
    /(?:recordOpenRouterUsage|captureExternalAuditArtifacts)\(\s*ctx,\s*{(?:(?!\n\s*}\s*\);)[\s\S])*\n\s*auditId,\n/,
    "Helper callsites should spread auditId only when defined.",
  );
  assert.doesNotMatch(
    actionSource,
    /recordAuditUsageEvent\(\s*ctx,\s*{(?:(?!\n\s*}\s*\);)[\s\S])*\n\s*auditId:\s*args\.auditId,\n/,
    "recordAuditUsageEvent callsites should spread args.auditId only when defined.",
  );
});

test("persistAuditStage callsites avoid nested maybe-undefined usage objects", () => {
  assert.doesNotMatch(
    actionSource,
    /persistAuditStage\(\s*{(?:(?!\n\s*}\s*\);)[\s\S])*usage:\s*{[\s\S]*?(?:inputTokens|outputTokens|totalTokens|cacheReadTokens):/,
    "persistAuditStage callsites should use a usage helper or conditional spreads, not inline maybe-undefined usage objects.",
  );
});

test("classification stage uses v3 audit classification schema", () => {
  assert.equal(
    hasPattern(actionSource, /auditClassificationSchema/),
    true,
    "Action should reference the v3 auditClassificationSchema.",
  );
  assert.equal(
    hasStageCall("auditClassificationSchema"),
    true,
    "Classification generateObject call should validate v3 finding payloads.",
  );
  assert.equal(
    hasStageCall("internalFindingsSchema"),
    false,
    "Classification should no longer validate against legacy-only internalFindingsSchema.",
  );
});

test("action uses multimodal file parts with mediaType image/* when screenshots are available", () => {
  assert.equal(
    hasPattern(
      actionSource,
      /type:\s*["']file["'][\s\S]*mediaType:\s*(?:getValidMediaType|["']image\/)/,
    ),
    true,
    "Multimodal call should include AI file parts with image mediaType",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /ctx\.storage\.(get|getUrl)\(/,
    ),
    true,
    "Multimodal call should try to fetch screenshots from Convex storage",
  );
});

test("action handles missing screenshots with warning event fallback", () => {
  assert.equal(
    hasPattern(actionSource, /level:\s*["']warning["'][\s\S]*Screenshot|Vorschaubild/),
    true,
    "Action should append warning event when multimodal screenshot input is unavailable",
  );
  assert.equal(
    hasPattern(actionSource, /messages:\s*\[[\s\S]*type:\s*["']text["'][\s\S]*\]/),
    true,
    "Action should fall back to text-only multimodal calls when required parts are missing",
  );
});

test("action keeps German copy guard as telemetry without blocking outreach-ready", () => {
  assert.equal(
    hasPattern(actionSource, /validateCustomerFacingCopy/),
    true,
    "Action should still run German copy validation for telemetry.",
  );
  assert.doesNotMatch(
    actionSource,
    /guardResult\.passed[\s\S]{0,500}finishAuditGenerationRun[\s\S]{0,250}status:\s*["']failed["']/,
    "German copy guard findings should not finish the audit generation as failed.",
  );
  assert.match(
    actionSource,
    /guardTelemetry|deterministicGuard/,
    "German copy guard output should be persisted as telemetry in the quality payload.",
  );
  assert.equal(
    hasPattern(actionSource, /internal\.leads\.reviewUpdateInternal/),
    true,
    "Action should patch lead via internal.leads.reviewUpdateInternal",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /isTerminalLeadContactStatus/,
    ),
    true,
    "Action should set contactStatus to outreach_ready only when terminal guard allows it.",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /do_not_contact|contacted|replied/i,
    ),
    true,
    "Action should explicitly guard against terminal lead statuses before outreach-ready.",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /Lead-Status wurde nicht auf outreach_ready gesetzt/,
    ),
    true,
    "Action should emit warning event when outreach-ready cannot be set.",
  );
});

test("action persists audit and outreach outputs before finishing succeeded run", () => {
  assert.equal(
    hasPattern(
      actionSource,
      /internal\.audits\.upsertFromAuditGeneration/,
    ),
    true,
    "Action should persist audit output via internal.audits.upsertFromAuditGeneration",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /internal\.outreach\.upsertFromAuditGeneration/,
    ),
    true,
    "Action should persist outreach output via internal.outreach.upsertFromAuditGeneration",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /internal\.audits\.upsertFromAuditGeneration[\s\S]*internal\.outreach\.upsertFromAuditGeneration[\s\S]*internal\.auditGeneration\.finishAuditGenerationRun[\s\S]*status:\s*["']succeeded["']/,
    ),
    true,
    "Action should finish success after persisted outputs",
  );
});

test("action uses model profiles for generation parameters", () => {
  assert.equal(
    hasPattern(actionSource, /resolveModelProfile\("classification"\)/),
    true,
    "classification generation should use resolveModelProfile.",
  );
  assert.equal(
    hasPattern(actionSource, /resolveModelProfile\("multimodalAudit"\)/),
    true,
    "multimodal generation should use resolveModelProfile.",
  );
  assert.equal(
    hasPattern(actionSource, /resolveModelProfile\("germanCopy"\)/),
    true,
    "german copy generation should use resolveModelProfile.",
  );
  assert.equal(
    hasPattern(actionSource, /resolveModelProfile\("qualityReview"\)/),
    true,
    "quality review generation should use resolveModelProfile.",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /temperature:\s*classificationProfile\.temperature[\s\S]*maxOutputTokens:\s*classificationProfile\.maxTokens/,
    ),
    true,
    "classification stage should use profile temperature/maxTokens.",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /temperature:\s*germanCopyProfile\.temperature[\s\S]*maxOutputTokens:\s*germanCopyProfile\.maxTokens/,
    ),
    true,
    "german copy stages should use profile temperature/maxTokens.",
  );
  assert.equal(
    hasPattern(
      actionSource,
      /temperature:\s*qualityReviewProfile\.temperature[\s\S]*maxOutputTokens:\s*qualityReviewProfile\.maxTokens/,
    ),
    true,
    "quality review stage should use profile temperature/maxTokens.",
  );
});

test("action sanitization masks env-backed secrets", () => {
  assert.equal(
    hasPattern(
      actionSource,
      /sanitizeSecretCandidates\([\s\S]*process\.env/,
    ),
    true,
    "sanitize logic should include env-backed secret masking.",
  );
  assert.equal(
    hasPattern(actionSource, /OPENROUTER_API_KEY/),
    true,
    "sanitizer should include OPENROUTER_API_KEY in secret hints.",
  );
});

test("auditGeneration scheduler reference in queueLeadAuditGeneration is typed", () => {
  assert.equal(
    hasPattern(
      generationSource,
      /internal\.auditGenerationAction\.processAuditGeneration/,
    ),
    true,
    "queueLeadAuditGeneration should reference internal.auditGenerationAction.processAuditGeneration",
  );
  assert.equal(
    hasPattern(
      generationSource,
      /internal as any/,
    ),
    false,
    "No temporary internal cast should remain for the processAuditGeneration schedule",
  );
});