webdev-pipeline/tests/audit-generation-persistence-source.test.ts

import assert from "node:assert/strict";
import { existsSync, readFileSync } from "node:fs";
import { join } from "node:path";
import test from "node:test";
import ts from "typescript";

const auditGenerationPath = join(process.cwd(), "convex", "auditGeneration.ts");
const auditGenerationSource = existsSync(auditGenerationPath)
  ? readFileSync(auditGenerationPath, "utf8")
  : "";

const sourceFile = ts.createSourceFile(
  "auditGeneration.ts",
  auditGenerationSource,
  ts.ScriptTarget.ES2022,
  true,
  ts.ScriptKind.TS,
);

function getExportedConstNames(file: ts.SourceFile) {
  const names = new Set<string>();

  const visit = (node: ts.Node) => {
    if (ts.isVariableStatement(node)) {
      const isExported = node.modifiers?.some(
        (mod) => mod.kind === ts.SyntaxKind.ExportKeyword,
      );
      if (!isExported) {
        ts.forEachChild(node, visit);
        return;
      }

      const isConst = node.declarationList.flags & ts.NodeFlags.Const;
      if (!isConst) {
        ts.forEachChild(node, visit);
        return;
      }

      for (const declaration of node.declarationList.declarations) {
        if (ts.isIdentifier(declaration.name)) {
          names.add(declaration.name.text);
        }
      }
    }

    ts.forEachChild(node, visit);
  };

  ts.forEachChild(file, visit);
  return names;
}

function hasPattern(source: string, pattern: RegExp) {
  return pattern.test(source);
}

function extractExportSource(name: string) {
  const marker = `export const ${name} = `;
  const declarationIndex = auditGenerationSource.indexOf(marker);
  assert.notEqual(
    declarationIndex,
    -1,
    `Expected declaration for ${name}`,
  );

  const openBraceIndex = auditGenerationSource.indexOf("{", declarationIndex);
  let depth = 0;
  let end = -1;

  for (let index = openBraceIndex; index < auditGenerationSource.length; index += 1) {
    const char = auditGenerationSource[index];
    if (char === "{") {
      depth += 1;
    } else if (char === "}") {
      depth -= 1;
      if (depth === 0) {
        end = index;
        break;
      }
    }
  }

  assert.notEqual(end, -1, `Expected balanced braces for ${name}`);
  return auditGenerationSource.slice(openBraceIndex, end + 1);
}

test("auditGeneration module exports required mutation contracts", () => {
  assert.equal(
    existsSync(auditGenerationPath),
    true,
    "auditGeneration.ts should be present",
  );

  const exports = getExportedConstNames(sourceFile);
  const required = [
    "queueLeadAuditGeneration",
    "startAuditGenerationRun",
    "persistAuditGenerationResult",
    "replaceAuditFindings",
    "finishAuditGenerationRun",
  ];

  for (const exportName of required) {
    assert.equal(
      exports.has(exportName),
      true,
      `Expected export: ${exportName}`,
    );
  }
});

test("auditGeneration module registers internalMutation contracts", () => {
  for (const name of [
    "queueLeadAuditGeneration",
    "startAuditGenerationRun",
    "persistAuditGenerationResult",
    "replaceAuditFindings",
    "finishAuditGenerationRun",
  ]) {
    assert.equal(
      hasPattern(
        auditGenerationSource,
        new RegExp(`export const ${name} = internalMutation\\s*\\(`),
      ),
      true,
      `${name} should be registered as internalMutation.`,
    );
  }
});

test("replaceAuditFindings replaces persisted audit findings with evidence refs", () => {
  const replaceSource = extractExportSource("replaceAuditFindings");

  assert.equal(
    hasPattern(replaceSource, /query\("auditFindings"\)/),
    true,
    "replaceAuditFindings should query auditFindings.",
  );
  assert.equal(
    hasPattern(replaceSource, /withIndex\("by_auditId"/),
    true,
    "replaceAuditFindings should query existing findings by auditId.",
  );
  assert.equal(
    hasPattern(replaceSource, /ctx\.db\.delete\(/),
    true,
    "replaceAuditFindings should delete stale findings before inserting replacements.",
  );
  assert.equal(
    hasPattern(replaceSource, /ctx\.db\.insert\(\s*"auditFindings"/),
    true,
    "replaceAuditFindings should insert into auditFindings.",
  );
  for (const field of [
    "skillId",
    "claim",
    "recommendation",
    "customerBenefit",
    "severity",
    "confidence",
    "evidenceRefs",
    "reviewStatus",
  ]) {
    assert.equal(
      hasPattern(replaceSource, new RegExp(`${field}:\\s*finding\\.${field}`)),
      true,
      `replaceAuditFindings should persist ${field}.`,
    );
  }
});

test("queueLeadAuditGeneration dedupes pending/running runs and schedules action", () => {
  const queueSource = extractExportSource("queueLeadAuditGeneration");

  assert.equal(
    hasPattern(
      queueSource,
      /withIndex\("by_type_and_status_and_leadId"[\s\S]*?eq\("type",\s*"audit_generation"\)[\s\S]*?eq\("status",\s*"pending"\)[\s\S]*?eq\("leadId",\s*args\.leadId\)/,
    ),
    true,
    "Queue should dedupe pending runs with by_type_and_status_and_leadId for type audit_generation.",
  );
  assert.equal(
    hasPattern(
      queueSource,
      /withIndex\("by_type_and_status_and_leadId"[\s\S]*?eq\("type",\s*"audit_generation"\)[\s\S]*?eq\("status",\s*"running"\)[\s\S]*?eq\("leadId",\s*args\.leadId\)/,
    ),
    true,
    "Queue should dedupe running runs with by_type_and_status_and_leadId for type audit_generation.",
  );
  assert.equal(
    hasPattern(
      queueSource,
      /ctx\.scheduler\.runAfter\(\s*0,\s*internal\.auditGenerationAction\.processAuditGeneration,[\s\S]*?runId/,
    ),
    true,
    "Queue should schedule internal.auditGenerationAction.processAuditGeneration.",
  );
  assert.equal(
    hasPattern(queueSource, /Audit-Generierung wurde in die Warteschlange gesetzt\./),
    true,
    "Queue should emit a queue event message.",
  );
});

test("startAuditGenerationRun validates and marks run as running", () => {
  const startSource = extractExportSource("startAuditGenerationRun");

  assert.equal(
    hasPattern(startSource, /run\.type\s*!==\s*"audit_generation"/),
    true,
    "start should validate audit_generation run type.",
  );
  assert.equal(
    hasPattern(startSource, /run\.status\s*!==\s*"pending"/),
    true,
    "start should require pending status.",
  );
  assert.equal(
    hasPattern(startSource, /!run\.leadId[\s\S]*status:\s*"failed"/),
    true,
    "start should fail clearly when leadId missing.",
  );
  assert.equal(
    hasPattern(startSource, /!lead[\s\S]*status:\s*"failed"/),
    true,
    "start should fail clearly when lead cannot be loaded.",
  );
  assert.equal(
    hasPattern(
      startSource,
      /ctx\.db\.patch\(\s*args\.runId,[\s\S]*status:\s*"running"/,
    ),
    true,
    "start should set run status running.",
  );
  assert.equal(
    hasPattern(startSource, /message:\s*"[^"]*konnte nicht gestartet werden[^"]*"/i),
    true,
    "start should emit clear failure events when starting fails.",
  );
});

test("persistAuditGenerationResult inserts into auditGenerations", () => {
  const persistSource = extractExportSource("persistAuditGenerationResult");

  assert.equal(
    hasPattern(persistSource, /ctx\.db\.insert\(\s*"auditGenerations"/),
    true,
    "persistAuditGenerationResult should insert into auditGenerations.",
  );
  assert.equal(
    hasPattern(
      persistSource,
      /prompt:\s*sanitizeAndCapString\(args\.prompt,\s*MAX_PROMPT_BYTES\)/,
    ),
    true,
    "persist function should sanitize prompt before persisting to avoid secrets.",
  );
  assert.equal(
    hasPattern(
      persistSource,
      /rawResponse:\s*sanitizeAndCapString\(args\.rawResponse,\s*MAX_RAW_RESPONSE_BYTES\)/,
    ),
    true,
    "persist function should sanitize rawResponse before persisting to avoid secrets.",
  );
});

test("getAuditGenerationEvidence loads latest successful website enrichment evidence by lead", () => {
  const evidenceSource = extractExportSource("getAuditGenerationEvidence");

  assert.equal(
    hasPattern(
      evidenceSource,
      /query\("agentRuns"\)[\s\S]*withIndex\("by_type_and_status_and_leadId"[\s\S]*eq\("type",\s*"website_enrichment"\)[\s\S]*eq\("status",\s*"succeeded"\)[\s\S]*eq\("leadId",\s*lead\._id\)[\s\S]*order\("desc"\)[\s\S]*take\(1\)/,
    ),
    true,
    "Evidence query should locate the latest successful website_enrichment run for the same lead.",
  );
  assert.equal(
    hasPattern(
      evidenceSource,
      /const\s+enrichmentEvidenceRunId\s*=\s*latestSuccessfulEnrichmentRun\[0\]\?\._id\s*\?\?\s*args\.runId/,
    ),
    true,
    "Evidence query should fall back to the audit run only when no enrichment run exists.",
  );
  for (const table of [
    "websiteCrawlPages",
    "websiteTechnicalChecks",
  ]) {
    assert.equal(
      hasPattern(
        evidenceSource,
        new RegExp(
          `query\\("${table}"\\)[\\s\\S]*withIndex\\("by_runId"[\\s\\S]*eq\\("runId",\\s*enrichmentEvidenceRunId\\)`,
        ),
      ),
      true,
      `${table} should be loaded from the enrichment evidence run.`,
    );
  }
  assert.equal(
    hasPattern(
      evidenceSource,
      /const\s+screenshots\s*=\s*\[\s*\.\.\.auditCaptureScreenshotsByRun,\s*\.\.\.enrichmentScreenshotsByRun\s*\]/,
    ),
    true,
    "Evidence query should include audit-run ScreenshotOne captures and enrichment screenshots.",
  );
});

test("truncateWithMarker is byte-capped and marker-safe in persistence", () => {
  assert.equal(
    hasPattern(auditGenerationSource, /const markerBytes = byteLength\(TRUNCATION_MARKER\);/),
    true,
    "truncateWithMarker should calculate marker bytes explicitly.",
  );
  assert.equal(
    hasPattern(
      auditGenerationSource,
      /if\s*\(byteLength\(value\)\s*<=\s*maxBytes\)\s*\{\s*return\s*value;\s*\}/,
    ),
    true,
    "truncateWithMarker should return early when already within byte limit.",
  );
  assert.equal(
    hasPattern(
      auditGenerationSource,
      /if\s*\(markerBytes\s*>=\s*maxBytes\)/,
    ),
    true,
    "truncateWithMarker should handle marker length edge cases.",
  );
  assert.equal(
    hasPattern(
      auditGenerationSource,
      /new TextDecoder\(\)\.decode\(markerBytesBuffer\.slice\(0,\s*maxBytes\)\)/,
    ),
    true,
    "truncateWithMarker should trim marker bytes with decoder slice fallback.",
  );
  assert.equal(
    hasPattern(
      auditGenerationSource,
      /TRUNCATION_MARKER\\.slice\(0,\s*maxBytes\)/,
    ),
    false,
    "truncateWithMarker should not use unbounded marker slicing by bytes.",
  );
});

test("sanitizer masks env-backed secret values in persistence", () => {
  assert.equal(
    hasPattern(auditGenerationSource, /function\s+sanitizeSecretCandidates/),
    true,
    "Persistence should expose secret candidate sanitizer.",
  );
  assert.equal(
    hasPattern(auditGenerationSource, /OPENROUTER_API_KEY/),
    true,
    "Persistence sanitizer should know OPENROUTER_API_KEY.",
  );
  assert.equal(
    hasPattern(
      auditGenerationSource,
      /return\s+sanitized\s*\r?\n\s*\.replace\(/,
    ),
    true,
    "Persistence sanitizer should apply regex secret-masking patterns.",
  );
});

test("persistence sanitizer handles external service secrets with regex metacharacters", () => {
  for (const secretKey of ["SCREENSHOTONE_API_KEY", "JINA_API_KEY"]) {
    assert.equal(
      hasPattern(auditGenerationSource, new RegExp(`["']${secretKey}["']`)),
      true,
      `Persistence sanitizer should redact ${secretKey}.`,
    );
  }

  assert.equal(
    auditGenerationSource.includes(
      'return value.replace(/[.*+?^${}()|[\\]\\\\]/g, "\\\\$&");',
    ),
    true,
    "escapeRegExp should escape regex metacharacters with the canonical character class.",
  );
  assert.equal(
    auditGenerationSource.includes("/[.*+?^${}()|[\\\\]\\\\]/g"),
    false,
    "escapeRegExp should not keep the malformed bracket/backslash character class.",
  );
});

test("finishAuditGenerationRun updates run status/counters/currentStep", () => {
  const finishSource = extractExportSource("finishAuditGenerationRun");

  assert.equal(
    hasPattern(
      finishSource,
      /ctx\.db\.patch\(\s*args\.runId,[\s\S]*?status:\s*args\.status/,
    ),
    true,
    "finish should set run status.",
  );
  assert.equal(
    hasPattern(
      finishSource,
      /status:\s*args\.status[\s\S]*finishedAt:\s*now/,
    ),
    true,
    "finish should set finishedAt.",
  );
  assert.equal(
    hasPattern(
      finishSource,
      /counters:\s*\{[\s\S]*errors:\s*args\.errors/,
    ),
    true,
    "finish should update counters with errors.",
  );
  assert.equal(
    hasPattern(
      finishSource,
      /currentStep:\s*args\.currentStep\s*(\|\||\?\?)\s*"audit_generation"/,
    ),
    true,
    "finish should update currentStep.",
  );
});