import assert from "node:assert/strict"; import { existsSync, readFileSync } from "node:fs"; import path from "node:path"; import test from "node:test"; const actionPath = path.join(process.cwd(), "convex", "auditGenerationAction.ts"); const actionSource = existsSync(actionPath) ? readFileSync(actionPath, "utf8") : ""; const generationPath = path.join(process.cwd(), "convex", "auditGeneration.ts"); const generationSource = existsSync(generationPath) ? readFileSync(generationPath, "utf8") : ""; function extractFunctionSource(functionName: string) { const declarationPattern = new RegExp( `(?:async\\s+)?function\\s+${functionName}\\s*\\([\\s\\S]*?\\n\\)\\s*(?::\\s*[^\\{]+)?\\{`, ); const match = declarationPattern.exec(actionSource); assert.notEqual(match, null, `Expected function ${functionName}.`); const openBraceIndex = match!.index + match![0].lastIndexOf("{"); let depth = 0; let end = -1; for (let index = openBraceIndex; index < actionSource.length; index += 1) { const char = actionSource[index]; if (char === "{") { depth += 1; } else if (char === "}") { depth -= 1; if (depth === 0) { end = index; break; } } } assert.notEqual(end, -1, `Expected balanced braces for ${functionName}.`); return actionSource.slice(match!.index, end + 1); } test("audit generation action orchestrates external capture helpers when legacy crawl artifacts are absent", () => { assert.match( actionSource, /buildScreenshotOneRequests[\s\S]*buildJinaReaderAuditInput[\s\S]*estimateExternalAuditCostUsd|estimateExternalAuditCostUsd[\s\S]*buildScreenshotOneRequests[\s\S]*buildJinaReaderAuditInput/, "Action should import and use the approved external audit service helpers.", ); assert.match( actionSource, /SCREENSHOTONE_API_KEY/, "ScreenshotOne capture should be guarded by the managed SCREENSHOTONE_API_KEY env key.", ); assert.match( actionSource, /JINA_API_KEY/, "Jina capture should be compatible with the optional managed JINA_API_KEY env key.", ); assert.match( actionSource, /evidence\.screenshots\.length\s*===\s*0[\s\S]*(started\.lead\.websiteUrl|started\.lead\.websiteDomain)/, "External capture should be prepared from the started lead URL/domain when legacy screenshots are missing.", ); }); test("audit generation action records provider usage events for capture and OpenRouter generation", () => { assert.match( actionSource, /internal\.usageEvents\.recordUsageEvent/, "Action should record usage through internal.usageEvents.recordUsageEvent.", ); for (const provider of ["screenshotone", "jina", "openrouter"]) { assert.match( actionSource, new RegExp(`provider:\\s*["']${provider}["']`), `Action should record ${provider} usage.`, ); } assert.match( actionSource, /provider:\s*["']openrouter["'][\s\S]*operation:\s*["']audit_generation["']/, "OpenRouter usage should be recorded as audit_generation.", ); assert.match( actionSource, /provider:\s*["']screenshotone["'][\s\S]*operation:\s*["']audit_capture["']/, "ScreenshotOne usage should be recorded as audit_capture.", ); assert.match( actionSource, /provider:\s*["']jina["'][\s\S]*operation:\s*["']audit_capture["']/, "Jina usage should be recorded as audit_capture.", ); }); test("Jina markdown joins the evidence prompt without requiring Playwright crawl pages", () => { assert.match( actionSource, /jina(?:Reader)?AuditInput[\s\S]*markdown/, "Action should keep Jina reader markdown as an audit evidence input.", ); assert.match( actionSource, /buildAuditEvidenceInput\(\{[\s\S]*externalMarkdown|externalMarkdown[\s\S]*buildAuditEvidenceInput\(\{/, "Action should pass external markdown into the evidence builder.", ); assert.match( generationSource, /externalMarkdown/, "Audit generation evidence types should expose external markdown for prompts.", ); }); test("external capture fetches use timeout, abort signal, and bounded response readers", () => { for (const constantName of [ "EXTERNAL_CAPTURE_TIMEOUT_MS", "MAX_SCREENSHOT_BYTES", "MAX_JINA_MARKDOWN_BYTES", "MAX_JINA_MARKDOWN_CHARS", ]) { assert.match( actionSource, new RegExp(`const\\s+${constantName}\\s*=`), `Action should define ${constantName}.`, ); } assert.match( actionSource, /AbortController/, "External fetches should use AbortController for per-request timeouts.", ); assert.match( actionSource, /fetch\([\s\S]*signal:/, "External fetches should pass an AbortSignal.", ); assert.doesNotMatch( actionSource, /response\.blob\(\)/, "ScreenshotOne capture should not call unbounded response.blob().", ); assert.doesNotMatch( actionSource, /response\.text\(\)/, "Jina capture should not call unbounded response.text().", ); }); test("audit generation action sanitizes raw errors before run events and run failure summaries", () => { assert.match( actionSource, /function messageFromError[\s\S]*sanitizeSecretCandidates/, "messageFromError should sanitize/redact before returning error text.", ); for (const secretName of ["SCREENSHOTONE_API_KEY", "JINA_API_KEY"]) { assert.match( actionSource, new RegExp(`["']${secretName}["']`), `Secret sanitizer should know ${secretName}.`, ); } assert.doesNotMatch( actionSource, /value:\s*messageFromError\(error\)/, "Run event details should not receive raw messageFromError calls inline.", ); assert.doesNotMatch( actionSource, /errorSummary\s*=\s*messageFromError\(error\)/, "Failure summaries should not assign unsanitized raw errors inline.", ); }); test("german-copy OpenRouter usage event aggregates all six generation calls", () => { assert.match( actionSource, /aggregateOpenRouterUsage/, "Action should expose an aggregation helper for stage-level OpenRouter usage.", ); assert.match( actionSource, /aggregateOpenRouterUsage\(\[[\s\S]*publicSummaryResult\.usage[\s\S]*germanBodyResult\.usage[\s\S]*germanSubjectResult\.usage[\s\S]*germanEmailResult\.usage[\s\S]*germanCallScriptResult\.usage[\s\S]*germanFollowUpResult\.usage[\s\S]*\]\)/, "German-copy usage should aggregate public summary, body, subject, email, call script, and follow-up calls.", ); }); test("usage event recording is best-effort and cannot fail audit generation", () => { const usageRecorder = extractFunctionSource("recordAuditUsageEvent"); assert.match( usageRecorder, /try\s*\{[\s\S]*await ctx\.runMutation\(internal\.usageEvents\.recordUsageEvent/, "Usage recorder should isolate recordUsageEvent in a try block.", ); assert.match( usageRecorder, /catch\s*\(error\)\s*\{[\s\S]*messageFromError\(error\)[\s\S]*level:\s*["']warning["']/, "Usage recorder should sanitize/log failures as warnings.", ); assert.match( usageRecorder, /catch\s*\(error\)\s*\{[\s\S]*try\s*\{[\s\S]*appendRunEvent[\s\S]*\}\s*catch/, "Warning logging for usage failures should also be best-effort.", ); }); test("external capture timeout covers body streaming and cancels readers", () => { const fetcher = extractFunctionSource("fetchExternalCapture"); const reader = extractFunctionSource("readLimitedResponseBytes"); assert.match( fetcher, /return\s*\{[\s\S]*response[\s\S]*abortController:\s*controller[\s\S]*timeout[\s\S]*\}/, "fetchExternalCapture should return the active deadline context for body reads.", ); assert.doesNotMatch( fetcher, /finally\s*\{[\s\S]*clearTimeout\(timeout\)/, "fetchExternalCapture should not clear the timeout before body streaming completes.", ); assert.match( reader, /signal\??:\s*AbortSignal/, "Bounded response reader should accept an AbortSignal.", ); assert.match( reader, /signal\?\.addEventListener\(\s*["']abort["'][\s\S]*reader\.cancel/, "Bounded response reader should cancel the reader on timeout/abort.", ); assert.match( reader, /totalBytes\s*>\s*maxBytes[\s\S]*await reader\.cancel\(/, "Bounded response reader should cancel the stream when the byte cap is exceeded.", ); assert.match( actionSource, /readLimitedResponseBytes\([\s\S]*MAX_SCREENSHOT_BYTES[\s\S]*abortController\.signal/, "Screenshot body reads should use the active timeout signal.", ); assert.match( actionSource, /readLimitedMarkdown\([\s\S]*abortController\.signal/, "Jina markdown body reads should use the active timeout signal.", ); assert.match( actionSource, /finally\s*\{[\s\S]*clearExternalCaptureTimeout/, "Capture loops should clear the external timeout after fetch and body streaming finish.", ); }); test("external capture request builders are provider-level best-effort", () => { const capture = extractFunctionSource("captureExternalAuditArtifacts"); assert.match( capture, /if\s*\(args\.needsScreenshots\)[\s\S]*try\s*\{[\s\S]*buildScreenshotOneRequests/, "ScreenshotOne request construction should be inside a provider-level try block.", ); assert.match( capture, /buildScreenshotOneRequests[\s\S]*catch\s*\(error\)[\s\S]*messageFromError\(error\)[\s\S]*level:\s*["']warning["']/, "ScreenshotOne request construction failures should degrade to sanitized warnings.", ); assert.match( capture, /if\s*\(args\.needsMarkdown\)[\s\S]*try\s*\{[\s\S]*buildJinaReaderAuditInput/, "Jina reader input construction should be inside a provider-level try block.", ); assert.match( capture, /buildJinaReaderAuditInput[\s\S]*catch\s*\(error\)[\s\S]*messageFromError\(error\)[\s\S]*level:\s*["']warning["']/, "Jina reader input construction failures should degrade to sanitized warnings.", ); }); test("ScreenshotOne missing-key skip emits best-effort warning only when screenshots are needed", () => { const capture = extractFunctionSource("captureExternalAuditArtifacts"); const needsScreenshotsIndex = capture.indexOf("if (args.needsScreenshots)"); const needsMarkdownIndex = capture.indexOf("if (args.needsMarkdown)"); const missingKeyWarningIndex = capture.indexOf( "ScreenshotOne ist nicht konfiguriert; Screenshot-Erfassung wurde übersprungen.", ); assert.notEqual( needsScreenshotsIndex, -1, "External capture should branch on needsScreenshots.", ); assert.notEqual( needsMarkdownIndex, -1, "External capture should keep the later needsMarkdown branch.", ); assert.notEqual( missingKeyWarningIndex, -1, "Missing ScreenshotOne config should emit a clear warning message.", ); assert.equal( missingKeyWarningIndex > needsScreenshotsIndex && missingKeyWarningIndex < needsMarkdownIndex, true, "Missing-key warning should live inside the needsScreenshots branch, so legacy screenshots do not warn.", ); assert.match( capture, /if\s*\(!screenshotOneApiKey\)\s*\{[\s\S]*try\s*\{[\s\S]*await appendRunEvent\(ctx,\s*\{[\s\S]*level:\s*["']warning["'][\s\S]*ScreenshotOne ist nicht konfiguriert; Screenshot-Erfassung wurde übersprungen\.[\s\S]*\}\s*\);[\s\S]*\}\s*catch\s*\{[\s\S]*\}/, "Missing-key warning logging should be best-effort and unable to fail the audit run.", ); }); test("external capture non-OK responses cancel bodies before continuing", () => { const capture = extractFunctionSource("captureExternalAuditArtifacts"); const nonOkCancelCount = [ ...capture.matchAll( /if\s*\(!response\.ok\)\s*\{[\s\S]*?await cancelExternalResponseBody\(response\);[\s\S]*?continue;/g, ), ].length; assert.match( actionSource, /async function cancelExternalResponseBody/, "Action should centralize best-effort body cancellation for non-OK responses.", ); assert.equal( nonOkCancelCount, 2, "Both ScreenshotOne and Jina non-OK branches should cancel bodies before continue.", ); });