Externalize audit pipeline services
This commit is contained in:
335
tests/external-audit-pipeline-source.test.ts
Normal file
335
tests/external-audit-pipeline-source.test.ts
Normal file
@@ -0,0 +1,335 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
|
||||
const actionPath = path.join(process.cwd(), "convex", "auditGenerationAction.ts");
|
||||
const actionSource = existsSync(actionPath) ? readFileSync(actionPath, "utf8") : "";
|
||||
const generationPath = path.join(process.cwd(), "convex", "auditGeneration.ts");
|
||||
const generationSource = existsSync(generationPath)
|
||||
? readFileSync(generationPath, "utf8")
|
||||
: "";
|
||||
|
||||
function extractFunctionSource(functionName: string) {
|
||||
const declarationPattern = new RegExp(
|
||||
`(?:async\\s+)?function\\s+${functionName}\\s*\\([\\s\\S]*?\\n\\)\\s*(?::\\s*[^\\{]+)?\\{`,
|
||||
);
|
||||
const match = declarationPattern.exec(actionSource);
|
||||
assert.notEqual(match, null, `Expected function ${functionName}.`);
|
||||
|
||||
const openBraceIndex = match!.index + match![0].lastIndexOf("{");
|
||||
let depth = 0;
|
||||
let end = -1;
|
||||
|
||||
for (let index = openBraceIndex; index < actionSource.length; index += 1) {
|
||||
const char = actionSource[index];
|
||||
if (char === "{") {
|
||||
depth += 1;
|
||||
} else if (char === "}") {
|
||||
depth -= 1;
|
||||
if (depth === 0) {
|
||||
end = index;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert.notEqual(end, -1, `Expected balanced braces for ${functionName}.`);
|
||||
return actionSource.slice(match!.index, end + 1);
|
||||
}
|
||||
|
||||
test("audit generation action orchestrates external capture helpers when legacy crawl artifacts are absent", () => {
|
||||
assert.match(
|
||||
actionSource,
|
||||
/buildScreenshotOneRequests[\s\S]*buildJinaReaderAuditInput[\s\S]*estimateExternalAuditCostUsd|estimateExternalAuditCostUsd[\s\S]*buildScreenshotOneRequests[\s\S]*buildJinaReaderAuditInput/,
|
||||
"Action should import and use the approved external audit service helpers.",
|
||||
);
|
||||
assert.match(
|
||||
actionSource,
|
||||
/SCREENSHOTONE_API_KEY/,
|
||||
"ScreenshotOne capture should be guarded by the managed SCREENSHOTONE_API_KEY env key.",
|
||||
);
|
||||
assert.match(
|
||||
actionSource,
|
||||
/JINA_API_KEY/,
|
||||
"Jina capture should be compatible with the optional managed JINA_API_KEY env key.",
|
||||
);
|
||||
assert.match(
|
||||
actionSource,
|
||||
/evidence\.screenshots\.length\s*===\s*0[\s\S]*(started\.lead\.websiteUrl|started\.lead\.websiteDomain)/,
|
||||
"External capture should be prepared from the started lead URL/domain when legacy screenshots are missing.",
|
||||
);
|
||||
});
|
||||
|
||||
test("audit generation action records provider usage events for capture and OpenRouter generation", () => {
|
||||
assert.match(
|
||||
actionSource,
|
||||
/internal\.usageEvents\.recordUsageEvent/,
|
||||
"Action should record usage through internal.usageEvents.recordUsageEvent.",
|
||||
);
|
||||
|
||||
for (const provider of ["screenshotone", "jina", "openrouter"]) {
|
||||
assert.match(
|
||||
actionSource,
|
||||
new RegExp(`provider:\\s*["']${provider}["']`),
|
||||
`Action should record ${provider} usage.`,
|
||||
);
|
||||
}
|
||||
|
||||
assert.match(
|
||||
actionSource,
|
||||
/provider:\s*["']openrouter["'][\s\S]*operation:\s*["']audit_generation["']/,
|
||||
"OpenRouter usage should be recorded as audit_generation.",
|
||||
);
|
||||
assert.match(
|
||||
actionSource,
|
||||
/provider:\s*["']screenshotone["'][\s\S]*operation:\s*["']audit_capture["']/,
|
||||
"ScreenshotOne usage should be recorded as audit_capture.",
|
||||
);
|
||||
assert.match(
|
||||
actionSource,
|
||||
/provider:\s*["']jina["'][\s\S]*operation:\s*["']audit_capture["']/,
|
||||
"Jina usage should be recorded as audit_capture.",
|
||||
);
|
||||
});
|
||||
|
||||
test("Jina markdown joins the evidence prompt without requiring Playwright crawl pages", () => {
|
||||
assert.match(
|
||||
actionSource,
|
||||
/jina(?:Reader)?AuditInput[\s\S]*markdown/,
|
||||
"Action should keep Jina reader markdown as an audit evidence input.",
|
||||
);
|
||||
assert.match(
|
||||
actionSource,
|
||||
/buildAuditEvidenceInput\(\{[\s\S]*externalMarkdown|externalMarkdown[\s\S]*buildAuditEvidenceInput\(\{/,
|
||||
"Action should pass external markdown into the evidence builder.",
|
||||
);
|
||||
assert.match(
|
||||
generationSource,
|
||||
/externalMarkdown/,
|
||||
"Audit generation evidence types should expose external markdown for prompts.",
|
||||
);
|
||||
});
|
||||
|
||||
test("external capture fetches use timeout, abort signal, and bounded response readers", () => {
|
||||
for (const constantName of [
|
||||
"EXTERNAL_CAPTURE_TIMEOUT_MS",
|
||||
"MAX_SCREENSHOT_BYTES",
|
||||
"MAX_JINA_MARKDOWN_BYTES",
|
||||
"MAX_JINA_MARKDOWN_CHARS",
|
||||
]) {
|
||||
assert.match(
|
||||
actionSource,
|
||||
new RegExp(`const\\s+${constantName}\\s*=`),
|
||||
`Action should define ${constantName}.`,
|
||||
);
|
||||
}
|
||||
|
||||
assert.match(
|
||||
actionSource,
|
||||
/AbortController/,
|
||||
"External fetches should use AbortController for per-request timeouts.",
|
||||
);
|
||||
assert.match(
|
||||
actionSource,
|
||||
/fetch\([\s\S]*signal:/,
|
||||
"External fetches should pass an AbortSignal.",
|
||||
);
|
||||
assert.doesNotMatch(
|
||||
actionSource,
|
||||
/response\.blob\(\)/,
|
||||
"ScreenshotOne capture should not call unbounded response.blob().",
|
||||
);
|
||||
assert.doesNotMatch(
|
||||
actionSource,
|
||||
/response\.text\(\)/,
|
||||
"Jina capture should not call unbounded response.text().",
|
||||
);
|
||||
});
|
||||
|
||||
test("audit generation action sanitizes raw errors before run events and run failure summaries", () => {
|
||||
assert.match(
|
||||
actionSource,
|
||||
/function messageFromError[\s\S]*sanitizeSecretCandidates/,
|
||||
"messageFromError should sanitize/redact before returning error text.",
|
||||
);
|
||||
|
||||
for (const secretName of ["SCREENSHOTONE_API_KEY", "JINA_API_KEY"]) {
|
||||
assert.match(
|
||||
actionSource,
|
||||
new RegExp(`["']${secretName}["']`),
|
||||
`Secret sanitizer should know ${secretName}.`,
|
||||
);
|
||||
}
|
||||
|
||||
assert.doesNotMatch(
|
||||
actionSource,
|
||||
/value:\s*messageFromError\(error\)/,
|
||||
"Run event details should not receive raw messageFromError calls inline.",
|
||||
);
|
||||
assert.doesNotMatch(
|
||||
actionSource,
|
||||
/errorSummary\s*=\s*messageFromError\(error\)/,
|
||||
"Failure summaries should not assign unsanitized raw errors inline.",
|
||||
);
|
||||
});
|
||||
|
||||
test("german-copy OpenRouter usage event aggregates all six generation calls", () => {
|
||||
assert.match(
|
||||
actionSource,
|
||||
/aggregateOpenRouterUsage/,
|
||||
"Action should expose an aggregation helper for stage-level OpenRouter usage.",
|
||||
);
|
||||
assert.match(
|
||||
actionSource,
|
||||
/aggregateOpenRouterUsage\(\[[\s\S]*publicSummaryResult\.usage[\s\S]*germanBodyResult\.usage[\s\S]*germanSubjectResult\.usage[\s\S]*germanEmailResult\.usage[\s\S]*germanCallScriptResult\.usage[\s\S]*germanFollowUpResult\.usage[\s\S]*\]\)/,
|
||||
"German-copy usage should aggregate public summary, body, subject, email, call script, and follow-up calls.",
|
||||
);
|
||||
});
|
||||
|
||||
test("usage event recording is best-effort and cannot fail audit generation", () => {
|
||||
const usageRecorder = extractFunctionSource("recordAuditUsageEvent");
|
||||
|
||||
assert.match(
|
||||
usageRecorder,
|
||||
/try\s*\{[\s\S]*await ctx\.runMutation\(internal\.usageEvents\.recordUsageEvent/,
|
||||
"Usage recorder should isolate recordUsageEvent in a try block.",
|
||||
);
|
||||
assert.match(
|
||||
usageRecorder,
|
||||
/catch\s*\(error\)\s*\{[\s\S]*messageFromError\(error\)[\s\S]*level:\s*["']warning["']/,
|
||||
"Usage recorder should sanitize/log failures as warnings.",
|
||||
);
|
||||
assert.match(
|
||||
usageRecorder,
|
||||
/catch\s*\(error\)\s*\{[\s\S]*try\s*\{[\s\S]*appendRunEvent[\s\S]*\}\s*catch/,
|
||||
"Warning logging for usage failures should also be best-effort.",
|
||||
);
|
||||
});
|
||||
|
||||
test("external capture timeout covers body streaming and cancels readers", () => {
|
||||
const fetcher = extractFunctionSource("fetchExternalCapture");
|
||||
const reader = extractFunctionSource("readLimitedResponseBytes");
|
||||
|
||||
assert.match(
|
||||
fetcher,
|
||||
/return\s*\{[\s\S]*response[\s\S]*abortController:\s*controller[\s\S]*timeout[\s\S]*\}/,
|
||||
"fetchExternalCapture should return the active deadline context for body reads.",
|
||||
);
|
||||
assert.doesNotMatch(
|
||||
fetcher,
|
||||
/finally\s*\{[\s\S]*clearTimeout\(timeout\)/,
|
||||
"fetchExternalCapture should not clear the timeout before body streaming completes.",
|
||||
);
|
||||
assert.match(
|
||||
reader,
|
||||
/signal\??:\s*AbortSignal/,
|
||||
"Bounded response reader should accept an AbortSignal.",
|
||||
);
|
||||
assert.match(
|
||||
reader,
|
||||
/signal\?\.addEventListener\(\s*["']abort["'][\s\S]*reader\.cancel/,
|
||||
"Bounded response reader should cancel the reader on timeout/abort.",
|
||||
);
|
||||
assert.match(
|
||||
reader,
|
||||
/totalBytes\s*>\s*maxBytes[\s\S]*await reader\.cancel\(/,
|
||||
"Bounded response reader should cancel the stream when the byte cap is exceeded.",
|
||||
);
|
||||
assert.match(
|
||||
actionSource,
|
||||
/readLimitedResponseBytes\([\s\S]*MAX_SCREENSHOT_BYTES[\s\S]*abortController\.signal/,
|
||||
"Screenshot body reads should use the active timeout signal.",
|
||||
);
|
||||
assert.match(
|
||||
actionSource,
|
||||
/readLimitedMarkdown\([\s\S]*abortController\.signal/,
|
||||
"Jina markdown body reads should use the active timeout signal.",
|
||||
);
|
||||
assert.match(
|
||||
actionSource,
|
||||
/finally\s*\{[\s\S]*clearExternalCaptureTimeout/,
|
||||
"Capture loops should clear the external timeout after fetch and body streaming finish.",
|
||||
);
|
||||
});
|
||||
|
||||
test("external capture request builders are provider-level best-effort", () => {
|
||||
const capture = extractFunctionSource("captureExternalAuditArtifacts");
|
||||
|
||||
assert.match(
|
||||
capture,
|
||||
/if\s*\(args\.needsScreenshots\)[\s\S]*try\s*\{[\s\S]*buildScreenshotOneRequests/,
|
||||
"ScreenshotOne request construction should be inside a provider-level try block.",
|
||||
);
|
||||
assert.match(
|
||||
capture,
|
||||
/buildScreenshotOneRequests[\s\S]*catch\s*\(error\)[\s\S]*messageFromError\(error\)[\s\S]*level:\s*["']warning["']/,
|
||||
"ScreenshotOne request construction failures should degrade to sanitized warnings.",
|
||||
);
|
||||
assert.match(
|
||||
capture,
|
||||
/if\s*\(args\.needsMarkdown\)[\s\S]*try\s*\{[\s\S]*buildJinaReaderAuditInput/,
|
||||
"Jina reader input construction should be inside a provider-level try block.",
|
||||
);
|
||||
assert.match(
|
||||
capture,
|
||||
/buildJinaReaderAuditInput[\s\S]*catch\s*\(error\)[\s\S]*messageFromError\(error\)[\s\S]*level:\s*["']warning["']/,
|
||||
"Jina reader input construction failures should degrade to sanitized warnings.",
|
||||
);
|
||||
});
|
||||
|
||||
test("ScreenshotOne missing-key skip emits best-effort warning only when screenshots are needed", () => {
|
||||
const capture = extractFunctionSource("captureExternalAuditArtifacts");
|
||||
const needsScreenshotsIndex = capture.indexOf("if (args.needsScreenshots)");
|
||||
const needsMarkdownIndex = capture.indexOf("if (args.needsMarkdown)");
|
||||
const missingKeyWarningIndex = capture.indexOf(
|
||||
"ScreenshotOne ist nicht konfiguriert; Screenshot-Erfassung wurde übersprungen.",
|
||||
);
|
||||
|
||||
assert.notEqual(
|
||||
needsScreenshotsIndex,
|
||||
-1,
|
||||
"External capture should branch on needsScreenshots.",
|
||||
);
|
||||
assert.notEqual(
|
||||
needsMarkdownIndex,
|
||||
-1,
|
||||
"External capture should keep the later needsMarkdown branch.",
|
||||
);
|
||||
assert.notEqual(
|
||||
missingKeyWarningIndex,
|
||||
-1,
|
||||
"Missing ScreenshotOne config should emit a clear warning message.",
|
||||
);
|
||||
assert.equal(
|
||||
missingKeyWarningIndex > needsScreenshotsIndex &&
|
||||
missingKeyWarningIndex < needsMarkdownIndex,
|
||||
true,
|
||||
"Missing-key warning should live inside the needsScreenshots branch, so legacy screenshots do not warn.",
|
||||
);
|
||||
assert.match(
|
||||
capture,
|
||||
/if\s*\(!screenshotOneApiKey\)\s*\{[\s\S]*try\s*\{[\s\S]*await appendRunEvent\(ctx,\s*\{[\s\S]*level:\s*["']warning["'][\s\S]*ScreenshotOne ist nicht konfiguriert; Screenshot-Erfassung wurde übersprungen\.[\s\S]*\}\s*\);[\s\S]*\}\s*catch\s*\{[\s\S]*\}/,
|
||||
"Missing-key warning logging should be best-effort and unable to fail the audit run.",
|
||||
);
|
||||
});
|
||||
|
||||
test("external capture non-OK responses cancel bodies before continuing", () => {
|
||||
const capture = extractFunctionSource("captureExternalAuditArtifacts");
|
||||
const nonOkCancelCount = [
|
||||
...capture.matchAll(
|
||||
/if\s*\(!response\.ok\)\s*\{[\s\S]*?await cancelExternalResponseBody\(response\);[\s\S]*?continue;/g,
|
||||
),
|
||||
].length;
|
||||
|
||||
assert.match(
|
||||
actionSource,
|
||||
/async function cancelExternalResponseBody/,
|
||||
"Action should centralize best-effort body cancellation for non-OK responses.",
|
||||
);
|
||||
assert.equal(
|
||||
nonOkCancelCount,
|
||||
2,
|
||||
"Both ScreenshotOne and Jina non-OK branches should cancel bodies before continue.",
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user