feat: add OpenRouter audit generation pipeline
This commit is contained in:
335
tests/audit-generation-action-source.test.ts
Normal file
335
tests/audit-generation-action-source.test.ts
Normal file
@@ -0,0 +1,335 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
|
||||
const actionPath = path.join(process.cwd(), "convex", "auditGenerationAction.ts");
|
||||
const actionSource = existsSync(actionPath) ? readFileSync(actionPath, "utf8") : "";
|
||||
const generationSourcePath = path.join(process.cwd(), "convex", "auditGeneration.ts");
|
||||
const generationSource = existsSync(generationSourcePath)
|
||||
? readFileSync(generationSourcePath, "utf8")
|
||||
: "";
|
||||
|
||||
function hasPattern(source: string, pattern: RegExp) {
|
||||
return pattern.test(source);
|
||||
}
|
||||
|
||||
function hasExportedInternalAction(exportName: string) {
|
||||
const pattern = new RegExp(
|
||||
`export const ${exportName}\\s*=\\s*internalAction\\s*\\(`,
|
||||
);
|
||||
return hasPattern(actionSource, pattern);
|
||||
}
|
||||
|
||||
function hasStageCall(schema: string) {
|
||||
const escaped = schema.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
return hasPattern(
|
||||
actionSource,
|
||||
new RegExp(
|
||||
`generateObject\\([\\s\\S]*schema:\\s*${escaped}[\\s\\S]*\\)`,
|
||||
"m",
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
test("auditGenerationAction module exists and is a Node action file", () => {
|
||||
assert.equal(existsSync(actionPath), true, "auditGenerationAction.ts should exist");
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /^"use node";/m),
|
||||
true,
|
||||
"auditGenerationAction.ts should start with \"use node\"",
|
||||
);
|
||||
});
|
||||
|
||||
test("auditGenerationAction exports processAuditGeneration with runId validator", () => {
|
||||
assert.equal(
|
||||
hasExportedInternalAction("processAuditGeneration"),
|
||||
true,
|
||||
"processAuditGeneration should be an internalAction",
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/processAuditGeneration\s*=\s*internalAction\(\s*{\s*args:\s*{\s*runId:\s*v\.id\(\s*["']agentRuns["']\s*\)\s*,?\s*}/,
|
||||
),
|
||||
true,
|
||||
"processAuditGeneration should validate runId: v.id(\"agentRuns\")",
|
||||
);
|
||||
});
|
||||
|
||||
test("action starts, queries evidence, and runs stage pipeline", () => {
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/internal\.auditGeneration\.startAuditGenerationRun/,
|
||||
),
|
||||
true,
|
||||
"Action should start the run via internal.auditGeneration.startAuditGenerationRun",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/internal\.auditGeneration\.getAuditGenerationEvidence/,
|
||||
),
|
||||
true,
|
||||
"Action should load evidence via internal.auditGeneration.getAuditGenerationEvidence",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/internal\.auditGeneration\.persistAuditGenerationResult/,
|
||||
),
|
||||
true,
|
||||
"Action should persist each stage result",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/internal\.auditGeneration\.finishAuditGenerationRun/,
|
||||
),
|
||||
true,
|
||||
"Action should finish run via internal.auditGeneration.finishAuditGenerationRun",
|
||||
);
|
||||
});
|
||||
|
||||
test("action includes all required audit stages", () => {
|
||||
for (const stage of [
|
||||
"classification",
|
||||
"multimodalAudit",
|
||||
"germanCopy",
|
||||
"qualityReview",
|
||||
]) {
|
||||
const token = new RegExp(`stage:\\s*["']${stage}["']`);
|
||||
assert.equal(
|
||||
hasPattern(actionSource, token),
|
||||
true,
|
||||
`Action should reference ${stage} stage`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("action handles post-start failure paths in action-level catch", () => {
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/try\s*{[\s\S]*internal\.auditGeneration\.getAuditGenerationEvidence[\s\S]*const provider = createOpenRouterProvider\(\)/,
|
||||
),
|
||||
true,
|
||||
"Action should include evidence query and provider init inside catch-covered flow.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/catch\s*\(error\)\s*{[\s\S]*appendRunEvent[\s\S]*finishAuditGenerationRun[\s\S]*"failed"/,
|
||||
),
|
||||
true,
|
||||
"Action-level error handler should emit run events.",
|
||||
);
|
||||
});
|
||||
|
||||
test("action calls generateObject with required schemas", () => {
|
||||
const requiredSchemas = [
|
||||
"internalFindingsSchema",
|
||||
"auditSummarySchema",
|
||||
"publicAuditTextSchema",
|
||||
"emailDraftSchema",
|
||||
"emailSubjectSchema",
|
||||
"callScriptSchema",
|
||||
"followUpDraftSchema",
|
||||
"qualityReviewSchema",
|
||||
];
|
||||
|
||||
for (const requiredSchema of requiredSchemas) {
|
||||
assert.equal(
|
||||
hasStageCall(requiredSchema),
|
||||
true,
|
||||
`Action should call generateObject with schema ${requiredSchema}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("action uses multimodal file parts with mediaType image/* when screenshots are available", () => {
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/type:\s*["']file["'][\s\S]*mediaType:\s*(?:getValidMediaType|["']image\/)/,
|
||||
),
|
||||
true,
|
||||
"Multimodal call should include AI file parts with image mediaType",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/ctx\.storage\.(get|getUrl)\(/,
|
||||
),
|
||||
true,
|
||||
"Multimodal call should try to fetch screenshots from Convex storage",
|
||||
);
|
||||
});
|
||||
|
||||
test("action handles missing screenshots with warning event fallback", () => {
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /level:\s*["']warning["'][\s\S]*Screenshot|Vorschaubild/),
|
||||
true,
|
||||
"Action should append warning event when multimodal screenshot input is unavailable",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /messages:\s*\[[\s\S]*type:\s*["']text["'][\s\S]*\]/),
|
||||
true,
|
||||
"Action should fall back to text-only multimodal calls when required parts are missing",
|
||||
);
|
||||
});
|
||||
|
||||
test("action runs german copy guard and blocks outreach-ready on validation failure", () => {
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /validateCustomerFacingCopy/),
|
||||
true,
|
||||
"Action should run German copy validation",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/guardResult\.passed|qualityPassed\s*=\s*qualityResult\.object\.isValid\s*&&\s*guardResult\.passed/,
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /api\.leads\.reviewUpdate/),
|
||||
true,
|
||||
"Action should patch lead via api.leads.reviewUpdate",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/isTerminalLeadContactStatus/,
|
||||
),
|
||||
true,
|
||||
"Action should set contactStatus to outreach_ready only when terminal guard allows it.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/do_not_contact|contacted|replied/i,
|
||||
),
|
||||
true,
|
||||
"Action should explicitly guard against terminal lead statuses before outreach-ready.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/Lead-Status wurde nicht auf outreach_ready gesetzt/,
|
||||
),
|
||||
true,
|
||||
"Action should emit warning event when outreach-ready cannot be set.",
|
||||
);
|
||||
});
|
||||
|
||||
test("action persists audit and outreach outputs before finishing succeeded run", () => {
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/internal\.audits\.upsertFromAuditGeneration/,
|
||||
),
|
||||
true,
|
||||
"Action should persist audit output via internal.audits.upsertFromAuditGeneration",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/internal\.outreach\.upsertFromAuditGeneration/,
|
||||
),
|
||||
true,
|
||||
"Action should persist outreach output via internal.outreach.upsertFromAuditGeneration",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/internal\.audits\.upsertFromAuditGeneration[\s\S]*internal\.outreach\.upsertFromAuditGeneration[\s\S]*internal\.auditGeneration\.finishAuditGenerationRun[\s\S]*status:\s*["']succeeded["']/,
|
||||
),
|
||||
true,
|
||||
"Action should finish success after persisted outputs",
|
||||
);
|
||||
});
|
||||
|
||||
test("action uses model profiles for generation parameters", () => {
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /resolveModelProfile\("classification"\)/),
|
||||
true,
|
||||
"classification generation should use resolveModelProfile.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /resolveModelProfile\("multimodalAudit"\)/),
|
||||
true,
|
||||
"multimodal generation should use resolveModelProfile.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /resolveModelProfile\("germanCopy"\)/),
|
||||
true,
|
||||
"german copy generation should use resolveModelProfile.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /resolveModelProfile\("qualityReview"\)/),
|
||||
true,
|
||||
"quality review generation should use resolveModelProfile.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/temperature:\s*classificationProfile\.temperature[\s\S]*maxOutputTokens:\s*classificationProfile\.maxTokens/,
|
||||
),
|
||||
true,
|
||||
"classification stage should use profile temperature/maxTokens.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/temperature:\s*germanCopyProfile\.temperature[\s\S]*maxOutputTokens:\s*germanCopyProfile\.maxTokens/,
|
||||
),
|
||||
true,
|
||||
"german copy stages should use profile temperature/maxTokens.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/temperature:\s*qualityReviewProfile\.temperature[\s\S]*maxOutputTokens:\s*qualityReviewProfile\.maxTokens/,
|
||||
),
|
||||
true,
|
||||
"quality review stage should use profile temperature/maxTokens.",
|
||||
);
|
||||
});
|
||||
|
||||
test("action sanitization masks env-backed secrets", () => {
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/sanitizeSecretCandidates\([\s\S]*process\.env/,
|
||||
),
|
||||
true,
|
||||
"sanitize logic should include env-backed secret masking.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /OPENROUTER_API_KEY/),
|
||||
true,
|
||||
"sanitizer should include OPENROUTER_API_KEY in secret hints.",
|
||||
);
|
||||
});
|
||||
|
||||
test("auditGeneration scheduler reference in queueLeadAuditGeneration is typed", () => {
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
generationSource,
|
||||
/internal\.auditGenerationAction\.processAuditGeneration/,
|
||||
),
|
||||
true,
|
||||
"queueLeadAuditGeneration should reference internal.auditGenerationAction.processAuditGeneration",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
generationSource,
|
||||
/internal as any/,
|
||||
),
|
||||
false,
|
||||
"No temporary internal cast should remain for the processAuditGeneration schedule",
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user