Fix audit generation and enrichment fallback
This commit is contained in:
@@ -32,6 +32,39 @@ function hasStageCall(schema: string) {
|
||||
);
|
||||
}
|
||||
|
||||
function extractFunctionSource(functionName: string) {
|
||||
const marker = `function ${functionName}`;
|
||||
const asyncMarker = `async function ${functionName}`;
|
||||
const declarationIndex = actionSource.indexOf(marker) === -1
|
||||
? actionSource.indexOf(asyncMarker)
|
||||
: actionSource.indexOf(marker);
|
||||
assert.notEqual(
|
||||
declarationIndex,
|
||||
-1,
|
||||
`Expected function ${functionName} to exist.`,
|
||||
);
|
||||
|
||||
const openBraceIndex = actionSource.indexOf("{", declarationIndex);
|
||||
let depth = 0;
|
||||
let end = -1;
|
||||
|
||||
for (let index = openBraceIndex; index < actionSource.length; index += 1) {
|
||||
const char = actionSource[index];
|
||||
if (char === "{") {
|
||||
depth += 1;
|
||||
} else if (char === "}") {
|
||||
depth -= 1;
|
||||
if (depth === 0) {
|
||||
end = index;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert.notEqual(end, -1, `Expected balanced braces for ${functionName}.`);
|
||||
return actionSource.slice(declarationIndex, end + 1);
|
||||
}
|
||||
|
||||
test("auditGenerationAction module exists and is a Node action file", () => {
|
||||
assert.equal(existsSync(actionPath), true, "auditGenerationAction.ts should exist");
|
||||
assert.equal(
|
||||
@@ -130,7 +163,7 @@ test("action handles post-start failure paths in action-level catch", () => {
|
||||
|
||||
test("action calls generateObject with required schemas", () => {
|
||||
const requiredSchemas = [
|
||||
"internalFindingsSchema",
|
||||
"auditClassificationSchema",
|
||||
"auditSummarySchema",
|
||||
"publicAuditTextSchema",
|
||||
"emailDraftSchema",
|
||||
@@ -149,6 +182,155 @@ test("action calls generateObject with required schemas", () => {
|
||||
}
|
||||
});
|
||||
|
||||
test("action loads v3 skill registry from v2 source for evidence input", () => {
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /import\s*{[\s\S]*loadSkillsRegistry[\s\S]*}\s*from\s*["']\.\.\/lib\/skills-registry["']/),
|
||||
true,
|
||||
"Action should import loadSkillsRegistry from the shared registry parser.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /loadSkillsRegistry\(\s*(?:join\()?[\s\S]*v2_elemente[\s\S]*skills\.md[\s\S]*\)/),
|
||||
true,
|
||||
"Action should load the v3 registry from v2_elemente/skills.md.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /skillRegistry:\s*\[\s*\]/),
|
||||
false,
|
||||
"Action should not pass an always-empty skillRegistry to buildAuditEvidenceInput.",
|
||||
);
|
||||
});
|
||||
|
||||
test("registry load warning logging is isolated from fallback return", () => {
|
||||
const loadRegistrySource = extractFunctionSource("loadAuditSkillRegistry");
|
||||
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
loadRegistrySource,
|
||||
/catch\s*\(error\)\s*{[\s\S]*try\s*{[\s\S]*appendRunEvent[\s\S]*}\s*catch\s*{[\s\S]*}\s*return\s*\[\s*\]/,
|
||||
),
|
||||
true,
|
||||
"Registry load fallback should return [] even when warning event logging fails.",
|
||||
);
|
||||
});
|
||||
|
||||
test("persistAuditStage omits undefined fields from Convex mutation args", () => {
|
||||
const persistSource = extractFunctionSource("persistAuditStage");
|
||||
const mutationPayloadSource = persistSource.slice(
|
||||
persistSource.indexOf("await ctx.runMutation"),
|
||||
);
|
||||
|
||||
assert.doesNotMatch(
|
||||
actionSource,
|
||||
/persistAuditStage\(\s*{(?:(?!\n\s*}\s*\);)[\s\S])*(?:parsedJson|rawResponse|usage|finishReason|errorSummary):\s*undefined/,
|
||||
"Call sites should not pass explicit undefined stage payload fields.",
|
||||
);
|
||||
assert.doesNotMatch(
|
||||
persistSource,
|
||||
/usage:\s*usage\s*\?\s*toPersistedUsage\(usage\)\s*:\s*undefined/,
|
||||
"persistAuditStage should not emit usage: undefined.",
|
||||
);
|
||||
|
||||
for (const field of [
|
||||
"systemPrompt",
|
||||
"rawResponse",
|
||||
"parsedJson",
|
||||
"finishReason",
|
||||
"errorSummary",
|
||||
]) {
|
||||
assert.doesNotMatch(
|
||||
mutationPayloadSource,
|
||||
new RegExp(`\\n\\s*${field},`),
|
||||
`persistAuditStage should conditionally spread ${field}.`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("OpenRouter usage payloads omit undefined token fields", () => {
|
||||
const recordUsageSource = extractFunctionSource("recordOpenRouterUsage");
|
||||
|
||||
assert.match(
|
||||
actionSource,
|
||||
/function toPersistedUsage[\s\S]*usage\.inputTokens\s*!==\s*undefined[\s\S]*promptTokens:\s*usage\.inputTokens/,
|
||||
"toPersistedUsage should omit promptTokens when inputTokens is undefined.",
|
||||
);
|
||||
assert.doesNotMatch(
|
||||
recordUsageSource,
|
||||
/tokens:\s*{[\s\S]*inputTokens:\s*args\.usage\.inputTokens/,
|
||||
"recordOpenRouterUsage should not build token payloads with undefined properties.",
|
||||
);
|
||||
});
|
||||
|
||||
test("appendRunEvent omits undefined details from Convex mutation args", () => {
|
||||
assert.doesNotMatch(
|
||||
actionSource,
|
||||
/ctx\.runMutation\(internal\.runs\.appendEventInternal,\s*{[\s\S]*\n\s*details:\s*args\.details,\n/,
|
||||
"appendRunEvent should conditionally include details only when defined.",
|
||||
);
|
||||
});
|
||||
|
||||
test("success finishAuditGenerationRun omits undefined errorSummary", () => {
|
||||
assert.doesNotMatch(
|
||||
actionSource,
|
||||
/finishAuditGenerationRun,\s*{[\s\S]*status:\s*["']succeeded["'][\s\S]*errorSummary:\s*qualityPassed\s*\?\s*undefined/,
|
||||
"Succeeded finishAuditGenerationRun payload should not send errorSummary: undefined.",
|
||||
);
|
||||
});
|
||||
|
||||
test("quality review stage does not pass explicit undefined optional fields", () => {
|
||||
assert.doesNotMatch(
|
||||
actionSource,
|
||||
/persistAuditStage\(\s*{[\s\S]*stage:\s*["']qualityReview["'][\s\S]*errorSummary:\s*qualityPassed\s*\?\s*undefined/,
|
||||
"Quality persistAuditStage callsite should conditionally include errorSummary.",
|
||||
);
|
||||
});
|
||||
|
||||
test("persistAuditStage callsites conditionally include optional auditId", () => {
|
||||
assert.doesNotMatch(
|
||||
actionSource,
|
||||
/await\s+persistAuditStage\(\s*{(?:(?!\n\s*}\s*\);)[\s\S])*\n\s*auditId,\n/,
|
||||
"persistAuditStage callsites should spread auditId only when defined.",
|
||||
);
|
||||
});
|
||||
|
||||
test("audit generation helper callsites conditionally include optional auditId", () => {
|
||||
assert.doesNotMatch(
|
||||
actionSource,
|
||||
/(?:recordOpenRouterUsage|captureExternalAuditArtifacts)\(\s*ctx,\s*{(?:(?!\n\s*}\s*\);)[\s\S])*\n\s*auditId,\n/,
|
||||
"Helper callsites should spread auditId only when defined.",
|
||||
);
|
||||
assert.doesNotMatch(
|
||||
actionSource,
|
||||
/recordAuditUsageEvent\(\s*ctx,\s*{(?:(?!\n\s*}\s*\);)[\s\S])*\n\s*auditId:\s*args\.auditId,\n/,
|
||||
"recordAuditUsageEvent callsites should spread args.auditId only when defined.",
|
||||
);
|
||||
});
|
||||
|
||||
test("persistAuditStage callsites avoid nested maybe-undefined usage objects", () => {
|
||||
assert.doesNotMatch(
|
||||
actionSource,
|
||||
/persistAuditStage\(\s*{(?:(?!\n\s*}\s*\);)[\s\S])*usage:\s*{[\s\S]*?(?:inputTokens|outputTokens|totalTokens|cacheReadTokens):/,
|
||||
"persistAuditStage callsites should use a usage helper or conditional spreads, not inline maybe-undefined usage objects.",
|
||||
);
|
||||
});
|
||||
|
||||
test("classification stage uses v3 audit classification schema", () => {
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /auditClassificationSchema/),
|
||||
true,
|
||||
"Action should reference the v3 auditClassificationSchema.",
|
||||
);
|
||||
assert.equal(
|
||||
hasStageCall("auditClassificationSchema"),
|
||||
true,
|
||||
"Classification generateObject call should validate v3 finding payloads.",
|
||||
);
|
||||
assert.equal(
|
||||
hasStageCall("internalFindingsSchema"),
|
||||
false,
|
||||
"Classification should no longer validate against legacy-only internalFindingsSchema.",
|
||||
);
|
||||
});
|
||||
|
||||
test("action uses multimodal file parts with mediaType image/* when screenshots are available", () => {
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
@@ -190,14 +372,23 @@ test("action runs german copy guard and blocks outreach-ready on validation fail
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/guardResult\.passed|qualityPassed\s*=\s*qualityResult\.object\.isValid\s*&&\s*guardResult\.passed/,
|
||||
/qualityPassed\s*=\s*guardResult\.passed/,
|
||||
),
|
||||
true,
|
||||
"Only deterministic German copy guard failures should hard-block the audit run.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /api\.leads\.reviewUpdate/),
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/qualityPassed\s*=\s*qualityResult\.object\.isValid\s*&&\s*guardResult\.passed/,
|
||||
),
|
||||
false,
|
||||
"Subjective model QA warnings should not be combined with guardResult for terminal failure.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /internal\.leads\.reviewUpdateInternal/),
|
||||
true,
|
||||
"Action should patch lead via api.leads.reviewUpdate",
|
||||
"Action should patch lead via internal.leads.reviewUpdateInternal",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
|
||||
Reference in New Issue
Block a user