Fix audit generation and enrichment fallback

This commit is contained in:
2026-06-07 23:03:57 +02:00
parent e9463e8ef2
commit 470fb0f348
10 changed files with 2190 additions and 138 deletions

View File

@@ -32,6 +32,39 @@ function hasStageCall(schema: string) {
);
}
function extractFunctionSource(functionName: string) {
const marker = `function ${functionName}`;
const asyncMarker = `async function ${functionName}`;
const declarationIndex = actionSource.indexOf(marker) === -1
? actionSource.indexOf(asyncMarker)
: actionSource.indexOf(marker);
assert.notEqual(
declarationIndex,
-1,
`Expected function ${functionName} to exist.`,
);
const openBraceIndex = actionSource.indexOf("{", declarationIndex);
let depth = 0;
let end = -1;
for (let index = openBraceIndex; index < actionSource.length; index += 1) {
const char = actionSource[index];
if (char === "{") {
depth += 1;
} else if (char === "}") {
depth -= 1;
if (depth === 0) {
end = index;
break;
}
}
}
assert.notEqual(end, -1, `Expected balanced braces for ${functionName}.`);
return actionSource.slice(declarationIndex, end + 1);
}
test("auditGenerationAction module exists and is a Node action file", () => {
assert.equal(existsSync(actionPath), true, "auditGenerationAction.ts should exist");
assert.equal(
@@ -130,7 +163,7 @@ test("action handles post-start failure paths in action-level catch", () => {
test("action calls generateObject with required schemas", () => {
const requiredSchemas = [
"internalFindingsSchema",
"auditClassificationSchema",
"auditSummarySchema",
"publicAuditTextSchema",
"emailDraftSchema",
@@ -149,6 +182,155 @@ test("action calls generateObject with required schemas", () => {
}
});
test("action loads v3 skill registry from v2 source for evidence input", () => {
assert.equal(
hasPattern(actionSource, /import\s*{[\s\S]*loadSkillsRegistry[\s\S]*}\s*from\s*["']\.\.\/lib\/skills-registry["']/),
true,
"Action should import loadSkillsRegistry from the shared registry parser.",
);
assert.equal(
hasPattern(actionSource, /loadSkillsRegistry\(\s*(?:join\()?[\s\S]*v2_elemente[\s\S]*skills\.md[\s\S]*\)/),
true,
"Action should load the v3 registry from v2_elemente/skills.md.",
);
assert.equal(
hasPattern(actionSource, /skillRegistry:\s*\[\s*\]/),
false,
"Action should not pass an always-empty skillRegistry to buildAuditEvidenceInput.",
);
});
test("registry load warning logging is isolated from fallback return", () => {
const loadRegistrySource = extractFunctionSource("loadAuditSkillRegistry");
assert.equal(
hasPattern(
loadRegistrySource,
/catch\s*\(error\)\s*{[\s\S]*try\s*{[\s\S]*appendRunEvent[\s\S]*}\s*catch\s*{[\s\S]*}\s*return\s*\[\s*\]/,
),
true,
"Registry load fallback should return [] even when warning event logging fails.",
);
});
test("persistAuditStage omits undefined fields from Convex mutation args", () => {
const persistSource = extractFunctionSource("persistAuditStage");
const mutationPayloadSource = persistSource.slice(
persistSource.indexOf("await ctx.runMutation"),
);
assert.doesNotMatch(
actionSource,
/persistAuditStage\(\s*{(?:(?!\n\s*}\s*\);)[\s\S])*(?:parsedJson|rawResponse|usage|finishReason|errorSummary):\s*undefined/,
"Call sites should not pass explicit undefined stage payload fields.",
);
assert.doesNotMatch(
persistSource,
/usage:\s*usage\s*\?\s*toPersistedUsage\(usage\)\s*:\s*undefined/,
"persistAuditStage should not emit usage: undefined.",
);
for (const field of [
"systemPrompt",
"rawResponse",
"parsedJson",
"finishReason",
"errorSummary",
]) {
assert.doesNotMatch(
mutationPayloadSource,
new RegExp(`\\n\\s*${field},`),
`persistAuditStage should conditionally spread ${field}.`,
);
}
});
test("OpenRouter usage payloads omit undefined token fields", () => {
const recordUsageSource = extractFunctionSource("recordOpenRouterUsage");
assert.match(
actionSource,
/function toPersistedUsage[\s\S]*usage\.inputTokens\s*!==\s*undefined[\s\S]*promptTokens:\s*usage\.inputTokens/,
"toPersistedUsage should omit promptTokens when inputTokens is undefined.",
);
assert.doesNotMatch(
recordUsageSource,
/tokens:\s*{[\s\S]*inputTokens:\s*args\.usage\.inputTokens/,
"recordOpenRouterUsage should not build token payloads with undefined properties.",
);
});
test("appendRunEvent omits undefined details from Convex mutation args", () => {
assert.doesNotMatch(
actionSource,
/ctx\.runMutation\(internal\.runs\.appendEventInternal,\s*{[\s\S]*\n\s*details:\s*args\.details,\n/,
"appendRunEvent should conditionally include details only when defined.",
);
});
test("success finishAuditGenerationRun omits undefined errorSummary", () => {
assert.doesNotMatch(
actionSource,
/finishAuditGenerationRun,\s*{[\s\S]*status:\s*["']succeeded["'][\s\S]*errorSummary:\s*qualityPassed\s*\?\s*undefined/,
"Succeeded finishAuditGenerationRun payload should not send errorSummary: undefined.",
);
});
test("quality review stage does not pass explicit undefined optional fields", () => {
assert.doesNotMatch(
actionSource,
/persistAuditStage\(\s*{[\s\S]*stage:\s*["']qualityReview["'][\s\S]*errorSummary:\s*qualityPassed\s*\?\s*undefined/,
"Quality persistAuditStage callsite should conditionally include errorSummary.",
);
});
test("persistAuditStage callsites conditionally include optional auditId", () => {
assert.doesNotMatch(
actionSource,
/await\s+persistAuditStage\(\s*{(?:(?!\n\s*}\s*\);)[\s\S])*\n\s*auditId,\n/,
"persistAuditStage callsites should spread auditId only when defined.",
);
});
test("audit generation helper callsites conditionally include optional auditId", () => {
assert.doesNotMatch(
actionSource,
/(?:recordOpenRouterUsage|captureExternalAuditArtifacts)\(\s*ctx,\s*{(?:(?!\n\s*}\s*\);)[\s\S])*\n\s*auditId,\n/,
"Helper callsites should spread auditId only when defined.",
);
assert.doesNotMatch(
actionSource,
/recordAuditUsageEvent\(\s*ctx,\s*{(?:(?!\n\s*}\s*\);)[\s\S])*\n\s*auditId:\s*args\.auditId,\n/,
"recordAuditUsageEvent callsites should spread args.auditId only when defined.",
);
});
test("persistAuditStage callsites avoid nested maybe-undefined usage objects", () => {
assert.doesNotMatch(
actionSource,
/persistAuditStage\(\s*{(?:(?!\n\s*}\s*\);)[\s\S])*usage:\s*{[\s\S]*?(?:inputTokens|outputTokens|totalTokens|cacheReadTokens):/,
"persistAuditStage callsites should use a usage helper or conditional spreads, not inline maybe-undefined usage objects.",
);
});
test("classification stage uses v3 audit classification schema", () => {
assert.equal(
hasPattern(actionSource, /auditClassificationSchema/),
true,
"Action should reference the v3 auditClassificationSchema.",
);
assert.equal(
hasStageCall("auditClassificationSchema"),
true,
"Classification generateObject call should validate v3 finding payloads.",
);
assert.equal(
hasStageCall("internalFindingsSchema"),
false,
"Classification should no longer validate against legacy-only internalFindingsSchema.",
);
});
test("action uses multimodal file parts with mediaType image/* when screenshots are available", () => {
assert.equal(
hasPattern(
@@ -190,14 +372,23 @@ test("action runs german copy guard and blocks outreach-ready on validation fail
assert.equal(
hasPattern(
actionSource,
/guardResult\.passed|qualityPassed\s*=\s*qualityResult\.object\.isValid\s*&&\s*guardResult\.passed/,
/qualityPassed\s*=\s*guardResult\.passed/,
),
true,
"Only deterministic German copy guard failures should hard-block the audit run.",
);
assert.equal(
hasPattern(actionSource, /api\.leads\.reviewUpdate/),
hasPattern(
actionSource,
/qualityPassed\s*=\s*qualityResult\.object\.isValid\s*&&\s*guardResult\.passed/,
),
false,
"Subjective model QA warnings should not be combined with guardResult for terminal failure.",
);
assert.equal(
hasPattern(actionSource, /internal\.leads\.reviewUpdateInternal/),
true,
"Action should patch lead via api.leads.reviewUpdate",
"Action should patch lead via internal.leads.reviewUpdateInternal",
);
assert.equal(
hasPattern(