feat: add OpenRouter audit generation pipeline
This commit is contained in:
130
tests/ai-model-profiles.test.ts
Normal file
130
tests/ai-model-profiles.test.ts
Normal file
@@ -0,0 +1,130 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import {
|
||||
MODEL_PROFILE_KEYS,
|
||||
MODEL_PROFILES,
|
||||
resolveModelProfile,
|
||||
resolveModelId,
|
||||
} from "../lib/ai/model-profiles";
|
||||
import type { ModelProfileKey } from "../lib/ai/model-profiles";
|
||||
|
||||
type AssertNoExtraProfiles = Array<
|
||||
(typeof MODEL_PROFILE_KEYS)[number]
|
||||
>;
|
||||
|
||||
const assertNoExtraProfiles: AssertNoExtraProfiles = [
|
||||
"classification",
|
||||
"multimodalAudit",
|
||||
"germanCopy",
|
||||
"qualityReview",
|
||||
];
|
||||
|
||||
test("all required model profiles exist", () => {
|
||||
const keys = Object.keys(MODEL_PROFILES).sort();
|
||||
|
||||
assert.deepEqual(keys, [...assertNoExtraProfiles].sort());
|
||||
});
|
||||
|
||||
test("each profile includes required contract fields", () => {
|
||||
const profileEntries = Object.entries(MODEL_PROFILES) as Array<
|
||||
[ModelProfileKey, unknown]
|
||||
>;
|
||||
|
||||
for (const [key, rawProfile] of profileEntries) {
|
||||
const profile = rawProfile as {
|
||||
modelId: string;
|
||||
temperature: number;
|
||||
maxTokens: number;
|
||||
supportsImages: boolean;
|
||||
stage: string;
|
||||
envOverrideKey: string;
|
||||
};
|
||||
|
||||
assert.equal(typeof profile.modelId, "string", `${key} modelId`);
|
||||
assert.equal(
|
||||
profile.modelId.length > 0,
|
||||
true,
|
||||
`${key} modelId should be non-empty`,
|
||||
);
|
||||
assert.equal(typeof profile.temperature, "number", `${key} temperature`);
|
||||
assert.equal(typeof profile.maxTokens, "number", `${key} maxTokens`);
|
||||
assert.equal(
|
||||
Number.isFinite(profile.temperature),
|
||||
true,
|
||||
`${key} temperature numeric`,
|
||||
);
|
||||
assert.equal(
|
||||
Number.isInteger(profile.maxTokens),
|
||||
true,
|
||||
`${key} maxTokens integer`,
|
||||
);
|
||||
assert.equal(
|
||||
typeof profile.supportsImages,
|
||||
"boolean",
|
||||
`${key} supportsImages`,
|
||||
);
|
||||
assert.equal(typeof profile.stage, "string", `${key} stage`);
|
||||
assert.equal(profile.stage.length > 0, true, `${key} stage label`);
|
||||
assert.equal(typeof profile.envOverrideKey, "string", `${key} env override`);
|
||||
assert.equal(profile.envOverrideKey.length > 0, true, `${key} env key`);
|
||||
}
|
||||
});
|
||||
|
||||
test("multimodal profile explicitly supports images", () => {
|
||||
assert.equal(MODEL_PROFILES.multimodalAudit.supportsImages, true);
|
||||
});
|
||||
|
||||
test("non-multimodal profiles disable image support", () => {
|
||||
assert.equal(MODEL_PROFILES.classification.supportsImages, false);
|
||||
assert.equal(MODEL_PROFILES.germanCopy.supportsImages, false);
|
||||
assert.equal(MODEL_PROFILES.qualityReview.supportsImages, false);
|
||||
});
|
||||
|
||||
test("model IDs can be overridden via dedicated env variables", () => {
|
||||
assert.equal(
|
||||
resolveModelId("classification", {
|
||||
OPENROUTER_MODEL_CLASSIFICATION: "custom/classification",
|
||||
}),
|
||||
"custom/classification",
|
||||
);
|
||||
assert.equal(
|
||||
resolveModelId("multimodalAudit", {
|
||||
OPENROUTER_MODEL_MULTIMODAL_AUDIT: "custom/multimodal",
|
||||
}),
|
||||
"custom/multimodal",
|
||||
);
|
||||
assert.equal(
|
||||
resolveModelId("germanCopy", {
|
||||
OPENROUTER_MODEL_GERMAN_COPY: "custom/german",
|
||||
}),
|
||||
"custom/german",
|
||||
);
|
||||
assert.equal(
|
||||
resolveModelId("qualityReview", {
|
||||
OPENROUTER_MODEL_QUALITY_REVIEW: "custom/quality",
|
||||
}),
|
||||
"custom/quality",
|
||||
);
|
||||
});
|
||||
|
||||
test("ENV overrides are ignored when empty", () => {
|
||||
assert.equal(
|
||||
resolveModelId("classification", {
|
||||
OPENROUTER_MODEL_CLASSIFICATION: "",
|
||||
}),
|
||||
MODEL_PROFILES.classification.modelId,
|
||||
);
|
||||
});
|
||||
|
||||
test("resolveModelProfile returns profile config including runtime values", () => {
|
||||
const profile = resolveModelProfile("qualityReview", {
|
||||
OPENROUTER_MODEL_QUALITY_REVIEW: "custom/quality-review-profile",
|
||||
});
|
||||
|
||||
assert.equal(profile.stage, "qualityReview");
|
||||
assert.equal(profile.maxTokens, MODEL_PROFILES.qualityReview.maxTokens);
|
||||
assert.equal(profile.temperature, MODEL_PROFILES.qualityReview.temperature);
|
||||
assert.equal(profile.supportsImages, MODEL_PROFILES.qualityReview.supportsImages);
|
||||
assert.equal(profile.modelId, "custom/quality-review-profile");
|
||||
});
|
||||
137
tests/ai-schemas.test.ts
Normal file
137
tests/ai-schemas.test.ts
Normal file
@@ -0,0 +1,137 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import {
|
||||
callScriptSchema,
|
||||
emailDraftSchema,
|
||||
emailSubjectSchema,
|
||||
followUpDraftSchema,
|
||||
auditSummarySchema,
|
||||
qualityReviewSchema,
|
||||
publicAuditTextSchema,
|
||||
internalFindingsSchema,
|
||||
type CallScript,
|
||||
type EmailDraft,
|
||||
type EmailSubject,
|
||||
type FollowUpDraft,
|
||||
type AuditSummary,
|
||||
type PublicAuditText,
|
||||
type QualityReview,
|
||||
type InternalFindings,
|
||||
} from "../lib/ai/schemas";
|
||||
|
||||
test("internal findings schema accepts task-focused evidence", () => {
|
||||
const parsed = internalFindingsSchema.parse({
|
||||
findings: [
|
||||
{
|
||||
section: "UX",
|
||||
finding: "Landingpage is not responsive on mobile viewport.",
|
||||
suggestion: "Add responsive breakpoints for cards and typography.",
|
||||
},
|
||||
],
|
||||
summary: "One high-priority UX gap was found.",
|
||||
});
|
||||
|
||||
assert.equal(parsed.findings.length, 1);
|
||||
assert.equal(parsed.findings[0].section, "UX");
|
||||
});
|
||||
|
||||
test("audit summary and public text schemas remain intentionally lightweight", () => {
|
||||
const summaryParsed = auditSummarySchema.parse({
|
||||
summary: "Kurze Zusammenfassung mit den wichtigsten Verbesserungen.",
|
||||
keyFindings: ["Fehlende Kontaktmöglichkeit.", "Langsame Ladezeiten."],
|
||||
});
|
||||
const publicParsed = publicAuditTextSchema.parse({
|
||||
publicText: "Dein Shop ist sauber, aber der erste Eindruck lässt Potenzial erkennen.",
|
||||
});
|
||||
|
||||
assert.equal(summaryParsed.keyFindings.length, 2);
|
||||
assert.equal(typeof publicParsed.publicText, "string");
|
||||
});
|
||||
|
||||
test("outreach schemas parse German customer-facing payloads", () => {
|
||||
const emailDraftParsed = emailDraftSchema.parse({
|
||||
body: "Hallo, ich habe mir euer Angebot angesehen...",
|
||||
});
|
||||
const subjectParsed = emailSubjectSchema.parse({
|
||||
subject: "Kurznotiz zu eurem Webauftritt",
|
||||
});
|
||||
const callParsed = callScriptSchema.parse({
|
||||
openingLine: "Guten Tag, ich bin ...",
|
||||
callScript: [
|
||||
"Euer Fokus auf Terminbuchung ist stark.",
|
||||
"Wie läuft eure aktuelle Lead-Generierung?",
|
||||
],
|
||||
closeLine: "Ich schicke im Anschluss kurz die wichtigsten Beobachtungen.",
|
||||
});
|
||||
const followParsed = followUpDraftSchema.parse({
|
||||
message: "Kurzer Follow-up-Hinweis für nächste Woche.",
|
||||
followInDays: 4,
|
||||
goals: ["Antwort auf Rückmeldung erhalten"],
|
||||
});
|
||||
const qualityParsed = qualityReviewSchema.parse({
|
||||
isValid: true,
|
||||
issues: [],
|
||||
suggestions: ["Mehr Kundennutzen konkret beschreiben."],
|
||||
});
|
||||
|
||||
assert.equal(typeof emailDraftParsed.body, "string");
|
||||
assert.equal(typeof subjectParsed.subject, "string");
|
||||
assert.equal(Array.isArray(callParsed.callScript), true);
|
||||
assert.equal(typeof followParsed.message, "string");
|
||||
assert.equal(Array.isArray(qualityParsed.suggestions), true);
|
||||
});
|
||||
|
||||
test("schema-inferred types are exported for Convex action wiring", () => {
|
||||
const typedFindings: InternalFindings = {
|
||||
findings: [
|
||||
{
|
||||
section: "Homepage",
|
||||
finding: "No visible Datenschutzhinweis.",
|
||||
suggestion: "Bitte Hinweis ergänzen.",
|
||||
},
|
||||
],
|
||||
summary: "One finding identified.",
|
||||
};
|
||||
|
||||
const typedSummary: AuditSummary = {
|
||||
summary: "Kernbefund mit 2 Punkten.",
|
||||
keyFindings: ["Kontaktseite fehlt."],
|
||||
};
|
||||
|
||||
const typedPublicText: PublicAuditText = {
|
||||
publicText: "Starker Start, aber optimierungsfähig.",
|
||||
};
|
||||
|
||||
const typedEmail: EmailDraft = {
|
||||
body: "Text mit Ich-Perspektive und konkretem Vorschlag.",
|
||||
};
|
||||
|
||||
const typedSubject: EmailSubject = {
|
||||
subject: "Kurzer Betreff",
|
||||
};
|
||||
|
||||
const typedCall: CallScript = {
|
||||
openingLine: "Hallo, ich habe euer Shop geprüft.",
|
||||
callScript: ["Wie gehen die Leads aktuell rein?"],
|
||||
closeLine: "Ich melde mich nach der Rückfrage erneut.",
|
||||
};
|
||||
|
||||
const typedFollowUp: FollowUpDraft = {
|
||||
message: "Kurzes Follow-up ohne harte Floskel.",
|
||||
};
|
||||
|
||||
const typedQuality: QualityReview = {
|
||||
isValid: true,
|
||||
issues: [],
|
||||
suggestions: [],
|
||||
};
|
||||
|
||||
assert.equal(typedFindings.findings.length, 1);
|
||||
assert.equal(typedSummary.keyFindings.length, 1);
|
||||
assert.equal(typedPublicText.publicText.length > 0, true);
|
||||
assert.equal(typeof typedEmail.body, "string");
|
||||
assert.equal(typeof typedSubject.subject, "string");
|
||||
assert.equal(typedCall.callScript.length, 1);
|
||||
assert.equal(typedFollowUp.message.length > 0, true);
|
||||
assert.equal(typedQuality.isValid, true);
|
||||
});
|
||||
337
tests/audit-evidence.test.ts
Normal file
337
tests/audit-evidence.test.ts
Normal file
@@ -0,0 +1,337 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import {
|
||||
buildAuditEvidenceInput,
|
||||
type SkillRegistryEntryEvidence,
|
||||
} from "../lib/ai/audit-evidence";
|
||||
|
||||
const SAMPLE_SKILL_REGISTRY: SkillRegistryEntryEvidence[] = [
|
||||
{
|
||||
name: "Design Audit",
|
||||
purpose: "Designqualität prüfen.",
|
||||
whenToUse:
|
||||
"Nutze diesen Skill, wenn Seitenhierarchie, visuelle Klarheit und visuelle UX bewertet werden sollen.",
|
||||
whenNotToUse: "Nicht für technische Fehlerlisten.",
|
||||
requiredInput: "URL, Seitentypen und Screenshots.",
|
||||
expectedOutput: "Praktische Design-Prioritäten.",
|
||||
category: "design",
|
||||
version: "1.0",
|
||||
source: "skills/design-audit.md",
|
||||
},
|
||||
{
|
||||
name: "UX Friction Review",
|
||||
purpose: "Nutzerfluss prüfen.",
|
||||
whenToUse:
|
||||
"Nutze diesen Skill bei Formularen, Kontaktwegen und ersten Nutzeraktionen.",
|
||||
whenNotToUse: "Nicht ohne Nutzerfluss.",
|
||||
requiredInput: "URL, Kontaktflüsse, Aktionen.",
|
||||
expectedOutput: "Priorisierte UX-Senkungen.",
|
||||
category: "ux",
|
||||
version: "1.0",
|
||||
source: "skills/ux-friction-review.md",
|
||||
},
|
||||
{
|
||||
name: "Copy Clarity",
|
||||
purpose: "Textklarheit prüfen.",
|
||||
whenToUse:
|
||||
"Nutze diesen Skill bei unklaren, langen oder abstrakten Website-Texten.",
|
||||
whenNotToUse: "Nicht für technische Datenlisten.",
|
||||
requiredInput: "Textbausteine, Zielgruppe, Tonalität.",
|
||||
expectedOutput: "Klarere Formulierungen.",
|
||||
category: "copy",
|
||||
version: "1.0",
|
||||
source: "skills/copy-clarity.md",
|
||||
},
|
||||
{
|
||||
name: "Local SEO",
|
||||
purpose: "Lokale Auffindbarkeit prüfen.",
|
||||
whenToUse:
|
||||
"Nutze diesen Skill bei lokaler Relevanz, Impressum, Kontakt- und Google-Nähe.",
|
||||
whenNotToUse: "Nicht ohne lokalen Kontext.",
|
||||
requiredInput: "Ort, Nische, Seitenstruktur.",
|
||||
expectedOutput: "Sichtbarkeits-Verbesserungen.",
|
||||
category: "seo",
|
||||
version: "1.0",
|
||||
source: "skills/local-seo.md",
|
||||
},
|
||||
{
|
||||
name: "Offer Writing",
|
||||
purpose: "Angebotsstruktur liefern.",
|
||||
whenToUse: "Nutze diesen Skill, wenn ein Angebotsentwurf gebraucht wird.",
|
||||
whenNotToUse: "Nicht bei reinen Buglisten.",
|
||||
requiredInput: "Projektumfang und Umfang.",
|
||||
expectedOutput: "Konkrete Angebotsform.",
|
||||
category: "offer",
|
||||
version: "1.0",
|
||||
source: "skills/offer-writing.md",
|
||||
},
|
||||
];
|
||||
|
||||
test("buildAuditEvidenceInput sanitizes and caps lead/company context", () => {
|
||||
const actual = buildAuditEvidenceInput({
|
||||
lead: {
|
||||
companyName: "Bäckerei <strong>Muster</strong>",
|
||||
niche: "Bäckerei & Kaffeehaus",
|
||||
websiteUrl: "https://example.com/kontakt?ref=ad",
|
||||
address: "Musterstraße 1, 10115 Berlin",
|
||||
city: "Berlin",
|
||||
contactPerson: "<b>Anna</b> Hoffmann",
|
||||
},
|
||||
crawlPages: [
|
||||
{
|
||||
sourceUrl: "https://example.com",
|
||||
finalUrl: "https://example.com",
|
||||
pageKind: "homepage",
|
||||
title: "Startseite",
|
||||
},
|
||||
{
|
||||
sourceUrl: "https://example.com/kontakt",
|
||||
finalUrl: "https://example.com/kontakt",
|
||||
pageKind: "contact",
|
||||
title: "Kontakt",
|
||||
},
|
||||
{
|
||||
sourceUrl: "https://example.com/kontakt?x=1",
|
||||
finalUrl: "https://example.com/kontakt?x=1",
|
||||
pageKind: "contact",
|
||||
title: "Kontakt",
|
||||
},
|
||||
],
|
||||
skillRegistry: SAMPLE_SKILL_REGISTRY,
|
||||
});
|
||||
|
||||
assert.equal(actual.companyContext.length > 0, true);
|
||||
assert.equal(actual.companyContext.some((line) => /https?:\/\//.test(line)), false);
|
||||
assert.equal(actual.companyContext.some((line) => /<[^>]+>/.test(line)), false);
|
||||
assert.equal(
|
||||
actual.companyContext.some((line) => line.includes("Bäckerei Muster")),
|
||||
true,
|
||||
);
|
||||
assert.equal(actual.checkedPages.length >= 2, true);
|
||||
});
|
||||
|
||||
test("buildAuditEvidenceInput deduplicates and caps checked pages", () => {
|
||||
const pages = Array.from({ length: 28 }, (_, index) => ({
|
||||
sourceUrl:
|
||||
`https://example.com/seite-${Math.floor(index / 4)}?cache=${index}`,
|
||||
finalUrl:
|
||||
`https://example.com/seite-${Math.floor(index / 4)}?cache=${index}`,
|
||||
pageKind: index % 2 === 0 ? "other" : "services",
|
||||
title: `Seite ${index}`,
|
||||
}));
|
||||
|
||||
const actual = buildAuditEvidenceInput({
|
||||
crawlPages: pages,
|
||||
skillRegistry: SAMPLE_SKILL_REGISTRY,
|
||||
});
|
||||
|
||||
const uniqueCount = new Set(actual.checkedPages).size;
|
||||
assert.equal(actual.checkedPages.length, uniqueCount);
|
||||
assert.equal(actual.checkedPages.length <= 8, true);
|
||||
});
|
||||
|
||||
test("buildAuditEvidenceInput builds observed UX/content/technical signals and sanitizes long text", () => {
|
||||
const actual = buildAuditEvidenceInput({
|
||||
crawlPages: [
|
||||
{
|
||||
sourceUrl: "https://example.com",
|
||||
finalUrl: "https://example.com",
|
||||
pageKind: "homepage",
|
||||
title: "Wir lieben guten Kaffee",
|
||||
hasContactFormSignal: true,
|
||||
hasContactCtaSignal: true,
|
||||
},
|
||||
{
|
||||
sourceUrl: "https://example.com/ueber-uns",
|
||||
finalUrl: "https://example.com/ueber-uns",
|
||||
pageKind: "about",
|
||||
title: "Über uns",
|
||||
},
|
||||
],
|
||||
technicalChecks: [
|
||||
{
|
||||
sourceUrl: "https://example.com",
|
||||
finalUrl: "https://example.com",
|
||||
usesHttps: false,
|
||||
missingTitle: true,
|
||||
missingMetaDescription: true,
|
||||
hasVisibleContactPath: true,
|
||||
brokenInternalLinkCount: 3,
|
||||
},
|
||||
],
|
||||
skillRegistry: SAMPLE_SKILL_REGISTRY,
|
||||
});
|
||||
|
||||
assert.equal(actual.observedUxSignals.length > 0, true);
|
||||
assert.equal(actual.observedContentSignals.length > 0, true);
|
||||
assert.equal(actual.observedTechnicalSignals.length > 0, true);
|
||||
assert.equal(
|
||||
actual.observedUxSignals.every((line) => !/https?:\/\//.test(line)),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
actual.observedContentSignals.every((line) => !/https?:\/\//.test(line)),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test("buildAuditEvidenceInput preserves screenshot references without base64 payloads", () => {
|
||||
const actual = buildAuditEvidenceInput({
|
||||
screenshots: [
|
||||
{
|
||||
storageId: "storage-1",
|
||||
sourceUrl: "https://example.com",
|
||||
viewport: "desktop",
|
||||
width: 1200,
|
||||
height: 3000,
|
||||
mimeType: "image/png",
|
||||
capturedAt: 1_700_000_000_000,
|
||||
// builder must ignore any binary-like fields if they exist
|
||||
imageBase64: "iVBORw0KGgoAAAANSUhEUgAAAAUA",
|
||||
},
|
||||
{
|
||||
storageId: "storage-2",
|
||||
sourceUrl: "https://example.com",
|
||||
viewport: "mobile",
|
||||
width: 390,
|
||||
height: 844,
|
||||
mimeType: "image/png",
|
||||
capturedAt: 1_700_000_001_000,
|
||||
},
|
||||
] as const,
|
||||
skillRegistry: SAMPLE_SKILL_REGISTRY,
|
||||
}) as {
|
||||
screenshotReferences: Array<
|
||||
Record<string, unknown> & {
|
||||
storageId: string;
|
||||
sourceUrl: string;
|
||||
viewport: string;
|
||||
width: number;
|
||||
height: number;
|
||||
mimeType: string;
|
||||
capturedAt: number;
|
||||
}
|
||||
>;
|
||||
};
|
||||
|
||||
assert.equal(actual.screenshotReferences.length, 2);
|
||||
for (const reference of actual.screenshotReferences) {
|
||||
assert.equal(reference.storageId.startsWith("storage-"), true);
|
||||
assert.equal("imageBase64" in reference, false);
|
||||
assert.equal(typeof reference.sourceUrl, "string");
|
||||
assert.equal(reference.width > 0, true);
|
||||
assert.equal(reference.height > 0, true);
|
||||
}
|
||||
});
|
||||
|
||||
test("buildAuditEvidenceInput converts PageSpeed implications into sanitized customer-facing text", () => {
|
||||
const actual = buildAuditEvidenceInput({
|
||||
pageSpeedInputs: [
|
||||
{
|
||||
strategy: "mobile",
|
||||
status: "succeeded",
|
||||
sourceUrl: "https://example.com",
|
||||
normalized: {
|
||||
implications: [
|
||||
"Score 0.42: Erster Inhalt liegt deutlich hinter Standards.",
|
||||
"Die Seite zeigt das wichtigste Bild zu langsam.",
|
||||
"Weitere Infos: https://example.com/psi",
|
||||
"{ \"pagespeed\": 0.92, \"score\": 88 }",
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
strategy: "desktop",
|
||||
status: "failed",
|
||||
sourceUrl: "https://example.com",
|
||||
errorType: "api_error",
|
||||
errorSummary: "Score 0.22: timeout",
|
||||
},
|
||||
],
|
||||
skillRegistry: SAMPLE_SKILL_REGISTRY,
|
||||
});
|
||||
|
||||
assert.equal(actual.pageSpeedCustomerImplications.length >= 1, true);
|
||||
assert.equal(
|
||||
actual.pageSpeedCustomerImplications.includes(
|
||||
"Score 0.42: Erster Inhalt liegt deutlich hinter Standards.",
|
||||
),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
actual.pageSpeedCustomerImplications.every(
|
||||
(line) =>
|
||||
!/https?:\/\/|pagespeed|score|lighthouse|raw storage|rawStorage/i.test(line),
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal(actual.pageSpeedCustomerImplications.length <= 8, true);
|
||||
});
|
||||
|
||||
test("buildAuditEvidenceInput selects deterministic skills and supports design/ux/copy/seo", () => {
|
||||
const input = {
|
||||
lead: {
|
||||
companyName: "Bäckerei Muster",
|
||||
niche: "Bäckerei",
|
||||
city: "Berlin",
|
||||
websiteDomain: "example.com",
|
||||
},
|
||||
crawlPages: [
|
||||
{
|
||||
sourceUrl: "https://example.com",
|
||||
finalUrl: "https://example.com",
|
||||
pageKind: "homepage",
|
||||
title: "Willkommen bei Bäckerei Muster",
|
||||
hasContactFormSignal: true,
|
||||
hasContactCtaSignal: true,
|
||||
},
|
||||
],
|
||||
technicalChecks: [
|
||||
{
|
||||
sourceUrl: "https://example.com",
|
||||
finalUrl: "https://example.com",
|
||||
usesHttps: false,
|
||||
missingMetaDescription: true,
|
||||
missingTitle: false,
|
||||
hasVisibleContactPath: true,
|
||||
brokenInternalLinkCount: 1,
|
||||
},
|
||||
],
|
||||
screenshots: [
|
||||
{
|
||||
storageId: "storage-1",
|
||||
sourceUrl: "https://example.com",
|
||||
viewport: "desktop",
|
||||
width: 1200,
|
||||
height: 3000,
|
||||
mimeType: "image/png",
|
||||
capturedAt: 1700000000000,
|
||||
},
|
||||
],
|
||||
skillRegistry: SAMPLE_SKILL_REGISTRY,
|
||||
};
|
||||
|
||||
const first = buildAuditEvidenceInput(input);
|
||||
const second = buildAuditEvidenceInput({
|
||||
...input,
|
||||
skillRegistry: [...SAMPLE_SKILL_REGISTRY].reverse(),
|
||||
});
|
||||
|
||||
assert.equal(first.selectedSkills.length >= 4, true);
|
||||
assert.equal(first.selectedSkills.length, second.selectedSkills.length);
|
||||
assert.equal(
|
||||
first.selectedSkills.every((skill, index) => {
|
||||
const same = second.selectedSkills[index];
|
||||
return same?.name === skill.name && same?.category === skill.category;
|
||||
}),
|
||||
true,
|
||||
);
|
||||
const expectedCategories: Array<
|
||||
"design" | "ux" | "copy" | "seo"
|
||||
> = ["design", "ux", "copy", "seo"];
|
||||
const selectedCategories = new Set(first.selectedSkills.map((skill) => skill.category));
|
||||
for (const category of expectedCategories) {
|
||||
assert.equal(selectedCategories.has(category), true);
|
||||
}
|
||||
});
|
||||
335
tests/audit-generation-action-source.test.ts
Normal file
335
tests/audit-generation-action-source.test.ts
Normal file
@@ -0,0 +1,335 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
|
||||
const actionPath = path.join(process.cwd(), "convex", "auditGenerationAction.ts");
|
||||
const actionSource = existsSync(actionPath) ? readFileSync(actionPath, "utf8") : "";
|
||||
const generationSourcePath = path.join(process.cwd(), "convex", "auditGeneration.ts");
|
||||
const generationSource = existsSync(generationSourcePath)
|
||||
? readFileSync(generationSourcePath, "utf8")
|
||||
: "";
|
||||
|
||||
function hasPattern(source: string, pattern: RegExp) {
|
||||
return pattern.test(source);
|
||||
}
|
||||
|
||||
function hasExportedInternalAction(exportName: string) {
|
||||
const pattern = new RegExp(
|
||||
`export const ${exportName}\\s*=\\s*internalAction\\s*\\(`,
|
||||
);
|
||||
return hasPattern(actionSource, pattern);
|
||||
}
|
||||
|
||||
function hasStageCall(schema: string) {
|
||||
const escaped = schema.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
return hasPattern(
|
||||
actionSource,
|
||||
new RegExp(
|
||||
`generateObject\\([\\s\\S]*schema:\\s*${escaped}[\\s\\S]*\\)`,
|
||||
"m",
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
test("auditGenerationAction module exists and is a Node action file", () => {
|
||||
assert.equal(existsSync(actionPath), true, "auditGenerationAction.ts should exist");
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /^"use node";/m),
|
||||
true,
|
||||
"auditGenerationAction.ts should start with \"use node\"",
|
||||
);
|
||||
});
|
||||
|
||||
test("auditGenerationAction exports processAuditGeneration with runId validator", () => {
|
||||
assert.equal(
|
||||
hasExportedInternalAction("processAuditGeneration"),
|
||||
true,
|
||||
"processAuditGeneration should be an internalAction",
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/processAuditGeneration\s*=\s*internalAction\(\s*{\s*args:\s*{\s*runId:\s*v\.id\(\s*["']agentRuns["']\s*\)\s*,?\s*}/,
|
||||
),
|
||||
true,
|
||||
"processAuditGeneration should validate runId: v.id(\"agentRuns\")",
|
||||
);
|
||||
});
|
||||
|
||||
test("action starts, queries evidence, and runs stage pipeline", () => {
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/internal\.auditGeneration\.startAuditGenerationRun/,
|
||||
),
|
||||
true,
|
||||
"Action should start the run via internal.auditGeneration.startAuditGenerationRun",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/internal\.auditGeneration\.getAuditGenerationEvidence/,
|
||||
),
|
||||
true,
|
||||
"Action should load evidence via internal.auditGeneration.getAuditGenerationEvidence",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/internal\.auditGeneration\.persistAuditGenerationResult/,
|
||||
),
|
||||
true,
|
||||
"Action should persist each stage result",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/internal\.auditGeneration\.finishAuditGenerationRun/,
|
||||
),
|
||||
true,
|
||||
"Action should finish run via internal.auditGeneration.finishAuditGenerationRun",
|
||||
);
|
||||
});
|
||||
|
||||
test("action includes all required audit stages", () => {
|
||||
for (const stage of [
|
||||
"classification",
|
||||
"multimodalAudit",
|
||||
"germanCopy",
|
||||
"qualityReview",
|
||||
]) {
|
||||
const token = new RegExp(`stage:\\s*["']${stage}["']`);
|
||||
assert.equal(
|
||||
hasPattern(actionSource, token),
|
||||
true,
|
||||
`Action should reference ${stage} stage`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("action handles post-start failure paths in action-level catch", () => {
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/try\s*{[\s\S]*internal\.auditGeneration\.getAuditGenerationEvidence[\s\S]*const provider = createOpenRouterProvider\(\)/,
|
||||
),
|
||||
true,
|
||||
"Action should include evidence query and provider init inside catch-covered flow.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/catch\s*\(error\)\s*{[\s\S]*appendRunEvent[\s\S]*finishAuditGenerationRun[\s\S]*"failed"/,
|
||||
),
|
||||
true,
|
||||
"Action-level error handler should emit run events.",
|
||||
);
|
||||
});
|
||||
|
||||
test("action calls generateObject with required schemas", () => {
|
||||
const requiredSchemas = [
|
||||
"internalFindingsSchema",
|
||||
"auditSummarySchema",
|
||||
"publicAuditTextSchema",
|
||||
"emailDraftSchema",
|
||||
"emailSubjectSchema",
|
||||
"callScriptSchema",
|
||||
"followUpDraftSchema",
|
||||
"qualityReviewSchema",
|
||||
];
|
||||
|
||||
for (const requiredSchema of requiredSchemas) {
|
||||
assert.equal(
|
||||
hasStageCall(requiredSchema),
|
||||
true,
|
||||
`Action should call generateObject with schema ${requiredSchema}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("action uses multimodal file parts with mediaType image/* when screenshots are available", () => {
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/type:\s*["']file["'][\s\S]*mediaType:\s*(?:getValidMediaType|["']image\/)/,
|
||||
),
|
||||
true,
|
||||
"Multimodal call should include AI file parts with image mediaType",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/ctx\.storage\.(get|getUrl)\(/,
|
||||
),
|
||||
true,
|
||||
"Multimodal call should try to fetch screenshots from Convex storage",
|
||||
);
|
||||
});
|
||||
|
||||
test("action handles missing screenshots with warning event fallback", () => {
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /level:\s*["']warning["'][\s\S]*Screenshot|Vorschaubild/),
|
||||
true,
|
||||
"Action should append warning event when multimodal screenshot input is unavailable",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /messages:\s*\[[\s\S]*type:\s*["']text["'][\s\S]*\]/),
|
||||
true,
|
||||
"Action should fall back to text-only multimodal calls when required parts are missing",
|
||||
);
|
||||
});
|
||||
|
||||
test("action runs german copy guard and blocks outreach-ready on validation failure", () => {
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /validateCustomerFacingCopy/),
|
||||
true,
|
||||
"Action should run German copy validation",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/guardResult\.passed|qualityPassed\s*=\s*qualityResult\.object\.isValid\s*&&\s*guardResult\.passed/,
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /api\.leads\.reviewUpdate/),
|
||||
true,
|
||||
"Action should patch lead via api.leads.reviewUpdate",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/isTerminalLeadContactStatus/,
|
||||
),
|
||||
true,
|
||||
"Action should set contactStatus to outreach_ready only when terminal guard allows it.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/do_not_contact|contacted|replied/i,
|
||||
),
|
||||
true,
|
||||
"Action should explicitly guard against terminal lead statuses before outreach-ready.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/Lead-Status wurde nicht auf outreach_ready gesetzt/,
|
||||
),
|
||||
true,
|
||||
"Action should emit warning event when outreach-ready cannot be set.",
|
||||
);
|
||||
});
|
||||
|
||||
test("action persists audit and outreach outputs before finishing succeeded run", () => {
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/internal\.audits\.upsertFromAuditGeneration/,
|
||||
),
|
||||
true,
|
||||
"Action should persist audit output via internal.audits.upsertFromAuditGeneration",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/internal\.outreach\.upsertFromAuditGeneration/,
|
||||
),
|
||||
true,
|
||||
"Action should persist outreach output via internal.outreach.upsertFromAuditGeneration",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/internal\.audits\.upsertFromAuditGeneration[\s\S]*internal\.outreach\.upsertFromAuditGeneration[\s\S]*internal\.auditGeneration\.finishAuditGenerationRun[\s\S]*status:\s*["']succeeded["']/,
|
||||
),
|
||||
true,
|
||||
"Action should finish success after persisted outputs",
|
||||
);
|
||||
});
|
||||
|
||||
test("action uses model profiles for generation parameters", () => {
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /resolveModelProfile\("classification"\)/),
|
||||
true,
|
||||
"classification generation should use resolveModelProfile.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /resolveModelProfile\("multimodalAudit"\)/),
|
||||
true,
|
||||
"multimodal generation should use resolveModelProfile.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /resolveModelProfile\("germanCopy"\)/),
|
||||
true,
|
||||
"german copy generation should use resolveModelProfile.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /resolveModelProfile\("qualityReview"\)/),
|
||||
true,
|
||||
"quality review generation should use resolveModelProfile.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/temperature:\s*classificationProfile\.temperature[\s\S]*maxOutputTokens:\s*classificationProfile\.maxTokens/,
|
||||
),
|
||||
true,
|
||||
"classification stage should use profile temperature/maxTokens.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/temperature:\s*germanCopyProfile\.temperature[\s\S]*maxOutputTokens:\s*germanCopyProfile\.maxTokens/,
|
||||
),
|
||||
true,
|
||||
"german copy stages should use profile temperature/maxTokens.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/temperature:\s*qualityReviewProfile\.temperature[\s\S]*maxOutputTokens:\s*qualityReviewProfile\.maxTokens/,
|
||||
),
|
||||
true,
|
||||
"quality review stage should use profile temperature/maxTokens.",
|
||||
);
|
||||
});
|
||||
|
||||
test("action sanitization masks env-backed secrets", () => {
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/sanitizeSecretCandidates\([\s\S]*process\.env/,
|
||||
),
|
||||
true,
|
||||
"sanitize logic should include env-backed secret masking.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /OPENROUTER_API_KEY/),
|
||||
true,
|
||||
"sanitizer should include OPENROUTER_API_KEY in secret hints.",
|
||||
);
|
||||
});
|
||||
|
||||
test("auditGeneration scheduler reference in queueLeadAuditGeneration is typed", () => {
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
generationSource,
|
||||
/internal\.auditGenerationAction\.processAuditGeneration/,
|
||||
),
|
||||
true,
|
||||
"queueLeadAuditGeneration should reference internal.auditGenerationAction.processAuditGeneration",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
generationSource,
|
||||
/internal as any/,
|
||||
),
|
||||
false,
|
||||
"No temporary internal cast should remain for the processAuditGeneration schedule",
|
||||
);
|
||||
});
|
||||
323
tests/audit-generation-persistence-source.test.ts
Normal file
323
tests/audit-generation-persistence-source.test.ts
Normal file
@@ -0,0 +1,323 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import ts from "typescript";
|
||||
|
||||
const auditGenerationPath = join(process.cwd(), "convex", "auditGeneration.ts");
|
||||
const auditGenerationSource = existsSync(auditGenerationPath)
|
||||
? readFileSync(auditGenerationPath, "utf8")
|
||||
: "";
|
||||
|
||||
const sourceFile = ts.createSourceFile(
|
||||
"auditGeneration.ts",
|
||||
auditGenerationSource,
|
||||
ts.ScriptTarget.ES2022,
|
||||
true,
|
||||
ts.ScriptKind.TS,
|
||||
);
|
||||
|
||||
function getExportedConstNames(file: ts.SourceFile) {
|
||||
const names = new Set<string>();
|
||||
|
||||
const visit = (node: ts.Node) => {
|
||||
if (ts.isVariableStatement(node)) {
|
||||
const isExported = node.modifiers?.some(
|
||||
(mod) => mod.kind === ts.SyntaxKind.ExportKeyword,
|
||||
);
|
||||
if (!isExported) {
|
||||
ts.forEachChild(node, visit);
|
||||
return;
|
||||
}
|
||||
|
||||
const isConst = node.declarationList.flags & ts.NodeFlags.Const;
|
||||
if (!isConst) {
|
||||
ts.forEachChild(node, visit);
|
||||
return;
|
||||
}
|
||||
|
||||
for (const declaration of node.declarationList.declarations) {
|
||||
if (ts.isIdentifier(declaration.name)) {
|
||||
names.add(declaration.name.text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ts.forEachChild(node, visit);
|
||||
};
|
||||
|
||||
ts.forEachChild(file, visit);
|
||||
return names;
|
||||
}
|
||||
|
||||
function hasPattern(source: string, pattern: RegExp) {
|
||||
return pattern.test(source);
|
||||
}
|
||||
|
||||
function extractExportSource(name: string) {
|
||||
const marker = `export const ${name} = `;
|
||||
const declarationIndex = auditGenerationSource.indexOf(marker);
|
||||
assert.notEqual(
|
||||
declarationIndex,
|
||||
-1,
|
||||
`Expected declaration for ${name}`,
|
||||
);
|
||||
|
||||
const openBraceIndex = auditGenerationSource.indexOf("{", declarationIndex);
|
||||
let depth = 0;
|
||||
let end = -1;
|
||||
|
||||
for (let index = openBraceIndex; index < auditGenerationSource.length; index += 1) {
|
||||
const char = auditGenerationSource[index];
|
||||
if (char === "{") {
|
||||
depth += 1;
|
||||
} else if (char === "}") {
|
||||
depth -= 1;
|
||||
if (depth === 0) {
|
||||
end = index;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert.notEqual(end, -1, `Expected balanced braces for ${name}`);
|
||||
return auditGenerationSource.slice(openBraceIndex, end + 1);
|
||||
}
|
||||
|
||||
test("auditGeneration module exports required mutation contracts", () => {
|
||||
assert.equal(
|
||||
existsSync(auditGenerationPath),
|
||||
true,
|
||||
"auditGeneration.ts should be present",
|
||||
);
|
||||
|
||||
const exports = getExportedConstNames(sourceFile);
|
||||
const required = [
|
||||
"queueLeadAuditGeneration",
|
||||
"startAuditGenerationRun",
|
||||
"persistAuditGenerationResult",
|
||||
"finishAuditGenerationRun",
|
||||
];
|
||||
|
||||
for (const exportName of required) {
|
||||
assert.equal(
|
||||
exports.has(exportName),
|
||||
true,
|
||||
`Expected export: ${exportName}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("auditGeneration module registers internalMutation contracts", () => {
|
||||
for (const name of [
|
||||
"queueLeadAuditGeneration",
|
||||
"startAuditGenerationRun",
|
||||
"persistAuditGenerationResult",
|
||||
"finishAuditGenerationRun",
|
||||
]) {
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
auditGenerationSource,
|
||||
new RegExp(`export const ${name} = internalMutation\\s*\\(`),
|
||||
),
|
||||
true,
|
||||
`${name} should be registered as internalMutation.`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("queueLeadAuditGeneration dedupes pending/running runs and schedules action", () => {
|
||||
const queueSource = extractExportSource("queueLeadAuditGeneration");
|
||||
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
queueSource,
|
||||
/withIndex\("by_type_and_status_and_leadId"[\s\S]*?eq\("type",\s*"audit_generation"\)[\s\S]*?eq\("status",\s*"pending"\)[\s\S]*?eq\("leadId",\s*args\.leadId\)/,
|
||||
),
|
||||
true,
|
||||
"Queue should dedupe pending runs with by_type_and_status_and_leadId for type audit_generation.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
queueSource,
|
||||
/withIndex\("by_type_and_status_and_leadId"[\s\S]*?eq\("type",\s*"audit_generation"\)[\s\S]*?eq\("status",\s*"running"\)[\s\S]*?eq\("leadId",\s*args\.leadId\)/,
|
||||
),
|
||||
true,
|
||||
"Queue should dedupe running runs with by_type_and_status_and_leadId for type audit_generation.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
queueSource,
|
||||
/ctx\.scheduler\.runAfter\(\s*0,\s*internal\.auditGenerationAction\.processAuditGeneration,[\s\S]*?runId/,
|
||||
),
|
||||
true,
|
||||
"Queue should schedule internal.auditGenerationAction.processAuditGeneration.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(queueSource, /Audit-Generierung wurde in die Warteschlange gesetzt\./),
|
||||
true,
|
||||
"Queue should emit a queue event message.",
|
||||
);
|
||||
});
|
||||
|
||||
test("startAuditGenerationRun validates and marks run as running", () => {
|
||||
const startSource = extractExportSource("startAuditGenerationRun");
|
||||
|
||||
assert.equal(
|
||||
hasPattern(startSource, /run\.type\s*!==\s*"audit_generation"/),
|
||||
true,
|
||||
"start should validate audit_generation run type.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(startSource, /run\.status\s*!==\s*"pending"/),
|
||||
true,
|
||||
"start should require pending status.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(startSource, /!run\.leadId[\s\S]*status:\s*"failed"/),
|
||||
true,
|
||||
"start should fail clearly when leadId missing.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(startSource, /!lead[\s\S]*status:\s*"failed"/),
|
||||
true,
|
||||
"start should fail clearly when lead cannot be loaded.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
startSource,
|
||||
/ctx\.db\.patch\(\s*args\.runId,[\s\S]*status:\s*"running"/,
|
||||
),
|
||||
true,
|
||||
"start should set run status running.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(startSource, /message:\s*"[^"]*konnte nicht gestartet werden[^"]*"/i),
|
||||
true,
|
||||
"start should emit clear failure events when starting fails.",
|
||||
);
|
||||
});
|
||||
|
||||
test("persistAuditGenerationResult inserts into auditGenerations", () => {
|
||||
const persistSource = extractExportSource("persistAuditGenerationResult");
|
||||
|
||||
assert.equal(
|
||||
hasPattern(persistSource, /ctx\.db\.insert\(\s*"auditGenerations"/),
|
||||
true,
|
||||
"persistAuditGenerationResult should insert into auditGenerations.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
persistSource,
|
||||
/prompt:\s*sanitizeAndCapString\(args\.prompt,\s*MAX_PROMPT_BYTES\)/,
|
||||
),
|
||||
true,
|
||||
"persist function should sanitize prompt before persisting to avoid secrets.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
persistSource,
|
||||
/rawResponse:\s*sanitizeAndCapString\(args\.rawResponse,\s*MAX_RAW_RESPONSE_BYTES\)/,
|
||||
),
|
||||
true,
|
||||
"persist function should sanitize rawResponse before persisting to avoid secrets.",
|
||||
);
|
||||
});
|
||||
|
||||
test("truncateWithMarker is byte-capped and marker-safe in persistence", () => {
|
||||
assert.equal(
|
||||
hasPattern(auditGenerationSource, /const markerBytes = byteLength\(TRUNCATION_MARKER\);/),
|
||||
true,
|
||||
"truncateWithMarker should calculate marker bytes explicitly.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
auditGenerationSource,
|
||||
/if\s*\(byteLength\(value\)\s*<=\s*maxBytes\)\s*\{\s*return\s*value;\s*\}/,
|
||||
),
|
||||
true,
|
||||
"truncateWithMarker should return early when already within byte limit.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
auditGenerationSource,
|
||||
/if\s*\(markerBytes\s*>=\s*maxBytes\)/,
|
||||
),
|
||||
true,
|
||||
"truncateWithMarker should handle marker length edge cases.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
auditGenerationSource,
|
||||
/new TextDecoder\(\)\.decode\(markerBytesBuffer\.slice\(0,\s*maxBytes\)\)/,
|
||||
),
|
||||
true,
|
||||
"truncateWithMarker should trim marker bytes with decoder slice fallback.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
auditGenerationSource,
|
||||
/TRUNCATION_MARKER\\.slice\(0,\s*maxBytes\)/,
|
||||
),
|
||||
false,
|
||||
"truncateWithMarker should not use unbounded marker slicing by bytes.",
|
||||
);
|
||||
});
|
||||
|
||||
test("sanitizer masks env-backed secret values in persistence", () => {
|
||||
assert.equal(
|
||||
hasPattern(auditGenerationSource, /function\s+sanitizeSecretCandidates/),
|
||||
true,
|
||||
"Persistence should expose secret candidate sanitizer.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(auditGenerationSource, /OPENROUTER_API_KEY/),
|
||||
true,
|
||||
"Persistence sanitizer should know OPENROUTER_API_KEY.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
auditGenerationSource,
|
||||
/return\s+sanitized\s*\r?\n\s*\.replace\(/,
|
||||
),
|
||||
true,
|
||||
"Persistence sanitizer should apply regex secret-masking patterns.",
|
||||
);
|
||||
});
|
||||
|
||||
test("finishAuditGenerationRun updates run status/counters/currentStep", () => {
|
||||
const finishSource = extractExportSource("finishAuditGenerationRun");
|
||||
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
finishSource,
|
||||
/ctx\.db\.patch\(\s*args\.runId,[\s\S]*?status:\s*args\.status/,
|
||||
),
|
||||
true,
|
||||
"finish should set run status.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
finishSource,
|
||||
/status:\s*args\.status[\s\S]*finishedAt:\s*now/,
|
||||
),
|
||||
true,
|
||||
"finish should set finishedAt.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
finishSource,
|
||||
/counters:\s*\{[\s\S]*errors:\s*args\.errors/,
|
||||
),
|
||||
true,
|
||||
"finish should update counters with errors.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
finishSource,
|
||||
/currentStep:\s*args\.currentStep\s*(\|\||\?\?)\s*"audit_generation"/,
|
||||
),
|
||||
true,
|
||||
"finish should update currentStep.",
|
||||
);
|
||||
});
|
||||
204
tests/audit-generation-schema.test.ts
Normal file
204
tests/audit-generation-schema.test.ts
Normal file
@@ -0,0 +1,204 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
|
||||
const schemaSource = readFileSync(
|
||||
join(process.cwd(), "convex", "schema.ts"),
|
||||
"utf8",
|
||||
);
|
||||
const domainSource = readFileSync(
|
||||
join(process.cwd(), "convex", "domain.ts"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
function extractTableSection(tableName: string) {
|
||||
const marker = `${tableName}: defineTable({`;
|
||||
const markerIndex = schemaSource.indexOf(marker);
|
||||
assert.notEqual(
|
||||
markerIndex,
|
||||
-1,
|
||||
`Expected schema table definition for ${tableName}.`,
|
||||
);
|
||||
|
||||
const objectStart = schemaSource.indexOf("{", markerIndex);
|
||||
let depth = 0;
|
||||
let objectEnd = -1;
|
||||
|
||||
for (let index = objectStart; index < schemaSource.length; index += 1) {
|
||||
if (schemaSource[index] === "{") {
|
||||
depth += 1;
|
||||
} else if (schemaSource[index] === "}") {
|
||||
depth -= 1;
|
||||
if (depth === 0) {
|
||||
objectEnd = index;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert.notEqual(objectEnd, -1, `Could not parse schema object for ${tableName}.`);
|
||||
|
||||
const remainder = schemaSource.slice(objectEnd + 1);
|
||||
const nextTableMatch = remainder.match(
|
||||
/^\s*[a-zA-Z_][\w]*:\s*defineTable\(/m,
|
||||
);
|
||||
|
||||
const sectionEnd =
|
||||
nextTableMatch === null
|
||||
? schemaSource.length
|
||||
: objectEnd + 1 + nextTableMatch.index!;
|
||||
|
||||
const section = schemaSource.slice(markerIndex, sectionEnd);
|
||||
const objectBlock = schemaSource.slice(markerIndex, objectEnd + 1);
|
||||
|
||||
return { section, objectBlock };
|
||||
}
|
||||
|
||||
function assertHas(pattern: RegExp, source: string, message: string) {
|
||||
assert.equal(pattern.test(source), true, message);
|
||||
}
|
||||
|
||||
test("auditGenerations table has contract fields", () => {
|
||||
const { section, objectBlock } = extractTableSection("auditGenerations");
|
||||
|
||||
assertHas(
|
||||
/leadId:\s*v\.id\(["']leads["']\)/,
|
||||
objectBlock,
|
||||
"auditGenerations.leadId must be required lead id.",
|
||||
);
|
||||
assertHas(
|
||||
/auditId:\s*v\.optional\(\s*v\.id\(["']audits["']\)\s*\)/,
|
||||
objectBlock,
|
||||
"auditGenerations.auditId should be optional audit id.",
|
||||
);
|
||||
assertHas(
|
||||
/runId:\s*v\.id\(["']agentRuns["']\)/,
|
||||
objectBlock,
|
||||
"auditGenerations.runId should be required agent run id.",
|
||||
);
|
||||
assertHas(
|
||||
/stage:\s*auditGenerationStage/,
|
||||
objectBlock,
|
||||
"auditGenerations.stage should use auditGenerationStage validator.",
|
||||
);
|
||||
assertHas(
|
||||
/modelProfile:\s*v\.string\(\)/,
|
||||
objectBlock,
|
||||
"auditGenerations.modelProfile should be required string.",
|
||||
);
|
||||
assertHas(
|
||||
/modelId:\s*v\.string\(\)/,
|
||||
objectBlock,
|
||||
"auditGenerations.modelId should be required string.",
|
||||
);
|
||||
assertHas(
|
||||
/prompt:\s*v\.string\(\)/,
|
||||
objectBlock,
|
||||
"auditGenerations.prompt should be required string.",
|
||||
);
|
||||
assertHas(
|
||||
/systemPrompt:\s*v\.optional\(\s*v\.string\(\)\s*\)/,
|
||||
objectBlock,
|
||||
"auditGenerations.systemPrompt should be optional string.",
|
||||
);
|
||||
assertHas(
|
||||
/rawResponse:\s*v\.optional\(\s*v\.string\(\)\s*\)/,
|
||||
objectBlock,
|
||||
"auditGenerations.rawResponse should be optional string.",
|
||||
);
|
||||
assertHas(
|
||||
/parsedJson:\s*v\.optional\(\s*auditGenerationParsedJson\s*\)/,
|
||||
objectBlock,
|
||||
"auditGenerations.parsedJson should allow string or structured object.",
|
||||
);
|
||||
assertHas(
|
||||
/usage:\s*v\.optional\(\s*auditGenerationUsage\s*\)/,
|
||||
objectBlock,
|
||||
"auditGenerations.usage should be optional token usage object.",
|
||||
);
|
||||
assertHas(
|
||||
/finishReason:\s*v\.optional\(\s*v\.string\(\)\s*\)/,
|
||||
objectBlock,
|
||||
"auditGenerations.finishReason should be optional string.",
|
||||
);
|
||||
assertHas(
|
||||
/status:\s*auditGenerationStatus/,
|
||||
objectBlock,
|
||||
"auditGenerations.status should use auditGenerationStatus validator.",
|
||||
);
|
||||
assertHas(
|
||||
/errorSummary:\s*v\.optional\(\s*v\.string\(\)\s*\)/,
|
||||
objectBlock,
|
||||
"auditGenerations.errorSummary should be optional string.",
|
||||
);
|
||||
assertHas(
|
||||
/createdAt:\s*v\.number\(\)/,
|
||||
objectBlock,
|
||||
"auditGenerations.createdAt should be required number.",
|
||||
);
|
||||
assertHas(
|
||||
/updatedAt:\s*v\.number\(\)/,
|
||||
objectBlock,
|
||||
"auditGenerations.updatedAt should be required number.",
|
||||
);
|
||||
|
||||
assertHas(
|
||||
/index\("by_leadId",\s*\["leadId"\]\)/,
|
||||
section,
|
||||
"auditGenerations should have by_leadId index.",
|
||||
);
|
||||
assertHas(
|
||||
/index\("by_auditId",\s*\["auditId"\]\)/,
|
||||
section,
|
||||
"auditGenerations should have by_auditId index.",
|
||||
);
|
||||
assertHas(
|
||||
/index\("by_runId",\s*\["runId"\]\)/,
|
||||
section,
|
||||
"auditGenerations should have by_runId index.",
|
||||
);
|
||||
assertHas(
|
||||
/index\("by_stage",\s*\["stage"\]\)/,
|
||||
section,
|
||||
"auditGenerations should have by_stage index.",
|
||||
);
|
||||
assertHas(
|
||||
/index\("by_leadId_and_stage",\s*\["leadId",\s*"stage"\]\)/,
|
||||
section,
|
||||
"auditGenerations should have by_leadId_and_stage index.",
|
||||
);
|
||||
});
|
||||
|
||||
test("audit-generation validators are declared", () => {
|
||||
assertHas(
|
||||
/const\s+auditGenerationStage\s*=\s*v\.union\([\s\S]*\)/,
|
||||
schemaSource,
|
||||
"schema should define auditGenerationStage union.",
|
||||
);
|
||||
assertHas(
|
||||
/const\s+auditGenerationStatus\s*=\s*v\.union\([\s\S]*\)/,
|
||||
schemaSource,
|
||||
"schema should define auditGenerationStatus union.",
|
||||
);
|
||||
assertHas(
|
||||
/AUDIT_GENERATION_STAGES\s*=\s*\[[\s\S]*["']classification["'][\s\S]*\]/,
|
||||
domainSource,
|
||||
"auditGenerationStage should include classification.",
|
||||
);
|
||||
assertHas(
|
||||
/AUDIT_GENERATION_STAGES\s*=\s*\[[\s\S]*["']multimodalAudit["'][\s\S]*\]/,
|
||||
domainSource,
|
||||
"auditGenerationStage should include multimodalAudit.",
|
||||
);
|
||||
assertHas(
|
||||
/AUDIT_GENERATION_STAGES\s*=\s*\[[\s\S]*["']germanCopy["'][\s\S]*\]/,
|
||||
domainSource,
|
||||
"auditGenerationStage should include germanCopy.",
|
||||
);
|
||||
assertHas(
|
||||
/AUDIT_GENERATION_STAGES\s*=\s*\[[\s\S]*["']qualityReview["'][\s\S]*\]/,
|
||||
domainSource,
|
||||
"auditGenerationStage should include qualityReview.",
|
||||
);
|
||||
});
|
||||
270
tests/german-copy-guard.test.ts
Normal file
270
tests/german-copy-guard.test.ts
Normal file
@@ -0,0 +1,270 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import {
|
||||
validateCallScriptCopy,
|
||||
validateCustomerFacingCopy,
|
||||
validateFollowUpCopy,
|
||||
} from "../lib/ai/german-copy-guard";
|
||||
|
||||
const validPayload = {
|
||||
auditSummary:
|
||||
"Ich habe euren Webauftritt geprüft. Mir ist aufgefallen, dass die Kontaktseite nicht klar erreichbar ist. Ich empfehle, den Kontaktbereich im Header sichtbar zu platzieren.",
|
||||
auditBody:
|
||||
"Mir ist aufgefallen, dass die Kontaktseite nur am Ende der Startseite eingebettet ist. Ich empfehle, sie im Kopfbereich direkt zu platzieren.",
|
||||
emailSubject: "Kurzes Feedback zu eurem Webauftritt",
|
||||
emailBody:
|
||||
"Hallo, ich habe eure Seite betrachtet und festgestellt, dass die Kontaktoptionen auf mobilen Geräten schwer zu finden sind. Ich empfehle, einen klar sichtbaren Button einzubauen.",
|
||||
callScript: {
|
||||
openingLine: "Hallo, ich bin Matthias von der Webberatung.",
|
||||
callScript: [
|
||||
"Ich habe eure Website geprüft und gesehen, dass der Kontaktbereich nicht sofort sichtbar ist.",
|
||||
"Ich schlage vor, den Kontakt-Button in den Header zu setzen und die Mobil-Ansicht anzupassen.",
|
||||
],
|
||||
closeLine: "Wenn das hilfreich klingt, kann ich euch in zwei Minuten die nächsten Schritte skizzieren.",
|
||||
},
|
||||
followUp:
|
||||
"Mir ist noch etwas aufgefallen: Auf der Mobilversion fehlt ein klarer Termin- oder Kontakthinweis. Ich schlage vor, diesen Bereich oberhalb der Leistungstexte deutlich zu markieren.",
|
||||
};
|
||||
|
||||
test("validateCustomerFacingCopy passes clean German outreach and audit copy", () => {
|
||||
const result = validateCustomerFacingCopy(validPayload);
|
||||
|
||||
assert.equal(result.passed, true);
|
||||
assert.equal(result.issues.length, 0);
|
||||
});
|
||||
|
||||
test("validateCustomerFacingCopy rejects likely non-German copy and reports language", () => {
|
||||
const result = validateCustomerFacingCopy({
|
||||
...validPayload,
|
||||
emailBody:
|
||||
"Your site looks very strong, and your performance score is 0.82 with good Lighthouse numbers.",
|
||||
});
|
||||
|
||||
assert.equal(result.passed, false);
|
||||
assert.equal(
|
||||
result.issues.some((issue) =>
|
||||
issue.field === "emailBody" && issue.rule === "not_german",
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test("validateCustomerFacingCopy flags short English artifact-like snippets in content fields", () => {
|
||||
const shortInputs: Array<{
|
||||
field: "auditSummary" | "auditBody" | "emailBody" | "followUp";
|
||||
value: string;
|
||||
}> = [
|
||||
{ field: "emailBody", value: "quick audit" },
|
||||
{ field: "auditBody", value: "bad website" },
|
||||
{ field: "followUp", value: "AI report" },
|
||||
];
|
||||
|
||||
for (const { field, value } of shortInputs) {
|
||||
const payload = { ...validPayload, [field]: value };
|
||||
const result = validateCustomerFacingCopy(payload as typeof validPayload);
|
||||
|
||||
assert.equal(result.passed, false);
|
||||
assert.equal(
|
||||
result.issues.some(
|
||||
(issue) => issue.field === field && issue.rule === "not_german",
|
||||
),
|
||||
true,
|
||||
`Expected ${field} short snippet "${value}" to fail german language check.`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("validateCustomerFacingCopy requires Ich-form in applicable customer-facing fields", () => {
|
||||
const result = validateCustomerFacingCopy({
|
||||
...validPayload,
|
||||
auditBody:
|
||||
"Ihre Seite hat eine gute Struktur. Der Kontaktbereich sollte klarer werden.",
|
||||
followUp: "Die Website sollte verbessert werden. Setzt bitte einen Kontaktbutton.",
|
||||
});
|
||||
|
||||
const hasAuditIssue = result.issues.some(
|
||||
(issue) => issue.field === "auditBody" && issue.rule === "missing_ich_form",
|
||||
);
|
||||
const hasFollowUpIssue = result.issues.some(
|
||||
(issue) => issue.field === "followUp" && issue.rule === "missing_ich_form",
|
||||
);
|
||||
|
||||
assert.equal(result.passed, false);
|
||||
assert.equal(hasAuditIssue, true);
|
||||
assert.equal(hasFollowUpIssue, true);
|
||||
});
|
||||
|
||||
test("validateCustomerFacingCopy blocks PageSpeed-like score artifacts in public text", () => {
|
||||
const result = validateCustomerFacingCopy({
|
||||
...validPayload,
|
||||
auditSummary:
|
||||
"Aus dem PageSpeed-Check ergibt sich ein score: 0.82 im Bereich Performance.",
|
||||
});
|
||||
|
||||
assert.equal(result.passed, false);
|
||||
assert.equal(
|
||||
result.issues.some(
|
||||
(issue) =>
|
||||
issue.field === "auditSummary" &&
|
||||
issue.rule === "pagespeed_score_artifact",
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test("validateCustomerFacingCopy blocks price/currency mention", () => {
|
||||
const result = validateCustomerFacingCopy({
|
||||
...validPayload,
|
||||
callScript: {
|
||||
...validPayload.callScript,
|
||||
callScript: [
|
||||
"Der Kontaktpunkt ist gut sichtbar.",
|
||||
"Ihr Paket kostet nur 99 € pro Monat.",
|
||||
"Ich habe den Kontaktpunkt geprüft und schlage vor, ihn in der Headerzeile zu fixieren.",
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(result.passed, false);
|
||||
assert.equal(
|
||||
result.issues.some(
|
||||
(issue) => issue.field === "callScript.callScript[1]" && issue.rule === "price_mention",
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test("validateCustomerFacingCopy rejects generic AI slop language", () => {
|
||||
const result = validateCustomerFacingCopy({
|
||||
...validPayload,
|
||||
emailBody:
|
||||
"Unsere maßgeschneiderte, nahtlose, innovative Lösung hebt Ihre Sichtbarkeit auf ein neues Level und ist wirklich disruptive.",
|
||||
});
|
||||
|
||||
assert.equal(result.passed, false);
|
||||
assert.equal(
|
||||
result.issues.some(
|
||||
(issue) =>
|
||||
issue.field === "emailBody" && issue.rule === "generic_ai_slop",
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test("validateCustomerFacingCopy flags accusatory tone", () => {
|
||||
const result = validateCustomerFacingCopy({
|
||||
...validPayload,
|
||||
auditBody:
|
||||
"Ihre Website ist katastrophal und wirkt absolut unprofessionell. Das sollte dringend geändert werden.",
|
||||
});
|
||||
|
||||
assert.equal(result.passed, false);
|
||||
assert.equal(
|
||||
result.issues.some(
|
||||
(issue) => issue.field === "auditBody" && issue.rule === "hostile_tone",
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test("validateCustomerFacingCopy strips technical artifacts like model ids and raw JSON", () => {
|
||||
const result = validateCustomerFacingCopy({
|
||||
...validPayload,
|
||||
followUp:
|
||||
'Ich habe folgende Diagnose: {"score": 0.8, "lighthouseResult": "ok", "storageId": "rawstorageid_abc123"}',
|
||||
});
|
||||
|
||||
assert.equal(result.passed, false);
|
||||
assert.equal(
|
||||
result.issues.some(
|
||||
(issue) =>
|
||||
issue.field === "followUp" &&
|
||||
issue.rule === "raw_technical_artifact",
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test("validateCustomerFacingCopy enforces observation + suggestion style", () => {
|
||||
const result = validateCustomerFacingCopy({
|
||||
...validPayload,
|
||||
emailBody:
|
||||
"Deine Website ist großartig, tolle Arbeit.",
|
||||
});
|
||||
|
||||
assert.equal(result.passed, false);
|
||||
assert.equal(
|
||||
result.issues.some(
|
||||
(issue) =>
|
||||
issue.field === "emailBody" &&
|
||||
issue.rule === "missing_observation_or_suggestion",
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test("validateCustomerFacingCopy is permissive for phone numbers and date values", () => {
|
||||
const result = validateCustomerFacingCopy({
|
||||
auditSummary:
|
||||
"Ich habe gesehen, dass eure Kontaktseite am 12.02.2026 aktualisiert wurde. Ich empfehle, den Kontaktbereich als Nächstes im Header zu verbessern.",
|
||||
auditBody:
|
||||
"Mir ist aufgefallen, dass die Telefonnummer 0201 123456 in der Fußzeile steht. Ich empfehle, sie zusätzlich im Header zu platzieren.",
|
||||
emailSubject: "Kurzes Feedback zu eurem Terminplan",
|
||||
emailBody:
|
||||
"Hallo, ich habe euren Webauftritt geprüft und habe gesehen, dass Termine auf der Seite mit dem Datum 12. Oktober erwähnt sind. Ich empfehle, diese Terminangabe im Header stärker hervorzuheben.",
|
||||
callScript: {
|
||||
openingLine:
|
||||
"Hallo, ich bin Matthias und ich habe eure Seite geprüft.",
|
||||
callScript: [
|
||||
"Ich habe auf eurer Seite gesehen, dass der Kontaktbutton erst sehr weit unten erscheint.",
|
||||
"Mir ist aufgefallen, dass hier noch eine kleine Verbesserung fehlt; ich schlage vor, den Kontaktbereich nach oben zu ziehen.",
|
||||
],
|
||||
closeLine: "Dann nehme ich das Thema in den nächsten Schritt mit auf.",
|
||||
},
|
||||
followUp:
|
||||
"Mir ist am 12. Oktober aufgefallen, dass die Telefonnummer 030 1234567 schon gut auffindbar ist; ich schlage vor, eine kleine Sichtbarkeitsanpassung vorzunehmen.",
|
||||
});
|
||||
|
||||
assert.equal(result.passed, true);
|
||||
});
|
||||
|
||||
test("validateCallScriptCopy validates each script line individually and returns field paths", () => {
|
||||
const result = validateCallScriptCopy({
|
||||
openingLine: "Hallo, ich bin Matthias.",
|
||||
callScript: [
|
||||
"{" +
|
||||
'"score": 0.82, "rawstorageid":"abc123"' +
|
||||
"}",
|
||||
"Ich habe auf der Seite gesehen, dass der Kontaktbutton fehlt.",
|
||||
"Mir fehlt noch ein konkreter Verbesserungsschritt.",
|
||||
],
|
||||
closeLine: "Schöne Grüße",
|
||||
});
|
||||
|
||||
assert.equal(result.passed, false);
|
||||
assert.equal(
|
||||
result.issues.some(
|
||||
(issue) =>
|
||||
issue.field === "callScript.callScript[0]" &&
|
||||
issue.rule === "raw_technical_artifact",
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test("validateFollowUpCopy enforces ich-form and guard output shape", () => {
|
||||
const result = validateFollowUpCopy({
|
||||
message: "Hier ist der Inhalt für das Follow-up.",
|
||||
});
|
||||
|
||||
assert.equal(result.passed, false);
|
||||
assert.equal(result.issues.length > 0, true);
|
||||
assert.equal(
|
||||
result.issues.some(
|
||||
(issue) =>
|
||||
issue.field === "followUp" && issue.rule === "missing_ich_form",
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
50
tests/openrouter-provider.test.ts
Normal file
50
tests/openrouter-provider.test.ts
Normal file
@@ -0,0 +1,50 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import assert from "node:assert/strict";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
|
||||
import { createOpenRouterProvider } from "../lib/ai/openrouter-provider";
|
||||
|
||||
const providerSource = readFileSync(
|
||||
join(process.cwd(), "lib", "ai", "openrouter-provider.ts"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
test("provider reads OPENROUTER_API_KEY from environment and requires it", () => {
|
||||
assert.equal(
|
||||
/OPENROUTER_API_KEY/.test(providerSource),
|
||||
true,
|
||||
"Provider should read OPENROUTER_API_KEY.",
|
||||
);
|
||||
assert.equal(
|
||||
/OPENROUTER_APP_NAME/.test(providerSource),
|
||||
true,
|
||||
"Provider should include optional OPENROUTER_APP_NAME.",
|
||||
);
|
||||
assert.equal(
|
||||
/OPENROUTER_APP_URL/.test(providerSource),
|
||||
true,
|
||||
"Provider should include optional OPENROUTER_APP_URL.",
|
||||
);
|
||||
|
||||
assert.throws(
|
||||
() =>
|
||||
createOpenRouterProvider({
|
||||
OPENROUTER_API_KEY: undefined,
|
||||
OPENROUTER_APP_NAME: "local-audit-tool",
|
||||
OPENROUTER_APP_URL: "https://example.local",
|
||||
}),
|
||||
/OPENROUTER_API_KEY is required/i,
|
||||
);
|
||||
});
|
||||
|
||||
test("provider forwards optional app metadata to createOpenRouter call", () => {
|
||||
const provider = createOpenRouterProvider({
|
||||
OPENROUTER_API_KEY: "dummy-key",
|
||||
OPENROUTER_APP_NAME: "local-audit-tool",
|
||||
OPENROUTER_APP_URL: "https://example.local",
|
||||
});
|
||||
|
||||
assert.equal(typeof provider, "function");
|
||||
assert.equal(provider !== null, true);
|
||||
});
|
||||
@@ -360,7 +360,8 @@ test("website enrichment action prepares Chromium AL2023 shared libraries for Co
|
||||
);
|
||||
|
||||
const executableIndex = actionSource.indexOf(
|
||||
"const executablePath = await resolveChromiumExecutablePath(",
|
||||
"resolveChromiumExecutablePath(",
|
||||
actionSource.indexOf("export const processLeadEnrichment"),
|
||||
);
|
||||
const launchIndex = actionSource.indexOf("chromium.launch({");
|
||||
const hasSetupIndex = Math.max(
|
||||
@@ -381,7 +382,7 @@ test("processLeadEnrichment wraps Playwright bootstrap in protected try/catch",
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
actionSource,
|
||||
/try\s*\{[\s\S]*?const \{ playwrightCore, serverlessChromium \}\s*=\s*await loadPlaywrightModules\(\);[\s\S]*?const executablePath = await resolveChromiumExecutablePath\(\s*serverlessChromium,\s*\);[\s\S]*?browser = await playwrightCore\.chromium\.launch\([\s\S]*?executablePath,[\s\S]*?desktopContext = await browser\.newContext\([\s\S]*?mobileContext = await browser\.newContext\(/,
|
||||
/try\s*\{[\s\S]*?const \{ playwrightCore, serverlessChromium \}\s*=[\s\S]*?loadPlaywrightModules\(\)[\s\S]*?const executablePath = await withActionTimeout\([\s\S]*?resolveChromiumExecutablePath\(\s*serverlessChromium\s*\)[\s\S]*?browser = await withActionTimeout\([\s\S]*?playwrightCore\.chromium\.launch\([\s\S]*?executablePath,[\s\S]*?desktopContext = await withActionTimeout\([\s\S]*?browser\.newContext\([\s\S]*?mobileContext = await withActionTimeout\([\s\S]*?browser\.newContext\(/,
|
||||
),
|
||||
true,
|
||||
"Playwright runtime bootstrap should use resolveChromiumExecutablePath() inside the action's try/catch-protected block",
|
||||
@@ -558,6 +559,77 @@ test("website enrichment enforces TASK-8 crawler limits and runtime timeboxes",
|
||||
);
|
||||
});
|
||||
|
||||
test("website enrichment guards long browser work before Convex action runtime aborts", () => {
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /DEFAULT_ACTION_BUDGET_MS\s*=\s*120_000/),
|
||||
true,
|
||||
"Action should keep an overall runtime budget below the observed Convex abort window.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /TASK8_ACTION_BUDGET_MS/),
|
||||
true,
|
||||
"Action runtime budget should be configurable for manual tuning.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /function actionBudgetMs\(\)/),
|
||||
true,
|
||||
"Action should resolve a bounded runtime budget.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /function remainingActionBudgetMs\(/),
|
||||
true,
|
||||
"Action should calculate remaining runtime before long awaits.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(actionSource, /async function withActionTimeout/),
|
||||
true,
|
||||
"Action should wrap long promises so JS catch runs before Convex kills the runtime.",
|
||||
);
|
||||
|
||||
const processBody = extractExportSource(actionSource, "processLeadEnrichment");
|
||||
assert.equal(
|
||||
hasPattern(processBody, /const actionStartedAt = Date\.now\(\)/),
|
||||
true,
|
||||
"processLeadEnrichment should track action start time.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(processBody, /const actionBudget = actionBudgetMs\(\)/),
|
||||
true,
|
||||
"processLeadEnrichment should resolve the action budget once.",
|
||||
);
|
||||
|
||||
const guardedPatterns = [
|
||||
/withActionTimeout\([\s\S]*loadPlaywrightModules\(\)/,
|
||||
/withActionTimeout\([\s\S]*resolveChromiumExecutablePath\(/,
|
||||
/withActionTimeout\([\s\S]*prepareChromiumSharedLibraries\(/,
|
||||
/withActionTimeout\([\s\S]*playwrightCore\.chromium\.launch\(/,
|
||||
/withActionTimeout\([\s\S]*crawlPage\(\s*desktopContext,\s*rootUrl/,
|
||||
/withActionTimeout\([\s\S]*captureHomepageScreenshot\(/,
|
||||
];
|
||||
|
||||
for (const pattern of guardedPatterns) {
|
||||
assert.equal(
|
||||
hasPattern(processBody, pattern),
|
||||
true,
|
||||
`Expected long await to be guarded by withActionTimeout: ${pattern}`,
|
||||
);
|
||||
}
|
||||
|
||||
assert.equal(
|
||||
hasPattern(processBody, /Math\.min\(\s*timeoutMs,\s*remainingActionBudgetMs\(/),
|
||||
true,
|
||||
"Per-page crawl timeout should be capped by remaining action budget.",
|
||||
);
|
||||
assert.equal(
|
||||
hasPattern(
|
||||
processBody,
|
||||
/desktopContext\.request\.get\([\s\S]*timeout:\s*Math\.min\([\s\S]*remainingActionBudgetMs\(/,
|
||||
),
|
||||
true,
|
||||
"Internal link checks should cap request timeouts by remaining action budget.",
|
||||
);
|
||||
});
|
||||
|
||||
test("processLeadEnrichment schedules PageSpeed audit jobs after successful enrichment", () => {
|
||||
const processBody = extractExportSource(actionSource, "processLeadEnrichment");
|
||||
const persistIndex = processBody.indexOf(
|
||||
|
||||
Reference in New Issue
Block a user