Improve audit pipeline and outreach review

tests/audit-specialist-schemas.test.ts

@@ -0,0 +1,181 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { zodSchema } from "ai";
+
+import {
+  auditEvidenceVerificationSchema,
+  auditSpecialistResultSchema,
+} from "../lib/ai/schemas";
+
+const validFinding = {
+  skillId: "local-seo-basics",
+  claim: "Die Startseite nennt den Standort im sichtbaren Bereich nicht klar.",
+  recommendation: "Ort und wichtigste Leistung in die erste Überschrift aufnehmen.",
+  customerBenefit: "Besucher erkennen schneller, ob das Angebot für sie passt.",
+  severity: 2,
+  confidence: 0.82,
+  evidenceRefs: [
+    {
+      id: "crawl_page:homepage:https-example-com",
+      type: "crawl_page",
+      label: "Startseite",
+      sourceUrl: "https://example.com",
+    },
+  ],
+  applies: true,
+  unknowns: [],
+};
+
+type JsonSchemaObject = {
+  type?: string;
+  properties?: Record<string, JsonSchemaObject>;
+  required?: string[];
+  items?: JsonSchemaObject;
+};
+
+function assertStrictRequiredProperties(schema: JsonSchemaObject, path = "schema") {
+  if (schema.type === "object" && schema.properties) {
+    const required = new Set(schema.required ?? []);
+    for (const key of Object.keys(schema.properties)) {
+      assert.equal(
+        required.has(key),
+        true,
+        `${path}.${key} must be required for Azure/OpenAI structured outputs`,
+      );
+      assertStrictRequiredProperties(schema.properties[key]!, `${path}.${key}`);
+    }
+  }
+
+  if (schema.type === "array" && schema.items) {
+    assertStrictRequiredProperties(schema.items, `${path}[]`);
+  }
+}
+
+test("specialist structured-output schemas require every declared property", () => {
+  assertStrictRequiredProperties(
+    zodSchema(auditSpecialistResultSchema).jsonSchema as JsonSchemaObject,
+  );
+  assertStrictRequiredProperties(
+    zodSchema(auditEvidenceVerificationSchema).jsonSchema as JsonSchemaObject,
+  );
+});
+
+test("auditSpecialistResultSchema accepts evidence-backed specialist findings", () => {
+  const parsed = auditSpecialistResultSchema.parse({
+    status: "success",
+    findings: [validFinding],
+    notes: ["Lokale Relevanz wurde anhand der Startseite geprüft."],
+  });
+
+  assert.equal(parsed.status, "success");
+  assert.equal(parsed.findings[0]?.evidenceRefs[0]?.type, "crawl_page");
+});
+
+test("auditSpecialistResultSchema rejects findings without evidence refs", () => {
+  assert.throws(
+    () =>
+      auditSpecialistResultSchema.parse({
+        status: "success",
+        findings: [{ ...validFinding, evidenceRefs: [] }],
+        notes: [],
+      }),
+    /evidenceRefs/,
+  );
+});
+
+test("auditSpecialistResultSchema rejects unsupported severity and confidence", () => {
+  assert.throws(
+    () =>
+      auditSpecialistResultSchema.parse({
+        status: "success",
+        findings: [{ ...validFinding, severity: 4 }],
+        notes: [],
+      }),
+    /severity/,
+  );
+  assert.throws(
+    () =>
+      auditSpecialistResultSchema.parse({
+        status: "success",
+        findings: [{ ...validFinding, confidence: 1.4 }],
+        notes: [],
+      }),
+    /confidence/,
+  );
+});
+
+test("auditSpecialistResultSchema rejects unknown-only findings", () => {
+  assert.throws(
+    () =>
+      auditSpecialistResultSchema.parse({
+        status: "success",
+        findings: [
+          {
+            ...validFinding,
+            claim: "Kontaktformular: Unbekannt",
+            recommendation: "Unbekannt prüfen.",
+            customerBenefit: "Unbekannt.",
+            evidenceRefs: [
+              {
+                id: "technical_check:unknown",
+                type: "technical_check",
+                label: "Kontaktformular unbekannt",
+                sourceUrl: "",
+              },
+            ],
+          },
+        ],
+        notes: [],
+      }),
+    /unknown/i,
+  );
+});
+
+test("auditEvidenceVerificationSchema returns compact verified ids and rejected decisions", () => {
+  const parsed = auditEvidenceVerificationSchema.parse({
+    verifiedFindingIds: ["finding-1"],
+    rejectedFindings: [
+      {
+        findingId: "finding-2",
+        skillId: validFinding.skillId,
+        claim: "Die Seite koennte moderner wirken.",
+        rejectionReason: "Zu generisch und nicht ausreichend belegt.",
+      },
+    ],
+    contradictions: [],
+    notes: ["Ein Finding wurde wegen generischer Sprache verworfen."],
+  });
+
+  assert.deepEqual(parsed.verifiedFindingIds, ["finding-1"]);
+  assert.equal(parsed.rejectedFindings[0]?.rejectionReason.includes("generisch"), true);
+});
+
+test("auditEvidenceVerificationSchema accepts rejected unknown-only claims", () => {
+  const parsed = auditEvidenceVerificationSchema.parse({
+    verifiedFindingIds: [],
+    rejectedFindings: [
+      {
+        findingId: "finding-1",
+        skillId: "contact-conversion",
+        claim: "Kontaktformular: Unbekannt",
+        rejectionReason: "Unknown-only Befunde duerfen nicht in die Kundencopy.",
+      },
+    ],
+    contradictions: [],
+    notes: [],
+  });
+
+  assert.equal(parsed.rejectedFindings.length, 1);
+});
+
+test("auditEvidenceVerificationSchema keeps verifier output compact for many findings", () => {
+  const parsed = auditEvidenceVerificationSchema.parse({
+    verifiedFindingIds: Array.from({ length: 12 }, (_, index) => `finding-${index + 1}`),
+    rejectedFindings: [],
+    contradictions: [],
+    notes: ["Full specialist findings stay in application state and are not echoed."],
+  });
+
+  assert.equal(parsed.verifiedFindingIds.length, 12);
+});