pitchfast/tests/audit-specialist-schemas.test.ts

import assert from "node:assert/strict";
import test from "node:test";

import { zodSchema } from "ai";

import {
  auditEvidenceVerificationSchema,
  auditSpecialistResultSchema,
} from "../lib/ai/schemas";

const validFinding = {
  skillId: "local-seo-basics",
  claim: "Die Startseite nennt den Standort im sichtbaren Bereich nicht klar.",
  recommendation: "Ort und wichtigste Leistung in die erste Überschrift aufnehmen.",
  customerBenefit: "Besucher erkennen schneller, ob das Angebot für sie passt.",
  severity: 2,
  confidence: 0.82,
  evidenceRefs: [
    {
      id: "crawl_page:homepage:https-example-com",
      type: "crawl_page",
      label: "Startseite",
      sourceUrl: "https://example.com",
    },
  ],
  applies: true,
  unknowns: [],
};

type JsonSchemaObject = {
  type?: string;
  properties?: Record<string, JsonSchemaObject>;
  required?: string[];
  items?: JsonSchemaObject;
};

function assertStrictRequiredProperties(schema: JsonSchemaObject, path = "schema") {
  if (schema.type === "object" && schema.properties) {
    const required = new Set(schema.required ?? []);
    for (const key of Object.keys(schema.properties)) {
      assert.equal(
        required.has(key),
        true,
        `${path}.${key} must be required for Azure/OpenAI structured outputs`,
      );
      assertStrictRequiredProperties(schema.properties[key]!, `${path}.${key}`);
    }
  }

  if (schema.type === "array" && schema.items) {
    assertStrictRequiredProperties(schema.items, `${path}[]`);
  }
}

test("specialist structured-output schemas require every declared property", () => {
  assertStrictRequiredProperties(
    zodSchema(auditSpecialistResultSchema).jsonSchema as JsonSchemaObject,
  );
  assertStrictRequiredProperties(
    zodSchema(auditEvidenceVerificationSchema).jsonSchema as JsonSchemaObject,
  );
});

test("auditSpecialistResultSchema accepts evidence-backed specialist findings", () => {
  const parsed = auditSpecialistResultSchema.parse({
    status: "success",
    findings: [validFinding],
    notes: ["Lokale Relevanz wurde anhand der Startseite geprüft."],
  });

  assert.equal(parsed.status, "success");
  assert.equal(parsed.findings[0]?.evidenceRefs[0]?.type, "crawl_page");
});

test("auditSpecialistResultSchema rejects findings without evidence refs", () => {
  assert.throws(
    () =>
      auditSpecialistResultSchema.parse({
        status: "success",
        findings: [{ ...validFinding, evidenceRefs: [] }],
        notes: [],
      }),
    /evidenceRefs/,
  );
});

test("auditSpecialistResultSchema rejects unsupported severity and confidence", () => {
  assert.throws(
    () =>
      auditSpecialistResultSchema.parse({
        status: "success",
        findings: [{ ...validFinding, severity: 4 }],
        notes: [],
      }),
    /severity/,
  );
  assert.throws(
    () =>
      auditSpecialistResultSchema.parse({
        status: "success",
        findings: [{ ...validFinding, confidence: 1.4 }],
        notes: [],
      }),
    /confidence/,
  );
});

test("auditSpecialistResultSchema rejects unknown-only findings", () => {
  assert.throws(
    () =>
      auditSpecialistResultSchema.parse({
        status: "success",
        findings: [
          {
            ...validFinding,
            claim: "Kontaktformular: Unbekannt",
            recommendation: "Unbekannt prüfen.",
            customerBenefit: "Unbekannt.",
            evidenceRefs: [
              {
                id: "technical_check:unknown",
                type: "technical_check",
                label: "Kontaktformular unbekannt",
                sourceUrl: "",
              },
            ],
          },
        ],
        notes: [],
      }),
    /unknown/i,
  );
});

test("auditEvidenceVerificationSchema returns compact verified ids and rejected decisions", () => {
  const parsed = auditEvidenceVerificationSchema.parse({
    verifiedFindingIds: ["finding-1"],
    rejectedFindings: [
      {
        findingId: "finding-2",
        skillId: validFinding.skillId,
        claim: "Die Seite koennte moderner wirken.",
        rejectionReason: "Zu generisch und nicht ausreichend belegt.",
      },
    ],
    contradictions: [],
    notes: ["Ein Finding wurde wegen generischer Sprache verworfen."],
  });

  assert.deepEqual(parsed.verifiedFindingIds, ["finding-1"]);
  assert.equal(parsed.rejectedFindings[0]?.rejectionReason.includes("generisch"), true);
});

test("auditEvidenceVerificationSchema accepts rejected unknown-only claims", () => {
  const parsed = auditEvidenceVerificationSchema.parse({
    verifiedFindingIds: [],
    rejectedFindings: [
      {
        findingId: "finding-1",
        skillId: "contact-conversion",
        claim: "Kontaktformular: Unbekannt",
        rejectionReason: "Unknown-only Befunde duerfen nicht in die Kundencopy.",
      },
    ],
    contradictions: [],
    notes: [],
  });

  assert.equal(parsed.rejectedFindings.length, 1);
});

test("auditEvidenceVerificationSchema keeps verifier output compact for many findings", () => {
  const parsed = auditEvidenceVerificationSchema.parse({
    verifiedFindingIds: Array.from({ length: 12 }, (_, index) => `finding-${index + 1}`),
    rejectedFindings: [],
    contradictions: [],
    notes: ["Full specialist findings stay in application state and are not echoed."],
  });

  assert.equal(parsed.verifiedFindingIds.length, 12);
});