From a45b92ea0ab16a9afc436ba95ca90f46b2a19453 Mon Sep 17 00:00:00 2001 From: Matthias Meister Date: Sun, 7 Jun 2026 23:06:31 +0200 Subject: [PATCH] Externalize audit pipeline services --- .env.example | 14 +- README.md | 28 +- ...31 - Require-auth-for-usage-event-reads.md | 43 +++ ...v3-skill-registry-into-audit-generation.md | 44 +++ ...-33 - Fix-v3-live-wiring-quality-issues.md | 44 +++ ...Harden-v3-selection-and-Convex-payloads.md | 42 +++ ...ing-undefined-audit-generation-payloads.md | 44 +++ ...- Remove-optional-helper-undefined-args.md | 44 +++ ...k-37 - Prioritize-v3-local-audit-skills.md | 44 +++ ...d-ScreenshotOne-missing-key-run-warning.md | 44 +++ .../task-39 - Secure-Convex-operator-APIs.md | 34 ++ ...40 - Behebe-abschliessende-Lint-Blocker.md | 40 ++ ...iere-Convex-Typecheck-fuer-Usage-Events.md | 47 +++ ...pe-v2-Referenzdateien-aus-dem-Typecheck.md | 43 +++ convex/_generated/api.d.ts | 2 + convex/auditGeneration.ts | 34 +- convex/audits.ts | 13 +- convex/domain.ts | 14 + convex/leads.ts | 350 +++++++++-------- convex/pageSpeedAction.ts | 14 +- convex/runs.ts | 74 +++- convex/schema.ts | 44 ++- convex/usageEvents.ts | 223 +++++++++++ docs/verification.md | 13 +- eslint.config.mjs | 2 + lib/ai/audit-evidence.ts | 128 ++++++- lib/external-audit-services.ts | 233 ++++++++++++ lib/operational-readiness.ts | 19 +- lib/skills-registry.ts | 183 ++++++++- tests/audit-evidence.test.ts | 159 ++++++++ ...udit-generation-persistence-source.test.ts | 23 ++ tests/audit-skill-registry-v3.test.ts | 87 +++++ tests/audit-skills-schema.test.ts | 13 +- tests/audits-auth-source.test.ts | 73 ++++ tests/external-audit-pipeline-source.test.ts | 335 ++++++++++++++++ tests/external-audit-services.test.ts | 184 +++++++++ tests/leads-runs-auth-source.test.ts | 195 ++++++++++ tests/operational-readiness.test.ts | 45 ++- tests/ops-quality-source.test.ts | 8 +- tests/pagespeed-action-source.test.ts | 4 +- tests/usage-events-source.test.ts | 356 ++++++++++++++++++ tsconfig.json | 5 +- 42 files changed, 3141 insertions(+), 247 deletions(-) create mode 100644 backlog/tasks/task-31 - Require-auth-for-usage-event-reads.md create mode 100644 backlog/tasks/task-32 - Wire-v3-skill-registry-into-audit-generation.md create mode 100644 backlog/tasks/task-33 - Fix-v3-live-wiring-quality-issues.md create mode 100644 backlog/tasks/task-34 - Harden-v3-selection-and-Convex-payloads.md create mode 100644 backlog/tasks/task-35 - Remove-remaining-undefined-audit-generation-payloads.md create mode 100644 backlog/tasks/task-36 - Remove-optional-helper-undefined-args.md create mode 100644 backlog/tasks/task-37 - Prioritize-v3-local-audit-skills.md create mode 100644 backlog/tasks/task-38 - Add-ScreenshotOne-missing-key-run-warning.md create mode 100644 backlog/tasks/task-39 - Secure-Convex-operator-APIs.md create mode 100644 backlog/tasks/task-40 - Behebe-abschliessende-Lint-Blocker.md create mode 100644 backlog/tasks/task-41 - Repariere-Convex-Typecheck-fuer-Usage-Events.md create mode 100644 backlog/tasks/task-42 - Scope-v2-Referenzdateien-aus-dem-Typecheck.md create mode 100644 convex/usageEvents.ts create mode 100644 lib/external-audit-services.ts create mode 100644 tests/audit-skill-registry-v3.test.ts create mode 100644 tests/audits-auth-source.test.ts create mode 100644 tests/external-audit-pipeline-source.test.ts create mode 100644 tests/external-audit-services.test.ts create mode 100644 tests/leads-runs-auth-source.test.ts create mode 100644 tests/usage-events-source.test.ts diff --git a/.env.example b/.env.example index 306fbdf..b50fc90 100644 --- a/.env.example +++ b/.env.example @@ -1,8 +1,12 @@ # App / Coolify APP_ENV=development -NEXT_PUBLIC_APP_URL=http://localhost:3000 +NEXT_PUBLIC_APP_URL=https://audit.matthias-meister-webdesign.de -# TASK-8 Playwright +# Personal deployment scope +# This repo currently targets audit.matthias-meister-webdesign.de with managed +# server-side provider keys. SaaS BYO keys, billing, and team roles come later. + +# Legacy TASK-8 Playwright enrichment (not required for the new external pipeline) TASK8_CRAWL_TIMEOUT_MS=60000 TASK8_CRAWL_MAX_PAGES=20 TASK8_BROWSER_ASSET_URL= @@ -31,6 +35,12 @@ OPENROUTER_MODEL_QUALITY_REVIEW= OPENROUTER_APP_NAME= OPENROUTER_APP_URL= +# ScreenshotOne +SCREENSHOTONE_API_KEY= + +# Jina (optional fallback; no key required for current readiness) +JINA_API_KEY= + # SMTP / Stalwart SMTP_HOST= SMTP_PORT=465 diff --git a/README.md b/README.md index 5cdb2d1..d9c1542 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # WebDev Pipeline -Interner Akquise-Agent fuer lokale Webdesign-Leads. Das MVP startet mit Next.js App Router, TypeScript, Tailwind CSS, shadcn/ui und Platzhalter-Routen fuer Dashboard, Login und oeffentliche Audit-Seiten. +Persoenlicher Akquise-Agent fuer lokale Webdesign-Leads auf `audit.matthias-meister-webdesign.de`. Das MVP startet mit Next.js App Router, TypeScript, Tailwind CSS, shadcn/ui und Platzhalter-Routen fuer Dashboard, Login und oeffentliche Audit-Seiten. + +Der aktuelle Scope ist bewusst persoenlich: Google, PageSpeed, OpenRouter, ScreenshotOne und optional Jina laufen ueber serverseitig verwaltete Keys. BYO-Keys, Billing und Teamrollen gehoeren zur spaeteren SaaS-Readiness, aber nicht zu dieser Welle. ## Getting Started @@ -23,12 +25,13 @@ Copy `.env.example` to `.env.local` for local development. Keep real secrets out - **App / Coolify:** `APP_ENV`, `NEXT_PUBLIC_APP_URL` - **Convex:** `NEXT_PUBLIC_CONVEX_URL`, `NEXT_PUBLIC_CONVEX_SITE_URL`, `CONVEX_DEPLOYMENT` -- **Google / Task-9 PageSpeed:** `GOOGLE_GEOCODING_API_KEY`, `GOOGLE_PLACES_API_KEY`, `PAGESPEED_API_KEY`, `PAGESPEED_TIMEOUT_MS` +- **Google / PageSpeed:** `GOOGLE_GEOCODING_API_KEY`, `GOOGLE_PLACES_API_KEY`, `PAGESPEED_API_KEY`, `PAGESPEED_TIMEOUT_MS` - **OpenRouter:** `OPENROUTER_API_KEY`, `OPENROUTER_MODEL_CLASSIFICATION`, `OPENROUTER_MODEL_MULTIMODAL_AUDIT`, `OPENROUTER_MODEL_GERMAN_COPY`, `OPENROUTER_MODEL_QUALITY_REVIEW`, optional: `OPENROUTER_APP_NAME`, `OPENROUTER_APP_URL` +- **ScreenshotOne:** `SCREENSHOTONE_API_KEY` +- **Jina:** optional `JINA_API_KEY` for future authenticated fallback usage; not required for current readiness. - **SMTP / Stalwart:** `SMTP_HOST`, `SMTP_PORT`, `SMTP_USER`, `SMTP_PASSWORD`, `SMTP_FROM` - **Rybbit:** `RYBBIT_API_URL`, `RYBBIT_API_KEY`, `NEXT_PUBLIC_RYBBIT_SITE_ID` - **Auth:** `BETTER_AUTH_SECRET` -- **TASK-8 enrichment:** `TASK8_BROWSER_ASSET_URL` Only variables prefixed with `NEXT_PUBLIC_` are intended for browser exposure. All API keys, SMTP credentials, and server-only URLs must stay server-side. @@ -50,24 +53,11 @@ Only variables prefixed with `NEXT_PUBLIC_` are intended for browser exposure. A Coolify should run `pnpm install`, `pnpm build`, and `pnpm start`. The current font setup uses `next/font/google`, so production builds need outbound access to Google Fonts unless fonts are later self-hosted. -TASK-8 enrichment uses `playwright-core` with `@sparticuz/chromium-min` in Convex. Local `npx playwright install` is a browser-testing helper only and does not affect the Convex runtime bundle. +The new audit pipeline expects managed server-side provider configuration for Google, PageSpeed, OpenRouter, ScreenshotOne, and optional Jina. Do not expose provider secrets in browser-prefixed variables. -TASK-8 requires a browser binary source URL configured on Convex. The preferred -variable is: +Playwright/TASK-8 is legacy enrichment context, not a required integration for the new external audit pipeline. Local `npx playwright install` remains a browser-testing helper only and does not affect the managed external-service readiness check. -- `TASK8_BROWSER_ASSET_URL` (for example your self-hosted or CDN Chromium bundle URL if you do not rely on package defaults). - -For backward compatibility, the action also supports: - -- `TASK8_CHROMIUM_EXECUTABLE_URL` -- `TASK8_CHROMIUM_EXECUTABLE` - -If none are set, enrichment deployment/startup will fail with a clear configuration -error so no silent fallback is used. - -If the URL is missing and no default is available in your environment, the enqueue action will throw a clear deploy/configuration error so enrichment does not silently fall back to a missing binary. - -For TASK-8 deployment updates, run Convex restart/deploy after code changes: +For Convex deployment updates, run restart/deploy after code changes: - Local: `pnpm exec convex dev` - Remote: `pnpm exec convex deploy` diff --git a/backlog/tasks/task-31 - Require-auth-for-usage-event-reads.md b/backlog/tasks/task-31 - Require-auth-for-usage-event-reads.md new file mode 100644 index 0000000..8cf174c --- /dev/null +++ b/backlog/tasks/task-31 - Require-auth-for-usage-event-reads.md @@ -0,0 +1,43 @@ +--- +id: TASK-31 +title: Require auth for usage event reads +status: In Progress +assignee: [] +created_date: '2026-06-06 20:27' +updated_date: '2026-06-06 20:31' +labels: [] +dependencies: [] +priority: high +ordinal: 33000 +--- + +## Description + + +Protect public Convex usageEvents read queries from unauthenticated access while preserving validators, bounded reads, and index usage. + + +## Acceptance Criteria + +- [x] #1 Source contracts assert every public usageEvents read query requires requireOperator auth +- [x] #2 usageEvents read queries call requireOperator before reading sensitive telemetry +- [x] #3 Focused usage-events source tests pass after the implementation + + +## Implementation Plan + + +1. Inspect usageEvents source tests and local auth patterns +2. Add RED source contracts for authenticated read queries +3. Run focused test and capture RED +4. Add minimal requireOperator guard to usageEvents reads +5. Run focused GREEN verification and self-review + + +## Implementation Notes + + +RED: pnpm test -- tests/usage-events-source.test.ts is blocked by pre-existing tests/ai-schemas.test.ts missing exports. Focused node --test tests/usage-events-source.test.ts fails as expected on missing usageEvents requireOperator auth guard. + +GREEN: node --test tests/usage-events-source.test.ts passes 6/6. pnpm test -- tests/usage-events-source.test.ts compiles and usageEvents tests pass, but the overall runner fails on existing external-audit-pipeline-source.test.js: audit generation action sanitizes raw errors before run events and run failure summaries, outside Worker F scope. + diff --git a/backlog/tasks/task-32 - Wire-v3-skill-registry-into-audit-generation.md b/backlog/tasks/task-32 - Wire-v3-skill-registry-into-audit-generation.md new file mode 100644 index 0000000..a1a0852 --- /dev/null +++ b/backlog/tasks/task-32 - Wire-v3-skill-registry-into-audit-generation.md @@ -0,0 +1,44 @@ +--- +id: TASK-32 +title: Wire v3 skill registry into audit generation +status: In Progress +assignee: [] +created_date: '2026-06-06 20:27' +updated_date: '2026-06-06 20:36' +labels: [] +dependencies: [] +priority: high +ordinal: 34000 +--- + +## Description + + +Fix the final review finding by using the v3 skills registry and v3 finding validation in the live audit generation path while preserving best-effort fallback behavior. + + +## Acceptance Criteria + +- [x] #1 auditGenerationAction loads and passes a non-empty v3 skill registry from v2_elemente/skills.md/loadSkillsRegistry when available +- [x] #2 Classification uses a v3 findings schema live instead of legacy-only internalFindingsSchema +- [x] #3 Audit persistence validators accept v3 usedSkills with id and optional category without forcing undefined category fields + + +## Implementation Plan + + +1. Read current audit generation, schemas, validators, and focused tests +2. Add RED source-contract/schema tests for v3 registry, v3 classification, and optional usedSkill category +3. Run focused tests and record failures +4. Implement minimal wiring and validator/schema changes +5. Run focused tests green plus relevant verification +6. Self-review scope and update task notes without closing + + +## Implementation Notes + + +RED: pnpm test tests/audit-generation-action-source.test.ts tests/ai-schemas.test.ts tests/audit-skills-schema.test.ts tests/audit-skill-registry-v3.test.ts failed in tsc because auditClassificationSchema and AuditClassification are not exported yet. This confirms the v3 classification schema is not wired. + +GREEN: pnpm exec tsc -p tsconfig.test.json exited 0. Focused compiled tests passed: node --test .test-output/tests/audit-generation-action-source.test.js .test-output/tests/ai-schemas.test.js .test-output/tests/audit-skills-schema.test.js .test-output/tests/audit-skill-registry-v3.test.js => 32/32 pass. Full pnpm test passed: 345/345. Self-review: no changes to convex/usageEvents.ts, no commit/staging; usedSkills optional fields are conditionally spread before persistence. + diff --git a/backlog/tasks/task-33 - Fix-v3-live-wiring-quality-issues.md b/backlog/tasks/task-33 - Fix-v3-live-wiring-quality-issues.md new file mode 100644 index 0000000..4d23693 --- /dev/null +++ b/backlog/tasks/task-33 - Fix-v3-live-wiring-quality-issues.md @@ -0,0 +1,44 @@ +--- +id: TASK-33 +title: Fix v3 live wiring quality issues +status: In Progress +assignee: [] +created_date: '2026-06-06 20:41' +updated_date: '2026-06-06 20:47' +labels: [] +dependencies: [] +priority: high +ordinal: 35000 +--- + +## Description + + +Address the two v3 live wiring review quality issues: select category-less v3 skills from the real registry and keep registry-load warning logging best-effort. + + +## Acceptance Criteria + +- [x] #1 Real v3 skills from v2_elemente/skills.md are selected from realistic audit evidence without fabricated categories +- [x] #2 Legacy category-based skill registry selection continues to work +- [x] #3 Registry load fallback returns an empty registry even when warning event logging fails + + +## Implementation Plan + + +1. Inspect current skill selection and action warning fallback +2. Add RED tests for real v3 registry selection and isolated warning logging +3. Run focused tests and record RED failures +4. Implement minimal selection and warning isolation fixes +5. Run focused tests green plus typecheck/relevant suite +6. Self-review scope and leave task In Progress + + +## Implementation Notes + + +RED: tsc passed. node --test .test-output/tests/audit-evidence.test.js .test-output/tests/audit-generation-action-source.test.js failed with 2 expected failures: real v3 registry selectedSkills was empty/missing ids, and loadAuditSkillRegistry warning logging lacked isolated try/catch fallback. + +GREEN: pnpm exec tsc -p tsconfig.test.json exited 0. Focused tests passed: node --test .test-output/tests/audit-evidence.test.js .test-output/tests/audit-generation-action-source.test.js => 23/23 pass. Full pnpm test passed: 347/347. Self-review: only touched audit-evidence skill selection, auditGenerationAction registry warning fallback, and focused tests; no staging/commit; no convex/usageEvents.ts changes. + diff --git a/backlog/tasks/task-34 - Harden-v3-selection-and-Convex-payloads.md b/backlog/tasks/task-34 - Harden-v3-selection-and-Convex-payloads.md new file mode 100644 index 0000000..645d0cc --- /dev/null +++ b/backlog/tasks/task-34 - Harden-v3-selection-and-Convex-payloads.md @@ -0,0 +1,42 @@ +--- +id: TASK-34 +title: Harden v3 selection and Convex payloads +status: In Progress +assignee: [] +created_date: '2026-06-06 20:54' +updated_date: '2026-06-06 21:03' +labels: [] +dependencies: [] +priority: high +ordinal: 36000 +--- + +## Description + + +Fix v3 quality review issues by removing explicit undefined values from Convex mutation payloads and making v3 skill selection registry-driven with negative applicability tests. + + +## Acceptance Criteria + +- [x] #1 Convex mutation payloads in auditGenerationAction omit undefined top-level and nested fields +- [x] #2 v3 skill selection is registry-driven by applies_when and declared inputs with deterministic capped output +- [x] #3 Negative v3 input/applicability tests and legacy category tests pass + + +## Implementation Plan + + +1. Inspect current Convex mutation payload construction and v3 selection +2. Add RED tests for no undefined payload patterns, negative v3 gating, and deterministic cap +3. Run focused tests and record RED failures +4. Implement minimal payload omission and registry-driven v3 selection +5. Run focused tests green plus pnpm test if fast +6. Self-review scope and leave task In Progress + + +## Implementation Notes + + +RED: tsc passed, focused node --test .test-output/tests/audit-evidence.test.js .test-output/tests/audit-generation-action-source.test.js failed as expected on registry-order v3 cap and explicit undefined stage payload contract. GREEN: tsc passed; focused tests passed 26/26; full pnpm test passed 350/350. Self-review: no commits/staging, no changes to convex/usageEvents.ts, no ScreenshotOne missing-key behavior changes. + diff --git a/backlog/tasks/task-35 - Remove-remaining-undefined-audit-generation-payloads.md b/backlog/tasks/task-35 - Remove-remaining-undefined-audit-generation-payloads.md new file mode 100644 index 0000000..d7fb84b --- /dev/null +++ b/backlog/tasks/task-35 - Remove-remaining-undefined-audit-generation-payloads.md @@ -0,0 +1,44 @@ +--- +id: TASK-35 +title: Remove remaining undefined audit generation payloads +status: In Progress +assignee: [] +created_date: '2026-06-06 21:06' +updated_date: '2026-06-06 21:13' +labels: [] +dependencies: [] +priority: high +ordinal: 37000 +--- + +## Description + + +Fix TASK-34 spec-review issues by preventing appendRunEvent, success finish, and quality stage calls from sending explicit undefined optional fields. + + +## Acceptance Criteria + +- [x] #1 appendRunEvent only sends details when defined +- [x] #2 success finishAuditGenerationRun omits errorSummary instead of sending undefined +- [x] #3 quality-stage persistAuditStage callsite does not pass explicit undefined optional fields + + +## Implementation Plan + + +1. Inspect appendRunEvent, quality persist stage, and success finish call +2. Add RED source contracts for remaining explicit undefined patterns +3. Run focused tests and record RED +4. Implement minimal conditional spreads +5. Run focused tests green and full pnpm test if fast +6. Self-review scope and leave task In Progress + + +## Implementation Notes + + +RED: tsc passed. Focused node --test .test-output/tests/audit-generation-action-source.test.js failed as expected on three contracts: appendRunEvent details sent as args.details, success finishAuditGenerationRun ternary errorSummary undefined, and qualityReview persistAuditStage callsite ternary errorSummary undefined. + +RED: tsc passed. Focused node --test .test-output/tests/audit-generation-action-source.test.js failed as expected on appendRunEvent details, success finishAuditGenerationRun errorSummary ternary, and qualityReview persistAuditStage errorSummary ternary. GREEN: focused source test passed 21/21; full pnpm test passed 353/353. Self-review: changed only convex/auditGenerationAction.ts and tests/audit-generation-action-source.test.ts in this turn; no commits/staging; no UsageEvents or ScreenshotOne behavior changes. + diff --git a/backlog/tasks/task-36 - Remove-optional-helper-undefined-args.md b/backlog/tasks/task-36 - Remove-optional-helper-undefined-args.md new file mode 100644 index 0000000..7570c28 --- /dev/null +++ b/backlog/tasks/task-36 - Remove-optional-helper-undefined-args.md @@ -0,0 +1,44 @@ +--- +id: TASK-36 +title: Remove optional helper undefined args +status: In Progress +assignee: [] +created_date: '2026-06-06 21:15' +updated_date: '2026-06-06 21:23' +labels: [] +dependencies: [] +priority: high +ordinal: 38000 +--- + +## Description + + +Fix remaining spec-review issues in auditGenerationAction by avoiding explicit undefined auditId and nested usage fields in helper call arguments. + + +## Acceptance Criteria + +- [x] #1 persistAuditStage callsites include auditId only by conditional spread +- [x] #2 recordOpenRouterUsage/recordAuditUsageEvent/capture helper callsites include optional auditId only by conditional spread +- [x] #3 stage usage helper args are built without explicit undefined token fields + + +## Implementation Plan + + +1. Inspect auditId and usage helper callsites +2. Add RED source contracts for optional auditId and nested usage args +3. Run focused test and record RED +4. Implement minimal conditional spreads and usage arg helper +5. Run focused tests green and full pnpm test if fast +6. Self-review scope and leave task In Progress + + +## Implementation Notes + + +RED: tsc passed. Focused node --test .test-output/tests/audit-generation-action-source.test.js failed as expected on persistAuditStage auditId callsites, helper auditId callsites, and inline nested usage objects. + +GREEN: pnpm exec tsc -p tsconfig.test.json passed. Focused node --test .test-output/tests/audit-generation-action-source.test.js passed 24/24. Full pnpm test passed 356/356. Implemented conditional auditId spreads at persist/helper callsites and stage usage builder for callsite usage objects. + diff --git a/backlog/tasks/task-37 - Prioritize-v3-local-audit-skills.md b/backlog/tasks/task-37 - Prioritize-v3-local-audit-skills.md new file mode 100644 index 0000000..fa79d68 --- /dev/null +++ b/backlog/tasks/task-37 - Prioritize-v3-local-audit-skills.md @@ -0,0 +1,44 @@ +--- +id: TASK-37 +title: Prioritize v3 local audit skills +status: In Progress +assignee: [] +created_date: '2026-06-06 21:30' +updated_date: '2026-06-06 21:38' +labels: [] +dependencies: [] +priority: high +ordinal: 39000 +--- + +## Description + + +Add a deterministic local-audit relevance rule before the v3 skill selection cap so core applicable skills are not displaced by registry order. + + +## Acceptance Criteria + +- [x] #1 Full-evidence v3 selection includes local-seo-basics and performance-experience within the cap +- [x] #2 v3 input/applicability gating remains enforced +- [x] #3 Legacy category-based skill selection remains supported + + +## Implementation Plan + + +1. Inspect current v3 selection and existing audit-evidence tests +2. Add RED tests against real v2_elemente/skills.md for full-evidence core skill inclusion and missing-input gating +3. Run focused test and record RED +4. Implement minimal deterministic local-audit relevance sort before cap +5. Run focused tests green and full pnpm test if fast +6. Self-review scope and leave task In Progress + + +## Implementation Notes + + +RED: pnpm exec tsc -p tsconfig.test.json passed. Focused node --test .test-output/tests/audit-evidence.test.js failed as expected: full-evidence v3 selection returned registry-order ids visual-design, first-impression-clarity, contact-conversion, mobile-usability, trust-signals, conversion-copy instead of including local-seo-basics and performance-experience before the cap. + +GREEN: pnpm exec tsc -p tsconfig.test.json passed. Focused node --test .test-output/tests/audit-evidence.test.js passed 8/8. Full pnpm test passed 356/356. Added deterministic v3 local-audit priority before cap while preserving applicability/input gating and legacy category selection. + diff --git a/backlog/tasks/task-38 - Add-ScreenshotOne-missing-key-run-warning.md b/backlog/tasks/task-38 - Add-ScreenshotOne-missing-key-run-warning.md new file mode 100644 index 0000000..e4ecdbd --- /dev/null +++ b/backlog/tasks/task-38 - Add-ScreenshotOne-missing-key-run-warning.md @@ -0,0 +1,44 @@ +--- +id: TASK-38 +title: Add ScreenshotOne missing-key run warning +status: In Progress +assignee: [] +created_date: '2026-06-06 21:41' +updated_date: '2026-06-06 21:46' +labels: [] +dependencies: [] +priority: high +ordinal: 40000 +--- + +## Description + + +Emit a best-effort warning run event when an external audit needs screenshots but SCREENSHOTONE_API_KEY is not configured, while keeping audit classification and AI stages running. + + +## Acceptance Criteria + +- [x] #1 needsScreenshots with missing SCREENSHOTONE_API_KEY writes a warning run event through appendRunEvent +- [x] #2 warning logging is best-effort and cannot fail the audit run +- [x] #3 needsScreenshots false does not emit the missing-key warning + + +## Implementation Plan + + +1. Inspect current ScreenshotOne skip path and source-contract style +2. Add RED source-contract for warning event and best-effort guard +3. Run focused test to capture RED +4. Implement minimal runtime warning inside needsScreenshots missing-key branch +5. Run focused tests green and broader tests if practical +6. Self-review and report without staging or commits + + +## Implementation Notes + + +RED verified: pnpm exec tsc -p tsconfig.test.json passed, then node --test .test-output/tests/external-audit-pipeline-source.test.js failed only on missing ScreenshotOne config warning message (actual index -1). + +GREEN verified: focused node --test .test-output/tests/external-audit-pipeline-source.test.js passed 11/11 after implementation. Full pnpm test passed 357/357 with exit 0. + diff --git a/backlog/tasks/task-39 - Secure-Convex-operator-APIs.md b/backlog/tasks/task-39 - Secure-Convex-operator-APIs.md new file mode 100644 index 0000000..1d2fdbc --- /dev/null +++ b/backlog/tasks/task-39 - Secure-Convex-operator-APIs.md @@ -0,0 +1,34 @@ +--- +id: TASK-39 +title: Secure Convex operator APIs +status: In Progress +assignee: [] +created_date: '2026-06-06 21:52' +updated_date: '2026-06-06 22:00' +labels: [] +dependencies: [] +priority: high +ordinal: 41000 +--- + +## Description + + +Guard non-public Convex audit, lead, and run APIs so sensitive operational data is not exposed or mutated without authentication while preserving internal pipeline calls. + + +## Acceptance Criteria + +- [x] #1 Audit admin reads and writes require operator auth while getPublicBySlug remains public +- [x] #2 Lead admin reads and review mutations require operator auth while internal audit-generation calls use internal functions +- [x] #3 Run admin reads/writes require operator auth while internal actions can append run events safely +- [x] #4 Source contracts and full tests pass + + +## Implementation Notes + + +Worker I audit slice: Added source-contract coverage for audit admin auth guards and preserved public getPublicBySlug. RED: node --test .test-output/tests/audits-auth-source.test.js failed on create missing requireOperator before ctx.db. GREEN: pnpm exec tsc -p tsconfig.test.json passed; node --test .test-output/tests/audits-auth-source.test.js passed (2/2). + +Worker J RED/GREEN: Added leads/runs source contracts; initial pnpm test failed on missing lead/run requireOperator guards and missing internal lead/run action refs. Implemented operator auth for public leads/runs APIs, added internal lead get/review update and run append event mutations, and switched auditGenerationAction/pageSpeedAction/websiteEnrichmentAction to internal refs. GREEN: pnpm test passed (363/363). Did not touch convex/audits.ts and did not stage/commit. + diff --git a/backlog/tasks/task-40 - Behebe-abschliessende-Lint-Blocker.md b/backlog/tasks/task-40 - Behebe-abschliessende-Lint-Blocker.md new file mode 100644 index 0000000..1e985a0 --- /dev/null +++ b/backlog/tasks/task-40 - Behebe-abschliessende-Lint-Blocker.md @@ -0,0 +1,40 @@ +--- +id: TASK-40 +title: Behebe abschliessende Lint-Blocker +status: In Progress +assignee: [] +created_date: '2026-06-06 22:10' +updated_date: '2026-06-06 22:15' +labels: [] +dependencies: [] +priority: high +ordinal: 42000 +--- + +## Description + + +Fix the final lint blockers after the v2 pipeline implementation without changing runtime behavior. Keep v2_elemente as planning/reference material unless production imports require otherwise. + + +## Acceptance Criteria + +- [x] #1 pnpm lint exits 0 or only documents unrelated pre-existing generated warnings with a scoped suppression decision +- [x] #2 pnpm test remains green +- [x] #3 git diff --check remains green + + +## Implementation Plan + + +1. Reproduce pnpm lint failures +2. Apply scoped minimal lint policy or test-file cleanup +3. Re-run pnpm lint, pnpm test, git diff --check +4. Leave task In Progress until Matthias confirms Done + + +## Implementation Notes + + +TASK-40 worker update: fixed final lint blockers by ignoring v2_elemente reference snippets in ESLint and removing an unused helper from tests/external-audit-pipeline-source.test.ts. Verification: pnpm lint exits 0 with only generated convex/betterAuth/_generated unused-disable warnings; pnpm test passes 363/363; git diff --check exits 0. Task intentionally left In Progress pending user confirmation. + diff --git a/backlog/tasks/task-41 - Repariere-Convex-Typecheck-fuer-Usage-Events.md b/backlog/tasks/task-41 - Repariere-Convex-Typecheck-fuer-Usage-Events.md new file mode 100644 index 0000000..b89d7e2 --- /dev/null +++ b/backlog/tasks/task-41 - Repariere-Convex-Typecheck-fuer-Usage-Events.md @@ -0,0 +1,47 @@ +--- +id: TASK-41 +title: Repariere Convex-Typecheck fuer Usage Events +status: In Progress +assignee: [] +created_date: '2026-06-06 22:13' +updated_date: '2026-06-06 22:16' +labels: [] +dependencies: [] +priority: high +ordinal: 43000 +--- + +## Description + + +Fix final Convex typecheck blockers after adding usageEvents and external screenshot persistence. This includes updating generated Convex API references if required and making screenshot blob storage type-valid without changing runtime behavior. + + +## Acceptance Criteria + +- [x] #1 pnpm exec convex codegen --dry-run --typecheck enable exits 0 +- [x] #2 pnpm exec tsc --noEmit exits 0 or reports only documented unrelated pre-existing issues +- [x] #3 pnpm test remains green + + +## Implementation Plan + + +1. Reproduce Convex typecheck/codegen failures +2. Regenerate Convex API if required +3. Fix screenshot Blob typing with minimal runtime-neutral change +4. Re-run Convex typecheck, tsc, pnpm test +5. Leave task In Progress until Matthias confirms Done + + +## Implementation Notes + + +Verification/results: +- Reproduced with `pnpm exec convex codegen --dry-run --typecheck enable` outside sandbox after pnpm sandbox DB failure; initial result failed with TS2339 `internal.usageEvents` missing and TS2322 `Uint8Array` not assignable to `BlobPart` in convex/auditGenerationAction.ts. +- Ran `pnpm exec convex codegen` outside sandbox; generated convex/_generated/api.d.ts now includes usageEvents. +- Applied minimal ownership-scoped Blob typing fix in convex/auditGenerationAction.ts by wrapping screenshotBytes with `new Uint8Array(screenshotBytes)` before Blob storage. +- `pnpm exec convex codegen --dry-run --typecheck enable` exits 0. +- `pnpm exec tsc --noEmit` exits 2 only because of unrelated pre-existing v2_elemente/* errors (missing local generated modules/imports and implicit any issues); no TASK-41/convex/auditGenerationAction.ts errors remain. Per user instruction, v2_elemente fixes were not touched. +- `pnpm test` exits 0: 363 tests passed, 0 failed. + diff --git a/backlog/tasks/task-42 - Scope-v2-Referenzdateien-aus-dem-Typecheck.md b/backlog/tasks/task-42 - Scope-v2-Referenzdateien-aus-dem-Typecheck.md new file mode 100644 index 0000000..3e08cc7 --- /dev/null +++ b/backlog/tasks/task-42 - Scope-v2-Referenzdateien-aus-dem-Typecheck.md @@ -0,0 +1,43 @@ +--- +id: TASK-42 +title: Scope v2 Referenzdateien aus dem Typecheck +status: In Progress +assignee: [] +created_date: '2026-06-06 22:16' +updated_date: '2026-06-06 22:18' +labels: [] +dependencies: [] +priority: high +ordinal: 44000 +--- + +## Description + + +Keep v2_elemente as PRD/reference snippets while ensuring the production TypeScript check is not broken by those exploratory files. + + +## Acceptance Criteria + +- [x] #1 pnpm exec tsc --noEmit exits 0 +- [x] #2 pnpm lint remains green +- [x] #3 pnpm test remains green +- [x] #4 v2_elemente content remains available as planning/reference material + + +## Implementation Plan + + +1. Reproduce tsc failures from v2_elemente snippets +2. Apply minimal production TypeScript scope fix +3. Re-run tsc, lint, tests, diff check +4. Leave task In Progress until Matthias confirms Done + + +## Implementation Notes + + +Reproduced pnpm exec tsc --noEmit failure: production tsconfig includes v2_elemente reference snippets via **/*.ts, while eslint already scopes them out as non-runtime material. + +Applied minimal scope fix: tsconfig.json now excludes v2_elemente/** from the production TypeScript program, matching the existing ESLint ignore for reference snippets. Verification passed: pnpm exec tsc --noEmit (exit 0), pnpm lint (exit 0 with two existing generated-file warnings), pnpm test (exit 0, 363 tests passed), git diff --check (exit 0). v2_elemente contents were not edited. + diff --git a/convex/_generated/api.d.ts b/convex/_generated/api.d.ts index ff9c5fc..2d6f048 100644 --- a/convex/_generated/api.d.ts +++ b/convex/_generated/api.d.ts @@ -28,6 +28,7 @@ import type * as runs from "../runs.js"; import type * as scheduledJobs from "../scheduledJobs.js"; import type * as settings from "../settings.js"; import type * as storage from "../storage.js"; +import type * as usageEvents from "../usageEvents.js"; import type * as websiteEnrichment from "../websiteEnrichment.js"; import type * as websiteEnrichmentAction from "../websiteEnrichmentAction.js"; @@ -58,6 +59,7 @@ declare const fullApi: ApiFromModules<{ scheduledJobs: typeof scheduledJobs; settings: typeof settings; storage: typeof storage; + usageEvents: typeof usageEvents; websiteEnrichment: typeof websiteEnrichment; websiteEnrichmentAction: typeof websiteEnrichmentAction; }>; diff --git a/convex/auditGeneration.ts b/convex/auditGeneration.ts index 25d8184..a613912 100644 --- a/convex/auditGeneration.ts +++ b/convex/auditGeneration.ts @@ -89,6 +89,7 @@ type AuditGenerationEvidence = { technicalChecks: AuditGenerationEvidenceTechnicalCheck[]; screenshots: AuditGenerationEvidenceScreenshot[]; pageSpeedInputs: PageSpeedMinimalAuditResult[]; + externalMarkdown?: string; }; function byteLength(value: string) { @@ -199,6 +200,8 @@ const secretHints = [ "SMTP_USER", "BETTER_AUTH_SECRET", "RYBBIT_API_KEY", + "SCREENSHOTONE_API_KEY", + "JINA_API_KEY", ]; function sanitizeSecretCandidates(value: string | undefined): string | undefined { @@ -226,7 +229,7 @@ function sanitizeSecretCandidates(value: string | undefined): string | undefined } function escapeRegExp(value: string) { - return value.replace(/[.*+?^${}()|[\\]\\]/g, "\\$&"); + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); } type StartLeadSnapshot = Pick< @@ -549,6 +552,35 @@ export const persistAuditGenerationResult = internalMutation({ }, }); +export const persistExternalCaptureScreenshot = internalMutation({ + args: { + leadId: v.id("leads"), + runId: v.id("agentRuns"), + storageId: v.id("_storage"), + viewport: v.union(v.literal("desktop"), v.literal("mobile")), + sourceUrl: v.string(), + capturedAt: v.number(), + width: v.number(), + height: v.number(), + mimeType: v.string(), + }, + returns: v.id("websiteCrawlScreenshots"), + handler: async (ctx, args): Promise> => { + return await ctx.db.insert("websiteCrawlScreenshots", { + leadId: args.leadId, + runId: args.runId, + storageId: args.storageId, + viewport: args.viewport, + sourceUrl: args.sourceUrl, + capturedAt: args.capturedAt, + width: args.width, + height: args.height, + mimeType: args.mimeType, + createdAt: Date.now(), + }); + }, +}); + export const finishAuditGenerationRun = internalMutation({ args: { runId: v.id("agentRuns"), diff --git a/convex/audits.ts b/convex/audits.ts index 1fa7f98..5ea3282 100644 --- a/convex/audits.ts +++ b/convex/audits.ts @@ -15,8 +15,9 @@ const auditStatus = v.union( ); const usedSkillsValidator = v.array( v.object({ + id: v.optional(v.string()), name: v.string(), - category: v.string(), + category: v.optional(v.string()), version: v.optional(v.string()), source: v.optional(v.string()), }), @@ -179,6 +180,8 @@ export const create = mutation({ ctaType: v.optional(v.string()), }, handler: async (ctx, args) => { + await requireOperator(ctx); + const now = Date.now(); const existing = await ctx.db .query("audits") @@ -201,6 +204,8 @@ export const create = mutation({ export const getDetail = query({ args: { id: v.id("audits") }, handler: async (ctx, args) => { + await requireOperator(ctx); + const audit = await ctx.db.get(args.id); if (!audit) { return null; @@ -214,6 +219,8 @@ export const getDetail = query({ export const get = query({ args: { id: v.id("audits") }, handler: async (ctx, args) => { + await requireOperator(ctx); + return await ctx.db.get(args.id); }, }); @@ -302,6 +309,8 @@ export const upsertFromAuditGeneration = internalMutation({ export const getBySlug = query({ args: { slug: v.string() }, handler: async (ctx, args) => { + await requireOperator(ctx); + const audits = await ctx.db .query("audits") .withIndex("by_slug", (q) => q.eq("slug", args.slug)) @@ -496,6 +505,8 @@ export const list = query({ limit: v.optional(v.number()), }, handler: async (ctx, args) => { + await requireOperator(ctx); + const limit = normalizeListLimit(args.limit); if (args.leadId) { diff --git a/convex/domain.ts b/convex/domain.ts index f5b835b..b8f6e96 100644 --- a/convex/domain.ts +++ b/convex/domain.ts @@ -119,6 +119,18 @@ export const PAGE_SPEED_ERROR_TYPES = [ "api_error", "unknown", ] as const; +export const USAGE_EVENT_PROVIDERS = [ + "openrouter", + "screenshotone", + "jina", + "pagespeed", + "google_places", +] as const; +export const USAGE_EVENT_OPERATIONS = [ + "audit_capture", + "audit_generation", + "lead_lookup", +] as const; export type CampaignStatus = (typeof CAMPAIGN_STATUSES)[number]; export type LeadPriority = (typeof LEAD_PRIORITIES)[number]; @@ -143,6 +155,8 @@ export type ScreenshotViewport = (typeof SCREENSHOT_VIEWPORTS)[number]; export type PageSpeedStrategy = (typeof PAGE_SPEED_STRATEGIES)[number]; export type PageSpeedResultStatus = (typeof PAGE_SPEED_RESULT_STATUSES)[number]; export type PageSpeedErrorType = (typeof PAGE_SPEED_ERROR_TYPES)[number]; +export type UsageEventProvider = (typeof USAGE_EVENT_PROVIDERS)[number]; +export type UsageEventOperation = (typeof USAGE_EVENT_OPERATIONS)[number]; export type SettingsRow = { key: string; diff --git a/convex/leads.ts b/convex/leads.ts index 11a5367..582a2bc 100644 --- a/convex/leads.ts +++ b/convex/leads.ts @@ -3,7 +3,13 @@ import { v } from "convex/values"; import { getUsableContactEmailFromEntries } from "../lib/lead-discovery-google"; import { normalizeListLimit } from "./domain"; import type { Doc, Id } from "./_generated/dataModel"; -import { mutation, query } from "./_generated/server"; +import { + internalMutation, + internalQuery, + mutation, + query, +} from "./_generated/server"; +import type { MutationCtx, QueryCtx } from "./_generated/server"; type LeadDoc = Doc<"leads">; @@ -37,6 +43,74 @@ type LeadReviewPatch = { contactPerson?: string; }; +type LeadReviewUpdateArgs = { + id: Id<"leads">; + priority?: LeadDoc["priority"]; + priorityReason?: string; + contactStatus?: LeadDoc["contactStatus"]; + contactStatusReason?: string; + notes?: string; + duplicateStatus?: LeadDoc["duplicateStatus"]; + duplicateReason?: string; + blacklistStatus?: LeadDoc["blacklistStatus"]; + blacklistReason?: string; + duplicateOfLeadId?: Id<"leads">; + applyBlacklist?: boolean; + reviewEmail?: string; + reviewEmailSource?: string; + reviewContactPerson?: string; + reviewIsBusinessContactAddress?: boolean; +}; + +const leadPriority = v.union( + v.literal("high"), + v.literal("medium"), + v.literal("low"), + v.literal("defer"), + v.literal("blocked"), +); +const leadContactStatus = v.union( + v.literal("new"), + v.literal("missing_contact"), + v.literal("audit_ready"), + v.literal("outreach_ready"), + v.literal("contacted"), + v.literal("replied"), + v.literal("do_not_contact"), +); +const leadDuplicateStatus = v.union( + v.literal("unchecked"), + v.literal("unique"), + v.literal("possible_duplicate"), + v.literal("duplicate"), +); +const leadBlacklistStatus = v.union(v.literal("clear"), v.literal("blocked")); +const reviewUpdateArgs = { + id: v.id("leads"), + priority: v.optional(leadPriority), + priorityReason: v.optional(v.string()), + contactStatus: v.optional(leadContactStatus), + contactStatusReason: v.optional(v.string()), + notes: v.optional(v.string()), + duplicateStatus: v.optional(leadDuplicateStatus), + duplicateReason: v.optional(v.string()), + blacklistStatus: v.optional(leadBlacklistStatus), + blacklistReason: v.optional(v.string()), + duplicateOfLeadId: v.optional(v.id("leads")), + applyBlacklist: v.optional(v.boolean()), + reviewEmail: v.optional(v.string()), + reviewEmailSource: v.optional(v.string()), + reviewContactPerson: v.optional(v.string()), + reviewIsBusinessContactAddress: v.optional(v.boolean()), +}; + +const requireOperator = async (ctx: MutationCtx | QueryCtx) => { + const identity = await ctx.auth.getUserIdentity(); + if (!identity) { + throw new Error("Nicht autorisiert."); + } +}; + function buildReviewContactPatch(args: { email?: string; emailSource?: string; @@ -88,6 +162,91 @@ function buildReviewContactPatch(args: { }); } +async function reviewUpdateLead(ctx: MutationCtx, args: LeadReviewUpdateArgs) { + const lead = await ctx.db.get(args.id); + + if (!lead) { + return null; + } + + const now = Date.now(); + const patch: LeadReviewPatch = { + updatedAt: now, + }; + + if (args.priority !== undefined) { + patch.priority = args.priority; + } + if (args.priorityReason !== undefined) { + patch.priorityReason = args.priorityReason; + } + if (args.contactStatus !== undefined) { + patch.contactStatus = args.contactStatus; + } + if (args.contactStatusReason !== undefined) { + patch.contactStatusReason = args.contactStatusReason; + } + if (args.notes !== undefined) { + patch.notes = args.notes; + } + if (args.duplicateStatus !== undefined) { + patch.duplicateStatus = args.duplicateStatus; + } + if (args.duplicateReason !== undefined) { + patch.duplicateReason = args.duplicateReason; + } + if (args.duplicateOfLeadId !== undefined) { + patch.duplicateOfLeadId = args.duplicateOfLeadId; + } + + if (args.applyBlacklist) { + patch.blacklistStatus = "blocked"; + if (args.blacklistReason !== undefined) { + patch.blacklistReason = args.blacklistReason; + } else if (lead.blacklistReason === undefined) { + patch.blacklistReason = "Manuell in der Review als Sperrgrund gesetzt."; + } + if (args.priority === undefined || args.priority !== "blocked") { + patch.priority = "blocked"; + } + } else if (args.applyBlacklist === false && args.blacklistStatus !== undefined) { + patch.blacklistStatus = args.blacklistStatus; + patch.blacklistReason = args.blacklistReason; + } else if (args.blacklistStatus !== undefined) { + patch.blacklistStatus = args.blacklistStatus; + patch.blacklistReason = args.blacklistReason; + } + + const reviewContactPatch = buildReviewContactPatch({ + email: args.reviewEmail, + emailSource: args.reviewEmailSource, + contactPerson: args.reviewContactPerson, + isBusinessContactAddress: args.reviewIsBusinessContactAddress, + explicitContactStatus: args.contactStatus !== undefined, + currentContactStatus: lead.contactStatus, + }); + + if (reviewContactPatch?.patch) { + Object.assign(patch, reviewContactPatch.patch); + } + + if ( + reviewContactPatch !== null && + reviewContactPatch.setContactStatus !== undefined && + args.contactStatus === undefined + ) { + patch.contactStatus = reviewContactPatch.setContactStatus; + } + + if (args.blacklistReason !== undefined && patch.blacklistStatus === undefined) { + patch.blacklistStatus = "blocked"; + patch.blacklistReason = args.blacklistReason; + } + + await ctx.db.patch(args.id, patch); + return args.id; +} + export const create = mutation({ args: { campaignId: v.optional(v.id("campaigns")), @@ -116,44 +275,20 @@ export const create = mutation({ email: v.optional(v.string()), emailSource: v.optional(v.string()), contactPerson: v.optional(v.string()), - priority: v.optional( - v.union( - v.literal("high"), - v.literal("medium"), - v.literal("low"), - v.literal("defer"), - v.literal("blocked"), - ), - ), + priority: v.optional(leadPriority), priorityReason: v.optional(v.string()), - contactStatus: v.optional( - v.union( - v.literal("new"), - v.literal("missing_contact"), - v.literal("audit_ready"), - v.literal("outreach_ready"), - v.literal("contacted"), - v.literal("replied"), - v.literal("do_not_contact"), - ), - ), + contactStatus: v.optional(leadContactStatus), contactStatusReason: v.optional(v.string()), - duplicateStatus: v.optional( - v.union( - v.literal("unchecked"), - v.literal("unique"), - v.literal("possible_duplicate"), - v.literal("duplicate"), - ), - ), + duplicateStatus: v.optional(leadDuplicateStatus), duplicateReason: v.optional(v.string()), blacklistReason: v.optional(v.string()), duplicateOfLeadId: v.optional(v.id("leads")), - blacklistStatus: v.optional(v.union(v.literal("clear"), v.literal("blocked"))), + blacklistStatus: v.optional(leadBlacklistStatus), normalizedGooglePlaceId: v.optional(v.string()), notes: v.optional(v.string()), }, handler: async (ctx, args) => { + await requireOperator(ctx); const now = Date.now(); return await ctx.db.insert("leads", { @@ -174,136 +309,29 @@ export const create = mutation({ }); export const reviewUpdate = mutation({ - args: { - id: v.id("leads"), - priority: v.optional( - v.union( - v.literal("high"), - v.literal("medium"), - v.literal("low"), - v.literal("defer"), - v.literal("blocked"), - ), - ), - priorityReason: v.optional(v.string()), - contactStatus: v.optional( - v.union( - v.literal("new"), - v.literal("missing_contact"), - v.literal("audit_ready"), - v.literal("outreach_ready"), - v.literal("contacted"), - v.literal("replied"), - v.literal("do_not_contact"), - ), - ), - contactStatusReason: v.optional(v.string()), - notes: v.optional(v.string()), - duplicateStatus: v.optional( - v.union( - v.literal("unchecked"), - v.literal("unique"), - v.literal("possible_duplicate"), - v.literal("duplicate"), - ), - ), - duplicateReason: v.optional(v.string()), - blacklistStatus: v.optional(v.union(v.literal("clear"), v.literal("blocked"))), - blacklistReason: v.optional(v.string()), - duplicateOfLeadId: v.optional(v.id("leads")), - applyBlacklist: v.optional(v.boolean()), - reviewEmail: v.optional(v.string()), - reviewEmailSource: v.optional(v.string()), - reviewContactPerson: v.optional(v.string()), - reviewIsBusinessContactAddress: v.optional(v.boolean()), - }, + args: reviewUpdateArgs, handler: async (ctx, args) => { - const lead = await ctx.db.get(args.id); + await requireOperator(ctx); + return await reviewUpdateLead(ctx, args); + }, +}); - if (!lead) { - return null; - } - - const now = Date.now(); - const patch: LeadReviewPatch = { - updatedAt: now, - }; - - if (args.priority !== undefined) { - patch.priority = args.priority; - } - if (args.priorityReason !== undefined) { - patch.priorityReason = args.priorityReason; - } - if (args.contactStatus !== undefined) { - patch.contactStatus = args.contactStatus; - } - if (args.contactStatusReason !== undefined) { - patch.contactStatusReason = args.contactStatusReason; - } - if (args.notes !== undefined) { - patch.notes = args.notes; - } - if (args.duplicateStatus !== undefined) { - patch.duplicateStatus = args.duplicateStatus; - } - if (args.duplicateReason !== undefined) { - patch.duplicateReason = args.duplicateReason; - } - if (args.duplicateOfLeadId !== undefined) { - patch.duplicateOfLeadId = args.duplicateOfLeadId; - } - - if (args.applyBlacklist) { - patch.blacklistStatus = "blocked"; - if (args.blacklistReason !== undefined) { - patch.blacklistReason = args.blacklistReason; - } else if (lead.blacklistReason === undefined) { - patch.blacklistReason = "Manuell in der Review als Sperrgrund gesetzt."; - } - if (args.priority === undefined || args.priority !== "blocked") { - patch.priority = "blocked"; - } - } else if (args.applyBlacklist === false && args.blacklistStatus !== undefined) { - patch.blacklistStatus = args.blacklistStatus; - patch.blacklistReason = args.blacklistReason; - } else if (args.blacklistStatus !== undefined) { - patch.blacklistStatus = args.blacklistStatus; - patch.blacklistReason = args.blacklistReason; - } - - const reviewContactPatch = buildReviewContactPatch({ - email: args.reviewEmail, - emailSource: args.reviewEmailSource, - contactPerson: args.reviewContactPerson, - isBusinessContactAddress: args.reviewIsBusinessContactAddress, - explicitContactStatus: args.contactStatus !== undefined, - currentContactStatus: lead.contactStatus, - }); - - if (reviewContactPatch?.patch) { - Object.assign(patch, reviewContactPatch.patch); - } - - if ( - reviewContactPatch !== null && - reviewContactPatch.setContactStatus !== undefined && - args.contactStatus === undefined - ) { - patch.contactStatus = reviewContactPatch.setContactStatus; - } - - if (args.blacklistReason !== undefined && patch.blacklistStatus === undefined) { - patch.blacklistStatus = "blocked"; - patch.blacklistReason = args.blacklistReason; - } - - await ctx.db.patch(args.id, patch); - return args.id; +export const reviewUpdateInternal = internalMutation({ + args: reviewUpdateArgs, + handler: async (ctx, args) => { + return await reviewUpdateLead(ctx, args); }, }); export const get = query({ + args: { id: v.id("leads") }, + handler: async (ctx, args) => { + await requireOperator(ctx); + return await ctx.db.get(args.id); + }, +}); + +export const getInternal = internalQuery({ args: { id: v.id("leads") }, handler: async (ctx, args) => { return await ctx.db.get(args.id); @@ -313,20 +341,11 @@ export const get = query({ export const list = query({ args: { campaignId: v.optional(v.id("campaigns")), - contactStatus: v.optional( - v.union( - v.literal("new"), - v.literal("missing_contact"), - v.literal("audit_ready"), - v.literal("outreach_ready"), - v.literal("contacted"), - v.literal("replied"), - v.literal("do_not_contact"), - ), - ), + contactStatus: v.optional(leadContactStatus), limit: v.optional(v.number()), }, handler: async (ctx, args) => { + await requireOperator(ctx); const limit = normalizeListLimit(args.limit); if (args.campaignId) { @@ -360,6 +379,7 @@ export const listFunnel = query({ limit: v.optional(v.number()), }, handler: async (ctx, args) => { + await requireOperator(ctx); const limit = normalizeListLimit(args.limit); const leads = await ctx.db.query("leads").order("desc").take(limit); diff --git a/convex/pageSpeedAction.ts b/convex/pageSpeedAction.ts index 2703e47..6c033b0 100644 --- a/convex/pageSpeedAction.ts +++ b/convex/pageSpeedAction.ts @@ -1,6 +1,6 @@ "use node"; -import { api, internal } from "./_generated/api"; +import { internal } from "./_generated/api"; import { internalAction } from "./_generated/server"; import type { Id } from "./_generated/dataModel"; import type { ActionCtx } from "./_generated/server"; @@ -122,7 +122,7 @@ async function queueAuditGenerationAfterPageSpeed( parentRunId: runId, }); } catch (auditQueueError) { - await ctx.runMutation(api.runs.appendEvent, { + await ctx.runMutation(internal.runs.appendEventInternal, { runId, level: "warning", message: "Audit-Generierung konnte nicht in die Warteschlange gesetzt werden.", @@ -164,7 +164,7 @@ export const processPageSpeedAudit = internalAction({ errorSummary, }); - await ctx.runMutation(api.runs.appendEvent, { + await ctx.runMutation(internal.runs.appendEventInternal, { runId: args.runId, level: "error", message: "PageSpeed-Analyse fehlgeschlagen.", @@ -210,7 +210,7 @@ export const processPageSpeedAudit = internalAction({ fetchedAt, }); - await ctx.runMutation(api.runs.appendEvent, { + await ctx.runMutation(internal.runs.appendEventInternal, { runId: args.runId, level: "warning", message: `PageSpeed-Analyse für ${strategy} fehlgeschlagen.`, @@ -248,7 +248,7 @@ export const processPageSpeedAudit = internalAction({ normalized: toPersistedPageSpeedNormalizedResult(normalized), }); - await ctx.runMutation(api.runs.appendEvent, { + await ctx.runMutation(internal.runs.appendEventInternal, { runId: args.runId, level: "info", message: `PageSpeed-Analyse für ${strategy} abgeschlossen.`, @@ -274,7 +274,7 @@ export const processPageSpeedAudit = internalAction({ fetchedAt, }); - await ctx.runMutation(api.runs.appendEvent, { + await ctx.runMutation(internal.runs.appendEventInternal, { runId: args.runId, level: "warning", message: `PageSpeed-Analyse für ${strategy} fehlgeschlagen.`, @@ -310,7 +310,7 @@ export const processPageSpeedAudit = internalAction({ errorSummary, }); - await ctx.runMutation(api.runs.appendEvent, { + await ctx.runMutation(internal.runs.appendEventInternal, { runId: args.runId, level: "error", message: "PageSpeed-Analyse fehlgeschlagen.", diff --git a/convex/runs.ts b/convex/runs.ts index 91f8f8e..e664631 100644 --- a/convex/runs.ts +++ b/convex/runs.ts @@ -6,13 +6,53 @@ import { RUN_TYPES, normalizeListLimit, } from "./domain"; -import { mutation, query } from "./_generated/server"; +import type { Id } from "./_generated/dataModel"; +import { internalMutation, mutation, query } from "./_generated/server"; +import type { MutationCtx, QueryCtx } from "./_generated/server"; const runType = v.union(...RUN_TYPES.map((type) => v.literal(type))); const runStatus = v.union(...RUN_STATUSES.map((status) => v.literal(status))); const eventLevel = v.union( ...RUN_EVENT_LEVELS.map((level) => v.literal(level)), ); +const appendEventArgs = { + runId: v.id("agentRuns"), + level: eventLevel, + message: v.string(), + details: v.optional( + v.array( + v.object({ + label: v.string(), + value: v.string(), + source: v.optional(v.string()), + }), + ), + ), +}; + +type AppendEventArgs = { + runId: Id<"agentRuns">; + level: (typeof RUN_EVENT_LEVELS)[number]; + message: string; + details?: { label: string; value: string; source?: string }[]; +}; + +const requireOperator = async (ctx: MutationCtx | QueryCtx) => { + const identity = await ctx.auth.getUserIdentity(); + if (!identity) { + throw new Error("Nicht autorisiert."); + } +}; + +async function appendRunEvent( + ctx: MutationCtx, + args: AppendEventArgs, +) { + return await ctx.db.insert("agentRunEvents", { + ...args, + createdAt: Date.now(), + }); +} export const create = mutation({ args: { @@ -24,6 +64,7 @@ export const create = mutation({ currentStep: v.optional(v.string()), }, handler: async (ctx, args) => { + await requireOperator(ctx); const now = Date.now(); return await ctx.db.insert("agentRuns", { @@ -50,6 +91,7 @@ export const updateStatus = mutation({ errorSummary: v.optional(v.string()), }, handler: async (ctx, args) => { + await requireOperator(ctx); const now = Date.now(); const patch: { status: typeof args.status; @@ -92,6 +134,7 @@ export const list = query({ limit: v.optional(v.number()), }, handler: async (ctx, args) => { + await requireOperator(ctx); const limit = normalizeListLimit(args.limit); if (args.type && args.status) { @@ -132,25 +175,17 @@ export const list = query({ }); export const appendEvent = mutation({ - args: { - runId: v.id("agentRuns"), - level: eventLevel, - message: v.string(), - details: v.optional( - v.array( - v.object({ - label: v.string(), - value: v.string(), - source: v.optional(v.string()), - }), - ), - ), - }, + args: appendEventArgs, handler: async (ctx, args) => { - return await ctx.db.insert("agentRunEvents", { - ...args, - createdAt: Date.now(), - }); + await requireOperator(ctx); + return await appendRunEvent(ctx, args); + }, +}); + +export const appendEventInternal = internalMutation({ + args: appendEventArgs, + handler: async (ctx, args) => { + return await appendRunEvent(ctx, args); }, }); @@ -160,6 +195,7 @@ export const listEvents = query({ limit: v.optional(v.number()), }, handler: async (ctx, args) => { + await requireOperator(ctx); const limit = normalizeListLimit(args.limit); return await ctx.db diff --git a/convex/schema.ts b/convex/schema.ts index 13921f6..2dd8bbf 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -7,6 +7,8 @@ import { RUN_EVENT_LEVELS, RUN_STATUSES, RUN_TYPES, + USAGE_EVENT_OPERATIONS, + USAGE_EVENT_PROVIDERS, } from "./domain"; const campaignStatus = v.union(v.literal("active"), v.literal("paused")); @@ -146,6 +148,12 @@ const pageSpeedErrorType = v.union( v.literal("api_error"), v.literal("unknown"), ); +const usageEventProvider = v.union( + ...USAGE_EVENT_PROVIDERS.map((provider) => v.literal(provider)), +); +const usageEventOperation = v.union( + ...USAGE_EVENT_OPERATIONS.map((operation) => v.literal(operation)), +); const settingsValue = v.union(v.string(), v.number(), v.boolean(), v.null()); const auditMetricSummary = v.object({ performanceScore: v.optional(v.number()), @@ -282,8 +290,9 @@ export default defineSchema({ usedSkills: v.optional( v.array( v.object({ + id: v.optional(v.string()), name: v.string(), - category: v.string(), + category: v.optional(v.string()), version: v.optional(v.string()), source: v.optional(v.string()), }), @@ -399,6 +408,39 @@ export default defineSchema({ .index("by_stage", ["stage"]) .index("by_leadId_and_stage", ["leadId", "stage"]), + usageEvents: defineTable({ + provider: usageEventProvider, + operation: usageEventOperation, + runId: v.optional(v.id("agentRuns")), + leadId: v.optional(v.id("leads")), + auditId: v.optional(v.id("audits")), + estimatedCostUsd: v.number(), + tokens: v.optional( + v.object({ + inputTokens: v.optional(v.number()), + outputTokens: v.optional(v.number()), + promptTokens: v.optional(v.number()), + completionTokens: v.optional(v.number()), + totalTokens: v.optional(v.number()), + cacheReadTokens: v.optional(v.number()), + }), + ), + callCounts: v.optional( + v.object({ + requests: v.optional(v.number()), + pages: v.optional(v.number()), + screenshots: v.optional(v.number()), + lookups: v.optional(v.number()), + }), + ), + createdAt: v.number(), + }) + .index("by_runId_and_createdAt", ["runId", "createdAt"]) + .index("by_leadId_and_createdAt", ["leadId", "createdAt"]) + .index("by_auditId_and_createdAt", ["auditId", "createdAt"]) + .index("by_provider_and_createdAt", ["provider", "createdAt"]) + .index("by_createdAt", ["createdAt"]), + websiteCrawlPages: defineTable({ leadId: v.id("leads"), runId: v.optional(v.id("agentRuns")), diff --git a/convex/usageEvents.ts b/convex/usageEvents.ts new file mode 100644 index 0000000..c83540e --- /dev/null +++ b/convex/usageEvents.ts @@ -0,0 +1,223 @@ +import type { Doc, Id } from "./_generated/dataModel"; +import { internalMutation, query } from "./_generated/server"; +import type { QueryCtx } from "./_generated/server"; +import { + normalizeListLimit, + USAGE_EVENT_OPERATIONS, + USAGE_EVENT_PROVIDERS, +} from "./domain"; +import { v } from "convex/values"; + +const usageEventProvider = v.union( + ...USAGE_EVENT_PROVIDERS.map((provider) => v.literal(provider)), +); +const usageEventOperation = v.union( + ...USAGE_EVENT_OPERATIONS.map((operation) => v.literal(operation)), +); +const usageEventTokens = v.object({ + inputTokens: v.optional(v.number()), + outputTokens: v.optional(v.number()), + promptTokens: v.optional(v.number()), + completionTokens: v.optional(v.number()), + totalTokens: v.optional(v.number()), + cacheReadTokens: v.optional(v.number()), +}); +const usageEventCallCounts = v.object({ + requests: v.optional(v.number()), + pages: v.optional(v.number()), + screenshots: v.optional(v.number()), + lookups: v.optional(v.number()), +}); +const usageEventDoc = v.object({ + _id: v.id("usageEvents"), + _creationTime: v.number(), + provider: usageEventProvider, + operation: usageEventOperation, + runId: v.optional(v.id("agentRuns")), + leadId: v.optional(v.id("leads")), + auditId: v.optional(v.id("audits")), + estimatedCostUsd: v.number(), + tokens: v.optional(usageEventTokens), + callCounts: v.optional(usageEventCallCounts), + createdAt: v.number(), +}); + +type UsageEventTokens = { + inputTokens?: number; + outputTokens?: number; + promptTokens?: number; + completionTokens?: number; + totalTokens?: number; + cacheReadTokens?: number; +}; + +type UsageEventCallCounts = { + requests?: number; + pages?: number; + screenshots?: number; + lookups?: number; +}; + +type UsageEventNumberArgs = { + estimatedCostUsd: number; + tokens?: UsageEventTokens; + callCounts?: UsageEventCallCounts; +}; + +const requireOperator = async (ctx: QueryCtx) => { + const identity = await ctx.auth.getUserIdentity(); + if (!identity) { + throw new Error("Nicht autorisiert."); + } +}; + +function assertFiniteNonNegativeNumber(value: number, fieldName: string) { + if (!Number.isFinite(value) || value < 0) { + throw new Error(`${fieldName} must be a finite non-negative number.`); + } +} + +function assertFiniteNonNegativeInteger( + value: number | undefined, + fieldName: string, +) { + if (value === undefined) { + return; + } + + if (!Number.isFinite(value) || value < 0 || !Number.isInteger(value)) { + throw new Error(`${fieldName} must be a finite non-negative integer.`); + } +} + +function assertValidUsageEventNumbers(args: UsageEventNumberArgs) { + assertFiniteNonNegativeNumber(args.estimatedCostUsd, "estimatedCostUsd"); + assertFiniteNonNegativeInteger(args.tokens?.inputTokens, "tokens.inputTokens"); + assertFiniteNonNegativeInteger(args.tokens?.outputTokens, "tokens.outputTokens"); + assertFiniteNonNegativeInteger(args.tokens?.promptTokens, "tokens.promptTokens"); + assertFiniteNonNegativeInteger(args.tokens?.completionTokens, "tokens.completionTokens"); + assertFiniteNonNegativeInteger(args.tokens?.totalTokens, "tokens.totalTokens"); + assertFiniteNonNegativeInteger(args.tokens?.cacheReadTokens, "tokens.cacheReadTokens"); + assertFiniteNonNegativeInteger(args.callCounts?.requests, "callCounts.requests"); + assertFiniteNonNegativeInteger(args.callCounts?.pages, "callCounts.pages"); + assertFiniteNonNegativeInteger(args.callCounts?.screenshots, "callCounts.screenshots"); + assertFiniteNonNegativeInteger(args.callCounts?.lookups, "callCounts.lookups"); +} + +export const recordUsageEvent = internalMutation({ + args: { + provider: usageEventProvider, + operation: usageEventOperation, + runId: v.optional(v.id("agentRuns")), + leadId: v.optional(v.id("leads")), + auditId: v.optional(v.id("audits")), + estimatedCostUsd: v.number(), + tokens: v.optional(usageEventTokens), + callCounts: v.optional(usageEventCallCounts), + createdAt: v.optional(v.number()), + }, + returns: v.id("usageEvents"), + handler: async (ctx, args): Promise> => { + assertValidUsageEventNumbers(args); + + const now = args.createdAt ?? Date.now(); + + return await ctx.db.insert("usageEvents", { + provider: args.provider, + operation: args.operation, + ...(args.runId ? { runId: args.runId } : {}), + ...(args.leadId ? { leadId: args.leadId } : {}), + ...(args.auditId ? { auditId: args.auditId } : {}), + estimatedCostUsd: args.estimatedCostUsd, + ...(args.tokens ? { tokens: args.tokens } : {}), + ...(args.callCounts ? { callCounts: args.callCounts } : {}), + createdAt: now, + }); + }, +}); + +export const listLatestUsageEvents = query({ + args: { + limit: v.optional(v.number()), + }, + returns: v.array(usageEventDoc), + handler: async (ctx, args): Promise[]> => { + await requireOperator(ctx); + + return await ctx.db + .query("usageEvents") + .withIndex("by_createdAt") + .order("desc") + .take(normalizeListLimit(args.limit)); + }, +}); + +export const listUsageEventsByRun = query({ + args: { + runId: v.id("agentRuns"), + limit: v.optional(v.number()), + }, + returns: v.array(usageEventDoc), + handler: async (ctx, args): Promise[]> => { + await requireOperator(ctx); + + return await ctx.db + .query("usageEvents") + .withIndex("by_runId_and_createdAt", (q) => q.eq("runId", args.runId)) + .order("desc") + .take(normalizeListLimit(args.limit)); + }, +}); + +export const listUsageEventsByLead = query({ + args: { + leadId: v.id("leads"), + limit: v.optional(v.number()), + }, + returns: v.array(usageEventDoc), + handler: async (ctx, args): Promise[]> => { + await requireOperator(ctx); + + return await ctx.db + .query("usageEvents") + .withIndex("by_leadId_and_createdAt", (q) => q.eq("leadId", args.leadId)) + .order("desc") + .take(normalizeListLimit(args.limit)); + }, +}); + +export const listUsageEventsByAudit = query({ + args: { + auditId: v.id("audits"), + limit: v.optional(v.number()), + }, + returns: v.array(usageEventDoc), + handler: async (ctx, args): Promise[]> => { + await requireOperator(ctx); + + return await ctx.db + .query("usageEvents") + .withIndex("by_auditId_and_createdAt", (q) => q.eq("auditId", args.auditId)) + .order("desc") + .take(normalizeListLimit(args.limit)); + }, +}); + +export const listUsageEventsByProvider = query({ + args: { + provider: usageEventProvider, + limit: v.optional(v.number()), + }, + returns: v.array(usageEventDoc), + handler: async (ctx, args): Promise[]> => { + await requireOperator(ctx); + + return await ctx.db + .query("usageEvents") + .withIndex("by_provider_and_createdAt", (q) => + q.eq("provider", args.provider), + ) + .order("desc") + .take(normalizeListLimit(args.limit)); + }, +}); diff --git a/docs/verification.md b/docs/verification.md index 774b6e9..fa0d91e 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -16,9 +16,16 @@ Diese Checkliste ist die wiederholbare manuelle Prüfung für die kritischen MVP ## Audit-Generierung -1. Lead mit Website durch Enrichment/PageSpeed laufen lassen. -2. Prüfen, dass PageSpeed-Erfolg oder -Fehler Audit-Generierung queued. -3. Im Outreach Review Workspace prüfen, dass Audit-Text, Quellen und Skills sichtbar sind. +1. Lead mit Website durch externe Audit-Services laufen lassen. +2. Prüfen, dass Google, PageSpeed, OpenRouter und ScreenshotOne als serverseitig verwaltete Provider konfiguriert sind. +3. Prüfen, dass fehlendes Jina keine Blockade auslöst. +4. Im Outreach Review Workspace prüfen, dass Audit-Text, Quellen und Skills sichtbar sind. + +## Operations Readiness + +1. `audit.matthias-meister-webdesign.de` als persönlichen Deployment-Scope prüfen. +2. Sicherstellen, dass BYO-Keys, Billing und Teamrollen nicht als aktuelle Voraussetzungen angezeigt werden. +3. Sicherstellen, dass Playwright/TASK-8 nicht als Pflichtintegration für die neue externe Pipeline angezeigt wird. ## Freigabe diff --git a/eslint.config.mjs b/eslint.config.mjs index 1fbfc71..a7fbf8a 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -13,6 +13,8 @@ const eslintConfig = defineConfig([ "build/**", ".test-output/**", "convex/_generated/**", + // v2_elemente contains PRD/reference snippets, not runtime source. + "v2_elemente/**", "next-env.d.ts", ]), ]); diff --git a/lib/ai/audit-evidence.ts b/lib/ai/audit-evidence.ts index 02ae34c..c8cc53b 100644 --- a/lib/ai/audit-evidence.ts +++ b/lib/ai/audit-evidence.ts @@ -60,6 +60,7 @@ export type AuditEvidenceInput = { observedUxSignals: string[]; observedContentSignals: string[]; observedTechnicalSignals: string[]; + externalMarkdown?: string; screenshotReferences: Array<{ storageId: string; sourceUrl: string; @@ -80,6 +81,7 @@ export type AuditEvidenceInputArgs = { screenshots?: readonly AuditScreenshotEvidence[]; pageSpeedInputs?: readonly PageSpeedMinimalAuditResult[]; skillRegistry?: readonly SkillRegistryEntryEvidence[]; + externalMarkdown?: string; }; const COMPANY_CONTEXT_LIMIT = 8; @@ -90,6 +92,20 @@ const TECHNICAL_SIGNAL_LIMIT = 6; const PAGESPEED_SIGNAL_LIMIT = 8; const SCREENSHOT_REFERENCE_LIMIT = 8; const SELECTED_SKILLS_LIMIT = 6; +const EXTERNAL_MARKDOWN_LIMIT = 4_000; +const V3_LOCAL_AUDIT_PRIORITY = new Map( + [ + "visual-design", + "contact-conversion", + "local-seo-basics", + "performance-experience", + "mobile-usability", + "conversion-copy", + "first-impression-clarity", + "trust-signals", + "accessibility-basics", + ].map((id, index) => [id, index] as const), +); const URL_PATTERN = /\bhttps?:\/\/[^\s<>"']+/i; const JSON_BRACKET_PATTERN = /\{[^}]*\}|\[[^\]]*\]/; @@ -140,6 +156,19 @@ function sanitizeCustomerText(value: unknown, maxLength = 180): string { return text; } +function sanitizeExternalMarkdown(value: unknown): string | undefined { + if (typeof value !== "string") { + return undefined; + } + + const markdown = value.replace(/\s+/g, " ").trim(); + if (!markdown) { + return undefined; + } + + return markdown.slice(0, EXTERNAL_MARKDOWN_LIMIT); +} + function addUniqueCapped( bucket: string[], input: string, @@ -233,6 +262,77 @@ function selectTopSkill( return toAuditUsedSkill(scored[0]!.candidate); } +type SkillInputAvailability = { + websiteExists: boolean; + hasDesktopScreenshot: boolean; + hasMobileScreenshot: boolean; + hasMarkdown: boolean; + hasPageSpeed: boolean; + hasDom: boolean; +}; + +function hasRequiredV3Input(input: string, availability: SkillInputAvailability) { + switch (input) { + case "desktop_screenshot": + return availability.hasDesktopScreenshot; + case "mobile_screenshot": + return availability.hasMobileScreenshot; + case "markdown": + return availability.hasMarkdown; + case "pagespeed": + return availability.hasPageSpeed; + case "dom": + return availability.hasDom; + default: + return false; + } +} + +function v3SkillApplies( + skill: SkillRegistryEntryEvidence, + availability: SkillInputAvailability, +) { + const appliesWhen = skill.appliesWhen ?? "website_exists"; + const applies = + appliesWhen === "always" || + (appliesWhen === "website_exists" && availability.websiteExists) || + (appliesWhen === "has_mobile_screenshot" && + availability.hasMobileScreenshot) || + (appliesWhen === "has_pagespeed" && availability.hasPageSpeed); + + if (!applies) { + return false; + } + + return (skill.inputs ?? []).every((input) => + hasRequiredV3Input(input, availability), + ); +} + +function selectV3Skills( + skillRegistry: readonly SkillRegistryEntryEvidence[], + availability: SkillInputAvailability, +) { + return skillRegistry + .map((skill, registryIndex) => ({ skill, registryIndex })) + .filter(({ skill }) => skill.id && !skill.category) + .filter(({ skill }) => v3SkillApplies(skill, availability)) + .sort((a, b) => { + // Keep core local-audit coverage inside the cap; otherwise preserve registry order. + const aPriority = V3_LOCAL_AUDIT_PRIORITY.get(a.skill.id ?? ""); + const bPriority = V3_LOCAL_AUDIT_PRIORITY.get(b.skill.id ?? ""); + if (aPriority !== undefined || bPriority !== undefined) { + return ( + (aPriority ?? Number.POSITIVE_INFINITY) - + (bPriority ?? Number.POSITIVE_INFINITY) + ); + } + return a.registryIndex - b.registryIndex; + }) + .slice(0, SELECTED_SKILLS_LIMIT) + .map(({ skill }) => toAuditUsedSkill(skill)); +} + function buildObservedSignals( crawlPages: readonly AuditCrawlPageEvidence[], technicalChecks: readonly AuditTechnicalCheckEvidence[], @@ -403,8 +503,12 @@ function extractSkills( marketing: boolean; offer: boolean; }, + availability: SkillInputAvailability, ): AuditUsedSkill[] { - const selected: AuditUsedSkill[] = []; + const selected: AuditUsedSkill[] = selectV3Skills( + skillRegistry, + availability, + ); const categoryOrder = ["design", "ux", "copy", "seo", "marketing", "offer"] as const; const evidenceText = { design: @@ -450,6 +554,7 @@ export function buildAuditEvidenceInput( const screenshots = args.screenshots ?? []; const pageSpeedInputs = args.pageSpeedInputs ?? []; const skillRegistry = args.skillRegistry ?? []; + const externalMarkdown = sanitizeExternalMarkdown(args.externalMarkdown); const companyContext: string[] = []; const checkedPages: string[] = []; @@ -542,6 +647,26 @@ export function buildAuditEvidenceInput( ...signals.evidenceText, marketing: false, offer: false, + }, { + websiteExists: + Boolean(lead.websiteDomain || lead.websiteUrl) || + crawlPages.length > 0 || + screenshots.length > 0, + hasDesktopScreenshot: screenshots.some( + (screenshot) => screenshot.viewport === "desktop", + ), + hasMobileScreenshot: screenshots.some( + (screenshot) => screenshot.viewport === "mobile", + ), + hasMarkdown: + Boolean(externalMarkdown) || + crawlPages.some((page) => + Boolean(page.visibleText || page.visibleTextExcerpt), + ), + hasPageSpeed: + pageSpeedInputsOutput.customerImplications.length > 0 || + pageSpeedInputs.some((input) => input.status === "succeeded"), + hasDom: crawlPages.length > 0 || technicalChecks.length > 0, }); return { @@ -550,6 +675,7 @@ export function buildAuditEvidenceInput( observedUxSignals: signals.ux, observedContentSignals: signals.content, observedTechnicalSignals: signals.technical, + ...(externalMarkdown ? { externalMarkdown } : {}), screenshotReferences: screenshotReferences.map((reference) => ({ ...reference, width: Math.max(reference.width, 0), diff --git a/lib/external-audit-services.ts b/lib/external-audit-services.ts new file mode 100644 index 0000000..2447e15 --- /dev/null +++ b/lib/external-audit-services.ts @@ -0,0 +1,233 @@ +export type ExternalAuditUsageInput = { + openRouter?: { + inputTokens?: number; + outputTokens?: number; + inputUsdPerMillionTokens?: number; + outputUsdPerMillionTokens?: number; + }; + screenshotOne?: { + screenshots?: number; + usdPerScreenshot?: number; + }; + jina?: { + requests?: number; + pages?: number; + usdPerRequest?: number; + usdPerPage?: number; + }; + pageSpeed?: { + requests?: number; + }; +}; + +export type ExternalAuditCostEstimate = { + byProvider: { + openRouter: number; + screenshotOne: number; + jina: number; + pageSpeed: number; + }; + totalUsd: number; +}; + +export type ScreenshotOneViewport = "desktop" | "mobile"; + +export type ScreenshotOneRequest = { + viewport: ScreenshotOneViewport; + url: string; +}; + +export type BuildScreenshotOneRequestsInput = { + accessKey: string; + targetUrl: string; + endpoint?: string; +}; + +export type JinaReaderPagePath = "/" | "/kontakt" | "/impressum" | "/leistungen" | "/ueber-uns"; + +export type JinaReaderPageInput = { + url: string; + markdown: string; +}; + +export type JinaReaderAuditInput = { + pages: Array<{ + path: JinaReaderPagePath; + sourceUrl: string; + readerUrl: string; + }>; + readerUrls: string[]; + markdown: string; +}; + +export type BuildJinaReaderAuditInputOptions = { + baseUrl: string; + pages?: JinaReaderPageInput[]; + maxMarkdownChars: number; +}; + +const SCREENSHOT_ONE_ENDPOINT = "https://api.screenshotone.com/take"; +const JINA_READER_PREFIX = "https://r.jina.ai/"; +const JINA_PAGE_PATHS: JinaReaderPagePath[] = [ + "/", + "/kontakt", + "/impressum", + "/leistungen", + "/ueber-uns", +]; + +function roundUsd(value: number): number { + return Math.round((value + Number.EPSILON) * 1_000_000) / 1_000_000; +} + +function nonNegativeOrZero(value: number | undefined): number { + return typeof value === "number" && Number.isFinite(value) ? Math.max(0, value) : 0; +} + +export function estimateExternalAuditCostUsd( + usage: ExternalAuditUsageInput, +): ExternalAuditCostEstimate { + const openRouter = roundUsd( + (nonNegativeOrZero(usage.openRouter?.inputTokens) / 1_000_000) * + nonNegativeOrZero(usage.openRouter?.inputUsdPerMillionTokens) + + (nonNegativeOrZero(usage.openRouter?.outputTokens) / 1_000_000) * + nonNegativeOrZero(usage.openRouter?.outputUsdPerMillionTokens), + ); + const screenshotOne = roundUsd( + nonNegativeOrZero(usage.screenshotOne?.screenshots) * + nonNegativeOrZero(usage.screenshotOne?.usdPerScreenshot), + ); + const jina = roundUsd( + nonNegativeOrZero(usage.jina?.requests) * nonNegativeOrZero(usage.jina?.usdPerRequest) + + nonNegativeOrZero(usage.jina?.pages) * nonNegativeOrZero(usage.jina?.usdPerPage), + ); + const pageSpeed = 0; + + return { + byProvider: { + openRouter, + screenshotOne, + jina, + pageSpeed, + }, + totalUsd: roundUsd(openRouter + screenshotOne + jina + pageSpeed), + }; +} + +export function buildScreenshotOneRequests({ + accessKey, + targetUrl, + endpoint = SCREENSHOT_ONE_ENDPOINT, +}: BuildScreenshotOneRequestsInput): ScreenshotOneRequest[] { + let normalizedTargetUrl: string; + try { + const parsedTargetUrl = parseWebUrl(targetUrl, "target URL"); + normalizedTargetUrl = parsedTargetUrl.toString(); + } catch { + throw new Error("Invalid target URL for ScreenshotOne request. Only http and https URLs are supported."); + } + + const viewports: Array<{ + viewport: ScreenshotOneViewport; + width: number; + height: number; + scale: number; + }> = [ + { viewport: "desktop", width: 1280, height: 900, scale: 1 }, + { viewport: "mobile", width: 390, height: 844, scale: 2 }, + ]; + + return viewports.map(({ viewport, width, height, scale }) => { + const requestUrl = new URL(endpoint); + requestUrl.searchParams.set("access_key", accessKey); + requestUrl.searchParams.set("url", normalizedTargetUrl); + requestUrl.searchParams.set("viewport_width", String(width)); + requestUrl.searchParams.set("viewport_height", String(height)); + requestUrl.searchParams.set("device_scale_factor", String(scale)); + requestUrl.searchParams.set("full_page", "true"); + requestUrl.searchParams.set("block_cookie_banners", "true"); + requestUrl.searchParams.set("block_ads", "true"); + requestUrl.searchParams.set("block_trackers", "true"); + + return { + viewport, + url: requestUrl.toString(), + }; + }); +} + +export function buildJinaReaderAuditInput({ + baseUrl, + pages = [], + maxMarkdownChars, +}: BuildJinaReaderAuditInputOptions): JinaReaderAuditInput { + const normalizedBaseUrl = normalizeBaseUrl(baseUrl); + const pagesByUrl = new Map( + pages.map((page) => [normalizeComparableUrl(page.url), page.markdown]), + ); + const preparedPages = JINA_PAGE_PATHS.map((path) => { + const sourceUrl = new URL(path, normalizedBaseUrl).toString(); + const readerUrl = toJinaReaderUrl(sourceUrl); + return { + path, + sourceUrl, + readerUrl, + }; + }); + const markdown = preparedPages + .map((page) => { + const pageMarkdown = pagesByUrl.get(normalizeComparableUrl(page.sourceUrl)) ?? ""; + return `Source: ${page.sourceUrl}\n\n${pageMarkdown.trim()}`; + }) + .join("\n\n---\n\n"); + + return { + pages: preparedPages, + readerUrls: preparedPages.map((page) => page.readerUrl), + markdown: capMarkdown(markdown, maxMarkdownChars), + }; +} + +function normalizeBaseUrl(baseUrl: string): URL { + try { + const url = parseWebUrl(baseUrl, "base URL"); + url.hash = ""; + url.search = ""; + url.pathname = "/"; + return url; + } catch { + throw new Error("Invalid base URL for Jina Reader input. Only http and https URLs are supported."); + } +} + +function normalizeComparableUrl(url: string): string { + const normalized = parseWebUrl(url, "page URL"); + normalized.hash = ""; + if (normalized.pathname !== "/" && normalized.pathname.endsWith("/")) { + normalized.pathname = normalized.pathname.slice(0, -1); + } + return normalized.toString(); +} + +function toJinaReaderUrl(sourceUrl: string): string { + const url = parseWebUrl(sourceUrl, "source URL"); + return `${JINA_READER_PREFIX}${url.protocol}//${url.host}${url.pathname}${url.search}`; +} + +function parseWebUrl(value: string, label: string): URL { + const url = new URL(value); + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new Error(`Invalid ${label}. Only http and https URLs are supported.`); + } + return url; +} + +function capMarkdown(markdown: string, maxMarkdownChars: number): string { + if (markdown.length <= maxMarkdownChars) { + return markdown; + } + + const suffix = `[truncated to ${maxMarkdownChars} chars]`; + const availableChars = Math.max(0, maxMarkdownChars - suffix.length); + return `${markdown.slice(0, availableChars)}${suffix}`; +} diff --git a/lib/operational-readiness.ts b/lib/operational-readiness.ts index 5d0caf1..69cc400 100644 --- a/lib/operational-readiness.ts +++ b/lib/operational-readiness.ts @@ -5,10 +5,11 @@ export type IntegrationReadinessDefinition = { | "google" | "pagespeed" | "openrouter" - | "playwright" + | "screenshotone" | "smtp" | "convex_jobs" - | "rybbit"; + | "rybbit" + | "jina"; label: string; requiredEnv: string[]; errorSurface: string; @@ -39,10 +40,10 @@ export const integrationReadinessDefinitions: IntegrationReadinessDefinition[] = errorSurface: "Audit-Generierungsruns zeigen Modell- und Guard-Fehler.", }, { - id: "playwright", - label: "Playwright", - requiredEnv: ["TASK8_BROWSER_ASSET_URL"], - errorSurface: "Website-Enrichment-Runs zeigen Browser- und Crawl-Fehler.", + id: "screenshotone", + label: "ScreenshotOne", + requiredEnv: ["SCREENSHOTONE_API_KEY"], + errorSurface: "Screenshot-Erfassung zeigt API-, Quota- und Rendering-Fehler.", }, { id: "smtp", @@ -62,6 +63,12 @@ export const integrationReadinessDefinitions: IntegrationReadinessDefinition[] = requiredEnv: ["RYBBIT_API_URL", "RYBBIT_API_KEY", "NEXT_PUBLIC_RYBBIT_SITE_ID"], errorSurface: "Analytics zeigt API-Fehler als nicht blockierende Meldung.", }, + { + id: "jina", + label: "Jina", + requiredEnv: [], + errorSurface: "Optionaler Fetch-/Reader-Fallback zeigt Fehler im Audit-Quellenkontext.", + }, ]; export function getIntegrationReadiness( diff --git a/lib/skills-registry.ts b/lib/skills-registry.ts index 663fe76..1c866d6 100644 --- a/lib/skills-registry.ts +++ b/lib/skills-registry.ts @@ -13,20 +13,27 @@ export const SKILL_CATEGORIES = [ export type SkillCategory = (typeof SKILL_CATEGORIES)[number]; export type SkillRegistryEntry = { + id?: string; name: string; + title?: string; purpose: string; whenToUse: string; whenNotToUse: string; requiredInput: string; expectedOutput: string; - category: SkillCategory; + category?: SkillCategory; + appliesWhen?: string; + inputs?: string[]; + outputs?: string; + instructions?: string; version?: string; source?: string; }; export type AuditUsedSkill = { + id?: string; name: string; - category: SkillCategory; + category?: SkillCategory; version?: string; source?: string; }; @@ -51,6 +58,7 @@ const REQUIRED_FIELDS: ParsedFieldName[] = [ ]; const FIELD_LABELS_RE = /^(Purpose|When to use|When not to use|Required input|Expected output|Category|Version|Source):\s*(.*?)\s*$/; +const V3_META_BLOCK_RE = /```yaml\s*\n([\s\S]*?)\n```\s*\n?([\s\S]*)$/; function normalizeCategory(value: string): SkillCategory { const normalized = value.toLowerCase(); @@ -129,6 +137,108 @@ function parseSection(lines: string[], sectionIndex: number): SkillRegistryEntry }; } +function parseV3List(value: string): string[] { + const trimmed = value.trim(); + if (!trimmed.startsWith("[") || !trimmed.endsWith("]")) { + return trimmed ? [trimmed] : []; + } + + return trimmed + .slice(1, -1) + .split(",") + .map((entry) => entry.trim()) + .filter(Boolean); +} + +function parseV3MetaBlock(metaSource: string): Record { + const values: Record = {}; + + for (const line of metaSource.split("\n")) { + const match = line.trim().match(/^([a-z_]+):\s*(.*?)\s*$/); + if (match) { + values[match[1]] = match[2].trim(); + } + } + + return values; +} + +function parseV3Section( + rawBody: string, + sectionIndex: number, +): SkillRegistryEntry | null { + const match = rawBody.match(V3_META_BLOCK_RE); + if (!match) { + return null; + } + + const values = parseV3MetaBlock(match[1]); + if (!values.id) { + return null; + } + + const requiredFields = ["id", "title", "applies_when", "inputs", "outputs"]; + for (const field of requiredFields) { + if (!values[field]) { + throw new Error( + `Missing required v3 field "${field}" for skill section ${sectionIndex}.`, + ); + } + } + + const id = values.id; + const title = values.title; + const inputs = parseV3List(values.inputs); + const instructions = match[2].trim(); + + if (instructions.length === 0) { + throw new Error(`Missing instructions for v3 skill "${id}".`); + } + + return { + id, + name: title, + title, + purpose: instructions, + whenToUse: values.applies_when, + whenNotToUse: "Use only when applies_when and inputs match.", + requiredInput: inputs.join(", "), + expectedOutput: values.outputs, + appliesWhen: values.applies_when, + inputs, + outputs: values.outputs, + instructions, + }; +} + +function addParsedEntry( + entries: SkillRegistryEntry[], + names: Set, + ids: Set, + parsed: SkillRegistryEntry, +) { + const normalizedName = parsed.name.trim().toLowerCase(); + if (names.has(normalizedName)) { + throw new Error(`Duplicate skill name "${parsed.name}" in skills registry.`); + } + if (parsed.id) { + const normalizedId = parsed.id.trim().toLowerCase(); + if (ids.has(normalizedId)) { + throw new Error(`Duplicate skill id "${parsed.id}" in skills registry.`); + } + ids.add(normalizedId); + } + + names.add(normalizedName); + entries.push(parsed); +} + +function hasLegacyFieldLabels(source: string): boolean { + return source + .split("\n") + .some((line) => FIELD_LABELS_RE.test(line.trim())); +} + export function parseSkillsRegistry(source: string): SkillRegistryEntry[] { const normalized = source.replace(/\r\n/g, "\n"); const rawSections = normalized @@ -138,6 +248,45 @@ export function parseSkillsRegistry(source: string): SkillRegistryEntry[] { const entries: SkillRegistryEntry[] = []; const names = new Set(); + const ids = new Set(); + const v3Entries: SkillRegistryEntry[] = []; + + for (let index = 0; index < rawSections.length; index += 1) { + const rawSection = rawSections[index]; + const lines = rawSection + .split("\n") + .map((line) => line.trimEnd()) + .filter((line, lineIndex) => line.length > 0 || lineIndex === 0); + const sectionBody = lines.slice(1).join("\n"); + const parsed = parseV3Section(sectionBody, index + 1); + if (parsed && parsed.id !== "kebab-case-id") { + v3Entries.push(parsed); + } + } + + if (v3Entries.length > 0) { + for (let index = 0; index < rawSections.length; index += 1) { + const rawSection = rawSections[index]; + const lines = rawSection + .split("\n") + .map((line) => line.trimEnd()) + .filter((line, lineIndex) => line.length > 0 || lineIndex === 0); + const sectionTitle = lines.at(0) ?? ""; + const sectionBody = lines.slice(1).join("\n"); + const sectionLines = [`## ${sectionTitle}`, ...lines.slice(1)]; + const parsed = parseV3Section(sectionBody, index + 1); + if (parsed) { + if (parsed.id !== "kebab-case-id") { + addParsedEntry(entries, names, ids, parsed); + } + continue; + } + if (hasLegacyFieldLabels(sectionBody)) { + addParsedEntry(entries, names, ids, parseSection(sectionLines, index + 1)); + } + } + return entries; + } for (let index = 0; index < rawSections.length; index += 1) { const rawSection = rawSections[index]; @@ -146,16 +295,10 @@ export function parseSkillsRegistry(source: string): SkillRegistryEntry[] { .map((line) => line.trimEnd()) .filter((line, lineIndex) => line.length > 0 || lineIndex === 0); - const sectionLines = [`## ${lines.at(0) ?? ""}`, ...lines.slice(1)]; + const sectionTitle = lines.at(0) ?? ""; + const sectionLines = [`## ${sectionTitle}`, ...lines.slice(1)]; const parsed = parseSection(sectionLines, index + 1); - - const normalizedName = parsed.name.trim().toLowerCase(); - if (names.has(normalizedName)) { - throw new Error(`Duplicate skill name "${parsed.name}" in skills registry.`); - } - - names.add(normalizedName); - entries.push(parsed); + addParsedEntry(entries, names, ids, parsed); } return entries; @@ -169,10 +312,24 @@ export async function loadSkillsRegistry( } export function toAuditUsedSkill(skill: SkillRegistryEntry): AuditUsedSkill { - return { + const usedSkill: AuditUsedSkill = { name: skill.name, - category: skill.category, version: skill.version, source: skill.source, }; + + if (skill.id) { + usedSkill.id = skill.id; + } + if (skill.category) { + usedSkill.category = skill.category; + } + if (!skill.version) { + delete usedSkill.version; + } + if (!skill.source) { + delete usedSkill.source; + } + + return usedSkill; } diff --git a/tests/audit-evidence.test.ts b/tests/audit-evidence.test.ts index c04e5b5..c361fdc 100644 --- a/tests/audit-evidence.test.ts +++ b/tests/audit-evidence.test.ts @@ -1,10 +1,13 @@ import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; import test from "node:test"; import { buildAuditEvidenceInput, type SkillRegistryEntryEvidence, } from "../lib/ai/audit-evidence"; +import { parseSkillsRegistry } from "../lib/skills-registry"; const SAMPLE_SKILL_REGISTRY: SkillRegistryEntryEvidence[] = [ { @@ -335,3 +338,159 @@ test("buildAuditEvidenceInput selects deterministic skills and supports design/u assert.equal(selectedCategories.has(category), true); } }); + +test("buildAuditEvidenceInput prioritizes local-audit v3 skills before cap", () => { + const source = readFileSync( + join(process.cwd(), "v2_elemente", "skills.md"), + "utf8", + ); + const skillRegistry = parseSkillsRegistry(source); + + assert.equal( + skillRegistry.some((skill) => skill.id === "visual-design" && !skill.category), + true, + ); + + const actual = buildAuditEvidenceInput({ + lead: { + companyName: "Bäckerei Muster", + niche: "Bäckerei", + city: "Berlin", + websiteDomain: "example.com", + }, + crawlPages: [ + { + sourceUrl: "https://example.com", + finalUrl: "https://example.com", + pageKind: "homepage", + title: "Bäckerei Muster Berlin", + visibleTextExcerpt: + "Frische Backwaren in Berlin. Rufen Sie uns an oder schreiben Sie uns fuer eine Bestellung.", + hasContactCtaSignal: true, + }, + { + sourceUrl: "https://example.com/kontakt", + finalUrl: "https://example.com/kontakt", + pageKind: "contact", + title: "Kontakt", + visibleTextExcerpt: + "Telefon 030 123456, E-Mail hallo@example.com, Öffnungszeiten und Kontaktformular.", + hasContactFormSignal: true, + hasContactCtaSignal: true, + }, + ], + technicalChecks: [ + { + sourceUrl: "https://example.com", + finalUrl: "https://example.com", + usesHttps: true, + missingMetaDescription: true, + hasVisibleContactPath: true, + }, + ], + screenshots: [ + { + storageId: "desktop-storage", + sourceUrl: "https://example.com", + viewport: "desktop", + width: 1280, + height: 900, + mimeType: "image/png", + capturedAt: 1700000000000, + }, + { + storageId: "mobile-storage", + sourceUrl: "https://example.com", + viewport: "mobile", + width: 390, + height: 844, + mimeType: "image/png", + capturedAt: 1700000001000, + }, + ], + pageSpeedInputs: [ + { + strategy: "mobile", + status: "succeeded", + sourceUrl: "https://example.com", + normalized: { + implications: [ + "Die wichtigsten Inhalte erscheinen auf dem Smartphone spürbar verzögert.", + ], + }, + }, + ], + skillRegistry, + }); + + const selectedIds = new Set(actual.selectedSkills.map((skill) => skill.id)); + assert.deepEqual(actual.selectedSkills.map((skill) => skill.id), [ + "visual-design", + "contact-conversion", + "local-seo-basics", + "performance-experience", + "mobile-usability", + "conversion-copy", + ]); + assert.equal(actual.selectedSkills.length, 6); + for (const id of [ + "visual-design", + "contact-conversion", + "local-seo-basics", + "performance-experience", + ]) { + assert.equal(selectedIds.has(id), true, `${id} should be inside the cap.`); + } + assert.equal( + actual.selectedSkills.every((skill) => skill.category === undefined), + true, + ); +}); + +test("buildAuditEvidenceInput gates v3 skills when declared inputs are missing", () => { + const source = readFileSync( + join(process.cwd(), "v2_elemente", "skills.md"), + "utf8", + ); + const skillRegistry = parseSkillsRegistry(source); + + const actual = buildAuditEvidenceInput({ + lead: { + companyName: "Bäckerei Muster", + websiteDomain: "example.com", + }, + crawlPages: [ + { + sourceUrl: "https://example.com", + finalUrl: "https://example.com", + pageKind: "homepage", + title: "Bäckerei Muster", + }, + ], + screenshots: [ + { + storageId: "desktop-storage", + sourceUrl: "https://example.com", + viewport: "desktop", + width: 1280, + height: 900, + mimeType: "image/png", + capturedAt: 1700000000000, + }, + ], + skillRegistry, + }); + + const selectedIds = new Set(actual.selectedSkills.map((skill) => skill.id)); + for (const id of [ + "visual-design", + "first-impression-clarity", + "contact-conversion", + "mobile-usability", + "conversion-copy", + "performance-experience", + ]) { + assert.equal(selectedIds.has(id), false, `${id} should require missing inputs.`); + } + assert.equal(selectedIds.has("accessibility-basics"), true); +}); diff --git a/tests/audit-generation-persistence-source.test.ts b/tests/audit-generation-persistence-source.test.ts index b3f4204..d476068 100644 --- a/tests/audit-generation-persistence-source.test.ts +++ b/tests/audit-generation-persistence-source.test.ts @@ -285,6 +285,29 @@ test("sanitizer masks env-backed secret values in persistence", () => { ); }); +test("persistence sanitizer handles external service secrets with regex metacharacters", () => { + for (const secretKey of ["SCREENSHOTONE_API_KEY", "JINA_API_KEY"]) { + assert.equal( + hasPattern(auditGenerationSource, new RegExp(`["']${secretKey}["']`)), + true, + `Persistence sanitizer should redact ${secretKey}.`, + ); + } + + assert.equal( + auditGenerationSource.includes( + 'return value.replace(/[.*+?^${}()|[\\]\\\\]/g, "\\\\$&");', + ), + true, + "escapeRegExp should escape regex metacharacters with the canonical character class.", + ); + assert.equal( + auditGenerationSource.includes("/[.*+?^${}()|[\\\\]\\\\]/g"), + false, + "escapeRegExp should not keep the malformed bracket/backslash character class.", + ); +}); + test("finishAuditGenerationRun updates run status/counters/currentStep", () => { const finishSource = extractExportSource("finishAuditGenerationRun"); diff --git a/tests/audit-skill-registry-v3.test.ts b/tests/audit-skill-registry-v3.test.ts new file mode 100644 index 0000000..7acb58b --- /dev/null +++ b/tests/audit-skill-registry-v3.test.ts @@ -0,0 +1,87 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import test from "node:test"; + +import { parseSkillsRegistry, toAuditUsedSkill } from "../lib/skills-registry"; + +test("parseSkillsRegistry parses v3 yaml metablocks from v2 source", async () => { + const source = await readFile(join(process.cwd(), "v2_elemente", "skills.md"), "utf8"); + + const parsed = parseSkillsRegistry(source); + + assert.equal(parsed.length, 9); + const visualDesign = parsed.find((entry) => entry.id === "visual-design"); + assert.ok(visualDesign); + assert.equal(visualDesign.title, "Visueller Gesamteindruck & Zeitgemäßheit"); + assert.equal(visualDesign.name, "Visueller Gesamteindruck & Zeitgemäßheit"); + assert.equal(visualDesign.appliesWhen, "website_exists"); + assert.deepEqual(visualDesign.inputs, [ + "desktop_screenshot", + "mobile_screenshot", + ]); + assert.equal(visualDesign.outputs, "findings"); + const instructions = visualDesign.instructions; + if (typeof instructions !== "string") { + assert.fail("Expected visual-design instructions to be parsed."); + } + assert.match(instructions, /Beurteile den ersten visuellen Eindruck/); +}); + +test("toAuditUsedSkill exposes stable ids for v3 registry entries", async () => { + const source = await readFile(join(process.cwd(), "v2_elemente", "skills.md"), "utf8"); + const parsed = parseSkillsRegistry(source); + const skill = parsed.find((entry) => entry.id === "contact-conversion"); + + assert.ok(skill); + assert.deepEqual(toAuditUsedSkill(skill), { + id: "contact-conversion", + name: "Kontaktaufnahme & Handlungsaufforderung", + }); +}); + +test("parseSkillsRegistry does not infer categories for v3 entries without explicit metadata", async () => { + const source = await readFile(join(process.cwd(), "v2_elemente", "skills.md"), "utf8"); + const parsed = parseSkillsRegistry(source); + const skill = parsed.find((entry) => entry.id === "performance-experience"); + + assert.ok(skill); + assert.equal(skill.category, undefined); + assert.deepEqual(toAuditUsedSkill(skill), { + id: "performance-experience", + name: "Tempo & Ladeerlebnis", + }); +}); + +test("parseSkillsRegistry can read legacy and v3 skill sections from one registry", () => { + const source = ` +## Legacy Copy Skill +Purpose: Improve customer-facing copy. +When to use: Use when page text is unclear. +When not to use: Skip when copy is not available. +Required input: Markdown copy. +Expected output: Copy recommendations. +Category: copy + +## mobile-usability + +\`\`\`yaml +id: mobile-usability +title: Mobile Nutzbarkeit +applies_when: has_mobile_screenshot +inputs: [mobile_screenshot, pagespeed] +outputs: findings +\`\`\` + +Pruefe mobile Lesbarkeit und Tap-Ziele. +`; + + const parsed = parseSkillsRegistry(source); + + assert.equal(parsed.length, 2); + assert.equal(parsed[0].name, "Legacy Copy Skill"); + assert.equal(parsed[0].category, "copy"); + assert.equal(parsed[1].id, "mobile-usability"); + assert.equal(parsed[1].category, undefined); + assert.deepEqual(parsed[1].inputs, ["mobile_screenshot", "pagespeed"]); +}); diff --git a/tests/audit-skills-schema.test.ts b/tests/audit-skills-schema.test.ts index 1278437..53fe3c0 100644 --- a/tests/audit-skills-schema.test.ts +++ b/tests/audit-skills-schema.test.ts @@ -127,8 +127,13 @@ test("audits schema stores compact usedSkills metadata", () => { ); hasPattern( usedSkillsSection, - /category:\s*v\.string\(\)/, - "usedSkills.category should be string.", + /id:\s*v\.optional\(\s*v\.string\(\)\s*\)/, + "usedSkills.id should be optional string.", + ); + hasPattern( + usedSkillsSection, + /category:\s*v\.optional\(\s*v\.string\(\)\s*\)/, + "usedSkills.category should be optional string.", ); hasPattern( usedSkillsSection, @@ -179,8 +184,8 @@ test("audits.create accepts usedSkills validator and persists metadata payloads" ); hasPattern( auditsSource, - /v\.object\([\s\S]*?name:\s*v\.string\(\)[\s\S]*?category:\s*v\.string\(\)[\s\S]*?version:\s*v\.optional\(\s*v\.string\(\)\s*\)[\s\S]*?source:\s*v\.optional\(\s*v\.string\(\)\s*\)/, - "audits.ts should define a reusable usedSkillsValidator.", + /v\.object\([\s\S]*?id:\s*v\.optional\(\s*v\.string\(\)\s*\)[\s\S]*?name:\s*v\.string\(\)[\s\S]*?category:\s*v\.optional\(\s*v\.string\(\)\s*\)[\s\S]*?version:\s*v\.optional\(\s*v\.string\(\)\s*\)[\s\S]*?source:\s*v\.optional\(\s*v\.string\(\)\s*\)/, + "audits.ts should define reusable v3-compatible usedSkillsValidator fields.", ); hasPattern( diff --git a/tests/audits-auth-source.test.ts b/tests/audits-auth-source.test.ts new file mode 100644 index 0000000..8251bfc --- /dev/null +++ b/tests/audits-auth-source.test.ts @@ -0,0 +1,73 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import test from "node:test"; + +const source = async (relativePath: string) => { + return await readFile( + join(process.cwd(), ...relativePath.split("/")), + "utf8", + ); +}; + +function extractExportSource(sourceText: string, name: string) { + const marker = `export const ${name} = `; + const declarationIndex = sourceText.indexOf(marker); + assert.notEqual(declarationIndex, -1, `Expected declaration for ${name}.`); + + const openBraceIndex = sourceText.indexOf("{", declarationIndex); + let depth = 0; + let end = -1; + + for (let index = openBraceIndex; index < sourceText.length; index += 1) { + const char = sourceText[index]; + if (char === "{") { + depth += 1; + } else if (char === "}") { + depth -= 1; + if (depth === 0) { + end = index; + break; + } + } + } + + assert.notEqual(end, -1, `Expected balanced braces for ${name}.`); + return sourceText.slice(openBraceIndex, end + 1); +} + +test("audit admin APIs require operator auth before database access", async () => { + const auditsSource = await source("convex/audits.ts"); + + assert.match( + auditsSource, + /const requireOperator\s*=\s*async\s*\(\s*ctx:\s*(?:MutationCtx\s*\|\s*QueryCtx|QueryCtx\s*\|\s*MutationCtx)\s*\)[\s\S]*ctx\.auth\.getUserIdentity\(\)[\s\S]*throw new Error\(["']Nicht autorisiert\.["']\)/, + "audits should define the local requireOperator auth guard.", + ); + + for (const name of ["create", "getDetail", "get", "getBySlug", "list"]) { + const exportSource = extractExportSource(auditsSource, name); + const authIndex = exportSource.indexOf("await requireOperator(ctx)"); + const dbIndex = exportSource.indexOf("ctx.db"); + + assert.notEqual(authIndex, -1, `${name} should require operator auth.`); + assert.notEqual(dbIndex, -1, `${name} should access ctx.db.`); + assert.equal( + authIndex < dbIndex, + true, + `${name} should require operator auth before accessing ctx.db.`, + ); + } +}); + +test("public audit slug lookup remains unauthenticated", async () => { + const auditsSource = await source("convex/audits.ts"); + const publicSource = extractExportSource(auditsSource, "getPublicBySlug"); + + assert.match(publicSource, /ctx\.db/, "public audit lookup should keep reading public audit data."); + assert.doesNotMatch( + publicSource, + /requireOperator\(ctx\)/, + "getPublicBySlug should remain public and unauthenticated.", + ); +}); diff --git a/tests/external-audit-pipeline-source.test.ts b/tests/external-audit-pipeline-source.test.ts new file mode 100644 index 0000000..8b245d5 --- /dev/null +++ b/tests/external-audit-pipeline-source.test.ts @@ -0,0 +1,335 @@ +import assert from "node:assert/strict"; +import { existsSync, readFileSync } from "node:fs"; +import path from "node:path"; +import test from "node:test"; + +const actionPath = path.join(process.cwd(), "convex", "auditGenerationAction.ts"); +const actionSource = existsSync(actionPath) ? readFileSync(actionPath, "utf8") : ""; +const generationPath = path.join(process.cwd(), "convex", "auditGeneration.ts"); +const generationSource = existsSync(generationPath) + ? readFileSync(generationPath, "utf8") + : ""; + +function extractFunctionSource(functionName: string) { + const declarationPattern = new RegExp( + `(?:async\\s+)?function\\s+${functionName}\\s*\\([\\s\\S]*?\\n\\)\\s*(?::\\s*[^\\{]+)?\\{`, + ); + const match = declarationPattern.exec(actionSource); + assert.notEqual(match, null, `Expected function ${functionName}.`); + + const openBraceIndex = match!.index + match![0].lastIndexOf("{"); + let depth = 0; + let end = -1; + + for (let index = openBraceIndex; index < actionSource.length; index += 1) { + const char = actionSource[index]; + if (char === "{") { + depth += 1; + } else if (char === "}") { + depth -= 1; + if (depth === 0) { + end = index; + break; + } + } + } + + assert.notEqual(end, -1, `Expected balanced braces for ${functionName}.`); + return actionSource.slice(match!.index, end + 1); +} + +test("audit generation action orchestrates external capture helpers when legacy crawl artifacts are absent", () => { + assert.match( + actionSource, + /buildScreenshotOneRequests[\s\S]*buildJinaReaderAuditInput[\s\S]*estimateExternalAuditCostUsd|estimateExternalAuditCostUsd[\s\S]*buildScreenshotOneRequests[\s\S]*buildJinaReaderAuditInput/, + "Action should import and use the approved external audit service helpers.", + ); + assert.match( + actionSource, + /SCREENSHOTONE_API_KEY/, + "ScreenshotOne capture should be guarded by the managed SCREENSHOTONE_API_KEY env key.", + ); + assert.match( + actionSource, + /JINA_API_KEY/, + "Jina capture should be compatible with the optional managed JINA_API_KEY env key.", + ); + assert.match( + actionSource, + /evidence\.screenshots\.length\s*===\s*0[\s\S]*(started\.lead\.websiteUrl|started\.lead\.websiteDomain)/, + "External capture should be prepared from the started lead URL/domain when legacy screenshots are missing.", + ); +}); + +test("audit generation action records provider usage events for capture and OpenRouter generation", () => { + assert.match( + actionSource, + /internal\.usageEvents\.recordUsageEvent/, + "Action should record usage through internal.usageEvents.recordUsageEvent.", + ); + + for (const provider of ["screenshotone", "jina", "openrouter"]) { + assert.match( + actionSource, + new RegExp(`provider:\\s*["']${provider}["']`), + `Action should record ${provider} usage.`, + ); + } + + assert.match( + actionSource, + /provider:\s*["']openrouter["'][\s\S]*operation:\s*["']audit_generation["']/, + "OpenRouter usage should be recorded as audit_generation.", + ); + assert.match( + actionSource, + /provider:\s*["']screenshotone["'][\s\S]*operation:\s*["']audit_capture["']/, + "ScreenshotOne usage should be recorded as audit_capture.", + ); + assert.match( + actionSource, + /provider:\s*["']jina["'][\s\S]*operation:\s*["']audit_capture["']/, + "Jina usage should be recorded as audit_capture.", + ); +}); + +test("Jina markdown joins the evidence prompt without requiring Playwright crawl pages", () => { + assert.match( + actionSource, + /jina(?:Reader)?AuditInput[\s\S]*markdown/, + "Action should keep Jina reader markdown as an audit evidence input.", + ); + assert.match( + actionSource, + /buildAuditEvidenceInput\(\{[\s\S]*externalMarkdown|externalMarkdown[\s\S]*buildAuditEvidenceInput\(\{/, + "Action should pass external markdown into the evidence builder.", + ); + assert.match( + generationSource, + /externalMarkdown/, + "Audit generation evidence types should expose external markdown for prompts.", + ); +}); + +test("external capture fetches use timeout, abort signal, and bounded response readers", () => { + for (const constantName of [ + "EXTERNAL_CAPTURE_TIMEOUT_MS", + "MAX_SCREENSHOT_BYTES", + "MAX_JINA_MARKDOWN_BYTES", + "MAX_JINA_MARKDOWN_CHARS", + ]) { + assert.match( + actionSource, + new RegExp(`const\\s+${constantName}\\s*=`), + `Action should define ${constantName}.`, + ); + } + + assert.match( + actionSource, + /AbortController/, + "External fetches should use AbortController for per-request timeouts.", + ); + assert.match( + actionSource, + /fetch\([\s\S]*signal:/, + "External fetches should pass an AbortSignal.", + ); + assert.doesNotMatch( + actionSource, + /response\.blob\(\)/, + "ScreenshotOne capture should not call unbounded response.blob().", + ); + assert.doesNotMatch( + actionSource, + /response\.text\(\)/, + "Jina capture should not call unbounded response.text().", + ); +}); + +test("audit generation action sanitizes raw errors before run events and run failure summaries", () => { + assert.match( + actionSource, + /function messageFromError[\s\S]*sanitizeSecretCandidates/, + "messageFromError should sanitize/redact before returning error text.", + ); + + for (const secretName of ["SCREENSHOTONE_API_KEY", "JINA_API_KEY"]) { + assert.match( + actionSource, + new RegExp(`["']${secretName}["']`), + `Secret sanitizer should know ${secretName}.`, + ); + } + + assert.doesNotMatch( + actionSource, + /value:\s*messageFromError\(error\)/, + "Run event details should not receive raw messageFromError calls inline.", + ); + assert.doesNotMatch( + actionSource, + /errorSummary\s*=\s*messageFromError\(error\)/, + "Failure summaries should not assign unsanitized raw errors inline.", + ); +}); + +test("german-copy OpenRouter usage event aggregates all six generation calls", () => { + assert.match( + actionSource, + /aggregateOpenRouterUsage/, + "Action should expose an aggregation helper for stage-level OpenRouter usage.", + ); + assert.match( + actionSource, + /aggregateOpenRouterUsage\(\[[\s\S]*publicSummaryResult\.usage[\s\S]*germanBodyResult\.usage[\s\S]*germanSubjectResult\.usage[\s\S]*germanEmailResult\.usage[\s\S]*germanCallScriptResult\.usage[\s\S]*germanFollowUpResult\.usage[\s\S]*\]\)/, + "German-copy usage should aggregate public summary, body, subject, email, call script, and follow-up calls.", + ); +}); + +test("usage event recording is best-effort and cannot fail audit generation", () => { + const usageRecorder = extractFunctionSource("recordAuditUsageEvent"); + + assert.match( + usageRecorder, + /try\s*\{[\s\S]*await ctx\.runMutation\(internal\.usageEvents\.recordUsageEvent/, + "Usage recorder should isolate recordUsageEvent in a try block.", + ); + assert.match( + usageRecorder, + /catch\s*\(error\)\s*\{[\s\S]*messageFromError\(error\)[\s\S]*level:\s*["']warning["']/, + "Usage recorder should sanitize/log failures as warnings.", + ); + assert.match( + usageRecorder, + /catch\s*\(error\)\s*\{[\s\S]*try\s*\{[\s\S]*appendRunEvent[\s\S]*\}\s*catch/, + "Warning logging for usage failures should also be best-effort.", + ); +}); + +test("external capture timeout covers body streaming and cancels readers", () => { + const fetcher = extractFunctionSource("fetchExternalCapture"); + const reader = extractFunctionSource("readLimitedResponseBytes"); + + assert.match( + fetcher, + /return\s*\{[\s\S]*response[\s\S]*abortController:\s*controller[\s\S]*timeout[\s\S]*\}/, + "fetchExternalCapture should return the active deadline context for body reads.", + ); + assert.doesNotMatch( + fetcher, + /finally\s*\{[\s\S]*clearTimeout\(timeout\)/, + "fetchExternalCapture should not clear the timeout before body streaming completes.", + ); + assert.match( + reader, + /signal\??:\s*AbortSignal/, + "Bounded response reader should accept an AbortSignal.", + ); + assert.match( + reader, + /signal\?\.addEventListener\(\s*["']abort["'][\s\S]*reader\.cancel/, + "Bounded response reader should cancel the reader on timeout/abort.", + ); + assert.match( + reader, + /totalBytes\s*>\s*maxBytes[\s\S]*await reader\.cancel\(/, + "Bounded response reader should cancel the stream when the byte cap is exceeded.", + ); + assert.match( + actionSource, + /readLimitedResponseBytes\([\s\S]*MAX_SCREENSHOT_BYTES[\s\S]*abortController\.signal/, + "Screenshot body reads should use the active timeout signal.", + ); + assert.match( + actionSource, + /readLimitedMarkdown\([\s\S]*abortController\.signal/, + "Jina markdown body reads should use the active timeout signal.", + ); + assert.match( + actionSource, + /finally\s*\{[\s\S]*clearExternalCaptureTimeout/, + "Capture loops should clear the external timeout after fetch and body streaming finish.", + ); +}); + +test("external capture request builders are provider-level best-effort", () => { + const capture = extractFunctionSource("captureExternalAuditArtifacts"); + + assert.match( + capture, + /if\s*\(args\.needsScreenshots\)[\s\S]*try\s*\{[\s\S]*buildScreenshotOneRequests/, + "ScreenshotOne request construction should be inside a provider-level try block.", + ); + assert.match( + capture, + /buildScreenshotOneRequests[\s\S]*catch\s*\(error\)[\s\S]*messageFromError\(error\)[\s\S]*level:\s*["']warning["']/, + "ScreenshotOne request construction failures should degrade to sanitized warnings.", + ); + assert.match( + capture, + /if\s*\(args\.needsMarkdown\)[\s\S]*try\s*\{[\s\S]*buildJinaReaderAuditInput/, + "Jina reader input construction should be inside a provider-level try block.", + ); + assert.match( + capture, + /buildJinaReaderAuditInput[\s\S]*catch\s*\(error\)[\s\S]*messageFromError\(error\)[\s\S]*level:\s*["']warning["']/, + "Jina reader input construction failures should degrade to sanitized warnings.", + ); +}); + +test("ScreenshotOne missing-key skip emits best-effort warning only when screenshots are needed", () => { + const capture = extractFunctionSource("captureExternalAuditArtifacts"); + const needsScreenshotsIndex = capture.indexOf("if (args.needsScreenshots)"); + const needsMarkdownIndex = capture.indexOf("if (args.needsMarkdown)"); + const missingKeyWarningIndex = capture.indexOf( + "ScreenshotOne ist nicht konfiguriert; Screenshot-Erfassung wurde übersprungen.", + ); + + assert.notEqual( + needsScreenshotsIndex, + -1, + "External capture should branch on needsScreenshots.", + ); + assert.notEqual( + needsMarkdownIndex, + -1, + "External capture should keep the later needsMarkdown branch.", + ); + assert.notEqual( + missingKeyWarningIndex, + -1, + "Missing ScreenshotOne config should emit a clear warning message.", + ); + assert.equal( + missingKeyWarningIndex > needsScreenshotsIndex && + missingKeyWarningIndex < needsMarkdownIndex, + true, + "Missing-key warning should live inside the needsScreenshots branch, so legacy screenshots do not warn.", + ); + assert.match( + capture, + /if\s*\(!screenshotOneApiKey\)\s*\{[\s\S]*try\s*\{[\s\S]*await appendRunEvent\(ctx,\s*\{[\s\S]*level:\s*["']warning["'][\s\S]*ScreenshotOne ist nicht konfiguriert; Screenshot-Erfassung wurde übersprungen\.[\s\S]*\}\s*\);[\s\S]*\}\s*catch\s*\{[\s\S]*\}/, + "Missing-key warning logging should be best-effort and unable to fail the audit run.", + ); +}); + +test("external capture non-OK responses cancel bodies before continuing", () => { + const capture = extractFunctionSource("captureExternalAuditArtifacts"); + const nonOkCancelCount = [ + ...capture.matchAll( + /if\s*\(!response\.ok\)\s*\{[\s\S]*?await cancelExternalResponseBody\(response\);[\s\S]*?continue;/g, + ), + ].length; + + assert.match( + actionSource, + /async function cancelExternalResponseBody/, + "Action should centralize best-effort body cancellation for non-OK responses.", + ); + assert.equal( + nonOkCancelCount, + 2, + "Both ScreenshotOne and Jina non-OK branches should cancel bodies before continue.", + ); +}); diff --git a/tests/external-audit-services.test.ts b/tests/external-audit-services.test.ts new file mode 100644 index 0000000..3f3333c --- /dev/null +++ b/tests/external-audit-services.test.ts @@ -0,0 +1,184 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + buildJinaReaderAuditInput, + buildScreenshotOneRequests, + estimateExternalAuditCostUsd, +} from "../lib/external-audit-services"; + +test("estimateExternalAuditCostUsd totals managed provider usage", () => { + const estimate = estimateExternalAuditCostUsd({ + openRouter: { + inputTokens: 1_500_000, + outputTokens: 250_000, + inputUsdPerMillionTokens: 0.25, + outputUsdPerMillionTokens: 1.25, + }, + screenshotOne: { + screenshots: 2, + usdPerScreenshot: 0.01, + }, + jina: { + requests: 4, + pages: 4, + usdPerRequest: 0.001, + usdPerPage: 0.002, + }, + pageSpeed: { + requests: 2, + }, + }); + + assert.equal(estimate.totalUsd, 0.7195); + assert.deepEqual(estimate.byProvider, { + openRouter: 0.6875, + screenshotOne: 0.02, + jina: 0.012, + pageSpeed: 0, + }); +}); + +test("estimateExternalAuditCostUsd clamps negative usage and prices to zero", () => { + const estimate = estimateExternalAuditCostUsd({ + openRouter: { + inputTokens: -1_000_000, + outputTokens: 100_000, + inputUsdPerMillionTokens: 0.25, + outputUsdPerMillionTokens: -1.25, + }, + screenshotOne: { + screenshots: -2, + usdPerScreenshot: 0.01, + }, + jina: { + requests: 4, + pages: -4, + usdPerRequest: -0.001, + usdPerPage: 0.002, + }, + }); + + assert.deepEqual(estimate.byProvider, { + openRouter: 0, + screenshotOne: 0, + jina: 0, + pageSpeed: 0, + }); + assert.equal(estimate.totalUsd, 0); +}); + +test("buildScreenshotOneRequests creates stable desktop and mobile URLs", () => { + const requests = buildScreenshotOneRequests({ + accessKey: "sso_secret_key", + targetUrl: "https://example.com/landing?utm=abc", + }); + + assert.equal(requests.length, 2); + assert.deepEqual( + requests.map((request) => request.viewport), + ["desktop", "mobile"], + ); + + const desktop = new URL(requests[0]?.url ?? ""); + assert.equal(desktop.searchParams.get("access_key"), "sso_secret_key"); + assert.equal(desktop.searchParams.get("url"), "https://example.com/landing?utm=abc"); + assert.equal(desktop.searchParams.get("viewport_width"), "1280"); + assert.equal(desktop.searchParams.get("viewport_height"), "900"); + assert.equal(desktop.searchParams.get("device_scale_factor"), "1"); + assert.equal(desktop.searchParams.get("full_page"), "true"); + assert.equal(desktop.searchParams.get("block_cookie_banners"), "true"); + assert.equal(desktop.searchParams.get("block_ads"), "true"); + assert.equal(desktop.searchParams.get("block_trackers"), "true"); + + const mobile = new URL(requests[1]?.url ?? ""); + assert.equal(mobile.searchParams.get("viewport_width"), "390"); + assert.equal(mobile.searchParams.get("viewport_height"), "844"); + assert.equal(mobile.searchParams.get("device_scale_factor"), "2"); + assert.equal(mobile.searchParams.get("full_page"), "true"); +}); + +test("buildScreenshotOneRequests rejects non-web target URLs without leaking secrets", () => { + assert.throws( + () => + buildScreenshotOneRequests({ + accessKey: "sso_secret_key", + targetUrl: "ftp://example.com/landing", + }), + (error) => { + assert.equal(error instanceof Error, true); + assert.equal((error as Error).message.includes("sso_secret_key"), false); + assert.match((error as Error).message, /http.*https/i); + return true; + }, + ); +}); + +test("buildScreenshotOneRequests does not leak the access key in validation errors", () => { + assert.throws( + () => + buildScreenshotOneRequests({ + accessKey: "sso_secret_key", + targetUrl: "not a url", + }), + (error) => { + assert.equal(error instanceof Error, true); + assert.equal((error as Error).message.includes("sso_secret_key"), false); + assert.match((error as Error).message, /target url/i); + return true; + }, + ); +}); + +test("buildJinaReaderAuditInput rejects non-web base and page URLs", () => { + assert.throws( + () => + buildJinaReaderAuditInput({ + baseUrl: "file:///tmp/site.html", + maxMarkdownChars: 100, + }), + /http.*https/i, + ); + + assert.throws( + () => + buildJinaReaderAuditInput({ + baseUrl: "https://example.com", + pages: [{ url: "ftp://example.com/kontakt", markdown: "Kontakt" }], + maxMarkdownChars: 100, + }), + /http.*https/i, + ); +}); + +test("buildJinaReaderAuditInput prepares capped markdown for relevant pages", () => { + const input = buildJinaReaderAuditInput({ + baseUrl: "https://example.com", + pages: [ + { url: "https://example.com", markdown: "# Home\nWillkommen auf der Startseite." }, + { url: "https://example.com/kontakt", markdown: "Kontaktformular und Telefonnummer." }, + { url: "https://example.com/impressum", markdown: "Impressum mit Anbieterkennzeichnung." }, + { url: "https://example.com/leistungen", markdown: "Leistungen fuer Webdesign und SEO." }, + { url: "https://example.com/ueber-uns", markdown: "Ueber uns und Arbeitsweise." }, + ], + maxMarkdownChars: 95, + }); + + assert.deepEqual( + input.pages.map((page) => page.path), + ["/", "/kontakt", "/impressum", "/leistungen", "/ueber-uns"], + ); + assert.deepEqual( + input.readerUrls, + [ + "https://r.jina.ai/https://example.com/", + "https://r.jina.ai/https://example.com/kontakt", + "https://r.jina.ai/https://example.com/impressum", + "https://r.jina.ai/https://example.com/leistungen", + "https://r.jina.ai/https://example.com/ueber-uns", + ], + ); + assert.equal(input.markdown.length <= 95, true); + assert.match(input.markdown, /Source: https:\/\/example.com/); + assert.match(input.markdown, /\[truncated to 95 chars\]$/); +}); diff --git a/tests/leads-runs-auth-source.test.ts b/tests/leads-runs-auth-source.test.ts new file mode 100644 index 0000000..73e5629 --- /dev/null +++ b/tests/leads-runs-auth-source.test.ts @@ -0,0 +1,195 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import test from "node:test"; + +const source = async (relativePath: string) => { + return await readFile( + join(process.cwd(), ...relativePath.split("/")), + "utf8", + ); +}; + +function extractExportSource(sourceText: string, name: string) { + const marker = `export const ${name} = `; + const declarationIndex = sourceText.indexOf(marker); + assert.notEqual(declarationIndex, -1, `Expected declaration for ${name}.`); + + const openBraceIndex = sourceText.indexOf("{", declarationIndex); + let depth = 0; + let end = -1; + + for (let index = openBraceIndex; index < sourceText.length; index += 1) { + const char = sourceText[index]; + if (char === "{") { + depth += 1; + } else if (char === "}") { + depth -= 1; + if (depth === 0) { + end = index; + break; + } + } + } + + assert.notEqual(end, -1, `Expected balanced braces for ${name}.`); + return sourceText.slice(openBraceIndex, end + 1); +} + +function assertRequiresOperatorBeforeDataAccess( + moduleSource: string, + exportName: string, + helperName?: string, +) { + const functionSource = extractExportSource(moduleSource, exportName); + const authIndex = functionSource.indexOf("await requireOperator(ctx)"); + const dbIndex = functionSource.indexOf("ctx.db"); + const helperIndex = helperName === undefined + ? -1 + : functionSource.indexOf(helperName); + const dataAccessIndex = dbIndex === -1 ? helperIndex : dbIndex; + + assert.notEqual( + authIndex, + -1, + `${exportName} should call requireOperator before DB access.`, + ); + assert.notEqual( + dataAccessIndex, + -1, + `${exportName} should access ctx.db or call its DB helper.`, + ); + assert.ok( + authIndex < dataAccessIndex, + `${exportName} should require operator auth before its first data access.`, + ); +} + +test("lead public APIs require operator auth before DB access", async () => { + const leadsSource = await source("convex/leads.ts"); + + assert.match( + leadsSource, + /const requireOperator\s*=\s*async\s*\(\s*ctx:\s*(?:MutationCtx\s*\|\s*QueryCtx|QueryCtx\s*\|\s*MutationCtx)\s*\)/, + "leads.ts should define a local requireOperator helper.", + ); + assert.match( + leadsSource, + /ctx\.auth\.getUserIdentity\(\)[\s\S]*throw new Error\(["']Nicht autorisiert\.["']\)/, + "requireOperator should derive operator identity from Convex auth.", + ); + + for (const exportName of [ + "create", + "reviewUpdate", + "get", + "list", + "listFunnel", + ]) { + assertRequiresOperatorBeforeDataAccess( + leadsSource, + exportName, + exportName === "reviewUpdate" ? "reviewUpdateLead" : undefined, + ); + } +}); + +test("lead internal APIs exist for audit-generation action callsites", async () => { + const [leadsSource, actionSource] = await Promise.all([ + source("convex/leads.ts"), + source("convex/auditGenerationAction.ts"), + ]); + + assert.match( + leadsSource, + /import\s*{[\s\S]*internalMutation[\s\S]*internalQuery[\s\S]*}/, + "leads.ts should import internal Convex builders.", + ); + assert.match( + leadsSource, + /export const getInternal\s*=\s*internalQuery\(/, + "leads.ts should expose an internal lead get query for actions.", + ); + assert.match( + leadsSource, + /export const reviewUpdateInternal\s*=\s*internalMutation\(/, + "leads.ts should expose an internal lead review mutation for actions.", + ); + assert.match( + actionSource, + /internal\.leads\.getInternal/, + "auditGenerationAction should load leads through internal.leads.getInternal.", + ); + assert.match( + actionSource, + /internal\.leads\.reviewUpdateInternal/, + "auditGenerationAction should update leads through internal.leads.reviewUpdateInternal.", + ); + assert.doesNotMatch( + actionSource, + /api\.leads\.(get|reviewUpdate)/, + "auditGenerationAction should not use public lead APIs for internal calls.", + ); +}); + +test("run public APIs require operator auth before DB access", async () => { + const runsSource = await source("convex/runs.ts"); + + assert.match( + runsSource, + /const requireOperator\s*=\s*async\s*\(\s*ctx:\s*(?:MutationCtx\s*\|\s*QueryCtx|QueryCtx\s*\|\s*MutationCtx)\s*\)/, + "runs.ts should define a local requireOperator helper.", + ); + assert.match( + runsSource, + /ctx\.auth\.getUserIdentity\(\)[\s\S]*throw new Error\(["']Nicht autorisiert\.["']\)/, + "requireOperator should derive operator identity from Convex auth.", + ); + + for (const exportName of [ + "create", + "updateStatus", + "list", + "appendEvent", + "listEvents", + ]) { + assertRequiresOperatorBeforeDataAccess( + runsSource, + exportName, + exportName === "appendEvent" ? "appendRunEvent" : undefined, + ); + } +}); + +test("actions append run events through internal run mutation", async () => { + const [runsSource, auditAction, pageSpeedAction, enrichmentAction] = + await Promise.all([ + source("convex/runs.ts"), + source("convex/auditGenerationAction.ts"), + source("convex/pageSpeedAction.ts"), + source("convex/websiteEnrichmentAction.ts"), + ]); + + assert.match( + runsSource, + /export const appendEventInternal\s*=\s*internalMutation\(/, + "runs.ts should expose an internal append event mutation for actions.", + ); + + for (const [name, actionSource] of [ + ["auditGenerationAction", auditAction], + ["pageSpeedAction", pageSpeedAction], + ["websiteEnrichmentAction", enrichmentAction], + ] as const) { + assert.match( + actionSource, + /internal\.runs\.appendEventInternal/, + `${name} should append events through internal.runs.appendEventInternal.`, + ); + assert.doesNotMatch( + actionSource, + /api\.runs\.appendEvent/, + `${name} should not append events through the public runs API.`, + ); + } +}); diff --git a/tests/operational-readiness.test.ts b/tests/operational-readiness.test.ts index a32aeef..1509f12 100644 --- a/tests/operational-readiness.test.ts +++ b/tests/operational-readiness.test.ts @@ -13,10 +13,11 @@ test("integration readiness covers all MVP providers", () => { "google", "pagespeed", "openrouter", - "playwright", + "screenshotone", "smtp", "convex_jobs", "rybbit", + "jina", ], ); }); @@ -36,3 +37,45 @@ test("integration readiness reports missing configuration without leaking values assert.equal(JSON.stringify(rows).includes("secret-google"), false); assert.equal(JSON.stringify(rows).includes("secret-places"), false); }); + +test("integration readiness treats ScreenshotOne as required and Jina as optional", () => { + const rows = getIntegrationReadiness({ + GOOGLE_GEOCODING_API_KEY: "secret-google", + GOOGLE_PLACES_API_KEY: "secret-places", + PAGESPEED_API_KEY: "secret-pagespeed", + PAGESPEED_TIMEOUT_MS: "60000", + OPENROUTER_API_KEY: "secret-openrouter", + SMTP_HOST: "smtp.example.com", + SMTP_USER: "user", + SMTP_PASSWORD: "password", + SMTP_FROM: "Audit ", + NEXT_PUBLIC_CONVEX_URL: "https://example.convex.cloud", + CONVEX_DEPLOYMENT: "prod:example", + RYBBIT_API_URL: "https://analytics.example.com", + RYBBIT_API_KEY: "secret-rybbit", + NEXT_PUBLIC_RYBBIT_SITE_ID: "site-id", + }); + + const screenshotOne = rows.find((row) => row.id === "screenshotone"); + const jina = rows.find((row) => row.id === "jina"); + + assert.equal(screenshotOne?.status, "missing"); + assert.deepEqual(screenshotOne?.missingEnv, ["SCREENSHOTONE_API_KEY"]); + assert.equal(jina?.status, "configured"); + assert.deepEqual(jina?.missingEnv, []); +}); + +test("integration readiness no longer requires Playwright for the new pipeline", () => { + const definitionIds = integrationReadinessDefinitions.map((definition) => definition.id as string); + + assert.equal( + definitionIds.includes("playwright"), + false, + ); + assert.equal( + integrationReadinessDefinitions.some((definition) => + definition.requiredEnv.includes("TASK8_BROWSER_ASSET_URL"), + ), + false, + ); +}); diff --git a/tests/ops-quality-source.test.ts b/tests/ops-quality-source.test.ts index 3002682..c09aa5d 100644 --- a/tests/ops-quality-source.test.ts +++ b/tests/ops-quality-source.test.ts @@ -19,14 +19,20 @@ test("settings page surfaces integration status instead of a placeholder", () => "Google", "PageSpeed", "OpenRouter", - "Playwright", + "ScreenshotOne", "SMTP", "Convex Jobs", "Rybbit", + "Jina", "Konfiguration fehlt", ]) { assert.match(`${componentSource}\n${helperSource}`, new RegExp(label)); } + + assert.doesNotMatch(helperSource, /id: "playwright"/); + assert.doesNotMatch(helperSource, /requiredEnv: \["TASK8_BROWSER_ASSET_URL"\]/); + assert.match(helperSource, /requiredEnv: \["SCREENSHOTONE_API_KEY"\]/); + assert.match(helperSource, /requiredEnv: \[\]/); }); test("verification notes cover critical MVP flows", () => { diff --git a/tests/pagespeed-action-source.test.ts b/tests/pagespeed-action-source.test.ts index 840fa33..c6fa8ec 100644 --- a/tests/pagespeed-action-source.test.ts +++ b/tests/pagespeed-action-source.test.ts @@ -238,7 +238,7 @@ test("pageSpeedAction stores and persists results and writes events", () => { ); assert.equal( - /api\.runs\.appendEvent,\s*{\s*[\s\S]*runId:\s*args\.runId,\s*[\s\S]*level:\s*["']info["']/.test( + /internal\.runs\.appendEventInternal,\s*{\s*[\s\S]*runId:\s*args\.runId,\s*[\s\S]*level:\s*["']info["']/.test( actionSource, ), true, @@ -283,7 +283,7 @@ test("pageSpeedAction does not expose API key in event messages/details", () => assert.equal( hasPattern( actionSource, - /api\.runs\.appendEvent[\s\S]{0,500}PAGESPEED_API_KEY/, + /internal\.runs\.appendEventInternal[\s\S]{0,500}PAGESPEED_API_KEY/, ), false, "Action events should not include raw PAGESPEED_API_KEY", diff --git a/tests/usage-events-source.test.ts b/tests/usage-events-source.test.ts new file mode 100644 index 0000000..be5f3b6 --- /dev/null +++ b/tests/usage-events-source.test.ts @@ -0,0 +1,356 @@ +import assert from "node:assert/strict"; +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import test from "node:test"; +import ts from "typescript"; + +const schemaPath = join(process.cwd(), "convex", "schema.ts"); +const domainPath = join(process.cwd(), "convex", "domain.ts"); +const usageEventsPath = join(process.cwd(), "convex", "usageEvents.ts"); + +const schemaSource = readFileSync(schemaPath, "utf8"); +const domainSource = readFileSync(domainPath, "utf8"); +const usageEventsSource = existsSync(usageEventsPath) + ? readFileSync(usageEventsPath, "utf8") + : ""; + +const usageEventsSourceFile = ts.createSourceFile( + "usageEvents.ts", + usageEventsSource, + ts.ScriptTarget.ES2022, + true, + ts.ScriptKind.TS, +); + +function getExportedConstNames(file: ts.SourceFile) { + const names = new Set(); + + const visit = (node: ts.Node) => { + if (ts.isVariableStatement(node)) { + const isExported = node.modifiers?.some( + (mod) => mod.kind === ts.SyntaxKind.ExportKeyword, + ); + const isConst = node.declarationList.flags & ts.NodeFlags.Const; + + if (isExported && isConst) { + for (const declaration of node.declarationList.declarations) { + if (ts.isIdentifier(declaration.name)) { + names.add(declaration.name.text); + } + } + } + } + + ts.forEachChild(node, visit); + }; + + ts.forEachChild(file, visit); + return names; +} + +function extractTableSection(tableName: string) { + const marker = `${tableName}: defineTable({`; + const markerIndex = schemaSource.indexOf(marker); + assert.notEqual( + markerIndex, + -1, + `Expected schema table definition for ${tableName}.`, + ); + + const objectStart = schemaSource.indexOf("{", markerIndex); + let depth = 0; + let objectEnd = -1; + + for (let index = objectStart; index < schemaSource.length; index += 1) { + if (schemaSource[index] === "{") { + depth += 1; + } else if (schemaSource[index] === "}") { + depth -= 1; + if (depth === 0) { + objectEnd = index; + break; + } + } + } + + assert.notEqual(objectEnd, -1, `Could not parse schema object for ${tableName}.`); + + const remainder = schemaSource.slice(objectEnd + 1); + const nextTableMatch = remainder.match(/^\s*[a-zA-Z_][\w]*:\s*defineTable\(/m); + const sectionEnd = + nextTableMatch === null ? schemaSource.length : objectEnd + 1 + nextTableMatch.index!; + + return { + objectBlock: schemaSource.slice(markerIndex, objectEnd + 1), + section: schemaSource.slice(markerIndex, sectionEnd), + }; +} + +function extractExportSource(name: string) { + const marker = `export const ${name} = `; + const declarationIndex = usageEventsSource.indexOf(marker); + assert.notEqual(declarationIndex, -1, `Expected declaration for ${name}`); + + const openBraceIndex = usageEventsSource.indexOf("{", declarationIndex); + let depth = 0; + let end = -1; + + for (let index = openBraceIndex; index < usageEventsSource.length; index += 1) { + const char = usageEventsSource[index]; + if (char === "{") { + depth += 1; + } else if (char === "}") { + depth -= 1; + if (depth === 0) { + end = index; + break; + } + } + } + + assert.notEqual(end, -1, `Expected balanced braces for ${name}`); + return usageEventsSource.slice(openBraceIndex, end + 1); +} + +function assertHas(pattern: RegExp, source: string, message: string) { + assert.equal(pattern.test(source), true, message); +} + +const usageReadQueries = [ + { + name: "listLatestUsageEvents", + indexAssertion: + /withIndex\("by_createdAt"\)/, + message: "latest query should use by_createdAt.", + }, + { + name: "listUsageEventsByRun", + indexAssertion: + /withIndex\("by_runId_and_createdAt"[\s\S]*?eq\("runId",\s*args\.runId\)/, + message: "run query should use by_runId_and_createdAt with runId equality.", + }, + { + name: "listUsageEventsByLead", + indexAssertion: + /withIndex\("by_leadId_and_createdAt"[\s\S]*?eq\("leadId",\s*args\.leadId\)/, + message: "lead query should use by_leadId_and_createdAt with leadId equality.", + }, + { + name: "listUsageEventsByAudit", + indexAssertion: + /withIndex\("by_auditId_and_createdAt"[\s\S]*?eq\("auditId",\s*args\.auditId\)/, + message: "audit query should use by_auditId_and_createdAt with auditId equality.", + }, + { + name: "listUsageEventsByProvider", + indexAssertion: + /withIndex\("by_provider_and_createdAt"[\s\S]*?eq\("provider",\s*args\.provider\)/, + message: "provider query should use by_provider_and_createdAt with provider equality.", + }, +] as const; + +test("usage domain constants declare supported providers and operations", () => { + assertHas( + /USAGE_EVENT_PROVIDERS\s*=\s*\[[\s\S]*"openrouter"[\s\S]*"screenshotone"[\s\S]*"jina"[\s\S]*"pagespeed"[\s\S]*"google_places"[\s\S]*\]\s*as const/, + domainSource, + "Domain should declare usage providers for all managed external services.", + ); + assertHas( + /USAGE_EVENT_OPERATIONS\s*=\s*\[[\s\S]*"audit_capture"[\s\S]*"audit_generation"[\s\S]*"lead_lookup"[\s\S]*\]\s*as const/, + domainSource, + "Domain should declare usage operations for capture, generation, and lookup.", + ); +}); + +test("usageEvents schema stores cost and usage dimensions with bounded indexes", () => { + const { objectBlock, section } = extractTableSection("usageEvents"); + + assertHas(/provider:\s*usageEventProvider/, objectBlock, "provider should use the provider validator."); + assertHas(/operation:\s*usageEventOperation/, objectBlock, "operation should use the operation validator."); + assertHas( + /runId:\s*v\.optional\(\s*v\.id\(["']agentRuns["']\)\s*\)/, + objectBlock, + "runId should be optional for SaaS-ready attribution.", + ); + assertHas( + /leadId:\s*v\.optional\(\s*v\.id\(["']leads["']\)\s*\)/, + objectBlock, + "leadId should be optional for lead-level attribution.", + ); + assertHas( + /auditId:\s*v\.optional\(\s*v\.id\(["']audits["']\)\s*\)/, + objectBlock, + "auditId should be optional for audit-level attribution.", + ); + assertHas( + /estimatedCostUsd:\s*v\.number\(\)/, + objectBlock, + "estimatedCostUsd should be a required normalized number.", + ); + assertHas( + /tokens:\s*v\.optional\(\s*v\.object\([\s\S]*?inputTokens:\s*v\.optional\(v\.number\(\)\)[\s\S]*?outputTokens:\s*v\.optional\(v\.number\(\)\)[\s\S]*?promptTokens:\s*v\.optional\(v\.number\(\)\)[\s\S]*?completionTokens:\s*v\.optional\(v\.number\(\)\)[\s\S]*?totalTokens:\s*v\.optional\(v\.number\(\)\)[\s\S]*?\)\s*\)/, + objectBlock, + "tokens should capture OpenRouter-compatible token dimensions.", + ); + assertHas( + /callCounts:\s*v\.optional\(\s*v\.object\(/, + objectBlock, + "callCounts should be an optional object.", + ); + for (const countName of ["requests", "pages", "screenshots", "lookups"]) { + assertHas( + new RegExp(`${countName}:\\s*v\\.optional\\(v\\.number\\(\\)\\)`), + objectBlock, + `callCounts.${countName} should be optional number.`, + ); + } + assertHas(/createdAt:\s*v\.number\(\)/, objectBlock, "createdAt should be required."); + + for (const [indexName, fields] of [ + ["by_runId_and_createdAt", '"runId",\\s*"createdAt"'], + ["by_leadId_and_createdAt", '"leadId",\\s*"createdAt"'], + ["by_auditId_and_createdAt", '"auditId",\\s*"createdAt"'], + ["by_provider_and_createdAt", '"provider",\\s*"createdAt"'], + ["by_createdAt", '"createdAt"'], + ] as const) { + assertHas( + new RegExp(`index\\("${indexName}",\\s*\\[${fields}\\]\\)`), + section, + `usageEvents should define ${indexName}.`, + ); + } +}); + +test("usageEvents module exposes internal recorder and authenticated bounded read queries", () => { + assert.equal(existsSync(usageEventsPath), true, "usageEvents.ts should be present."); + + const exports = getExportedConstNames(usageEventsSourceFile); + for (const exportName of [ + "recordUsageEvent", + ...usageReadQueries.map((readQuery) => readQuery.name), + ]) { + assert.equal(exports.has(exportName), true, `Expected export: ${exportName}`); + } + + assertHas( + /export const recordUsageEvent = internalMutation\s*\(/, + usageEventsSource, + "recordUsageEvent should be an internalMutation.", + ); + for (const { name } of usageReadQueries) { + assertHas( + new RegExp(`export const ${name} = query\\s*\\(`), + usageEventsSource, + `${name} should remain a public authenticated bounded query.`, + ); + } +}); + +test("usageEvents queries use indexes and bounded take without filters or collect", () => { + const querySources = usageReadQueries.map((readQuery) => ({ + ...readQuery, + source: extractExportSource(readQuery.name), + })); + + for (const { source } of querySources) { + assertHas(/limit:\s*v\.optional\(v\.number\(\)\)/, source, "read query should validate limit."); + } + for (const { source, indexAssertion, message } of querySources) { + assertHas(indexAssertion, source, message); + } + + for (const source of [usageEventsSource, ...querySources.map((querySource) => querySource.source)]) { + assert.doesNotMatch(source, /\.filter\s*\(/, "usageEvents should not use query filters."); + assert.doesNotMatch(source, /\.collect\s*\(/, "usageEvents should not use unbounded collect."); + } + for (const { source } of querySources) { + assertHas( + /\.take\(\s*normalizeListLimit\(args\.limit\)\s*\)/, + source, + "read query should be bounded.", + ); + } +}); + +test("usageEvents read queries require operator auth before reading telemetry", () => { + assertHas( + /const\s+requireOperator\s*=\s*async\s*\(\s*ctx:\s*QueryCtx\s*\)[\s\S]*ctx\.auth\.getUserIdentity\(\)[\s\S]*throw new Error\(["']Nicht autorisiert\.["']\)/, + usageEventsSource, + "usageEvents should define the local requireOperator auth guard.", + ); + + for (const { name } of usageReadQueries) { + const source = extractExportSource(name); + const authIndex = source.indexOf("await requireOperator(ctx)"); + const readIndex = source.indexOf("ctx.db"); + + assert.notEqual(authIndex, -1, `${name} should require operator auth.`); + assert.notEqual(readIndex, -1, `${name} should read from ctx.db.`); + assert.equal( + authIndex < readIndex, + true, + `${name} should require auth before reading usage telemetry.`, + ); + } +}); + +test("recordUsageEvent guards finite non-negative usage numbers before insert", () => { + const recordSource = extractExportSource("recordUsageEvent"); + const guardCallIndex = recordSource.indexOf("assertValidUsageEventNumbers(args)"); + const insertIndex = recordSource.indexOf('ctx.db.insert("usageEvents"'); + + assert.notEqual( + guardCallIndex, + -1, + "recordUsageEvent should call assertValidUsageEventNumbers(args).", + ); + assert.notEqual(insertIndex, -1, "recordUsageEvent should insert usageEvents."); + assert.equal( + guardCallIndex < insertIndex, + true, + "recordUsageEvent should validate usage numbers before inserting.", + ); + + assertHas( + /function\s+assertFiniteNonNegativeNumber[\s\S]*Number\.isFinite\(value\)[\s\S]*value\s*<\s*0/, + usageEventsSource, + "Cost guard should reject NaN, Infinity, and negative numbers.", + ); + assertHas( + /function\s+assertFiniteNonNegativeInteger[\s\S]*Number\.isFinite\(value\)[\s\S]*value\s*<\s*0[\s\S]*Number\.isInteger\(value\)/, + usageEventsSource, + "Token/count guard should require finite non-negative integers.", + ); + assertHas( + /assertFiniteNonNegativeNumber\(args\.estimatedCostUsd,\s*["']estimatedCostUsd["']\)/, + usageEventsSource, + "estimatedCostUsd should use the finite non-negative number guard.", + ); + + for (const tokenName of [ + "inputTokens", + "outputTokens", + "promptTokens", + "completionTokens", + "totalTokens", + "cacheReadTokens", + ]) { + assertHas( + new RegExp( + `assertFiniteNonNegativeInteger\\(args\\.tokens\\?\\.${tokenName},\\s*["']tokens\\.${tokenName}["']\\)`, + ), + usageEventsSource, + `tokens.${tokenName} should use the finite non-negative integer guard.`, + ); + } + + for (const countName of ["requests", "pages", "screenshots", "lookups"]) { + assertHas( + new RegExp( + `assertFiniteNonNegativeInteger\\(args\\.callCounts\\?\\.${countName},\\s*["']callCounts\\.${countName}["']\\)`, + ), + usageEventsSource, + `callCounts.${countName} should use the finite non-negative integer guard.`, + ); + } +}); diff --git a/tsconfig.json b/tsconfig.json index 3a13f90..17be2ac 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -30,5 +30,8 @@ ".next/dev/types/**/*.ts", "**/*.mts" ], - "exclude": ["node_modules"] + "exclude": [ + "node_modules", + "v2_elemente/**" + ] }