diff --git a/app/dashboard/blacklist/page.tsx b/app/dashboard/blacklist/page.tsx index e6ac55a..095015f 100644 --- a/app/dashboard/blacklist/page.tsx +++ b/app/dashboard/blacklist/page.tsx @@ -1,10 +1,5 @@ -import { DashboardPlaceholderPage } from "@/components/dashboard-placeholder-page"; +import { BlacklistManager } from "@/components/blacklist/blacklist-manager"; export default function BlacklistPage() { - return ( - - ); + return ; } diff --git a/app/dashboard/leads/page.tsx b/app/dashboard/leads/page.tsx index 434a016..6f9f116 100644 --- a/app/dashboard/leads/page.tsx +++ b/app/dashboard/leads/page.tsx @@ -1,10 +1,5 @@ -import { DashboardPlaceholderPage } from "@/components/dashboard-placeholder-page"; +import { LeadsReviewTable } from "@/components/leads/leads-review-table"; export default function LeadsPage() { - return ( - - ); + return ; } diff --git a/backlog/archive/tasks/task-20 - Implement-TASK-7-slice-3-dashboard-UI.md b/backlog/archive/tasks/task-20 - Implement-TASK-7-slice-3-dashboard-UI.md new file mode 100644 index 0000000..2740abb --- /dev/null +++ b/backlog/archive/tasks/task-20 - Implement-TASK-7-slice-3-dashboard-UI.md @@ -0,0 +1,48 @@ +--- +id: TASK-20 +title: Implement TASK-7 slice 3 dashboard UI +status: In Progress +assignee: [] +created_date: '2026-06-04 13:54' +updated_date: '2026-06-04 13:58' +labels: [] +dependencies: [] +priority: high +ordinal: 22000 +--- + +## Description + + +Build dashboard leads review page and blacklist management UI for lead qualification and blacklist controls. + + +## Acceptance Criteria + +- [x] #1 Replace dashboard leads placeholder with inline lead review and review mutations +- [x] #2 Replace dashboard blacklist placeholder with blacklist create/edit/list/delete UI +- [ ] #3 Use shadcn-style dashboard components and keep TypeScript compile clean + + + + +## Implementation Plan + + +1. Build reusable lead-review helper-driven UI components under components/leads and components/blacklist +2. Replace dashboard placeholder pages for leads and blacklist +3. Extend dashboard-model label helpers where needed +4. Add/adjust dashboard-model tests for new helper mappings +5. Run lint/tests and report results + + +## Implementation Notes + + +1) Built lead-review model helpers and added dashboard-model tests +2) Replaced dashboard/leads and dashboard/blacklist placeholders with component-backed UI +3) Added lead review table controls for priority/contact, notes, duplicate/blacklist handling, and review email fields +4) Added blacklist manager with create/list/edit/delete and backend blocking note in UI + +Validation completed: pnpm -s exec tsc -p tsconfig.json --noEmit + pnpm -s test pass; targeted eslint on changed files pass; full `pnpm -s lint` currently fails on pre-existing blacklist.ts any-typed fields from prior task work + diff --git a/backlog/tasks/task-7 - Add-lead-qualification-deduplication-and-blacklist-handling.md b/backlog/tasks/task-7 - Add-lead-qualification-deduplication-and-blacklist-handling.md index f4d477a..f8428be 100644 --- a/backlog/tasks/task-7 - Add-lead-qualification-deduplication-and-blacklist-handling.md +++ b/backlog/tasks/task-7 - Add-lead-qualification-deduplication-and-blacklist-handling.md @@ -1,9 +1,10 @@ --- id: TASK-7 title: 'Add lead qualification, deduplication, and blacklist handling' -status: To Do +status: Done assignee: [] created_date: '2026-06-03 19:13' +updated_date: '2026-06-04 14:09' labels: - mvp - leads @@ -24,19 +25,57 @@ Implement the rules that turn raw business discoveries into usable lead states. ## Acceptance Criteria -- [ ] #1 Leads with no usable email are placed in Kontakt fehlt while preserving phone and source data -- [ ] #2 Generic business emails are preferred and named emails are accepted only when explicitly found as business contact addresses -- [ ] #3 Hard duplicates are detected by domain, Google Place ID, or email; probable duplicates are flagged by name plus address or phone -- [ ] #4 Manual blacklist entries for domain, email, phone, company name, and Place ID are enforced during discovery and review -- [ ] #5 Priority values Hoch, Mittel, Niedrig, Zurückstellen, and Gesperrt are assigned or editable with clear reasons +- [x] #1 Leads with no usable email are placed in Kontakt fehlt while preserving phone and source data +- [x] #2 Generic business emails are preferred and named emails are accepted only when explicitly found as business contact addresses +- [x] #3 Hard duplicates are detected by domain, Google Place ID, or email; probable duplicates are flagged by name plus address or phone +- [x] #4 Manual blacklist entries for domain, email, phone, company name, and Place ID are enforced during discovery and review +- [x] #5 Priority values Hoch, Mittel, Niedrig, Zurückstellen, and Gesperrt are assigned or editable with clear reasons ## Implementation Plan -1. Add blacklist CRUD in Convex and dashboard UI. -2. Implement email/contact extraction result fields and Kontakt fehlt transitions. -3. Add hard and probable duplicate matching rules. -4. Add priority assignment rules based on website/contact signals. -5. Surface reasons and source data in lead detail and run logs. +Subagent-driven TDD execution plan + +Orchestrator responsibilities: +1. Coordinate TASK-7 implementation end to end. +2. Use gpt-5.3-codex-spark subagents for implementation and review slices. +3. Enforce TDD: write failing tests first, verify red, implement minimal production code, verify green, then refactor. +4. Keep Backlog notes current and do not mark Done until user confirms manual testing. + +Implementation slices: +1. Rules/backend qualification: add tests and implementation for email usability, generic vs named email handling, hard duplicates by domain/place/email, probable duplicates by company+address or company+phone, blacklist normalization, and priority/status reason derivation. +2. Convex integration: extend schema/types/indexes and lead/blacklist APIs for qualification, editable priority/status/reasons, blacklist CRUD, and discovery/review enforcement. +3. Dashboard UI: replace Leads and Sperrliste placeholders with scan-friendly review tools that expose source data, duplicate/blacklist reasons, and editable priority/status controls. +4. Funnel/model polish: map blocked priority to Gesperrt and keep deferred/review funnel behavior coherent. +5. Verification: run targeted tests during each TDD slice, then pnpm test and pnpm lint at the end. + +Acceptance criteria mapping: +- AC1: contact qualification stores leads without usable email as Kontakt fehlt while preserving phone/source metadata. +- AC2: email rules prefer generic business addresses and only allow named emails when explicitly sourced as business contact addresses. +- AC3: duplicate rules distinguish hard duplicates and probable duplicates. +- AC4: blacklist entries for domain/email/phone/company/place ID apply during discovery and review. +- AC5: Hoch, Mittel, Niedrig, Zurückstellen, and Gesperrt are assignable/editable with clear reasons. + +## Implementation Notes + + +Execution started with subagent-driven TDD orchestration using gpt-5.3-codex-spark as requested. + +Aufgabe 7: implementiere Google-Places-Email-Review-Regeln, Sperrlisten-Enforcement für bestehende Leads, und korrigiere Firmen-Normalisierung in Blacklist-Matching. Beginne mit neuen TDD-Tests in lib/lead-discovery-google + Convex-Review-Pfad. + +TASK-7 implemented: added review-based email contact patch in convex/leads.ts, bounded blacklist enforcement on create/update in convex/blacklist.ts, company normalization fix in getBlacklistLookupValues/getBlacklistMatches, and schema support for new lead matching fields/reasons/blocked priority. Tests: pnpm -s test ✅ and pnpm -s tsc ✅. + +Progress: implementing code-quality fixes in convex/blacklist.ts, convex/leads.ts, convex/leadDiscovery.ts; running requested test/type/lint commands after changes. Plan: tighten mutation patch typing, bound blacklist propagation, split website signal, and avoid empty normalized writes. + +Executed requested TASK-7 backend quality fixes in scoped files and validated with pnpm -s test, pnpm -s tsc, and targeted eslint. Outstanding follow-up: keep an eye on very large blacklist match sets; enforcement currently remains batch-at-a-time by design. + +TASK-7 implementation verified by orchestrator. Added lead qualification helpers and Convex integration for usable email handling, hard/probable duplicate detection, blacklist enforcement with scheduled backfill/apply batches, blocked priority/reason fields, and dashboard Leads/Sperrliste review UI. Verified: pnpm -s test (67 pass), pnpm -s tsc (exit 0), pnpm -s lint (0 errors, 2 generated Better Auth warnings). Browser plugin could not open localhost due ERR_BLOCKED_BY_CLIENT; route HEAD checks redirect to /login as expected for protected dashboard pages. + + +## Final Summary + + +Implemented lead qualification, duplicate handling, blacklist enforcement, blocked priority/reason support, and dashboard review surfaces. Verified acceptance criteria #1-#5 with tests/typecheck/lint; user confirmed TASK-7 is done. + diff --git a/backlog/tasks/task-8 - Implement-Playwright-website-crawling-and-screenshot-capture.md b/backlog/tasks/task-8 - Implement-Playwright-website-crawling-and-screenshot-capture.md index bcdf86f..6fba9ec 100644 --- a/backlog/tasks/task-8 - Implement-Playwright-website-crawling-and-screenshot-capture.md +++ b/backlog/tasks/task-8 - Implement-Playwright-website-crawling-and-screenshot-capture.md @@ -4,6 +4,7 @@ title: Implement Playwright website crawling and screenshot capture status: To Do assignee: [] created_date: '2026-06-03 19:13' +updated_date: '2026-06-04 14:08' labels: - mvp - audit @@ -19,24 +20,37 @@ ordinal: 8000 ## Description -Build the website inspection layer using Playwright. For qualified leads, the system should load the company website, inspect the homepage and a small set of relevant subpages, capture desktop/mobile screenshots, extract visible text and contact signals, and store all raw evidence in Convex. +Build the website inspection and contact-enrichment layer using Playwright. For qualified leads, the system should load the company website, inspect the homepage and a small set of relevant subpages, capture desktop/mobile screenshots, extract visible text and contact signals, store all raw evidence in Convex, and feed found email candidates back into the TASK-7 qualification rules before a lead remains in Kontakt fehlt. Google Places does not provide business email fields, so website crawl evidence is the primary MVP source for usable business email addresses. ## Acceptance Criteria - [ ] #1 Playwright captures desktop and mobile screenshots for the homepage and stores them in Convex File Storage - [ ] #2 Crawler visits a bounded set of relevant subpages: Kontakt, Impressum, Leistungen/Angebot, Über uns/Team when discoverable -- [ ] #3 Crawler extracts visible text, page title, meta description, headings, links, phone numbers, email candidates, and CTA/contact-form signals -- [ ] #4 Simple technical checks include HTTPS/final URL, missing title/meta description, visible contact path, and obvious broken internal links within the crawl limit -- [ ] #5 Crawler failures produce useful dashboard-visible errors without blocking unrelated leads +- [ ] #3 Crawler extracts visible text, page title, meta description, headings, links, phone numbers, email candidates, email source URLs, contact-person context, and CTA/contact-form signals +- [ ] #4 Extracted email candidates are classified through the TASK-7 rules: generic business emails are preferred; named emails are accepted only when explicitly published as business contact addresses; no guessed addresses are generated +- [ ] #5 Leads discovered by Google Places with a website are automatically scheduled for contact enrichment before they remain in Kontakt fehlt; found usable email updates the lead contact fields and status while preserving phone and source data +- [ ] #6 Simple technical checks include HTTPS/final URL, missing title/meta description, visible contact path, and obvious broken internal links within the crawl limit +- [ ] #7 Crawler failures produce useful dashboard-visible errors without blocking unrelated leads + + ## Implementation Plan 1. Add Playwright runtime setup compatible with local development and Coolify container deployment. 2. Define crawl limits, viewports, timeout behavior, and allowed same-domain URL rules. -3. Capture homepage desktop/mobile screenshots and upload to Convex storage. +3. Capture homepage desktop/mobile screenshots and upload them to Convex storage. 4. Discover and inspect relevant subpages with bounded depth. -5. Persist extracted text, metadata, contact candidates, technical checks, screenshots, and errors. +5. Extract visible text, metadata, links, phone numbers, email candidates, contact-person context, CTA/contact-form signals, and source URLs. +6. Normalize and score email candidates, then call the existing TASK-7 lead review/contact qualification path so usable emails update lead contact fields and unqualified named emails do not. +7. Add contact-enrichment run state and dashboard-visible run events/errors for leads that still need manual contact research. +8. Persist extracted raw evidence, technical checks, screenshots, and crawler errors in Convex. + +## Implementation Notes + + +Expanded TASK-8 to cover website-based contact enrichment because Google Places does not provide business email fields. This keeps email handling evidence-based and reuses TASK-7 qualification rules instead of guessing addresses. + diff --git a/components/blacklist/blacklist-manager.tsx b/components/blacklist/blacklist-manager.tsx new file mode 100644 index 0000000..28f62fd --- /dev/null +++ b/components/blacklist/blacklist-manager.tsx @@ -0,0 +1,376 @@ +"use client"; + +import { useMemo, useState } from "react"; +import { useMutation, useQuery } from "convex/react"; +import { FunctionReturnType } from "convex/server"; +import { Id } from "@/convex/_generated/dataModel"; + +import { api } from "@/convex/_generated/api"; +import { Card } from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; + +type BlacklistResult = FunctionReturnType; +type BlacklistEntry = NonNullable[number]; + +type BlacklistType = + | "domain" + | "email" + | "phone" + | "company" + | "google_place_id"; + +const blacklistTypeOptions: BlacklistType[] = [ + "domain", + "email", + "phone", + "company", + "google_place_id", +]; + +function labelForType(type: BlacklistType): string { + if (type === "google_place_id") { + return "Google Place ID"; + } + + return type.charAt(0).toUpperCase() + type.slice(1); +} + +function formatDate(value: number): string { + return new Intl.DateTimeFormat("de-DE", { + dateStyle: "short", + timeStyle: "short", + }).format(new Date(value)); +} + +export function BlacklistManager() { + const entries = useQuery(api.blacklist.list, { limit: 150 }) as + | BlacklistResult + | undefined; + const createEntry = useMutation(api.blacklist.create); + const updateEntry = useMutation(api.blacklist.update); + const removeEntry = useMutation(api.blacklist.remove); + + const [type, setType] = useState("domain"); + const [value, setValue] = useState(""); + const [note, setNote] = useState(""); + const [rowBusyId, setRowBusyId] = useState | null>(null); + const [formBusy, setFormBusy] = useState(false); + const [statusMessage, setStatusMessage] = useState(null); + const [statusError, setStatusError] = useState(null); + + const entriesSorted = useMemo(() => { + if (!entries) { + return []; + } + + return [...entries].sort((a, b) => b.createdAt - a.createdAt); + }, [entries]); + + const submitNew = async () => { + if (!value.trim()) { + setStatusError("Bitte ein Sperrwert eintragen."); + return; + } + + setFormBusy(true); + setStatusError(null); + setStatusMessage(null); + + try { + await createEntry({ + type, + value: value.trim(), + note: note.trim().length > 0 ? note.trim() : undefined, + }); + setValue(""); + setNote(""); + setStatusMessage("Eintrag hinzugefügt."); + } catch { + setStatusError("Eintrag konnte nicht erstellt werden."); + } finally { + setFormBusy(false); + } + }; + + const remove = async (id: Id<"blacklistEntries">) => { + setRowBusyId(id); + setStatusError(null); + setStatusMessage(null); + + try { + await removeEntry({ id }); + setStatusMessage("Eintrag gelöscht."); + } catch { + setStatusError("Eintrag konnte nicht entfernt werden."); + } finally { + setRowBusyId(null); + } + }; + + return ( +
+
+

Blacklist-Verwaltung

+

Sperrliste

+
+ +
+ +

Neuen Eintrag anlegen

+

+ Neue Einträge wirken sofort: bestehende und neue Leads mit passendem + Typ werden automatisch blockiert. +

+ +
+ + + setValue(event.target.value)} + placeholder="Wert" + /> + + setNote(event.target.value)} + placeholder="Notiz (optional)" + /> + + +
+ + {statusError ? ( +

+ {statusError} +

+ ) : null} + {statusMessage ? ( +

+ {statusMessage} +

+ ) : null} +
+
+ +
+ +
+
+ + + + + + + + + + + + + {entries === undefined ? ( + + + + + + ) : entriesSorted.length === 0 ? ( + + + + + + ) : ( + + {entriesSorted.map((entry) => ( + { + setRowBusyId(nextEntry.id); + setStatusError(null); + setStatusMessage(null); + + try { + await updateEntry(nextEntry); + setStatusMessage("Eintrag aktualisiert."); + } catch { + setStatusError("Eintrag konnte nicht gespeichert werden."); + } finally { + setRowBusyId(null); + } + }} + isBusy={rowBusyId === entry._id} + /> + ))} + + )} +
TypWertNotizNormalisiertErstelltAktion
+

+ Sperrliste wird geladen… +

+
+

+ Noch keine Sperreinträge. +

+
+
+
+
+
+
+ ); +} + +function BlacklistEntryRow({ + entry, + onDelete, + onUpdate, + isBusy, +}: { + entry: BlacklistEntry; + onDelete: (id: Id<"blacklistEntries">) => Promise; + onUpdate: (next: { + id: Id<"blacklistEntries">; + type?: BlacklistType; + value?: string; + note?: string; + }) => Promise; + isBusy: boolean; +}) { + const [isEditing, setIsEditing] = useState(false); + const [type, setType] = useState(entry.type); + const [value, setValue] = useState(entry.value); + const [note, setNote] = useState(entry.note ?? ""); + const [rowMessage, setRowMessage] = useState(null); + + const submitUpdate = async () => { + if (!value.trim()) { + setRowMessage("Wert darf nicht leer sein."); + return; + } + + setRowMessage(null); + + await onUpdate({ + id: entry._id, + type, + value: value.trim(), + note: note.trim().length > 0 ? note.trim() : undefined, + }); + + setIsEditing(false); + setRowMessage("Gespeichert"); + }; + + return ( + + + {isEditing ? ( + + ) : ( + {labelForType(entry.type)} + )} + + + {isEditing ? ( + setValue(event.target.value)} /> + ) : ( +

{entry.value}

+ )} + + + {isEditing ? ( + setNote(event.target.value)} /> + ) : ( +

+ {entry.note ?? "—"} +

+ )} + + +

{entry.normalizedValue}

+ + +

{formatDate(entry.createdAt)}

+ + +
+ {isEditing ? ( + <> + + + + ) : ( + <> + + + + )} +
+ {rowMessage ? ( +

{rowMessage}

+ ) : null} + + + ); +} diff --git a/components/leads/leads-review-table.tsx b/components/leads/leads-review-table.tsx new file mode 100644 index 0000000..913c6cd --- /dev/null +++ b/components/leads/leads-review-table.tsx @@ -0,0 +1,576 @@ +"use client"; + +import { useMemo, useState } from "react"; +import { useMutation, useQuery } from "convex/react"; +import { FunctionReturnType } from "convex/server"; +import { Building2, Mail, MapPin, Phone, ShieldAlert } from "lucide-react"; + +import { api } from "@/convex/_generated/api"; +import { Id } from "@/convex/_generated/dataModel"; +import { + getLeadBlacklistStatusLabel, + getLeadContactStatusLabel, + getLeadDuplicateStatusLabel, + getLeadPriorityLabel, + leadBlacklistStatusOptions, + leadContactStatusOptions, + leadDuplicateStatusOptions, + leadPriorityOptions, + type LeadContactStatus, + type LeadDuplicateStatus, + type LeadPriority, + type LeadBlacklistStatus, +} from "@/lib/dashboard-model"; +import { Button } from "@/components/ui/button"; +import { Card } from "@/components/ui/card"; +import { Input } from "@/components/ui/input"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Badge } from "@/components/ui/badge"; +import { Switch } from "@/components/ui/switch"; + +type LeadsListResult = FunctionReturnType; +type LeadRow = NonNullable[number]; + +type LeadReviewDraft = { + priority: LeadPriority; + contactStatus: LeadContactStatus; + priorityReason: string; + contactStatusReason: string; + notes: string; + reviewEmail: string; + reviewEmailSource: string; + reviewContactPerson: string; + reviewIsBusinessContactAddress: boolean; + duplicateStatus: LeadDuplicateStatus; + blacklistStatus: LeadBlacklistStatus; +}; + +type LeadReviewPayload = { + id: Id<"leads">; + priority?: LeadPriority; + priorityReason?: string; + contactStatus?: LeadContactStatus; + contactStatusReason?: string; + notes?: string; + duplicateStatus?: LeadDuplicateStatus; + duplicateReason?: string; + blacklistStatus?: LeadBlacklistStatus; + blacklistReason?: string; + duplicateOfLeadId?: Id<"leads">; + applyBlacklist?: boolean; + reviewEmail?: string; + reviewEmailSource?: string; + reviewContactPerson?: string; + reviewIsBusinessContactAddress?: boolean; +}; + +function normalizeTextInput(value: string): string | undefined { + const next = value.trim(); + + return next.length > 0 ? next : undefined; +} + +function contactSourceLabel(lead: LeadRow): string { + if (lead.sourceProvider) { + return lead.sourceProvider; + } + + if (lead.emailSource) { + return lead.emailSource; + } + + return "Unbekannt"; +} + +function formatLocation(lead: LeadRow): string { + if (lead.postalCode && lead.city) { + return `${lead.postalCode} ${lead.city}`; + } + + if (lead.city || lead.address) { + return lead.city ?? lead.address ?? ""; + } + + return lead.address ?? "Ort offen"; +} + +function priorityBadgeClass(priority: LeadPriority): string { + switch (priority) { + case "high": + return "text-destructive border-destructive/30 bg-destructive/15"; + case "medium": + return "text-muted-foreground border-muted-foreground/30 bg-muted/20"; + case "low": + return "text-muted-foreground border-muted/40 bg-muted/35"; + case "defer": + return "text-muted-foreground border-secondary/50 bg-secondary/30"; + case "blocked": + return "text-destructive border-destructive/40 bg-destructive/15"; + default: + return "text-muted-foreground border-muted bg-muted/20"; + } +} + +function duplicateBadgeVariant( + duplicateStatus: LeadDuplicateStatus, +): "secondary" | "default" | "outline" | "destructive" { + if (duplicateStatus === "duplicate") { + return "destructive"; + } + + if (duplicateStatus === "possible_duplicate") { + return "outline"; + } + + if (duplicateStatus === "unique") { + return "secondary"; + } + + return "outline"; +} + +export function LeadsReviewTable() { + const leads = useQuery(api.leads.list, { limit: 120 }); + const [actionMessage, setActionMessage] = useState(null); + + const sortedLeads = useMemo(() => { + if (!leads) { + return []; + } + + return [...leads].sort((a, b) => b.createdAt - a.createdAt); + }, [leads]); + + return ( +
+
+

Leads Review

+

Leads prüfen

+
+ +
+ +
+
+ + + + + + + + + + + + + + {leads === undefined ? ( + + + + + + ) : sortedLeads.length === 0 ? ( + + + + + + ) : ( + + {sortedLeads.map((lead) => ( + + ))} + + )} +
Firma / OrtKontakt + QuellePrioritätKontaktstatusQualitätReview-FelderAktionen
+

+ Leads werden geladen… +

+
+

+ Keine Leads vorhanden. Bitte zuerst eine Kampagne starten + oder importieren. +

+
+
+
+
+
+ + {actionMessage ? ( +

+ {actionMessage} +

+ ) : null} +
+ ); +} + +function LeadReviewRow({ + lead, + onActionMessage, +}: { + lead: LeadRow; + onActionMessage: (value: string) => void; +}) { + const [draft, setDraft] = useState(() => ({ + priority: lead.priority, + contactStatus: lead.contactStatus, + priorityReason: lead.priorityReason ?? "", + contactStatusReason: lead.contactStatusReason ?? "", + notes: lead.notes ?? "", + reviewEmail: lead.email ?? "", + reviewEmailSource: lead.emailSource ?? "", + reviewContactPerson: lead.contactPerson ?? "", + reviewIsBusinessContactAddress: false, + duplicateStatus: (lead.duplicateStatus as LeadDuplicateStatus) ?? "unchecked", + blacklistStatus: lead.blacklistStatus, + })); + const [isSaving, setIsSaving] = useState(false); + const [isBlocking, setIsBlocking] = useState(false); + const [rowMessage, setRowMessage] = useState(null); + const reviewUpdate = useMutation(api.leads.reviewUpdate); + + const location = formatLocation(lead); + const reasonParts = [ + lead.priorityReason, + lead.contactStatusReason, + lead.duplicateReason, + lead.blacklistReason, + ].filter((item): item is string => Boolean(item)); + + const update = async ( + payload?: Omit, + ) => { + setIsSaving(true); + setRowMessage(null); + onActionMessage(""); + + try { + await reviewUpdate({ id: lead._id, ...payload } as LeadReviewPayload); + setRowMessage("Gespeichert"); + onActionMessage("Aktualisierung übernommen"); + } catch { + setRowMessage("Speichern fehlgeschlagen"); + } finally { + setIsSaving(false); + setTimeout(() => setRowMessage(null), 1400); + } + }; + + const saveRow = async () => { + const reviewEmail = normalizeTextInput(draft.reviewEmail); + const reviewEmailSource = normalizeTextInput(draft.reviewEmailSource); + const reviewContactPerson = draft.reviewContactPerson.trim(); + const shouldUpdateEmailReview = + reviewEmail !== normalizeTextInput(lead.email ?? "") || + reviewEmailSource !== normalizeTextInput(lead.emailSource ?? "") || + reviewContactPerson !== normalizeTextInput(lead.contactPerson ?? ""); + + if (shouldUpdateEmailReview && !reviewEmail && !lead.email) { + setRowMessage("Review-E-Mail setzen, um Kontaktinfos zu ändern."); + return; + } + + const payload = { + id: lead._id, + priority: draft.priority, + priorityReason: draft.priorityReason, + contactStatus: draft.contactStatus, + contactStatusReason: draft.contactStatusReason, + notes: draft.notes, + duplicateStatus: draft.duplicateStatus, + duplicateReason: lead.duplicateReason, + blacklistStatus: draft.blacklistStatus, + blacklistReason: lead.blacklistReason, + reviewIsBusinessContactAddress: draft.reviewIsBusinessContactAddress, + ...(shouldUpdateEmailReview ? { + reviewEmail: reviewEmail ?? lead.email, + reviewEmailSource: reviewEmailSource ?? lead.emailSource, + reviewContactPerson, + } : {}), + }; + + await update(payload); + }; + + const blockLead = async () => { + setIsBlocking(true); + await update({ applyBlacklist: true }); + setIsBlocking(false); + }; + + const updateDraft = ( + field: T, + value: LeadReviewDraft[T], + ) => { + setDraft((current) => ({ ...current, [field]: value })); + }; + + return ( + + +

{lead.companyName}

+

+ + {lead.niche ?? "Nische offen"} +

+

+ + {location} +

+ {lead.address ? ( +

+ {lead.address} +

+ ) : null} + + + +

+ + + {lead.email || "Keine E-Mail"} + +

+ {lead.phone ? ( +

+ + {lead.phone} +

+ ) : null} +

+ Quelle: {contactSourceLabel(lead)} +

+ {lead.websiteDomain ? ( +

+ Domain: {lead.websiteDomain} +

+ ) : null} + + + +

+ {getLeadPriorityLabel(draft.priority)} +

+
+ +
+ + + + + {getLeadContactStatusLabel(draft.contactStatus)} + +
+ +
+ + + +
+

Prioritätsgrund

+ { + updateDraft("priorityReason", event.target.value); + }} + /> +
+
+

Kontaktstatus-Notiz

+ { + updateDraft("contactStatusReason", event.target.value); + }} + /> +
+
+

Notiz

+ { + updateDraft("notes", event.target.value); + }} + /> +
+ +
+ + {getLeadDuplicateStatusLabel(draft.duplicateStatus)} + + + + {getLeadBlacklistStatusLabel(lead.blacklistStatus)} + +
+ +
+ {reasonParts.length === 0 ? ( +

Keine Zusatzhinweise

+ ) : ( + reasonParts.map((reason) =>

• {reason}

) + )} +
+ + + +
+

Review-E-Mail

+ { + updateDraft("reviewEmail", event.target.value); + }} + /> +
+ +
+

Review-Quelle

+ { + updateDraft("reviewEmailSource", event.target.value); + }} + /> +
+ +
+

Ansprechperson

+ { + updateDraft("reviewContactPerson", event.target.value); + }} + /> +
+ + + +
+

Duplikatstatus

+ +
+ +
+ + +
+ + + +
+ + + +
+ + {rowMessage ? ( +

{rowMessage}

+ ) : null} + + + ); +} diff --git a/convex/blacklist.ts b/convex/blacklist.ts index ac259eb..608dfc4 100644 --- a/convex/blacklist.ts +++ b/convex/blacklist.ts @@ -1,7 +1,15 @@ import { v } from "convex/values"; +import { + normalizeDomain, + normalizeEmailAddress, + normalizePhone, + normalizeText, +} from "../lib/lead-discovery-google"; import { normalizeListLimit } from "./domain"; -import { mutation, query } from "./_generated/server"; +import { internal } from "./_generated/api"; +import type { Doc } from "./_generated/dataModel"; +import { internalMutation, mutation, query, type MutationCtx } from "./_generated/server"; const blacklistType = v.union( v.literal("domain"), @@ -11,8 +19,193 @@ const blacklistType = v.union( v.literal("google_place_id"), ); -function normalizeBlacklistValue(value: string) { - return value.trim().toLowerCase(); +type BlacklistType = + | "domain" + | "email" + | "phone" + | "company" + | "google_place_id"; + +const BLACKLIST_APPLY_BATCH_SIZE = 100; +const BLACKLIST_REVIEW_NOTE_PREFIX = + "Lead automatisch durch Sperrlisteneintrag blockiert."; + +type BlacklistReason = { + type: BlacklistType; + normalizedValue: string; + reason: string; +}; + +type LeadIdAndBlacklistPatch = Pick< + Doc<"leads">, + "blacklistStatus" | "priority" | "contactStatus" | "blacklistReason" | "priorityReason" | "contactStatusReason" +> & { + updatedAt: number; +}; + +type LeadMatchingFieldsPatch = Partial< + Pick< + Doc<"leads">, + | "normalizedEmail" + | "normalizedPhone" + | "normalizedCompanyName" + | "normalizedAddress" + | "normalizedGooglePlaceId" + > +> & { + updatedAt: number; +}; + +type LeadIdRow = Pick, "_id">; + +type LeadMatchQuery = { + order: (direction: "asc" | "desc") => { + paginate: (args: { + numItems: number; + cursor: string | null; + }) => Promise<{ + page: LeadIdRow[]; + isDone: boolean; + continueCursor: string | null; + }>; + }; +}; + +function buildBlacklistReason(entry: { type: BlacklistType; value: string; note?: string }) { + const normalizedNote = entry.note?.trim(); + + return normalizedNote + ? `${BLACKLIST_REVIEW_NOTE_PREFIX} ${entry.type}: ${entry.value}. ${normalizedNote}` + : `${BLACKLIST_REVIEW_NOTE_PREFIX} ${entry.type}: ${entry.value}.`; +} + +function buildReasonPatch(reason: string) { + const patch: LeadIdAndBlacklistPatch = { + blacklistStatus: "blocked" as const, + priority: "blocked" as const, + contactStatus: "do_not_contact" as const, + blacklistReason: reason, + priorityReason: reason, + contactStatusReason: reason, + updatedAt: Date.now(), + }; + + return patch; +} + +function getLeadMatchQuery( + ctx: MutationCtx, + type: BlacklistType, + normalizedValue: string, +): (() => LeadMatchQuery) | null { + if (!normalizedValue) { + return null; + } + + switch (type) { + case "domain": + return () => + ctx.db + .query("leads") + .withIndex("by_websiteDomain", (q) => + q.eq("websiteDomain", normalizedValue), + ); + case "email": + return () => + ctx.db + .query("leads") + .withIndex("by_normalizedEmail", (q) => + q.eq("normalizedEmail", normalizedValue), + ); + case "phone": + return () => + ctx.db + .query("leads") + .withIndex("by_normalizedPhone", (q) => + q.eq("normalizedPhone", normalizedValue), + ); + case "company": + return () => + ctx.db + .query("leads") + .withIndex("by_normalizedCompanyName", (q) => + q.eq("normalizedCompanyName", normalizedValue), + ); + case "google_place_id": + return () => + ctx.db + .query("leads") + .withIndex("by_normalizedGooglePlaceId", (q) => + q.eq("normalizedGooglePlaceId", normalizedValue), + ); + default: + return null; + } +} + +function buildLeadMatchingFieldsPatch(lead: Doc<"leads">) { + const patch: LeadMatchingFieldsPatch = { + updatedAt: Date.now(), + }; + const normalizedEmail = normalizeEmailAddress(lead.email); + const normalizedPhone = normalizePhone(lead.phone); + const normalizedCompanyName = normalizeText(lead.companyName); + const normalizedAddress = normalizeText(lead.address); + const normalizedGooglePlaceId = normalizeDomain(lead.googlePlaceId); + + if (!lead.normalizedEmail && normalizedEmail) { + patch.normalizedEmail = normalizedEmail; + } + if (!lead.normalizedPhone && normalizedPhone) { + patch.normalizedPhone = normalizedPhone; + } + if (!lead.normalizedCompanyName && normalizedCompanyName) { + patch.normalizedCompanyName = normalizedCompanyName; + } + if (!lead.normalizedAddress && normalizedAddress) { + patch.normalizedAddress = normalizedAddress; + } + if (!lead.normalizedGooglePlaceId && normalizedGooglePlaceId) { + patch.normalizedGooglePlaceId = normalizedGooglePlaceId; + } + + return Object.keys(patch).length > 1 ? patch : null; +} + +async function scheduleBackfillThenBlacklistApply( + ctx: MutationCtx, + reason: BlacklistReason, +) { + await ctx.scheduler.runAfter( + 0, + internal.blacklist.backfillLeadMatchingFieldsForBlacklist, + { + ...reason, + cursor: null, + }, + ); +} + +function normalizeBlacklistValue(type: BlacklistType, value: string) { + const trimmed = value.trim(); + + if (!trimmed) { + return null; + } + + switch (type) { + case "email": + return normalizeEmailAddress(trimmed); + case "phone": + return normalizePhone(trimmed); + case "domain": + case "google_place_id": + return normalizeDomain(trimmed); + case "company": + return normalizeText(trimmed); + default: + return null; + } } export const create = mutation({ @@ -22,11 +215,238 @@ export const create = mutation({ note: v.optional(v.string()), }, handler: async (ctx, args) => { - return await ctx.db.insert("blacklistEntries", { - ...args, - normalizedValue: normalizeBlacklistValue(args.value), + const type = args.type as BlacklistType; + const normalizedValue = normalizeBlacklistValue(type, args.value); + + if (!normalizedValue) { + throw new Error("Blacklist-Wert ist ungültig."); + } + + const existing = await ctx.db + .query("blacklistEntries") + .withIndex("by_type_and_normalizedValue", (q) => + q.eq("type", type).eq("normalizedValue", normalizedValue), + ) + .take(1); + + if (existing[0]) { + await scheduleBackfillThenBlacklistApply(ctx, { + type, + normalizedValue, + reason: buildBlacklistReason({ + type, + value: existing[0].value, + note: existing[0].note, + }), + }); + return existing[0]._id; + } + + const created = await ctx.db.insert("blacklistEntries", { + type, + value: args.value.trim(), + normalizedValue, + note: args.note, createdAt: Date.now(), }); + + await scheduleBackfillThenBlacklistApply(ctx, { + type, + normalizedValue, + reason: buildBlacklistReason({ + type, + value: args.value.trim(), + note: args.note, + }), + }); + + return created; + }, +}); + +export const update = mutation({ + args: { + id: v.id("blacklistEntries"), + type: v.optional(blacklistType), + value: v.optional(v.string()), + note: v.optional(v.string()), + }, + handler: async (ctx, args) => { + const current = await ctx.db.get(args.id); + + if (!current) { + throw new Error("Blacklist-Eintrag nicht gefunden."); + } + + const nextType = (args.type ?? current.type) as BlacklistType; + const patch: { + type: BlacklistType; + value?: string; + normalizedValue?: string; + note?: string; + } = { + type: nextType, + }; + const nextNormalizedValueFromCurrent = normalizeBlacklistValue( + nextType, + current.value, + ); + + if (!nextNormalizedValueFromCurrent) { + throw new Error("Blacklist-Wert ist ungültig."); + } + + let nextValue = current.value; + let nextNormalizedValue = nextNormalizedValueFromCurrent; + + if (args.value !== undefined) { + const value = args.value.trim(); + const normalizedValue = normalizeBlacklistValue(nextType, value); + + if (!normalizedValue) { + throw new Error("Blacklist-Wert ist ungültig."); + } + + const existing = await ctx.db + .query("blacklistEntries") + .withIndex("by_type_and_normalizedValue", (q) => + q.eq("type", nextType).eq("normalizedValue", normalizedValue), + ) + .take(1); + + if (existing[0] && existing[0]._id !== args.id) { + return existing[0]._id; + } + + patch.value = value; + patch.normalizedValue = normalizedValue; + nextValue = value; + nextNormalizedValue = normalizedValue; + } + + if (args.note !== undefined) { + patch.note = args.note; + } + + await ctx.db.patch(args.id, patch); + await scheduleBackfillThenBlacklistApply(ctx, { + type: nextType, + normalizedValue: nextNormalizedValue, + reason: buildBlacklistReason({ + type: nextType, + value: nextValue, + note: patch.note ?? args.note ?? current.note, + }), + }); + return args.id; + }, +}); + +export const backfillLeadMatchingFieldsForBlacklist = internalMutation({ + args: { + type: blacklistType, + normalizedValue: v.string(), + reason: v.string(), + cursor: v.union(v.string(), v.null()), + }, + handler: async (ctx, args) => { + const page = await ctx.db + .query("leads") + .order("asc") + .paginate({ + numItems: BLACKLIST_APPLY_BATCH_SIZE, + cursor: args.cursor, + }); + + for (const lead of page.page) { + const patch = buildLeadMatchingFieldsPatch(lead); + + if (patch) { + await ctx.db.patch(lead._id, patch); + } + } + + if (!page.isDone) { + await ctx.scheduler.runAfter( + 0, + internal.blacklist.backfillLeadMatchingFieldsForBlacklist, + { + type: args.type, + normalizedValue: args.normalizedValue, + reason: args.reason, + cursor: page.continueCursor, + }, + ); + return null; + } + + await ctx.scheduler.runAfter( + 0, + internal.blacklist.applyBlacklistToMatchingLeadsBatch, + { + type: args.type, + normalizedValue: args.normalizedValue, + reason: args.reason, + cursor: null, + }, + ); + + return null; + }, +}); + +export const applyBlacklistToMatchingLeadsBatch = internalMutation({ + args: { + type: blacklistType, + normalizedValue: v.string(), + reason: v.string(), + cursor: v.union(v.string(), v.null()), + }, + handler: async (ctx, args) => { + const queryBuilder = getLeadMatchQuery( + ctx, + args.type as BlacklistType, + args.normalizedValue, + ); + + if (!queryBuilder) { + return null; + } + + const page = await queryBuilder() + .order("asc") + .paginate({ + numItems: BLACKLIST_APPLY_BATCH_SIZE, + cursor: args.cursor, + }); + const patch = buildReasonPatch(args.reason); + + for (const lead of page.page) { + await ctx.db.patch(lead._id, patch); + } + + if (!page.isDone) { + await ctx.scheduler.runAfter( + 0, + internal.blacklist.applyBlacklistToMatchingLeadsBatch, + { + type: args.type, + normalizedValue: args.normalizedValue, + reason: args.reason, + cursor: page.continueCursor, + }, + ); + } + + return null; + }, +}); + +export const remove = mutation({ + args: { id: v.id("blacklistEntries") }, + handler: async (ctx, args) => { + await ctx.db.delete(args.id); + return args.id; }, }); diff --git a/convex/domain.ts b/convex/domain.ts index 3536df9..f877b46 100644 --- a/convex/domain.ts +++ b/convex/domain.ts @@ -12,7 +12,7 @@ const SECRET_KEY_PATTERNS = [ ]; export const CAMPAIGN_STATUSES = ["active", "paused"] as const; -export const LEAD_PRIORITIES = ["high", "medium", "low", "defer"] as const; +export const LEAD_PRIORITIES = ["high", "medium", "low", "defer", "blocked"] as const; export const LEAD_CONTACT_STATUSES = [ "new", "missing_contact", diff --git a/convex/leadDiscovery.ts b/convex/leadDiscovery.ts index 7aca2ce..31fa279 100644 --- a/convex/leadDiscovery.ts +++ b/convex/leadDiscovery.ts @@ -5,13 +5,18 @@ import { buildGeocodingUrl, getBlacklistLookupValues, getBlacklistMatches, + getCandidateEmailValues, getPlacesSearchSpec, + normalizeDomain, + normalizePhone, + normalizeText, normalizePlacesResponse, parseGeocodingResponse, } from "../lib/lead-discovery-google"; import { buildLeadDiscoveryLeadRecord, buildLeadDiscoveryCounters, + getLeadDiscoveryPriority, } from "../lib/lead-discovery-run"; import { calculateNextRunAt } from "../lib/campaign-scheduling"; @@ -37,6 +42,20 @@ const candidateValidator = v.object({ googleTypes: v.array(v.string()), googlePrimaryType: nullableString, googleMapsUrl: nullableString, + email: v.optional(nullableString), + emailSource: v.optional(nullableString), + contactPerson: v.optional(nullableString), + isBusinessContactAddress: v.optional(v.boolean()), + contactEmails: v.optional( + v.array( + v.object({ + email: v.string(), + emailSource: v.optional(nullableString), + contactPerson: v.optional(nullableString), + isBusinessContactAddress: v.optional(v.boolean()), + }), + ), + ), sourceProvider: v.literal("google_places"), sourceFetchedAt: v.number(), }); @@ -396,23 +415,43 @@ export const persistDiscoveredLeads = internalMutation({ continue; } - const existingByPlaceId = await ctx.db - .query("leads") - .withIndex("by_googlePlaceId", (q) => - q.eq("googlePlaceId", candidate.placeId), - ) - .take(1); - const candidateDomain = candidate.websiteDomain; - const existingByDomain = candidateDomain + const normalizedPlaceId = normalizeDomain(candidate.placeId); + const normalizedDomain = normalizeDomain(candidate.websiteDomain); + const normalizedEmails = getCandidateEmailValues(candidate); + const normalizedPhone = normalizePhone(candidate.phone); + const normalizedCompanyName = normalizeText(candidate.businessName); + const normalizedAddress = normalizeText(candidate.address); + + const duplicateByPlaceId = normalizedPlaceId ? await ctx.db .query("leads") - .withIndex("by_websiteDomain", (q) => - q.eq("websiteDomain", candidateDomain), + .withIndex("by_normalizedGooglePlaceId", (q) => + q.eq("normalizedGooglePlaceId", normalizedPlaceId), ) .take(1) : []; - if (existingByPlaceId.length > 0 || existingByDomain.length > 0) { + const duplicateByDomain = normalizedDomain + ? await ctx.db + .query("leads") + .withIndex("by_websiteDomain", (q) => q.eq("websiteDomain", normalizedDomain)) + .take(1) + : []; + + const duplicateByEmailRows = []; + for (const email of normalizedEmails) { + const rows = await ctx.db + .query("leads") + .withIndex("by_normalizedEmail", (q) => q.eq("normalizedEmail", email)) + .take(1); + duplicateByEmailRows.push(...rows); + } + + if ( + duplicateByPlaceId.length > 0 || + duplicateByDomain.length > 0 || + duplicateByEmailRows.length > 0 + ) { skippedDuplicates += 1; await ctx.db.insert("agentRunEvents", { runId: args.runId, @@ -427,6 +466,29 @@ export const persistDiscoveredLeads = internalMutation({ continue; } + const probableDuplicateByPhone = normalizedPhone + ? await ctx.db + .query("leads") + .withIndex("by_normalizedPhone", (q) => + q.eq("normalizedPhone", normalizedPhone), + ) + .take(1) + : []; + + const probableDuplicateByAddress = normalizedCompanyName && normalizedAddress + ? await ctx.db + .query("leads") + .withIndex("by_normalizedCompanyName_and_normalizedAddress", (q) => + q + .eq("normalizedCompanyName", normalizedCompanyName) + .eq("normalizedAddress", normalizedAddress), + ) + .take(1) + : []; + + const probableDuplicateLead = + probableDuplicateByPhone[0] ?? probableDuplicateByAddress[0] ?? null; + const blacklistRows = []; for (const lookup of getBlacklistLookupValues(candidate)) { const rows = await ctx.db @@ -465,6 +527,34 @@ export const persistDiscoveredLeads = internalMutation({ candidate, now, }); + const hasWebsite = Boolean(candidate.websiteUrl ?? candidate.websiteDomain); + const priorityResult = getLeadDiscoveryPriority({ + isDuplicate: !!probableDuplicateLead, + hasWebsite, + hasWebsiteSignal: false, // plain Google-Places website hint maps to medium priority. + }); + const isDuplicateCandidate = !!probableDuplicateLead; + + if (normalizedPlaceId) { + lead.normalizedGooglePlaceId = normalizedPlaceId; + } + if (normalizedPhone !== "") { + lead.normalizedPhone = normalizedPhone; + } + if (normalizedCompanyName !== "") { + lead.normalizedCompanyName = normalizedCompanyName; + } + if (normalizedAddress !== "") { + lead.normalizedAddress = normalizedAddress; + } + lead.priority = priorityResult.priority; + lead.priorityReason = priorityResult.reason; + + if (isDuplicateCandidate) { + lead.duplicateStatus = "possible_duplicate"; + lead.duplicateReason = `Möglicher Dublettenkandidat zu Lead ${probableDuplicateLead._id}`; + lead.duplicateOfLeadId = probableDuplicateLead._id; + } await ctx.db.insert("leads", lead); leadsCreated += 1; diff --git a/convex/leads.ts b/convex/leads.ts index ce3d68e..08b1294 100644 --- a/convex/leads.ts +++ b/convex/leads.ts @@ -1,8 +1,93 @@ import { v } from "convex/values"; +import { getUsableContactEmailFromEntries } from "../lib/lead-discovery-google"; import { normalizeListLimit } from "./domain"; +import type { Doc, Id } from "./_generated/dataModel"; import { mutation, query } from "./_generated/server"; +type LeadDoc = Doc<"leads">; + +type LeadReviewContactPatch = { + email: string; + normalizedEmail: string; + emailSource?: string; + contactPerson?: string; +}; + +type BuildReviewContactPatchResult = { + patch?: LeadReviewContactPatch; + setContactStatus?: LeadDoc["contactStatus"]; +}; + +type LeadReviewPatch = { + updatedAt: number; + priority?: LeadDoc["priority"]; + priorityReason?: string; + contactStatus?: LeadDoc["contactStatus"]; + contactStatusReason?: string; + notes?: string; + duplicateStatus?: LeadDoc["duplicateStatus"]; + duplicateReason?: string; + duplicateOfLeadId?: Id<"leads">; + blacklistStatus?: LeadDoc["blacklistStatus"]; + blacklistReason?: string; + email?: string; + normalizedEmail?: string; + emailSource?: string; + contactPerson?: string; +}; + +function buildReviewContactPatch(args: { + email?: string; + emailSource?: string; + contactPerson?: string; + isBusinessContactAddress?: boolean; + explicitContactStatus?: boolean; + currentContactStatus?: "new" | "missing_contact" | "audit_ready" | "outreach_ready" | "contacted" | "replied" | "do_not_contact"; +}): BuildReviewContactPatchResult | null { + if (args.email === undefined) { + return null; + } + + const usable = getUsableContactEmailFromEntries([ + { + email: args.email, + emailSource: args.emailSource, + contactPerson: args.contactPerson, + isBusinessContactAddress: args.isBusinessContactAddress, + }, + ]); + + if (!usable) { + return { + setContactStatus: "missing_contact", + }; + } + + const patch: LeadReviewContactPatch = { + email: usable.email, + normalizedEmail: usable.email, + }; + + if (usable.emailSource !== null) { + patch.emailSource = usable.emailSource; + } + + if (usable.contactPerson !== null) { + patch.contactPerson = usable.contactPerson; + } + + const setContactStatus = + !args.explicitContactStatus && args.currentContactStatus === "missing_contact" + ? "new" + : undefined; + + return ({ + patch, + setContactStatus, + }); +} + export const create = mutation({ args: { campaignId: v.optional(v.id("campaigns")), @@ -24,6 +109,10 @@ export const create = mutation({ websiteUrl: v.optional(v.string()), websiteDomain: v.optional(v.string()), phone: v.optional(v.string()), + normalizedEmail: v.optional(v.string()), + normalizedPhone: v.optional(v.string()), + normalizedCompanyName: v.optional(v.string()), + normalizedAddress: v.optional(v.string()), email: v.optional(v.string()), emailSource: v.optional(v.string()), contactPerson: v.optional(v.string()), @@ -33,8 +122,10 @@ export const create = mutation({ v.literal("medium"), v.literal("low"), v.literal("defer"), + v.literal("blocked"), ), ), + priorityReason: v.optional(v.string()), contactStatus: v.optional( v.union( v.literal("new"), @@ -46,6 +137,20 @@ export const create = mutation({ v.literal("do_not_contact"), ), ), + contactStatusReason: v.optional(v.string()), + duplicateStatus: v.optional( + v.union( + v.literal("unchecked"), + v.literal("unique"), + v.literal("possible_duplicate"), + v.literal("duplicate"), + ), + ), + duplicateReason: v.optional(v.string()), + blacklistReason: v.optional(v.string()), + duplicateOfLeadId: v.optional(v.id("leads")), + blacklistStatus: v.optional(v.union(v.literal("clear"), v.literal("blocked"))), + normalizedGooglePlaceId: v.optional(v.string()), notes: v.optional(v.string()), }, handler: async (ctx, args) => { @@ -53,16 +158,151 @@ export const create = mutation({ return await ctx.db.insert("leads", { ...args, + normalizedEmail: args.normalizedEmail, + normalizedPhone: args.normalizedPhone, + normalizedCompanyName: args.normalizedCompanyName, + normalizedAddress: args.normalizedAddress, + normalizedGooglePlaceId: args.normalizedGooglePlaceId, priority: args.priority ?? "medium", contactStatus: args.contactStatus ?? "new", - duplicateStatus: "unchecked", - blacklistStatus: "clear", + duplicateStatus: args.duplicateStatus ?? "unchecked", + blacklistStatus: args.blacklistStatus ?? "clear", createdAt: now, updatedAt: now, }); }, }); +export const reviewUpdate = mutation({ + args: { + id: v.id("leads"), + priority: v.optional( + v.union( + v.literal("high"), + v.literal("medium"), + v.literal("low"), + v.literal("defer"), + v.literal("blocked"), + ), + ), + priorityReason: v.optional(v.string()), + contactStatus: v.optional( + v.union( + v.literal("new"), + v.literal("missing_contact"), + v.literal("audit_ready"), + v.literal("outreach_ready"), + v.literal("contacted"), + v.literal("replied"), + v.literal("do_not_contact"), + ), + ), + contactStatusReason: v.optional(v.string()), + notes: v.optional(v.string()), + duplicateStatus: v.optional( + v.union( + v.literal("unchecked"), + v.literal("unique"), + v.literal("possible_duplicate"), + v.literal("duplicate"), + ), + ), + duplicateReason: v.optional(v.string()), + blacklistStatus: v.optional(v.union(v.literal("clear"), v.literal("blocked"))), + blacklistReason: v.optional(v.string()), + duplicateOfLeadId: v.optional(v.id("leads")), + applyBlacklist: v.optional(v.boolean()), + reviewEmail: v.optional(v.string()), + reviewEmailSource: v.optional(v.string()), + reviewContactPerson: v.optional(v.string()), + reviewIsBusinessContactAddress: v.optional(v.boolean()), + }, + handler: async (ctx, args) => { + const lead = await ctx.db.get(args.id); + + if (!lead) { + return null; + } + + const now = Date.now(); + const patch: LeadReviewPatch = { + updatedAt: now, + }; + + if (args.priority !== undefined) { + patch.priority = args.priority; + } + if (args.priorityReason !== undefined) { + patch.priorityReason = args.priorityReason; + } + if (args.contactStatus !== undefined) { + patch.contactStatus = args.contactStatus; + } + if (args.contactStatusReason !== undefined) { + patch.contactStatusReason = args.contactStatusReason; + } + if (args.notes !== undefined) { + patch.notes = args.notes; + } + if (args.duplicateStatus !== undefined) { + patch.duplicateStatus = args.duplicateStatus; + } + if (args.duplicateReason !== undefined) { + patch.duplicateReason = args.duplicateReason; + } + if (args.duplicateOfLeadId !== undefined) { + patch.duplicateOfLeadId = args.duplicateOfLeadId; + } + + if (args.applyBlacklist) { + patch.blacklistStatus = "blocked"; + if (args.blacklistReason !== undefined) { + patch.blacklistReason = args.blacklistReason; + } else if (lead.blacklistReason === undefined) { + patch.blacklistReason = "Manuell in der Review als Sperrgrund gesetzt."; + } + if (args.priority === undefined || args.priority !== "blocked") { + patch.priority = "blocked"; + } + } else if (args.applyBlacklist === false && args.blacklistStatus !== undefined) { + patch.blacklistStatus = args.blacklistStatus; + patch.blacklistReason = args.blacklistReason; + } else if (args.blacklistStatus !== undefined) { + patch.blacklistStatus = args.blacklistStatus; + patch.blacklistReason = args.blacklistReason; + } + + const reviewContactPatch = buildReviewContactPatch({ + email: args.reviewEmail, + emailSource: args.reviewEmailSource, + contactPerson: args.reviewContactPerson, + isBusinessContactAddress: args.reviewIsBusinessContactAddress, + explicitContactStatus: args.contactStatus !== undefined, + currentContactStatus: lead.contactStatus, + }); + + if (reviewContactPatch?.patch) { + Object.assign(patch, reviewContactPatch.patch); + } + + if ( + reviewContactPatch !== null && + reviewContactPatch.setContactStatus !== undefined && + args.contactStatus === undefined + ) { + patch.contactStatus = reviewContactPatch.setContactStatus; + } + + if (args.blacklistReason !== undefined && patch.blacklistStatus === undefined) { + patch.blacklistStatus = "blocked"; + patch.blacklistReason = args.blacklistReason; + } + + await ctx.db.patch(args.id, patch); + return args.id; + }, +}); + export const get = query({ args: { id: v.id("leads") }, handler: async (ctx, args) => { diff --git a/convex/schema.ts b/convex/schema.ts index d7d9272..4449560 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -8,6 +8,7 @@ const leadPriority = v.union( v.literal("medium"), v.literal("low"), v.literal("defer"), + v.literal("blocked"), ); const leadContactStatus = v.union( v.literal("new"), @@ -158,6 +159,7 @@ export default defineSchema({ city: v.optional(v.string()), postalCode: v.optional(v.string()), googlePlaceId: v.optional(v.string()), + normalizedGooglePlaceId: v.optional(v.string()), googleMapsUrl: v.optional(v.string()), googlePrimaryType: v.optional(v.string()), googleTypes: v.optional(v.array(v.string())), @@ -169,9 +171,18 @@ export default defineSchema({ websiteUrl: v.optional(v.string()), websiteDomain: v.optional(v.string()), phone: v.optional(v.string()), + normalizedEmail: v.optional(v.string()), + normalizedPhone: v.optional(v.string()), + normalizedCompanyName: v.optional(v.string()), + normalizedAddress: v.optional(v.string()), email: v.optional(v.string()), emailSource: v.optional(v.string()), contactPerson: v.optional(v.string()), + priorityReason: v.optional(v.string()), + contactStatusReason: v.optional(v.string()), + duplicateReason: v.optional(v.string()), + blacklistReason: v.optional(v.string()), + duplicateOfLeadId: v.optional(v.id("leads")), priority: leadPriority, contactStatus: leadContactStatus, duplicateStatus: leadDuplicateStatus, @@ -183,8 +194,16 @@ export default defineSchema({ .index("by_campaignId", ["campaignId"]) .index("by_discoveryRunId", ["discoveryRunId"]) .index("by_contactStatus", ["contactStatus"]) + .index("by_normalizedEmail", ["normalizedEmail"]) + .index("by_normalizedPhone", ["normalizedPhone"]) + .index("by_normalizedCompanyName_and_normalizedAddress", [ + "normalizedCompanyName", + "normalizedAddress", + ]) + .index("by_normalizedGooglePlaceId", ["normalizedGooglePlaceId"]) .index("by_googlePlaceId", ["googlePlaceId"]) .index("by_websiteDomain", ["websiteDomain"]) + .index("by_normalizedCompanyName", ["normalizedCompanyName"]) .index("by_priority_and_contactStatus", ["priority", "contactStatus"]), audits: defineTable({ diff --git a/lib/dashboard-model.ts b/lib/dashboard-model.ts index 98cf7f1..7520a81 100644 --- a/lib/dashboard-model.ts +++ b/lib/dashboard-model.ts @@ -29,7 +29,7 @@ export type ReviewQueueItem = { detail: string; }; -export type LeadPriority = "high" | "medium" | "low" | "defer"; +export type LeadPriority = "high" | "medium" | "low" | "defer" | "blocked"; export type LeadContactStatus = | "new" @@ -41,6 +41,11 @@ export type LeadContactStatus = | "do_not_contact"; export type LeadBlacklistStatus = "clear" | "blocked"; +export type LeadDuplicateStatus = + | "unchecked" + | "unique" + | "possible_duplicate" + | "duplicate"; export type OutreachApprovalStatus = "draft" | "approved" | "rejected"; export type OutreachSendStatus = "not_sent" | "queued" | "sent" | "failed"; @@ -151,14 +156,15 @@ export const leadFunnelStages: LeadFunnelStage[] = [ }, ]; -const priorityLabels: Record = { +export const leadPriorityLabels: Record = { high: "Hoch", medium: "Mittel", low: "Niedrig", defer: "Zurückstellen", + blocked: "Gesperrt", }; -const contactStatusLabels: Record = { +export const leadContactStatusLabels: Record = { new: "Neu", missing_contact: "Kontakt fehlt", audit_ready: "Audit bereit", @@ -168,6 +174,61 @@ const contactStatusLabels: Record = { do_not_contact: "Nicht kontaktieren", }; +export const leadDuplicateStatusLabels: Record = { + unchecked: "Noch nicht geprüft", + unique: "Einzigartig", + possible_duplicate: "Möglicher Doppelter", + duplicate: "Duplikat", +}; + +export const leadBlacklistStatusLabels: Record = { + clear: "Offen", + blocked: "Gesperrt", +}; + +export const leadPriorityOptions: LeadPriority[] = [ + "high", + "medium", + "low", + "defer", + "blocked", +]; + +export const leadContactStatusOptions: LeadContactStatus[] = [ + "new", + "missing_contact", + "audit_ready", + "outreach_ready", + "contacted", + "replied", + "do_not_contact", +]; + +export const leadDuplicateStatusOptions: LeadDuplicateStatus[] = [ + "unchecked", + "unique", + "possible_duplicate", + "duplicate", +]; + +export const leadBlacklistStatusOptions: LeadBlacklistStatus[] = ["clear", "blocked"]; + +export function getLeadPriorityLabel(priority: LeadPriority): string { + return leadPriorityLabels[priority]; +} + +export function getLeadContactStatusLabel(status: LeadContactStatus): string { + return leadContactStatusLabels[status]; +} + +export function getLeadDuplicateStatusLabel(status: LeadDuplicateStatus): string { + return leadDuplicateStatusLabels[status]; +} + +export function getLeadBlacklistStatusLabel(status: LeadBlacklistStatus): string { + return leadBlacklistStatusLabels[status]; +} + export function toLeadFunnelCard(lead: LeadFunnelInput): LeadFunnelCard { return { id: lead.id, @@ -175,8 +236,8 @@ export function toLeadFunnelCard(lead: LeadFunnelInput): LeadFunnelCard { company: lead.companyName, niche: lead.niche ?? "Nische offen", location: formatLeadLocation(lead), - priorityLabel: priorityLabels[lead.priority], - contactStatusLabel: contactStatusLabels[lead.contactStatus], + priorityLabel: getLeadPriorityLabel(lead.priority), + contactStatusLabel: getLeadContactStatusLabel(lead.contactStatus), nextAction: getLeadNextAction(lead), websiteDomain: lead.websiteDomain, contactDetail: formatContactDetail(lead), @@ -198,6 +259,7 @@ function getLeadFunnelStageId(lead: LeadFunnelInput): LeadFunnelStageId { if ( lead.blacklistStatus === "blocked" || lead.priority === "defer" || + lead.priority === "blocked" || lead.contactStatus === "do_not_contact" ) { return "deferred"; diff --git a/lib/lead-discovery-google.ts b/lib/lead-discovery-google.ts index 3e5b035..2d658a7 100644 --- a/lib/lead-discovery-google.ts +++ b/lib/lead-discovery-google.ts @@ -228,6 +228,13 @@ type GooglePlaceDisplayName = text?: string; }; +type GooglePlaceContactEmailSource = { + email: string; + emailSource?: string | null; + contactPerson?: string | null; + isBusinessContactAddress?: boolean; +}; + type GooglePlaceApiPlace = { id?: string; displayName?: GooglePlaceDisplayName; @@ -254,6 +261,11 @@ export type GooglePlaceCandidate = { websiteUrl: string | null; websiteDomain: string | null; phone: string | null; + email?: string | null; + emailSource?: string | null; + contactPerson?: string | null; + isBusinessContactAddress?: boolean; + contactEmails?: GooglePlaceContactEmailSource[]; rating: number | null; userRatingCount: number | null; businessStatus: string | null; @@ -297,6 +309,163 @@ function normalizeWebsiteDomain(input?: string | null) { } } +const GENERIC_BUSINESS_EMAIL_LOCAL_PARTS = new Set([ + "info", + "kontakt", + "hello", + "hallo", + "office", + "post", + "service", + "team", + "anfrage", +]); + +export function normalizeText(value?: string | null) { + return value?.trim().toLowerCase().replace(/\s+/g, " ") ?? ""; +} + +export function normalizeEmailAddress(value?: string | null) { + const valueTrimmed = value?.trim().toLowerCase(); + + if (!valueTrimmed) { + return null; + } + + const [localPart, domain] = valueTrimmed.split("@"); + + if (!localPart || !domain) { + return null; + } + + if (!/^[a-z0-9._%+-]+$/.test(localPart)) { + return null; + } + + if (!/^[^\s@]+\.[^\s@]+$/.test(domain)) { + return null; + } + + return valueTrimmed; +} + +export type UsableContactEmail = { + email: string; + emailSource: string | null; + contactPerson: string | null; +}; + +type ParsedContactEmail = { + email: string; + emailSource: string | null; + contactPerson: string | null; + isBusinessContactAddress: boolean; + isGeneric: boolean; +}; + +type ContactEmailRuleInput = { + email: string; + emailSource?: string | null; + contactPerson?: string | null; + isBusinessContactAddress?: boolean; +}; + +export function getUsableContactEmailFromEntries( + entries: ContactEmailRuleInput[] | undefined, +) { + if (!Array.isArray(entries) || entries.length === 0) { + return null; + } + + const parsedEntries: ParsedContactEmail[] = []; + + for (const emailEntry of entries) { + const normalized = normalizeEmailAddress(emailEntry.email); + + if (!normalized) { + continue; + } + + parsedEntries.push({ + email: normalized, + emailSource: emailEntry.emailSource ?? null, + contactPerson: emailEntry.contactPerson ?? null, + isBusinessContactAddress: emailEntry.isBusinessContactAddress === true, + isGeneric: isGenericBusinessEmail(normalized), + }); + } + + const generic = parsedEntries.find((entry) => entry.isGeneric); + if (generic) { + return { + email: generic.email, + emailSource: generic.emailSource, + contactPerson: generic.contactPerson, + }; + } + + const named = parsedEntries.find((entry) => entry.isBusinessContactAddress); + if (!named) { + return null; + } + + return { + email: named.email, + emailSource: named.emailSource, + contactPerson: named.contactPerson, + }; +} + +function getCandidateEmailMetadata(candidate: GooglePlaceCandidate) { + const emails: GooglePlaceContactEmailSource[] = []; + + if (candidate.email) { + emails.push({ + email: candidate.email, + emailSource: candidate.emailSource, + contactPerson: candidate.contactPerson, + isBusinessContactAddress: candidate.isBusinessContactAddress, + }); + } + + if (Array.isArray(candidate.contactEmails)) { + emails.push(...candidate.contactEmails); + } + + return emails; +} + +export function getCandidateEmailValues(candidate: GooglePlaceCandidate) { + return getCandidateEmailMetadata(candidate) + .map((entry) => normalizeEmailAddress(entry.email)) + .filter((value): value is string => value !== null); +} + +function splitEmailLocalPart(email: string) { + const [localPart] = email.split("@"); + + return localPart?.split("+")[0] ?? ""; +} + +function isGenericBusinessEmail(email: string) { + const normalizedLocalPart = splitEmailLocalPart(email).toLowerCase(); + + return GENERIC_BUSINESS_EMAIL_LOCAL_PARTS.has(normalizedLocalPart); +} + +export function getUsableContactEmail( + candidate: GooglePlaceCandidate, +): UsableContactEmail | null { + return getUsableContactEmailFromEntries( + getCandidateEmailMetadata(candidate).map((entry) => ({ + email: entry.email, + emailSource: entry.emailSource, + contactPerson: entry.contactPerson, + isBusinessContactAddress: entry.isBusinessContactAddress, + })), + ); +} + export function normalizePlacesResponse( response: GooglePlacesApiResponse, fetchedAt: number, @@ -333,6 +502,10 @@ export function normalizePlacesResponse( export type ExistingLeadLike = { googlePlaceId?: string | null; websiteDomain?: string | null; + email?: string | null; + companyName?: string | null; + address?: string | null; + phone?: string | null; }; export type BlacklistRow = { @@ -342,20 +515,25 @@ export type BlacklistRow = { }; export type BlacklistLookupValue = { - type: "domain" | "phone" | "company" | "google_place_id"; + type: "domain" | "email" | "phone" | "company" | "google_place_id"; normalizedValue: string; }; -function normalizeDomain(value?: string | null) { +export function normalizeDomain(value?: string | null) { return value?.trim().toLowerCase().replace(/^www\./, "") ?? ""; } -function normalizePhone(value?: string | null) { +export function normalizePhone(value?: string | null) { if (!value) { return ""; } - return value.replace(/\D+/g, ""); + const digits = value.replace(/\D+/g, ""); + if (digits.startsWith("00")) { + return digits.slice(2); + } + + return digits; } function uniqueLookupValues(values: BlacklistLookupValue[]) { @@ -375,6 +553,8 @@ function uniqueLookupValues(values: BlacklistLookupValue[]) { export function getBlacklistLookupValues( candidate: GooglePlaceCandidate, ): BlacklistLookupValue[] { + const emailAddresses = getCandidateEmailValues(candidate); + return uniqueLookupValues([ { type: "google_place_id", @@ -386,7 +566,7 @@ export function getBlacklistLookupValues( }, { type: "company", - normalizedValue: normalizeDomain(candidate.businessName), + normalizedValue: normalizeText(candidate.businessName), }, { type: "phone", @@ -396,6 +576,10 @@ export function getBlacklistLookupValues( type: "phone", normalizedValue: normalizeDomain(candidate.phone), }, + ...emailAddresses.map((email) => ({ + type: "email" as const, + normalizedValue: email ?? "", + })), ]); } @@ -405,25 +589,57 @@ export function isDuplicateCandidate( ): boolean { const candidatePlaceId = normalizeDomain(candidate.placeId); const candidateDomain = normalizeDomain(candidate.websiteDomain); + const candidateEmails = getCandidateEmailValues(candidate); return existing.some((entry) => { const entryPlaceId = normalizeDomain(entry.googlePlaceId); const entryDomain = normalizeDomain(entry.websiteDomain); + const entryEmail = normalizeEmailAddress(entry.email); return ( (candidatePlaceId && entryPlaceId === candidatePlaceId) || - (candidateDomain && entryDomain === candidateDomain) + (candidateDomain && entryDomain === candidateDomain) || + candidateEmails.some( + (candidateEmail) => candidateEmail && entryEmail === candidateEmail, + ) ); }); } +export function isProbableDuplicateCandidate( + candidate: GooglePlaceCandidate, + existing: ExistingLeadLike[], +): boolean { + const candidateCompany = normalizeText(candidate.businessName); + const candidateAddress = normalizeText(candidate.address); + const candidatePhone = normalizePhone(candidate.phone); + + return existing.some((entry) => { + const entryCompany = normalizeText(entry.companyName); + const entryAddress = normalizeText(entry.address); + const entryPhone = normalizePhone(entry.phone); + + const isSameCompanyAndAddress = + candidateCompany && + candidateAddress && + entryCompany && + entryAddress && + candidateCompany === entryCompany && + candidateAddress === entryAddress; + + const isSamePhone = candidatePhone && entryPhone && candidatePhone === entryPhone; + + return isSameCompanyAndAddress || isSamePhone; + }); +} + export function getBlacklistMatches( candidate: GooglePlaceCandidate, blacklistRows: BlacklistRow[], ) { const candidatePlaceId = normalizeDomain(candidate.placeId); const candidateDomain = normalizeDomain(candidate.websiteDomain); - const candidateCompany = normalizeDomain(candidate.businessName); + const candidateCompany = normalizeText(candidate.businessName); const candidatePhone = normalizePhone(candidate.phone); return blacklistRows.filter((row) => { @@ -446,6 +662,10 @@ export function getBlacklistMatches( (row.normalizedValue === candidatePhone || normalizePhone(row.value) === candidatePhone) ); + case "email": + return getCandidateEmailValues(candidate).some( + (candidateEmail) => candidateEmail === row.normalizedValue, + ); default: return false; } diff --git a/lib/lead-discovery-run.ts b/lib/lead-discovery-run.ts index ae241d7..455563b 100644 --- a/lib/lead-discovery-run.ts +++ b/lib/lead-discovery-run.ts @@ -1,4 +1,10 @@ -import type { GooglePlaceCandidate } from "./lead-discovery-google"; +import { + normalizePhone, + normalizeText, + getUsableContactEmail, + type GooglePlaceCandidate, +} from "./lead-discovery-google"; +import type { Id } from "../convex/_generated/dataModel"; type AgentRunLike = { status: string; @@ -12,8 +18,16 @@ type LeadDiscoveryCounterInput = { }; type LeadDiscoveryContactInput = { - websiteDomain?: string | null; - phone?: string | null; + usableEmail?: string | null; +}; + +export type LeadDiscoveryPriority = "high" | "medium" | "low" | "defer" | "blocked"; + +type LeadDiscoveryPriorityInput = { + isBlacklisted?: boolean; + isDuplicate?: boolean; + hasWebsite?: boolean; + hasWebsiteSignal?: boolean; }; type LeadDiscoveryLeadRecordInput = { @@ -70,7 +84,7 @@ export function buildLeadDiscoveryCounters(input: LeadDiscoveryCounterInput) { export function getLeadDiscoveryContactStatus( input: LeadDiscoveryContactInput, ) { - if (input.websiteDomain || input.phone) { + if (input.usableEmail) { return "new"; } @@ -81,6 +95,14 @@ export function buildLeadDiscoveryLeadRecord< TCampaignId extends string, TRunId extends string, >(input: LeadDiscoveryLeadRecordInput) { + type LeadDiscoveryDuplicateStatus = + | "unchecked" + | "unique" + | "possible_duplicate" + | "duplicate"; + + const usableEmail = getUsableContactEmail(input.candidate); + const lead: { campaignId: TCampaignId; discoveryRunId: TRunId; @@ -100,9 +122,21 @@ export function buildLeadDiscoveryLeadRecord< websiteUrl?: string; websiteDomain?: string; phone?: string; - priority: "medium"; + normalizedGooglePlaceId?: string; + normalizedEmail?: string; + normalizedPhone?: string; + normalizedCompanyName?: string; + normalizedAddress?: string; + email?: string; + emailSource?: string; + contactPerson?: string; + priorityReason?: string; + duplicateReason?: string; + duplicateOfLeadId?: Id<"leads">; + blacklistReason?: string; + priority: LeadDiscoveryPriority; contactStatus: "new" | "missing_contact"; - duplicateStatus: "unique"; + duplicateStatus: LeadDiscoveryDuplicateStatus; blacklistStatus: "clear"; createdAt: number; updatedAt: number; @@ -119,8 +153,7 @@ export function buildLeadDiscoveryLeadRecord< sourceFetchedAt: input.candidate.sourceFetchedAt, priority: "medium", contactStatus: getLeadDiscoveryContactStatus({ - websiteDomain: input.candidate.websiteDomain, - phone: input.candidate.phone, + usableEmail: usableEmail?.email, }), duplicateStatus: "unique", blacklistStatus: "clear", @@ -136,6 +169,21 @@ export function buildLeadDiscoveryLeadRecord< const websiteUrl = optionalString(input.candidate.websiteUrl); const websiteDomain = optionalString(input.candidate.websiteDomain); const phone = optionalString(input.candidate.phone); + const normalizedPhone = normalizePhone(phone); + const normalizedCompanyName = normalizeText(input.candidate.businessName); + const normalizedAddress = normalizeText(input.candidate.address); + + if (normalizedCompanyName !== "") { + lead.normalizedCompanyName = normalizedCompanyName; + } + + if (normalizedAddress !== "") { + lead.normalizedAddress = normalizedAddress; + } + + if (normalizedPhone !== "") { + lead.normalizedPhone = normalizedPhone; + } if (googleMapsUrl !== undefined) { lead.googleMapsUrl = googleMapsUrl; @@ -161,6 +209,55 @@ export function buildLeadDiscoveryLeadRecord< if (phone !== undefined) { lead.phone = phone; } + if (usableEmail) { + lead.normalizedEmail = usableEmail.email; + lead.email = usableEmail.email; + if (usableEmail.emailSource !== null) { + lead.emailSource = usableEmail.emailSource; + } + if (usableEmail.contactPerson !== null) { + lead.contactPerson = usableEmail.contactPerson; + } + } else { + lead.contactStatus = "missing_contact"; + } return lead; } + +export function getLeadDiscoveryPriority( + input: LeadDiscoveryPriorityInput, +): { priority: LeadDiscoveryPriority; reason: string } { + if (input.isBlacklisted) { + return { + priority: "blocked", + reason: "Lead ist auf der Sperrliste.", + }; + } + + if (input.isDuplicate) { + return { + priority: "defer", + reason: "Dublettenprüfung oder Reviewpause.", + }; + } + + if (!input.hasWebsite) { + return { + priority: "high", + reason: "Kein Website-Indikator vorhanden.", + }; + } + + if (input.hasWebsiteSignal) { + return { + priority: "low", + reason: "Website vorhanden: geringer Kontaktaufwand.", + }; + } + + return { + priority: "medium", + reason: "Standardpriorität.", + }; +} diff --git a/tests/convex-domain.test.ts b/tests/convex-domain.test.ts index 34ec6ad..af6f3f0 100644 --- a/tests/convex-domain.test.ts +++ b/tests/convex-domain.test.ts @@ -4,6 +4,7 @@ import test from "node:test"; import { RUN_STATUSES, SCREENSHOT_VIEWPORTS, + LEAD_PRIORITIES, filterSafeSettingsRows, isSafeSettingsKey, normalizeListLimit, @@ -49,6 +50,10 @@ test("run statuses expose observable job lifecycle states", () => { ]); }); +test("lead priorities include manual blocking option", () => { + assert.deepEqual(LEAD_PRIORITIES, ["high", "medium", "low", "defer", "blocked"]); +}); + test("list limits are clamped to a positive integer range", () => { assert.equal(normalizeListLimit(undefined), 50); assert.equal(normalizeListLimit(-10), 1); diff --git a/tests/dashboard-model.test.ts b/tests/dashboard-model.test.ts index 400ddec..5be3ba8 100644 --- a/tests/dashboard-model.test.ts +++ b/tests/dashboard-model.test.ts @@ -2,6 +2,14 @@ import assert from "node:assert/strict"; import test from "node:test"; import { + getLeadBlacklistStatusLabel, + getLeadContactStatusLabel, + getLeadDuplicateStatusLabel, + getLeadPriorityLabel, + leadBlacklistStatusOptions, + leadContactStatusOptions, + leadDuplicateStatusOptions, + leadPriorityOptions, dashboardKpis, dashboardNavigation, groupLeadFunnelCards, @@ -138,6 +146,49 @@ test("groupLeadFunnelCards derives review, follow-up, and deferred columns witho ); }); +test("toLeadFunnelCard maps blocked priority to deferred stage with blocker label", () => { + const card = toLeadFunnelCard({ + id: "lead-blocked", + companyName: "Sperr Beispiel", + city: "Freiburg", + priority: "blocked", + contactStatus: "new", + blacklistStatus: "blocked", + }); + + assert.equal(card.stageId, "deferred"); + assert.equal(card.priorityLabel, "Gesperrt"); + assert.equal(card.nextAction, "Zurückstellung prüfen"); +}); + +test("dashboard-model exposes stable lead label helpers for UI mapping", () => { + assert.deepEqual(leadPriorityOptions, [ + "high", + "medium", + "low", + "defer", + "blocked", + ]); + assert.equal(getLeadPriorityLabel("high"), "Hoch"); + assert.equal(getLeadContactStatusLabel("missing_contact"), "Kontakt fehlt"); + assert.equal(getLeadBlacklistStatusLabel("blocked"), "Gesperrt"); +}); + +test("dashboard-model exposes duplicate status options and labels", () => { + assert.deepEqual(leadDuplicateStatusOptions, [ + "unchecked", + "unique", + "possible_duplicate", + "duplicate", + ]); + assert.equal(getLeadDuplicateStatusLabel("duplicate"), "Duplikat"); +}); + +test("dashboard-model exposes contact status options for lead review controls", () => { + assert.equal(leadContactStatusOptions[1], "missing_contact"); + assert.equal(leadBlacklistStatusOptions.length, 2); +}); + test("dashboardKpis and reviewQueue expose the above-the-fold dashboard summary", () => { assert.equal(dashboardKpis.length, 4); assert.equal(reviewQueue.length, 3); diff --git a/tests/lead-discovery-google.test.ts b/tests/lead-discovery-google.test.ts index 2f1fde0..3699e81 100644 --- a/tests/lead-discovery-google.test.ts +++ b/tests/lead-discovery-google.test.ts @@ -4,10 +4,14 @@ import test from "node:test"; import { GOOGLE_PLACES_FIELD_MASK, buildGeocodingUrl, + getUsableContactEmail, + getUsableContactEmailFromEntries, getBlacklistMatches, getBlacklistLookupValues, getPlacesSearchSpec, + isProbableDuplicateCandidate, isDuplicateCandidate, + normalizeEmailAddress, normalizePlacesResponse, parseGeocodingResponse, } from "../lib/lead-discovery-google"; @@ -205,8 +209,12 @@ test("places normalization maps source metadata and normalizes website domain", test("duplicate detection uses placeId and websiteDomain", () => { const existingLeads = [ - { googlePlaceId: "dup-1", websiteDomain: "other.de" }, - { googlePlaceId: "other-2", websiteDomain: "example.de" }, + { + googlePlaceId: "dup-1", + websiteDomain: "other.de", + email: "blocked@example.de", + }, + { googlePlaceId: "other-2", websiteDomain: "example.de", email: "blocked@example.de" }, ]; assert.equal( @@ -277,6 +285,158 @@ test("duplicate detection uses placeId and websiteDomain", () => { ), false, ); + + assert.equal( + isDuplicateCandidate( + { + placeId: "none", + businessName: "Test", + address: "A", + websiteUrl: "https://www.example.de", + websiteDomain: "new.de", + phone: null, + rating: null, + userRatingCount: null, + businessStatus: null, + googleTypes: [], + googlePrimaryType: null, + googleMapsUrl: null, + sourceProvider: "google_places", + sourceFetchedAt: 0, + contactEmails: [{ email: "Owner@Example.De", isBusinessContactAddress: false }], + }, + existingLeads, + ), + false, + ); + + assert.equal( + isDuplicateCandidate( + { + placeId: "none", + businessName: "Test", + address: "A", + websiteUrl: "https://www.new.de", + websiteDomain: "new.de", + phone: null, + rating: null, + userRatingCount: null, + businessStatus: null, + googleTypes: [], + googlePrimaryType: null, + googleMapsUrl: null, + sourceProvider: "google_places", + sourceFetchedAt: 0, + contactEmails: [{ email: "newlead@new.de" }], + }, + existingLeads, + ), + false, + ); + + assert.equal( + isDuplicateCandidate( + { + placeId: "none", + businessName: "Test", + address: "A", + websiteUrl: "https://www.example.de", + websiteDomain: "new.de", + phone: null, + rating: null, + userRatingCount: null, + businessStatus: null, + googleTypes: [], + googlePrimaryType: null, + googleMapsUrl: null, + sourceProvider: "google_places", + sourceFetchedAt: 0, + email: "Blocked@Example.De", + }, + existingLeads, + ), + true, + ); +}); + +test("probable duplicates are detected by normalized company+address or normalized phone", () => { + const existingLeads = [ + { + googlePlaceId: "dup-1", + companyName: "Muster GmbH", + address: "Hauptstraße 1, 60311 Frankfurt am Main", + phone: "+49 30 123456", + }, + ]; + + assert.equal( + isProbableDuplicateCandidate( + { + placeId: "none-1", + businessName: "Muster GmbH", + address: "Hauptstraße 1, 60311 Frankfurt am Main", + websiteUrl: null, + websiteDomain: null, + phone: null, + rating: null, + userRatingCount: null, + businessStatus: null, + googleTypes: [], + googlePrimaryType: null, + googleMapsUrl: null, + sourceProvider: "google_places", + sourceFetchedAt: 0, + }, + existingLeads, + ), + true, + ); + + assert.equal( + isProbableDuplicateCandidate( + { + placeId: "none-2", + businessName: "Other GmbH", + address: "Nebenstraße 9", + websiteUrl: null, + websiteDomain: null, + phone: "0049 30 123456", + rating: null, + userRatingCount: null, + businessStatus: null, + googleTypes: [], + googlePrimaryType: null, + googleMapsUrl: null, + sourceProvider: "google_places", + sourceFetchedAt: 0, + }, + existingLeads, + ), + true, + ); + + assert.equal( + isProbableDuplicateCandidate( + { + placeId: "none-3", + businessName: "Different GmbH", + address: "Musterallee 5", + websiteUrl: null, + websiteDomain: null, + phone: "+49 89 999999", + rating: null, + userRatingCount: null, + businessStatus: null, + googleTypes: [], + googlePrimaryType: null, + googleMapsUrl: null, + sourceProvider: "google_places", + sourceFetchedAt: 0, + }, + existingLeads, + ), + false, + ); }); test("blacklist matches include google_place_id, domain, company and phone", () => { @@ -287,6 +447,8 @@ test("blacklist matches include google_place_id, domain, company and phone", () websiteUrl: "https://www.Blocked.de", websiteDomain: "blocked.de", phone: "+49 30 555 123", + email: "Info@Blocked.De", + contactEmails: [{ email: "Hello@blocked.de", isBusinessContactAddress: false }], rating: null, userRatingCount: null, businessStatus: null, @@ -303,6 +465,8 @@ test("blacklist matches include google_place_id, domain, company and phone", () { type: "company", normalizedValue: "muster gmbh" }, { type: "phone", normalizedValue: "4930555123" }, { type: "phone", normalizedValue: "+49 30 555 123" }, + { type: "email", normalizedValue: "info@blocked.de" }, + { type: "email", normalizedValue: "hello@blocked.de" }, ]); const matches = getBlacklistMatches( @@ -323,12 +487,213 @@ test("blacklist matches include google_place_id, domain, company and phone", () }, { type: "email", value: "x@example.de", normalizedValue: "x@example.de" }, { type: "phone", value: "+49 30 999 999", normalizedValue: "4930999999" }, + { + type: "email", + value: "Info@Blocked.De", + normalizedValue: "info@blocked.de", + }, ], ); const matchTypes = matches.map((match) => match.type).sort(); assert.deepEqual( matchTypes, - ["company", "domain", "google_place_id", "phone", "phone"].sort(), + ["company", "domain", "google_place_id", "phone", "phone", "email"].sort(), ); }); + +test("company normalization for blacklist lookup uses text normalization", () => { + const candidate = { + placeId: "place-company-spaces", + businessName: "Muster GmbH", + address: "A", + websiteUrl: null, + websiteDomain: null, + phone: "+49 30 555 123", + rating: null, + userRatingCount: null, + businessStatus: null, + googleTypes: [], + googlePrimaryType: null, + googleMapsUrl: null, + sourceProvider: "google_places" as const, + sourceFetchedAt: 0, + }; + + assert.deepEqual(getBlacklistLookupValues(candidate), [ + { type: "google_place_id", normalizedValue: "place-company-spaces" }, + { type: "company", normalizedValue: "muster gmbh" }, + { type: "phone", normalizedValue: "4930555123" }, + { type: "phone", normalizedValue: "+49 30 555 123" }, + ]); +}); + +test("company blacklist matching supports whitespace-normalized names", () => { + const candidate = { + placeId: "place-company-spaces-2", + businessName: "Muster GmbH", + address: "A", + websiteUrl: null, + websiteDomain: null, + phone: null, + rating: null, + userRatingCount: null, + businessStatus: null, + googleTypes: [], + googlePrimaryType: null, + googleMapsUrl: null, + sourceProvider: "google_places" as const, + sourceFetchedAt: 0, + }; + + const matches = getBlacklistMatches(candidate, [ + { type: "company", value: "Muster GmbH", normalizedValue: "muster gmbh" }, + ]); + + assert.equal(matches.length, 1); + assert.equal(matches[0]!.normalizedValue, "muster gmbh"); +}); + +test("email normalization strips whitespace, lowercases, and rejects malformed addresses", () => { + assert.equal(normalizeEmailAddress(" INFO@Example.DE "), "info@example.de"); + assert.equal(normalizeEmailAddress("hello@domain"), null); + assert.equal(normalizeEmailAddress("no-at-symbol"), null); + assert.equal(normalizeEmailAddress("@missing-local.com"), null); + assert.equal(normalizeEmailAddress("name@"), null); + assert.equal(normalizeEmailAddress(""), null); + assert.equal(normalizeEmailAddress("näm@beispiel.de"), null); +}); + +test("usable email helper prefers generic business aliases and requires explicit metadata for named contacts", () => { + const genericPreferred = getUsableContactEmail({ + placeId: "place-1", + businessName: "Bäckerei", + address: "Musterweg 1", + websiteUrl: null, + websiteDomain: null, + phone: null, + rating: null, + userRatingCount: null, + businessStatus: null, + googleTypes: [], + googlePrimaryType: null, + googleMapsUrl: null, + sourceProvider: "google_places", + sourceFetchedAt: 0, + contactEmails: [ + { + email: "müller@bäckerei.de", + isBusinessContactAddress: false, + }, + { + email: "Hello@Bäckerei.De", + isBusinessContactAddress: false, + }, + { + email: "owner@Bäckerei.De", + isBusinessContactAddress: true, + }, + ], + }); + + assert.deepEqual(genericPreferred, { + email: "hello@bäckerei.de", + emailSource: null, + contactPerson: null, + }); + + const namedWithoutMetadata = getUsableContactEmail({ + placeId: "place-2", + businessName: "Bäckerei", + address: "Musterweg 2", + websiteUrl: null, + websiteDomain: null, + phone: null, + rating: null, + userRatingCount: null, + businessStatus: null, + googleTypes: [], + googlePrimaryType: null, + googleMapsUrl: null, + sourceProvider: "google_places", + sourceFetchedAt: 0, + contactEmails: [ + { + email: "owner@Bäckerei.De", + isBusinessContactAddress: false, + }, + ], + }); + + assert.equal(namedWithoutMetadata, null); + + const namedWithMetadata = getUsableContactEmail({ + placeId: "place-3", + businessName: "Bäckerei", + address: "Musterweg 3", + websiteUrl: null, + websiteDomain: null, + phone: null, + rating: null, + userRatingCount: null, + businessStatus: null, + googleTypes: [], + googlePrimaryType: null, + googleMapsUrl: null, + sourceProvider: "google_places", + sourceFetchedAt: 0, + contactEmails: [ + { + email: "owner@Bäckerei.De", + isBusinessContactAddress: true, + }, + ], + }); + + assert.deepEqual(namedWithMetadata, { + email: "owner@bäckerei.de", + emailSource: null, + contactPerson: null, + }); +}); + +test("standalone contact-email rule helper rejects invalid entries and prefers generic aliases", () => { + const validGeneric = getUsableContactEmailFromEntries([ + { + email: "owner@firma.de", + isBusinessContactAddress: false, + }, + { + email: "support@firma.de", + isBusinessContactAddress: false, + }, + { + email: "hello@firma.de", + isBusinessContactAddress: false, + }, + ]); + + assert.deepEqual(validGeneric, { + email: "hello@firma.de", + emailSource: null, + contactPerson: null, + }); + + const rejectedNamed = getUsableContactEmailFromEntries([ + { + email: "owner@firma.de", + isBusinessContactAddress: false, + }, + ]); + + assert.equal(rejectedNamed, null); + + const invalid = getUsableContactEmailFromEntries([ + { + email: "no-at-symbol", + isBusinessContactAddress: true, + }, + ]); + + assert.equal(invalid, null); +}); diff --git a/tests/lead-discovery-run.test.ts b/tests/lead-discovery-run.test.ts index baba9b8..7ed3d8e 100644 --- a/tests/lead-discovery-run.test.ts +++ b/tests/lead-discovery-run.test.ts @@ -7,6 +7,7 @@ import { canStartAgentRun, isStalePendingAgentRun, getLeadDiscoveryContactStatus, + getLeadDiscoveryPriority, } from "../lib/lead-discovery-run"; test("agent run guard ignores stale pending runs but blocks active runs", () => { @@ -62,19 +63,51 @@ test("lead discovery counters preserve audit and outreach counters", () => { test("lead discovery contact status separates leads without any contact route", () => { assert.equal( - getLeadDiscoveryContactStatus({ websiteDomain: null, phone: null }), + getLeadDiscoveryContactStatus({ usableEmail: null }), "missing_contact", ); assert.equal( - getLeadDiscoveryContactStatus({ websiteDomain: "example.de", phone: null }), + getLeadDiscoveryContactStatus({ usableEmail: "info@example.de" }), "new", ); assert.equal( - getLeadDiscoveryContactStatus({ websiteDomain: null, phone: "030 123" }), - "new", + getLeadDiscoveryContactStatus({ usableEmail: null }), + "missing_contact", ); }); +test("lead discovery lead record marks contact missing when no usable email exists", () => { + const record = buildLeadDiscoveryLeadRecord({ + campaignId: "campaign-1", + runId: "run-1", + niche: "Restaurant", + postalCode: "10115", + now: 1717480000000, + candidate: { + placeId: "place-2", + businessName: "Kontaktlos GmbH", + address: "Hauptstraße 2", + websiteUrl: "https://www.beispiel.de", + websiteDomain: "example.de", + phone: "+49 30 123", + rating: 3.9, + userRatingCount: 9, + businessStatus: "OPERATIONAL", + googleTypes: ["consulting"], + googlePrimaryType: "consulting", + googleMapsUrl: "https://maps.google.com/place-2", + sourceProvider: "google_places", + sourceFetchedAt: 1717480001000, + contactEmails: [{ email: "Herr.Bewerber@Beispiel.de", isBusinessContactAddress: false }], + }, + }); + + assert.equal(record.contactStatus, "missing_contact"); + assert.equal(record.phone, "+49 30 123"); + assert.equal(record.websiteDomain, "example.de"); + assert.equal(record.email, undefined); +}); + test("lead discovery lead record keeps raw website url and normalized domain", () => { const record = buildLeadDiscoveryLeadRecord({ campaignId: "campaign-1", @@ -106,3 +139,113 @@ test("lead discovery lead record keeps raw website url and normalized domain", ( assert.equal(record.googleUserRatingCount, 12); assert.equal(record.sourceFetchedAt, 1717480001000); }); + +test("lead discovery lead record stores valid email and sets contactStatus to new", () => { + const record = buildLeadDiscoveryLeadRecord({ + campaignId: "campaign-1", + runId: "run-1", + niche: "Restaurant", + postalCode: "10115", + now: 1717480000000, + candidate: { + placeId: "place-3", + businessName: "Beispiel GmbH", + address: "Hauptstraße 1", + websiteUrl: "https://www.example.de/path", + websiteDomain: "example.de", + phone: "+49 30 123", + rating: 4.5, + userRatingCount: 12, + businessStatus: "OPERATIONAL", + googleTypes: ["restaurant"], + googlePrimaryType: "restaurant", + googleMapsUrl: "https://maps.google.com/place-3", + sourceProvider: "google_places", + sourceFetchedAt: 1717480001000, + contactEmails: [ + { + email: "Herr@Beispiel.de", + isBusinessContactAddress: false, + }, + { + email: "info@beispiel.de", + isBusinessContactAddress: false, + }, + ], + }, + }); + + assert.equal(record.contactStatus, "new"); + assert.equal(record.email, "info@beispiel.de"); + assert.equal(record.contactPerson, undefined); +}); + +test("lead discovery lead record stores normalized matching fields", () => { + const record = buildLeadDiscoveryLeadRecord({ + campaignId: "campaign-1", + runId: "run-1", + niche: "Restaurant", + postalCode: "10115", + now: 1717480000000, + candidate: { + placeId: "place-4", + businessName: "Muster GmbH", + address: "Hauptstraße 1 60311 Berlin", + websiteUrl: "https://www.example.de/", + websiteDomain: "Example.de", + phone: "+49 30 123 456", + rating: 4.5, + userRatingCount: 12, + businessStatus: "OPERATIONAL", + googleTypes: ["restaurant"], + googlePrimaryType: "restaurant", + googleMapsUrl: "https://maps.google.com/place-4", + sourceProvider: "google_places", + sourceFetchedAt: 1717480001000, + email: "Info@Example.de", + contactEmails: [ + { + email: "Info@Example.de", + isBusinessContactAddress: false, + }, + ], + }, + }); + + assert.equal(record.normalizedEmail, "info@example.de"); + assert.equal(record.normalizedPhone, "4930123456"); + assert.equal(record.normalizedCompanyName, "muster gmbh"); + assert.equal(record.normalizedAddress, "hauptstraße 1 60311 berlin"); +}); + +test("lead discovery priority helper classifies blocked, deferred, and low-potential leads", () => { + assert.deepEqual(getLeadDiscoveryPriority({ isBlacklisted: true }), { + priority: "blocked", + reason: "Lead ist auf der Sperrliste.", + }); + + assert.deepEqual(getLeadDiscoveryPriority({ isDuplicate: true }), { + priority: "defer", + reason: "Dublettenprüfung oder Reviewpause.", + }); + + assert.deepEqual(getLeadDiscoveryPriority({ hasWebsite: false }), { + priority: "high", + reason: "Kein Website-Indikator vorhanden.", + }); + + assert.deepEqual(getLeadDiscoveryPriority({ hasWebsite: true, hasWebsiteSignal: true }), { + priority: "low", + reason: "Website vorhanden: geringer Kontaktaufwand.", + }); + + assert.deepEqual(getLeadDiscoveryPriority({ hasWebsite: true, hasWebsiteSignal: false }), { + priority: "medium", + reason: "Standardpriorität.", + }); + + assert.deepEqual(getLeadDiscoveryPriority({ hasWebsite: true }), { + priority: "medium", + reason: "Standardpriorität.", + }); +});