feat: add website enrichment crawler
This commit is contained in:
725
convex/websiteEnrichmentAction.ts
Normal file
725
convex/websiteEnrichmentAction.ts
Normal file
@@ -0,0 +1,725 @@
|
||||
"use node";
|
||||
|
||||
import type { Browser, BrowserContext } from "playwright-core";
|
||||
import { createHash } from "node:crypto";
|
||||
import { access, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { v } from "convex/values";
|
||||
import {
|
||||
buildTechnicalChecks,
|
||||
discoverRelevantSubpageUrls,
|
||||
extractContactSignalsFromHtmlLikeText,
|
||||
isSameRegistrableHostishDomain,
|
||||
normalizeCrawlUrl,
|
||||
} from "../lib/website-crawler";
|
||||
import {
|
||||
getUsableContactEmailFromEntries,
|
||||
normalizeEmailAddress,
|
||||
} from "../lib/lead-discovery-google";
|
||||
import { api, internal } from "./_generated/api";
|
||||
import type { Doc, Id } from "./_generated/dataModel";
|
||||
import { internalAction, type ActionCtx } from "./_generated/server";
|
||||
|
||||
const DEFAULT_CRAWL_TIMEOUT_MS = 60_000;
|
||||
const DEFAULT_CRAWL_MAX_PAGES = 5;
|
||||
const MAX_PERSISTED_LINKS = 120;
|
||||
const MAX_PERSISTED_EMAIL_CANDIDATES = 40;
|
||||
const SCREENSHOT_MIME_TYPE = "image/png";
|
||||
const CHROMIUM_SOURCE_MARKER_FILE = path.join(tmpdir(), "chromium-source.sha256");
|
||||
const CHROMIUM_EXECUTABLE_PATH = path.join(tmpdir(), "chromium");
|
||||
const CHROMIUM_PACK_PATH = path.join(tmpdir(), "chromium-pack");
|
||||
const GENERIC_EMAIL_LOCALS = new Set([
|
||||
"info",
|
||||
"kontakt",
|
||||
"contact",
|
||||
"sales",
|
||||
"team",
|
||||
"support",
|
||||
"service",
|
||||
"hello",
|
||||
"marketing",
|
||||
"admin",
|
||||
"office",
|
||||
"impressum",
|
||||
"post",
|
||||
]);
|
||||
const CHROMIUM_EXECUTABLE_SOURCE_ENV_VARS = [
|
||||
"TASK8_BROWSER_ASSET_URL",
|
||||
"TASK8_CHROMIUM_EXECUTABLE_URL",
|
||||
"TASK8_CHROMIUM_EXECUTABLE",
|
||||
];
|
||||
|
||||
type EnrichmentPageKind =
|
||||
| "homepage"
|
||||
| "contact"
|
||||
| "impressum"
|
||||
| "services"
|
||||
| "about"
|
||||
| "team"
|
||||
| "other";
|
||||
type CrawlPageLink = {
|
||||
href: string;
|
||||
text: string;
|
||||
isInternal: boolean;
|
||||
};
|
||||
type PersistedCrawlLink = CrawlPageLink & {
|
||||
pageUrl: string;
|
||||
};
|
||||
type PageResult = {
|
||||
sourceUrl: string;
|
||||
finalUrl: string;
|
||||
pageKind: EnrichmentPageKind;
|
||||
title: string;
|
||||
metaDescription: string;
|
||||
headings: string[];
|
||||
visibleText: string;
|
||||
links: CrawlPageLink[];
|
||||
emailCandidates: Array<{
|
||||
email: string;
|
||||
emailSource: string;
|
||||
contactPerson: string | null;
|
||||
isBusinessContactAddress: boolean;
|
||||
isGeneric: boolean;
|
||||
sourceUrl: string;
|
||||
accepted: boolean;
|
||||
normalizedEmail: string;
|
||||
}>;
|
||||
hasContactFormSignal: boolean;
|
||||
hasContactCtaSignal: boolean;
|
||||
};
|
||||
type StoredScreenshot = {
|
||||
storageId: Id<"_storage">;
|
||||
viewport: "desktop" | "mobile";
|
||||
sourceUrl: string;
|
||||
capturedAt: number;
|
||||
width: number;
|
||||
height: number;
|
||||
mimeType: string;
|
||||
};
|
||||
|
||||
type WebsiteLead = Pick<
|
||||
Doc<"leads">,
|
||||
"_id" | "websiteUrl" | "contactStatus"
|
||||
>;
|
||||
type StartedLead = {
|
||||
lead: WebsiteLead;
|
||||
};
|
||||
|
||||
type ServerlessChromiumModule = {
|
||||
args: string[];
|
||||
executablePath: (input?: string) => Promise<string>;
|
||||
inflate: (filePath: string) => Promise<string>;
|
||||
setupLambdaEnvironment: (baseLibPath: string) => void;
|
||||
};
|
||||
|
||||
function messageFromError(error: unknown) {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function readPositiveIntEnv(key: string, fallback: number) {
|
||||
const raw = process.env[key]?.trim();
|
||||
if (!raw) {
|
||||
return fallback;
|
||||
}
|
||||
const parsed = Number.parseInt(raw, 10);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
function crawlTimeoutMs() {
|
||||
return readPositiveIntEnv("TASK8_CRAWL_TIMEOUT_MS", DEFAULT_CRAWL_TIMEOUT_MS);
|
||||
}
|
||||
|
||||
function crawlMaxPages() {
|
||||
return Math.max(
|
||||
1,
|
||||
Math.min(
|
||||
DEFAULT_CRAWL_MAX_PAGES,
|
||||
readPositiveIntEnv("TASK8_CRAWL_MAX_PAGES", DEFAULT_CRAWL_MAX_PAGES),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function makePageKind(url: string, rootUrl: string): EnrichmentPageKind {
|
||||
const normalizedRoot = normalizeCrawlUrl(rootUrl);
|
||||
if (!normalizedRoot) {
|
||||
return "other";
|
||||
}
|
||||
|
||||
const homepagePath = new URL(normalizedRoot).pathname.replace(/\/$/, "") || "/";
|
||||
let pageUrl: string;
|
||||
try {
|
||||
pageUrl = new URL(url).pathname.toLowerCase();
|
||||
} catch {
|
||||
return "other";
|
||||
}
|
||||
|
||||
if (pageUrl === homepagePath || pageUrl === homepagePath.replace(/\/$/, "")) {
|
||||
return "homepage";
|
||||
}
|
||||
|
||||
const normalizedPath = pageUrl.toLowerCase();
|
||||
if (/(?:^|\/)(kontakt|contact)(?:[-/]|$)/.test(normalizedPath)) {
|
||||
return "contact";
|
||||
}
|
||||
if (/(?:^|\/)(impressum|imprint)(?:[-/]|$)/.test(normalizedPath)) {
|
||||
return "impressum";
|
||||
}
|
||||
if (/(?:^|\/)(leistungen|angebot|services?)(?:[-/]|$)/.test(normalizedPath)) {
|
||||
return "services";
|
||||
}
|
||||
if (/(?:^|\/)(ueber|über|about|team)(?:[-/]|$)/.test(normalizedPath)) {
|
||||
return "about";
|
||||
}
|
||||
|
||||
return "other";
|
||||
}
|
||||
|
||||
function trimExcerpt(value: string) {
|
||||
return value.replace(/\s+/g, " ").trim().slice(0, 1200);
|
||||
}
|
||||
|
||||
function isGenericBusinessEmail(email: string) {
|
||||
const local = email.split("@")[0]?.toLowerCase() ?? "";
|
||||
const base = local.split("+")[0] ?? "";
|
||||
return GENERIC_EMAIL_LOCALS.has(base);
|
||||
}
|
||||
|
||||
async function loadPlaywrightModules() {
|
||||
const [playwrightCore, chromiumPackage] = await Promise.all([
|
||||
import("playwright-core"),
|
||||
import("@sparticuz/chromium-min"),
|
||||
]);
|
||||
return {
|
||||
playwrightCore,
|
||||
serverlessChromium: {
|
||||
args: chromiumPackage.default.args,
|
||||
executablePath: chromiumPackage.default.executablePath,
|
||||
inflate: chromiumPackage.inflate,
|
||||
setupLambdaEnvironment: chromiumPackage.setupLambdaEnvironment,
|
||||
} as ServerlessChromiumModule,
|
||||
};
|
||||
}
|
||||
|
||||
function getChromiumExecutableSource() {
|
||||
for (const key of CHROMIUM_EXECUTABLE_SOURCE_ENV_VARS) {
|
||||
const value = process.env[key]?.trim();
|
||||
if (value) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function getChromiumSourceMarker(source: string) {
|
||||
return createHash("sha256").update(source).digest("hex");
|
||||
}
|
||||
|
||||
async function clearChromiumCacheForSourceMismatch(executableSource: string) {
|
||||
const nextMarker = getChromiumSourceMarker(executableSource);
|
||||
const marker = await readFile(CHROMIUM_SOURCE_MARKER_FILE, "utf8").catch(() => null);
|
||||
if ((marker ?? "").trim() === nextMarker) {
|
||||
return;
|
||||
}
|
||||
|
||||
await Promise.all([
|
||||
rm(CHROMIUM_EXECUTABLE_PATH, { force: true, recursive: true }),
|
||||
rm(CHROMIUM_PACK_PATH, { force: true, recursive: true }),
|
||||
]);
|
||||
}
|
||||
|
||||
async function resolveChromiumExecutablePath(
|
||||
chromium: ServerlessChromiumModule,
|
||||
) {
|
||||
const executableSource = getChromiumExecutableSource();
|
||||
if (!executableSource) {
|
||||
throw new Error(
|
||||
`Set TASK8_BROWSER_ASSET_URL (or legacy TASK8_CHROMIUM_EXECUTABLE_URL / TASK8_CHROMIUM_EXECUTABLE) to configure the Chromium source; no source is configured.`,
|
||||
);
|
||||
}
|
||||
|
||||
await clearChromiumCacheForSourceMismatch(executableSource);
|
||||
const executablePath = await chromium.executablePath(executableSource);
|
||||
await writeFile(
|
||||
CHROMIUM_SOURCE_MARKER_FILE,
|
||||
getChromiumSourceMarker(executableSource),
|
||||
);
|
||||
|
||||
return executablePath;
|
||||
}
|
||||
|
||||
async function captureHomepageScreenshot(
|
||||
ctx: ActionCtx,
|
||||
context: BrowserContext,
|
||||
homepageUrl: string,
|
||||
viewport: "desktop" | "mobile",
|
||||
timeoutMs: number,
|
||||
) {
|
||||
const page = await context.newPage();
|
||||
try {
|
||||
await page.goto(homepageUrl, {
|
||||
waitUntil: "domcontentloaded",
|
||||
timeout: timeoutMs,
|
||||
});
|
||||
const sourceUrl = page.url();
|
||||
const screenshot = await page.screenshot({
|
||||
fullPage: true,
|
||||
type: "png",
|
||||
});
|
||||
const storageId = await ctx.storage.store(
|
||||
new Blob([new Uint8Array(screenshot)], { type: SCREENSHOT_MIME_TYPE }),
|
||||
);
|
||||
const viewportSize = page.viewportSize() ?? { width: 0, height: 0 };
|
||||
|
||||
return {
|
||||
storageId,
|
||||
viewport,
|
||||
sourceUrl,
|
||||
capturedAt: Date.now(),
|
||||
width: viewportSize.width,
|
||||
height: viewportSize.height,
|
||||
mimeType: SCREENSHOT_MIME_TYPE,
|
||||
} satisfies StoredScreenshot;
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function crawlPage(
|
||||
context: BrowserContext,
|
||||
targetUrl: string,
|
||||
rootUrl: string,
|
||||
timeoutMs: number,
|
||||
) {
|
||||
const page = await context.newPage();
|
||||
try {
|
||||
const response = await page.goto(targetUrl, {
|
||||
waitUntil: "domcontentloaded",
|
||||
timeout: timeoutMs,
|
||||
});
|
||||
if (!response) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const finalUrl = page.url();
|
||||
const title = await page.title().catch(() => "");
|
||||
const metaDescription = await page
|
||||
.evaluate(() => {
|
||||
const meta = document.querySelector(
|
||||
"meta[name='description']",
|
||||
) as HTMLMetaElement | null;
|
||||
return meta?.content ?? "";
|
||||
})
|
||||
.catch(() => "");
|
||||
const content = await page.content();
|
||||
const signals = extractContactSignalsFromHtmlLikeText(content);
|
||||
const headings = await page
|
||||
.evaluate(() =>
|
||||
Array.from(document.querySelectorAll("h1, h2, h3"))
|
||||
.map((element) => element.textContent?.trim() ?? "")
|
||||
.filter((heading) => heading.length > 0),
|
||||
)
|
||||
.catch(() => []);
|
||||
const visibleText = await page.evaluate(() => {
|
||||
return document.body?.innerText ?? "";
|
||||
});
|
||||
const rawLinks = await page
|
||||
.evaluate(() =>
|
||||
Array.from(document.querySelectorAll("a[href]")).map((anchor) => ({
|
||||
href: anchor.getAttribute("href") ?? "",
|
||||
text: anchor.textContent?.trim() ?? "",
|
||||
})),
|
||||
)
|
||||
.catch(() => []);
|
||||
|
||||
const normalizedLinks = rawLinks
|
||||
.map((link) => {
|
||||
const normalizedHref = normalizeCrawlUrl(link.href, finalUrl);
|
||||
if (!normalizedHref) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
href: normalizedHref,
|
||||
text: link.text,
|
||||
isInternal: isSameRegistrableHostishDomain(normalizedHref, rootUrl),
|
||||
};
|
||||
})
|
||||
.filter(
|
||||
(entry): entry is { href: string; text: string; isInternal: boolean } =>
|
||||
entry !== null,
|
||||
);
|
||||
|
||||
const emailCandidates = signals.emailCandidates
|
||||
.map((entry) => {
|
||||
const normalizedEmail = normalizeEmailAddress(entry.email);
|
||||
if (!normalizedEmail) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
email: normalizedEmail,
|
||||
emailSource: finalUrl,
|
||||
contactPerson: entry.contactPerson ?? null,
|
||||
isBusinessContactAddress: entry.isBusinessContactAddress,
|
||||
isGeneric: isGenericBusinessEmail(normalizedEmail),
|
||||
sourceUrl: finalUrl,
|
||||
accepted: false,
|
||||
normalizedEmail,
|
||||
};
|
||||
})
|
||||
.filter((entry): entry is NonNullable<typeof entry> => entry !== null);
|
||||
|
||||
return {
|
||||
sourceUrl: finalUrl,
|
||||
finalUrl,
|
||||
pageKind: makePageKind(targetUrl, rootUrl),
|
||||
title,
|
||||
metaDescription,
|
||||
headings,
|
||||
visibleText,
|
||||
links: normalizedLinks,
|
||||
emailCandidates,
|
||||
hasContactFormSignal: signals.hasContactFormSignal,
|
||||
hasContactCtaSignal: signals.hasContactCtaSignal,
|
||||
} satisfies PageResult;
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
}
|
||||
|
||||
function deduplicateLeadEmailCandidates(
|
||||
candidates: PageResult["emailCandidates"],
|
||||
) {
|
||||
const unique = new Map<string, PageResult["emailCandidates"][number]>();
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (!unique.has(candidate.normalizedEmail)) {
|
||||
unique.set(candidate.normalizedEmail, candidate);
|
||||
}
|
||||
}
|
||||
|
||||
return [...unique.values()];
|
||||
}
|
||||
|
||||
function deduplicateCrawlLinks(links: PersistedCrawlLink[]) {
|
||||
const unique = new Map<string, PersistedCrawlLink>();
|
||||
|
||||
for (const link of links) {
|
||||
if (!unique.has(link.href)) {
|
||||
unique.set(link.href, link);
|
||||
}
|
||||
}
|
||||
|
||||
return [...unique.values()];
|
||||
}
|
||||
|
||||
export const processLeadEnrichment = internalAction({
|
||||
args: { runId: v.id("agentRuns") },
|
||||
handler: async (ctx, args) => {
|
||||
let started: StartedLead | null = null;
|
||||
const runId = args.runId;
|
||||
let browser: Browser | null = null;
|
||||
let desktopContext: BrowserContext | null = null;
|
||||
let mobileContext: BrowserContext | null = null;
|
||||
|
||||
try {
|
||||
started = await ctx.runMutation(internal.websiteEnrichment.startLeadEnrichmentRun, {
|
||||
runId,
|
||||
});
|
||||
|
||||
if (!started) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const rootUrl = normalizeCrawlUrl(started.lead.websiteUrl);
|
||||
if (!rootUrl) {
|
||||
await ctx.runMutation(internal.websiteEnrichment.finishLeadEnrichmentRun, {
|
||||
runId,
|
||||
status: "failed",
|
||||
currentStep: "website_enrichment",
|
||||
errorSummary: "Ungültige Website-URL.",
|
||||
errors: 1,
|
||||
});
|
||||
await ctx.runMutation(api.runs.appendEvent, {
|
||||
runId,
|
||||
level: "error",
|
||||
message: "Website-Enrichment fehlgeschlagen: Ungültige Website-URL.",
|
||||
details: [{ label: "Lead", value: started.lead._id }],
|
||||
});
|
||||
await ctx.runMutation(internal.websiteEnrichment.patchLeadFromWebsiteEnrichment, {
|
||||
leadId: started.lead._id,
|
||||
currentContactStatus: started.lead.contactStatus,
|
||||
contactStatusReason:
|
||||
"Website-Enrichment fehlgeschlagen: Ungültige Website-URL.",
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
const timeoutMs = crawlTimeoutMs();
|
||||
const maxPages = crawlMaxPages();
|
||||
|
||||
const { playwrightCore, serverlessChromium } =
|
||||
await loadPlaywrightModules();
|
||||
const executablePath = await resolveChromiumExecutablePath(
|
||||
serverlessChromium,
|
||||
);
|
||||
|
||||
const prepareChromiumSharedLibraries = async (
|
||||
chromiumRuntime: ServerlessChromiumModule,
|
||||
) => {
|
||||
const runtimeArchivePath = path.join(
|
||||
CHROMIUM_PACK_PATH,
|
||||
"al2023.tar.br",
|
||||
);
|
||||
await access(runtimeArchivePath).catch(() => {
|
||||
throw new Error(
|
||||
`AL2023 shared library archive not found at ${runtimeArchivePath}; cannot prepare Chromium shared libraries.`,
|
||||
);
|
||||
});
|
||||
|
||||
await chromiumRuntime.inflate(runtimeArchivePath);
|
||||
chromiumRuntime.setupLambdaEnvironment(path.join(tmpdir(), "al2023", "lib"));
|
||||
};
|
||||
|
||||
await prepareChromiumSharedLibraries(serverlessChromium);
|
||||
browser = await playwrightCore.chromium.launch({
|
||||
headless: true,
|
||||
executablePath,
|
||||
args: serverlessChromium.args,
|
||||
});
|
||||
const { devices } = playwrightCore;
|
||||
desktopContext = await browser.newContext({
|
||||
...devices["Desktop Chrome"],
|
||||
});
|
||||
mobileContext = await browser.newContext({
|
||||
...devices["iPhone 11"],
|
||||
});
|
||||
|
||||
const homepage = await crawlPage(desktopContext, rootUrl, rootUrl, timeoutMs);
|
||||
if (!homepage) {
|
||||
throw new Error("Homepage konnte nicht geladen werden.");
|
||||
}
|
||||
|
||||
const requestedPages = discoverRelevantSubpageUrls(
|
||||
homepage.links.map((link) => link.href),
|
||||
rootUrl,
|
||||
);
|
||||
const crawlTargets = requestedPages.slice(0, maxPages);
|
||||
const crawledPages: PageResult[] = [homepage];
|
||||
|
||||
for (const pageUrl of crawlTargets.slice(1)) {
|
||||
const crawled = await crawlPage(desktopContext, pageUrl, rootUrl, timeoutMs);
|
||||
if (crawled) {
|
||||
crawledPages.push(crawled);
|
||||
}
|
||||
}
|
||||
|
||||
const allLinks: PersistedCrawlLink[] = crawledPages.flatMap((page) =>
|
||||
page.links.map((link) => ({
|
||||
...link,
|
||||
pageUrl: page.finalUrl,
|
||||
})),
|
||||
);
|
||||
const internalLinks = allLinks.filter((link) => link.isInternal);
|
||||
const uniqueInternalLinks = [...new Set(internalLinks.map((link) => link.href))];
|
||||
|
||||
const checkMap = new Map<
|
||||
string,
|
||||
{ status: number | null; isBroken: boolean }
|
||||
>();
|
||||
|
||||
for (const href of uniqueInternalLinks.slice(0, 30)) {
|
||||
try {
|
||||
const response = await desktopContext.request.get(href, {
|
||||
timeout: Math.max(1_000, timeoutMs - 1_000),
|
||||
});
|
||||
const status = response.status();
|
||||
checkMap.set(href, {
|
||||
status,
|
||||
isBroken: status < 200 || status >= 400,
|
||||
});
|
||||
} catch {
|
||||
checkMap.set(href, {
|
||||
status: null,
|
||||
isBroken: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const desktopScreenshot = await captureHomepageScreenshot(
|
||||
ctx,
|
||||
desktopContext,
|
||||
homepage.finalUrl,
|
||||
"desktop",
|
||||
timeoutMs,
|
||||
);
|
||||
const mobileScreenshot = await captureHomepageScreenshot(
|
||||
ctx,
|
||||
mobileContext,
|
||||
homepage.finalUrl,
|
||||
"mobile",
|
||||
timeoutMs,
|
||||
);
|
||||
|
||||
const technicalInput = buildTechnicalChecks({
|
||||
rootUrl,
|
||||
finalUrl: homepage.finalUrl,
|
||||
title: homepage.title,
|
||||
metaDescription: homepage.metaDescription,
|
||||
visibleText: homepage.visibleText,
|
||||
checkedUrls: crawledPages.map((page) => page.finalUrl),
|
||||
links: allLinks.map((link) => {
|
||||
const check = checkMap.get(link.href);
|
||||
return {
|
||||
href: link.href,
|
||||
status: check?.status ?? undefined,
|
||||
statusCode: check?.status ?? undefined,
|
||||
isBroken: check?.isBroken,
|
||||
};
|
||||
}),
|
||||
});
|
||||
|
||||
const validCandidates = deduplicateLeadEmailCandidates(
|
||||
crawledPages.flatMap((page) => page.emailCandidates),
|
||||
);
|
||||
const persistedLinks = deduplicateCrawlLinks(allLinks).slice(
|
||||
0,
|
||||
MAX_PERSISTED_LINKS,
|
||||
);
|
||||
const persistedCandidates = validCandidates.slice(
|
||||
0,
|
||||
MAX_PERSISTED_EMAIL_CANDIDATES,
|
||||
);
|
||||
const usable = getUsableContactEmailFromEntries(
|
||||
validCandidates.map((candidate) => ({
|
||||
email: candidate.email,
|
||||
emailSource: candidate.emailSource,
|
||||
contactPerson: candidate.contactPerson,
|
||||
isBusinessContactAddress: candidate.isBusinessContactAddress,
|
||||
})),
|
||||
);
|
||||
|
||||
await ctx.runMutation(internal.websiteEnrichment.persistLeadEnrichmentResult, {
|
||||
runId,
|
||||
leadId: started.lead._id,
|
||||
pages: crawledPages.map((page) => ({
|
||||
sourceUrl: page.sourceUrl,
|
||||
finalUrl: page.finalUrl,
|
||||
pageKind: page.pageKind,
|
||||
title: page.title,
|
||||
metaDescription: page.metaDescription,
|
||||
headings: page.headings,
|
||||
visibleTextExcerpt: trimExcerpt(page.visibleText),
|
||||
hasContactFormSignal: page.hasContactFormSignal,
|
||||
hasContactCtaSignal: page.hasContactCtaSignal,
|
||||
})),
|
||||
links: persistedLinks.map((link) => ({
|
||||
pageUrl: link.pageUrl,
|
||||
href: link.href,
|
||||
text: link.text,
|
||||
isInternal: link.isInternal,
|
||||
isBroken: checkMap.get(link.href)?.isBroken,
|
||||
})),
|
||||
emailCandidates: persistedCandidates.map((candidate) => ({
|
||||
email: candidate.email,
|
||||
normalizedEmail: candidate.normalizedEmail,
|
||||
emailSource: candidate.emailSource,
|
||||
sourceUrl: candidate.sourceUrl,
|
||||
contactPerson: candidate.contactPerson ?? undefined,
|
||||
isBusinessContactAddress: candidate.isBusinessContactAddress,
|
||||
isGeneric: candidate.isGeneric,
|
||||
accepted:
|
||||
usable !== null && candidate.normalizedEmail === usable.email,
|
||||
})),
|
||||
screenshots: [
|
||||
...(desktopScreenshot ? [desktopScreenshot] : []),
|
||||
...(mobileScreenshot ? [mobileScreenshot] : []),
|
||||
],
|
||||
technicalChecks: [
|
||||
{
|
||||
sourceUrl: homepage.sourceUrl,
|
||||
finalUrl: homepage.finalUrl,
|
||||
usesHttps: technicalInput.https,
|
||||
missingTitle: technicalInput.missingTitle,
|
||||
missingMetaDescription: technicalInput.missingMetaDescription,
|
||||
hasVisibleContactPath: technicalInput.hasVisibleContactPath,
|
||||
brokenInternalLinkCount: technicalInput.brokenInternalLinks.length,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
if (usable) {
|
||||
await ctx.runMutation(internal.websiteEnrichment.patchLeadFromWebsiteEnrichment, {
|
||||
leadId: started.lead._id,
|
||||
email: usable.email,
|
||||
emailSource: usable.emailSource ?? undefined,
|
||||
contactPerson: usable.contactPerson ?? undefined,
|
||||
currentContactStatus: started.lead.contactStatus,
|
||||
});
|
||||
} else {
|
||||
await ctx.runMutation(internal.websiteEnrichment.patchLeadFromWebsiteEnrichment, {
|
||||
leadId: started.lead._id,
|
||||
currentContactStatus: started.lead.contactStatus,
|
||||
contactStatusReason:
|
||||
"Kein verwertbarer Kontakt auf der Website gefunden.",
|
||||
});
|
||||
}
|
||||
|
||||
await ctx.runMutation(internal.websiteEnrichment.finishLeadEnrichmentRun, {
|
||||
runId,
|
||||
status: "succeeded",
|
||||
currentStep: "website_enrichment",
|
||||
errors: 0,
|
||||
});
|
||||
|
||||
await ctx.runMutation(api.runs.appendEvent, {
|
||||
runId,
|
||||
level: "info",
|
||||
message: usable
|
||||
? "Website-Enrichment erfolgreich mit nutzbarer E-Mail abgeschlossen."
|
||||
: "Website-Enrichment abgeschlossen, aber ohne nutzbare E-Mail.",
|
||||
});
|
||||
|
||||
return runId;
|
||||
} catch (error) {
|
||||
const errorSummary = messageFromError(error);
|
||||
|
||||
await ctx.runMutation(internal.websiteEnrichment.finishLeadEnrichmentRun, {
|
||||
runId,
|
||||
status: "failed",
|
||||
currentStep: "website_enrichment",
|
||||
errorSummary,
|
||||
errors: 1,
|
||||
});
|
||||
|
||||
await ctx.runMutation(api.runs.appendEvent, {
|
||||
runId,
|
||||
level: "error",
|
||||
message: "Website-Enrichment fehlgeschlagen.",
|
||||
details: [
|
||||
{ label: "Fehler", value: errorSummary, source: "website_enrichment" },
|
||||
],
|
||||
});
|
||||
|
||||
if (started) {
|
||||
await ctx.runMutation(internal.websiteEnrichment.patchLeadFromWebsiteEnrichment, {
|
||||
leadId: started.lead._id,
|
||||
currentContactStatus: started.lead.contactStatus,
|
||||
contactStatusReason: `Website-Enrichment fehlgeschlagen: ${errorSummary}`,
|
||||
});
|
||||
}
|
||||
|
||||
return null;
|
||||
} finally {
|
||||
if (desktopContext) {
|
||||
await desktopContext.close();
|
||||
}
|
||||
if (mobileContext) {
|
||||
await mobileContext.close();
|
||||
}
|
||||
if (browser) {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user