420 lines
13 KiB
TypeScript
420 lines
13 KiB
TypeScript
/**
|
|
* convex/audits.ts
|
|
*
|
|
* Audit-Pipeline (M3). Dockt am Hook in campaigns.ts an: runAuditBatch.
|
|
* Ablauf je Lead: Struktur-Check → Screenshots → Markdown → PageSpeed →
|
|
* multimodale Auswertung gegen skills.md → strukturiertes Ergebnis.
|
|
*
|
|
* Convex-Pattern: Actions machen fetch + storage + Entschlüsselung; die DB
|
|
* läuft über runQuery/runMutation. Mandanten-Invariante: alles orgId-gescoped.
|
|
*/
|
|
import { v } from "convex/values";
|
|
import {
|
|
query,
|
|
mutation,
|
|
internalAction,
|
|
internalMutation,
|
|
internalQuery,
|
|
} from "./_generated/server";
|
|
import { internal } from "./_generated/api";
|
|
import type { Id } from "./_generated/dataModel";
|
|
import { decryptSecret } from "./lib/crypto";
|
|
import {
|
|
takeScreenshot,
|
|
toDataUrl,
|
|
fetchSiteMarkdown,
|
|
fetchPageSpeed,
|
|
parseStructuralSignals,
|
|
buildSystemPrompt,
|
|
runMultimodalAnalysis,
|
|
estimateAuditCostUsd,
|
|
} from "./lib/audit";
|
|
// Build-Step erzeugt diese Datei aus skills.md (z. B. prebuild-Skript):
|
|
// echo "export const SKILLS_REGISTRY = \`$(cat skills.md)\`;" > convex/lib/skillsRegistry.ts
|
|
import { SKILLS_REGISTRY } from "./lib/skillsRegistry";
|
|
|
|
function normalizeUrl(domain: string): string {
|
|
return domain.startsWith("http") ? domain : `https://${domain}`;
|
|
}
|
|
|
|
function slugify(s: string): string {
|
|
return s
|
|
.toLowerCase()
|
|
.normalize("NFKD")
|
|
.replace(/[\u0300-\u036f]/g, "")
|
|
.replace(/ä/g, "ae").replace(/ö/g, "oe").replace(/ü/g, "ue").replace(/ß/g, "ss")
|
|
.replace(/[^a-z0-9]+/g, "-")
|
|
.replace(/^-+|-+$/g, "")
|
|
.slice(0, 60);
|
|
}
|
|
|
|
// ---- Interne DB-Helfer -------------------------------------------------------
|
|
|
|
export const getAuditContext = internalQuery({
|
|
args: { orgId: v.id("organizations"), leadId: v.id("leads") },
|
|
handler: async (ctx, { orgId, leadId }) => {
|
|
const lead = await ctx.db.get(leadId);
|
|
if (!lead || lead.orgId !== orgId) return null;
|
|
const org = await ctx.db.get(orgId);
|
|
|
|
const keyFor = async (provider: "google" | "openrouter" | "screenshotone") => {
|
|
const e = await ctx.db
|
|
.query("apiKeyVault")
|
|
.withIndex("by_org_provider", (q) =>
|
|
q.eq("orgId", orgId).eq("provider", provider),
|
|
)
|
|
.first();
|
|
return e?.encryptedKey ?? null;
|
|
};
|
|
|
|
return {
|
|
lead,
|
|
killSwitch: org?.killSwitch ?? false,
|
|
googleCipher: await keyFor("google"),
|
|
screenshotoneCipher: await keyFor("screenshotone"),
|
|
openrouterCipher: await keyFor("openrouter"),
|
|
// Jina ist optional; falls als eigener Provider geführt, hier ergänzen.
|
|
};
|
|
},
|
|
});
|
|
|
|
/** Hochpriorisierte Leads einer Kampagne ohne bestehendes Audit. */
|
|
export const getLeadsForAudit = internalQuery({
|
|
args: {
|
|
orgId: v.id("organizations"),
|
|
campaignId: v.id("campaigns"),
|
|
limit: v.number(),
|
|
},
|
|
handler: async (ctx, { orgId, campaignId, limit }) => {
|
|
const leads = await ctx.db
|
|
.query("leads")
|
|
.withIndex("by_campaign", (q) => q.eq("campaignId", campaignId))
|
|
.filter((q) => q.neq(q.field("priority"), "low"))
|
|
.collect();
|
|
|
|
const result: Id<"leads">[] = [];
|
|
for (const lead of leads) {
|
|
if (result.length >= limit) break;
|
|
if (lead.orgId !== orgId) continue;
|
|
if (!lead.domain) continue; // ohne Website kein Website-Audit
|
|
const existing = await ctx.db
|
|
.query("audits")
|
|
.withIndex("by_lead", (q) => q.eq("leadId", lead._id))
|
|
.first();
|
|
if (!existing) result.push(lead._id);
|
|
}
|
|
return result;
|
|
},
|
|
});
|
|
|
|
export const createAuditDraft = internalMutation({
|
|
args: { orgId: v.id("organizations"), leadId: v.id("leads") },
|
|
returns: v.object({
|
|
auditId: v.id("audits"),
|
|
alreadyExisted: v.boolean(),
|
|
}),
|
|
handler: async (ctx, { orgId, leadId }) => {
|
|
const existing = await ctx.db
|
|
.query("audits")
|
|
.withIndex("by_lead", (q) => q.eq("leadId", leadId))
|
|
.first();
|
|
if (existing) return { auditId: existing._id, alreadyExisted: true };
|
|
|
|
const lead = await ctx.db.get(leadId);
|
|
if (!lead || lead.orgId !== orgId) throw new Error("Lead nicht berechtigt");
|
|
|
|
// Deterministischer, aber nicht erratbarer Slug (Hash-Anteil aus der opaken _id).
|
|
const hash = leadId.toString().slice(-6);
|
|
const slug = `${slugify(lead.companyName)}-${slugify(lead.ort ?? "")}-${hash}`
|
|
.replace(/-+/g, "-");
|
|
|
|
const auditId = await ctx.db.insert("audits", {
|
|
orgId,
|
|
leadId,
|
|
status: "draft",
|
|
slug,
|
|
});
|
|
return { auditId, alreadyExisted: false };
|
|
},
|
|
});
|
|
|
|
export const saveAuditResult = internalMutation({
|
|
args: {
|
|
orgId: v.id("organizations"),
|
|
auditId: v.id("audits"),
|
|
leadId: v.id("leads"),
|
|
checkedDomain: v.string(),
|
|
checkedSubpages: v.array(v.string()),
|
|
desktopStorageId: v.optional(v.id("_storage")),
|
|
mobileStorageId: v.optional(v.id("_storage")),
|
|
pageSpeedRaw: v.optional(v.any()),
|
|
cheerioFindings: v.optional(v.any()),
|
|
jinaMarkdown: v.optional(v.string()),
|
|
result: v.any(), // geparste LLM-JSON
|
|
costUsd: v.number(),
|
|
tokensIn: v.number(),
|
|
tokensOut: v.number(),
|
|
},
|
|
handler: async (ctx, a) => {
|
|
const r = a.result ?? {};
|
|
await ctx.db.patch(a.auditId, {
|
|
checkedDomain: a.checkedDomain,
|
|
checkedSubpages: a.checkedSubpages,
|
|
screenshots: {
|
|
desktop: a.desktopStorageId,
|
|
mobile: a.mobileStorageId,
|
|
},
|
|
pageSpeedRaw: a.pageSpeedRaw,
|
|
cheerioFindings: a.cheerioFindings,
|
|
jinaMarkdown: a.jinaMarkdown,
|
|
usedSkills: r.usedSkills ?? [],
|
|
skillOutputs: r.findings ?? [],
|
|
multimodalAnalysis: r.findings ?? [],
|
|
finalSummary: r.finalSummary ?? "",
|
|
publicAuditText: r.publicAuditText ?? "",
|
|
ctaType: r.ctaType,
|
|
// bleibt "draft" — Freigabe ist manuell (Versand-Gate)
|
|
});
|
|
|
|
// Outreach-Entwurf anlegen (pending), Versand erst nach Freigabe.
|
|
await ctx.db.insert("outreach", {
|
|
orgId: a.orgId,
|
|
leadId: a.leadId,
|
|
auditId: a.auditId,
|
|
contactStrategy: "email_direct",
|
|
emailSubject: r.emailSubject,
|
|
emailBody: r.emailBody,
|
|
phoneScript: r.phoneScript,
|
|
approvalStatus: "pending",
|
|
replyStatus: "none",
|
|
});
|
|
|
|
await ctx.db.patch(a.leadId, { contactStatus: "audited" });
|
|
|
|
await ctx.db.insert("usageEvents", {
|
|
orgId: a.orgId,
|
|
type: "audit",
|
|
auditId: a.auditId,
|
|
leadId: a.leadId,
|
|
costEstimateUsd: a.costUsd,
|
|
tokensIn: a.tokensIn,
|
|
tokensOut: a.tokensOut,
|
|
callCounts: { screenshotone: 2, jina: 1, pagespeed: 1 },
|
|
});
|
|
},
|
|
});
|
|
|
|
// ---- Pipeline-Actions --------------------------------------------------------
|
|
|
|
export const runSingleAudit = internalAction({
|
|
args: { orgId: v.id("organizations"), leadId: v.id("leads") },
|
|
handler: async (ctx, { orgId, leadId }) => {
|
|
const c = await ctx.runQuery(internal.audits.getAuditContext, { orgId, leadId });
|
|
if (!c) throw new Error("Audit-Kontext nicht gefunden");
|
|
if (c.killSwitch) throw new Error("Kill-Switch aktiv");
|
|
if (!c.lead.domain) throw new Error("Lead ohne Domain");
|
|
if (!c.screenshotoneCipher || !c.openrouterCipher) {
|
|
throw new Error("Screenshot-/LLM-Key fehlt");
|
|
}
|
|
|
|
const { auditId, alreadyExisted } = await ctx.runMutation(
|
|
internal.audits.createAuditDraft,
|
|
{ orgId, leadId },
|
|
);
|
|
if (alreadyExisted) return; // schon auditiert
|
|
|
|
const url = normalizeUrl(c.lead.domain);
|
|
const screenshotKey = await decryptSecret(c.screenshotoneCipher);
|
|
const openRouterKey = await decryptSecret(c.openrouterCipher);
|
|
|
|
// 1) Struktur-Check (Roh-HTML)
|
|
let structural: any = null;
|
|
try {
|
|
const html = await (await fetch(url)).text();
|
|
structural = parseStructuralSignals(html, url);
|
|
} catch {
|
|
/* nicht erreichbar → Struktur bleibt null */
|
|
}
|
|
|
|
// 2) Screenshots → Convex File Storage
|
|
const desktopBytes = await takeScreenshot({ accessKey: screenshotKey, url, device: "desktop" });
|
|
const mobileBytes = await takeScreenshot({ accessKey: screenshotKey, url, device: "mobile" });
|
|
const desktopStorageId = await ctx.storage.store(
|
|
new Blob([desktopBytes], { type: "image/png" }),
|
|
);
|
|
const mobileStorageId = await ctx.storage.store(
|
|
new Blob([mobileBytes], { type: "image/png" }),
|
|
);
|
|
|
|
// 3) Copy als Markdown
|
|
const { markdown, pages } = await fetchSiteMarkdown({ baseUrl: url });
|
|
|
|
// 4) PageSpeed (optional, nur mit Google-Key)
|
|
let pageSpeed: any = null;
|
|
if (c.googleCipher) {
|
|
try {
|
|
const googleKey = await decryptSecret(c.googleCipher);
|
|
pageSpeed = await fetchPageSpeed({ googleKey, url });
|
|
} catch {
|
|
/* PageSpeed optional */
|
|
}
|
|
}
|
|
|
|
// 5) Multimodale Auswertung
|
|
const textContext = [
|
|
`Unternehmen: ${c.lead.companyName}`,
|
|
`Ort: ${c.lead.ort ?? c.lead.address ?? "unbekannt"}`,
|
|
`Branche: ${c.lead.niche ?? "unbekannt"}`,
|
|
`URL: ${url}`,
|
|
"",
|
|
`STRUKTUR-SIGNALE: ${JSON.stringify(structural)}`,
|
|
`PAGESPEED (mobil): ${JSON.stringify(pageSpeed)}`,
|
|
"",
|
|
"WEBSITE-INHALT (Markdown):",
|
|
markdown,
|
|
].join("\n");
|
|
|
|
const { json, inputTokens, outputTokens } = await runMultimodalAnalysis({
|
|
openRouterKey,
|
|
systemPrompt: buildSystemPrompt(SKILLS_REGISTRY),
|
|
textContext,
|
|
desktopDataUrl: toDataUrl(desktopBytes),
|
|
mobileDataUrl: toDataUrl(mobileBytes),
|
|
});
|
|
|
|
const costUsd = estimateAuditCostUsd({ inputTokens, outputTokens, screenshots: 2 });
|
|
|
|
await ctx.runMutation(internal.audits.saveAuditResult, {
|
|
orgId,
|
|
auditId,
|
|
leadId,
|
|
checkedDomain: url,
|
|
checkedSubpages: pages,
|
|
desktopStorageId,
|
|
mobileStorageId,
|
|
pageSpeedRaw: pageSpeed,
|
|
cheerioFindings: structural,
|
|
jinaMarkdown: markdown,
|
|
result: json,
|
|
costUsd,
|
|
tokensIn: inputTokens,
|
|
tokensOut: outputTokens,
|
|
});
|
|
},
|
|
});
|
|
|
|
/** Vom campaigns.ts-Hook aufgerufen: höchstpriorisierte neue Leads auditieren. */
|
|
export const runAuditBatch = internalAction({
|
|
args: {
|
|
orgId: v.id("organizations"),
|
|
campaignId: v.id("campaigns"),
|
|
limit: v.number(),
|
|
},
|
|
handler: async (ctx, { orgId, campaignId, limit }) => {
|
|
const leadIds = await ctx.runQuery(internal.audits.getLeadsForAudit, {
|
|
orgId,
|
|
campaignId,
|
|
limit,
|
|
});
|
|
// Sequenziell — schont Rate-Limits und hält die Kosten kalkulierbar.
|
|
for (const leadId of leadIds) {
|
|
try {
|
|
await ctx.runAction(internal.audits.runSingleAudit, { orgId, leadId });
|
|
} catch (err) {
|
|
console.error(`Audit fehlgeschlagen für ${leadId}:`, err);
|
|
}
|
|
}
|
|
},
|
|
});
|
|
|
|
// ---- Freigabe-Gate (manuell) -------------------------------------------------
|
|
|
|
async function requireOwnedAudit(ctx: any, auditId: Id<"audits">) {
|
|
const identity = await ctx.auth.getUserIdentity();
|
|
if (!identity) throw new Error("Nicht authentifiziert");
|
|
const user = await ctx.db
|
|
.query("users")
|
|
.withIndex("by_auth", (q: any) => q.eq("authRef", identity.subject))
|
|
.first();
|
|
const audit = await ctx.db.get(auditId);
|
|
if (!user || !audit || audit.orgId !== user.orgId) {
|
|
throw new Error("Audit nicht gefunden oder nicht berechtigt");
|
|
}
|
|
return audit;
|
|
}
|
|
|
|
export const approveAudit = mutation({
|
|
args: { auditId: v.id("audits") },
|
|
handler: async (ctx, { auditId }) => {
|
|
await requireOwnedAudit(ctx, auditId);
|
|
await ctx.db.patch(auditId, { status: "approved" });
|
|
},
|
|
});
|
|
|
|
export const publishAudit = mutation({
|
|
args: { auditId: v.id("audits") },
|
|
handler: async (ctx, { auditId }) => {
|
|
const audit = await requireOwnedAudit(ctx, auditId);
|
|
if (audit.status !== "approved") throw new Error("Erst freigeben");
|
|
await ctx.db.patch(auditId, { status: "published", publishedAt: Date.now() });
|
|
},
|
|
});
|
|
|
|
// ---- Öffentliche Audit-Seite -------------------------------------------------
|
|
|
|
export const getBySlug = query({
|
|
args: { slug: v.string() },
|
|
handler: async (ctx, { slug }) => {
|
|
const audit = await ctx.db
|
|
.query("audits")
|
|
.withIndex("by_slug", (q) => q.eq("slug", slug))
|
|
.first();
|
|
if (!audit) return null;
|
|
|
|
// Nicht freigegeben → extern nur Hinweis, keine Inhalte.
|
|
if (audit.status !== "published") {
|
|
return { status: audit.status, notReady: true as const };
|
|
}
|
|
const lead = await ctx.db.get(audit.leadId);
|
|
return {
|
|
status: audit.status,
|
|
notReady: false as const,
|
|
companyName: lead?.companyName ?? "",
|
|
publicAuditText: audit.publicAuditText ?? "",
|
|
ctaType: audit.ctaType ?? "anruf",
|
|
desktopUrl: audit.screenshots?.desktop
|
|
? await ctx.storage.getUrl(audit.screenshots.desktop)
|
|
: null,
|
|
mobileUrl: audit.screenshots?.mobile
|
|
? await ctx.storage.getUrl(audit.screenshots.mobile)
|
|
: null,
|
|
};
|
|
},
|
|
});
|
|
|
|
// ---- Lifecycle (von crons.ts, täglich) ---------------------------------------
|
|
|
|
const DAY = 24 * 60 * 60 * 1000;
|
|
|
|
export const processLifecycle = internalMutation({
|
|
args: {},
|
|
handler: async (ctx) => {
|
|
const now = Date.now();
|
|
// Hinweis: bei Volumen einen dedizierten by_status-Index ergänzen.
|
|
const published = await ctx.db
|
|
.query("audits")
|
|
.filter((q) => q.eq(q.field("status"), "published"))
|
|
.collect();
|
|
|
|
for (const a of published) {
|
|
if (!a.publishedAt) continue;
|
|
const age = now - a.publishedAt;
|
|
if (age >= 60 * DAY) {
|
|
await ctx.db.patch(a._id, { status: "expired", deactivatedAt: now });
|
|
} else if (age >= 30 * DAY && !a.noticeSentAt) {
|
|
await ctx.db.patch(a._id, { noticeSentAt: now }); // Dashboard-Hinweis
|
|
}
|
|
}
|
|
},
|
|
});
|