Surface audit generations on dashboard audits
This commit is contained in:
419
v2_elemente/audits.ts
Normal file
419
v2_elemente/audits.ts
Normal file
@@ -0,0 +1,419 @@
|
||||
/**
|
||||
* convex/audits.ts
|
||||
*
|
||||
* Audit-Pipeline (M3). Dockt am Hook in campaigns.ts an: runAuditBatch.
|
||||
* Ablauf je Lead: Struktur-Check → Screenshots → Markdown → PageSpeed →
|
||||
* multimodale Auswertung gegen skills.md → strukturiertes Ergebnis.
|
||||
*
|
||||
* Convex-Pattern: Actions machen fetch + storage + Entschlüsselung; die DB
|
||||
* läuft über runQuery/runMutation. Mandanten-Invariante: alles orgId-gescoped.
|
||||
*/
|
||||
import { v } from "convex/values";
|
||||
import {
|
||||
query,
|
||||
mutation,
|
||||
internalAction,
|
||||
internalMutation,
|
||||
internalQuery,
|
||||
} from "./_generated/server";
|
||||
import { internal } from "./_generated/api";
|
||||
import type { Id } from "./_generated/dataModel";
|
||||
import { decryptSecret } from "./lib/crypto";
|
||||
import {
|
||||
takeScreenshot,
|
||||
toDataUrl,
|
||||
fetchSiteMarkdown,
|
||||
fetchPageSpeed,
|
||||
parseStructuralSignals,
|
||||
buildSystemPrompt,
|
||||
runMultimodalAnalysis,
|
||||
estimateAuditCostUsd,
|
||||
} from "./lib/audit";
|
||||
// Build-Step erzeugt diese Datei aus skills.md (z. B. prebuild-Skript):
|
||||
// echo "export const SKILLS_REGISTRY = \`$(cat skills.md)\`;" > convex/lib/skillsRegistry.ts
|
||||
import { SKILLS_REGISTRY } from "./lib/skillsRegistry";
|
||||
|
||||
function normalizeUrl(domain: string): string {
|
||||
return domain.startsWith("http") ? domain : `https://${domain}`;
|
||||
}
|
||||
|
||||
function slugify(s: string): string {
|
||||
return s
|
||||
.toLowerCase()
|
||||
.normalize("NFKD")
|
||||
.replace(/[\u0300-\u036f]/g, "")
|
||||
.replace(/ä/g, "ae").replace(/ö/g, "oe").replace(/ü/g, "ue").replace(/ß/g, "ss")
|
||||
.replace(/[^a-z0-9]+/g, "-")
|
||||
.replace(/^-+|-+$/g, "")
|
||||
.slice(0, 60);
|
||||
}
|
||||
|
||||
// ---- Interne DB-Helfer -------------------------------------------------------
|
||||
|
||||
export const getAuditContext = internalQuery({
|
||||
args: { orgId: v.id("organizations"), leadId: v.id("leads") },
|
||||
handler: async (ctx, { orgId, leadId }) => {
|
||||
const lead = await ctx.db.get(leadId);
|
||||
if (!lead || lead.orgId !== orgId) return null;
|
||||
const org = await ctx.db.get(orgId);
|
||||
|
||||
const keyFor = async (provider: "google" | "openrouter" | "screenshotone") => {
|
||||
const e = await ctx.db
|
||||
.query("apiKeyVault")
|
||||
.withIndex("by_org_provider", (q) =>
|
||||
q.eq("orgId", orgId).eq("provider", provider),
|
||||
)
|
||||
.first();
|
||||
return e?.encryptedKey ?? null;
|
||||
};
|
||||
|
||||
return {
|
||||
lead,
|
||||
killSwitch: org?.killSwitch ?? false,
|
||||
googleCipher: await keyFor("google"),
|
||||
screenshotoneCipher: await keyFor("screenshotone"),
|
||||
openrouterCipher: await keyFor("openrouter"),
|
||||
// Jina ist optional; falls als eigener Provider geführt, hier ergänzen.
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
/** Hochpriorisierte Leads einer Kampagne ohne bestehendes Audit. */
|
||||
export const getLeadsForAudit = internalQuery({
|
||||
args: {
|
||||
orgId: v.id("organizations"),
|
||||
campaignId: v.id("campaigns"),
|
||||
limit: v.number(),
|
||||
},
|
||||
handler: async (ctx, { orgId, campaignId, limit }) => {
|
||||
const leads = await ctx.db
|
||||
.query("leads")
|
||||
.withIndex("by_campaign", (q) => q.eq("campaignId", campaignId))
|
||||
.filter((q) => q.neq(q.field("priority"), "low"))
|
||||
.collect();
|
||||
|
||||
const result: Id<"leads">[] = [];
|
||||
for (const lead of leads) {
|
||||
if (result.length >= limit) break;
|
||||
if (lead.orgId !== orgId) continue;
|
||||
if (!lead.domain) continue; // ohne Website kein Website-Audit
|
||||
const existing = await ctx.db
|
||||
.query("audits")
|
||||
.withIndex("by_lead", (q) => q.eq("leadId", lead._id))
|
||||
.first();
|
||||
if (!existing) result.push(lead._id);
|
||||
}
|
||||
return result;
|
||||
},
|
||||
});
|
||||
|
||||
export const createAuditDraft = internalMutation({
|
||||
args: { orgId: v.id("organizations"), leadId: v.id("leads") },
|
||||
returns: v.object({
|
||||
auditId: v.id("audits"),
|
||||
alreadyExisted: v.boolean(),
|
||||
}),
|
||||
handler: async (ctx, { orgId, leadId }) => {
|
||||
const existing = await ctx.db
|
||||
.query("audits")
|
||||
.withIndex("by_lead", (q) => q.eq("leadId", leadId))
|
||||
.first();
|
||||
if (existing) return { auditId: existing._id, alreadyExisted: true };
|
||||
|
||||
const lead = await ctx.db.get(leadId);
|
||||
if (!lead || lead.orgId !== orgId) throw new Error("Lead nicht berechtigt");
|
||||
|
||||
// Deterministischer, aber nicht erratbarer Slug (Hash-Anteil aus der opaken _id).
|
||||
const hash = leadId.toString().slice(-6);
|
||||
const slug = `${slugify(lead.companyName)}-${slugify(lead.ort ?? "")}-${hash}`
|
||||
.replace(/-+/g, "-");
|
||||
|
||||
const auditId = await ctx.db.insert("audits", {
|
||||
orgId,
|
||||
leadId,
|
||||
status: "draft",
|
||||
slug,
|
||||
});
|
||||
return { auditId, alreadyExisted: false };
|
||||
},
|
||||
});
|
||||
|
||||
export const saveAuditResult = internalMutation({
|
||||
args: {
|
||||
orgId: v.id("organizations"),
|
||||
auditId: v.id("audits"),
|
||||
leadId: v.id("leads"),
|
||||
checkedDomain: v.string(),
|
||||
checkedSubpages: v.array(v.string()),
|
||||
desktopStorageId: v.optional(v.id("_storage")),
|
||||
mobileStorageId: v.optional(v.id("_storage")),
|
||||
pageSpeedRaw: v.optional(v.any()),
|
||||
cheerioFindings: v.optional(v.any()),
|
||||
jinaMarkdown: v.optional(v.string()),
|
||||
result: v.any(), // geparste LLM-JSON
|
||||
costUsd: v.number(),
|
||||
tokensIn: v.number(),
|
||||
tokensOut: v.number(),
|
||||
},
|
||||
handler: async (ctx, a) => {
|
||||
const r = a.result ?? {};
|
||||
await ctx.db.patch(a.auditId, {
|
||||
checkedDomain: a.checkedDomain,
|
||||
checkedSubpages: a.checkedSubpages,
|
||||
screenshots: {
|
||||
desktop: a.desktopStorageId,
|
||||
mobile: a.mobileStorageId,
|
||||
},
|
||||
pageSpeedRaw: a.pageSpeedRaw,
|
||||
cheerioFindings: a.cheerioFindings,
|
||||
jinaMarkdown: a.jinaMarkdown,
|
||||
usedSkills: r.usedSkills ?? [],
|
||||
skillOutputs: r.findings ?? [],
|
||||
multimodalAnalysis: r.findings ?? [],
|
||||
finalSummary: r.finalSummary ?? "",
|
||||
publicAuditText: r.publicAuditText ?? "",
|
||||
ctaType: r.ctaType,
|
||||
// bleibt "draft" — Freigabe ist manuell (Versand-Gate)
|
||||
});
|
||||
|
||||
// Outreach-Entwurf anlegen (pending), Versand erst nach Freigabe.
|
||||
await ctx.db.insert("outreach", {
|
||||
orgId: a.orgId,
|
||||
leadId: a.leadId,
|
||||
auditId: a.auditId,
|
||||
contactStrategy: "email_direct",
|
||||
emailSubject: r.emailSubject,
|
||||
emailBody: r.emailBody,
|
||||
phoneScript: r.phoneScript,
|
||||
approvalStatus: "pending",
|
||||
replyStatus: "none",
|
||||
});
|
||||
|
||||
await ctx.db.patch(a.leadId, { contactStatus: "audited" });
|
||||
|
||||
await ctx.db.insert("usageEvents", {
|
||||
orgId: a.orgId,
|
||||
type: "audit",
|
||||
auditId: a.auditId,
|
||||
leadId: a.leadId,
|
||||
costEstimateUsd: a.costUsd,
|
||||
tokensIn: a.tokensIn,
|
||||
tokensOut: a.tokensOut,
|
||||
callCounts: { screenshotone: 2, jina: 1, pagespeed: 1 },
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
// ---- Pipeline-Actions --------------------------------------------------------
|
||||
|
||||
export const runSingleAudit = internalAction({
|
||||
args: { orgId: v.id("organizations"), leadId: v.id("leads") },
|
||||
handler: async (ctx, { orgId, leadId }) => {
|
||||
const c = await ctx.runQuery(internal.audits.getAuditContext, { orgId, leadId });
|
||||
if (!c) throw new Error("Audit-Kontext nicht gefunden");
|
||||
if (c.killSwitch) throw new Error("Kill-Switch aktiv");
|
||||
if (!c.lead.domain) throw new Error("Lead ohne Domain");
|
||||
if (!c.screenshotoneCipher || !c.openrouterCipher) {
|
||||
throw new Error("Screenshot-/LLM-Key fehlt");
|
||||
}
|
||||
|
||||
const { auditId, alreadyExisted } = await ctx.runMutation(
|
||||
internal.audits.createAuditDraft,
|
||||
{ orgId, leadId },
|
||||
);
|
||||
if (alreadyExisted) return; // schon auditiert
|
||||
|
||||
const url = normalizeUrl(c.lead.domain);
|
||||
const screenshotKey = await decryptSecret(c.screenshotoneCipher);
|
||||
const openRouterKey = await decryptSecret(c.openrouterCipher);
|
||||
|
||||
// 1) Struktur-Check (Roh-HTML)
|
||||
let structural: any = null;
|
||||
try {
|
||||
const html = await (await fetch(url)).text();
|
||||
structural = parseStructuralSignals(html, url);
|
||||
} catch {
|
||||
/* nicht erreichbar → Struktur bleibt null */
|
||||
}
|
||||
|
||||
// 2) Screenshots → Convex File Storage
|
||||
const desktopBytes = await takeScreenshot({ accessKey: screenshotKey, url, device: "desktop" });
|
||||
const mobileBytes = await takeScreenshot({ accessKey: screenshotKey, url, device: "mobile" });
|
||||
const desktopStorageId = await ctx.storage.store(
|
||||
new Blob([desktopBytes], { type: "image/png" }),
|
||||
);
|
||||
const mobileStorageId = await ctx.storage.store(
|
||||
new Blob([mobileBytes], { type: "image/png" }),
|
||||
);
|
||||
|
||||
// 3) Copy als Markdown
|
||||
const { markdown, pages } = await fetchSiteMarkdown({ baseUrl: url });
|
||||
|
||||
// 4) PageSpeed (optional, nur mit Google-Key)
|
||||
let pageSpeed: any = null;
|
||||
if (c.googleCipher) {
|
||||
try {
|
||||
const googleKey = await decryptSecret(c.googleCipher);
|
||||
pageSpeed = await fetchPageSpeed({ googleKey, url });
|
||||
} catch {
|
||||
/* PageSpeed optional */
|
||||
}
|
||||
}
|
||||
|
||||
// 5) Multimodale Auswertung
|
||||
const textContext = [
|
||||
`Unternehmen: ${c.lead.companyName}`,
|
||||
`Ort: ${c.lead.ort ?? c.lead.address ?? "unbekannt"}`,
|
||||
`Branche: ${c.lead.niche ?? "unbekannt"}`,
|
||||
`URL: ${url}`,
|
||||
"",
|
||||
`STRUKTUR-SIGNALE: ${JSON.stringify(structural)}`,
|
||||
`PAGESPEED (mobil): ${JSON.stringify(pageSpeed)}`,
|
||||
"",
|
||||
"WEBSITE-INHALT (Markdown):",
|
||||
markdown,
|
||||
].join("\n");
|
||||
|
||||
const { json, inputTokens, outputTokens } = await runMultimodalAnalysis({
|
||||
openRouterKey,
|
||||
systemPrompt: buildSystemPrompt(SKILLS_REGISTRY),
|
||||
textContext,
|
||||
desktopDataUrl: toDataUrl(desktopBytes),
|
||||
mobileDataUrl: toDataUrl(mobileBytes),
|
||||
});
|
||||
|
||||
const costUsd = estimateAuditCostUsd({ inputTokens, outputTokens, screenshots: 2 });
|
||||
|
||||
await ctx.runMutation(internal.audits.saveAuditResult, {
|
||||
orgId,
|
||||
auditId,
|
||||
leadId,
|
||||
checkedDomain: url,
|
||||
checkedSubpages: pages,
|
||||
desktopStorageId,
|
||||
mobileStorageId,
|
||||
pageSpeedRaw: pageSpeed,
|
||||
cheerioFindings: structural,
|
||||
jinaMarkdown: markdown,
|
||||
result: json,
|
||||
costUsd,
|
||||
tokensIn: inputTokens,
|
||||
tokensOut: outputTokens,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
/** Vom campaigns.ts-Hook aufgerufen: höchstpriorisierte neue Leads auditieren. */
|
||||
export const runAuditBatch = internalAction({
|
||||
args: {
|
||||
orgId: v.id("organizations"),
|
||||
campaignId: v.id("campaigns"),
|
||||
limit: v.number(),
|
||||
},
|
||||
handler: async (ctx, { orgId, campaignId, limit }) => {
|
||||
const leadIds = await ctx.runQuery(internal.audits.getLeadsForAudit, {
|
||||
orgId,
|
||||
campaignId,
|
||||
limit,
|
||||
});
|
||||
// Sequenziell — schont Rate-Limits und hält die Kosten kalkulierbar.
|
||||
for (const leadId of leadIds) {
|
||||
try {
|
||||
await ctx.runAction(internal.audits.runSingleAudit, { orgId, leadId });
|
||||
} catch (err) {
|
||||
console.error(`Audit fehlgeschlagen für ${leadId}:`, err);
|
||||
}
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
// ---- Freigabe-Gate (manuell) -------------------------------------------------
|
||||
|
||||
async function requireOwnedAudit(ctx: any, auditId: Id<"audits">) {
|
||||
const identity = await ctx.auth.getUserIdentity();
|
||||
if (!identity) throw new Error("Nicht authentifiziert");
|
||||
const user = await ctx.db
|
||||
.query("users")
|
||||
.withIndex("by_auth", (q: any) => q.eq("authRef", identity.subject))
|
||||
.first();
|
||||
const audit = await ctx.db.get(auditId);
|
||||
if (!user || !audit || audit.orgId !== user.orgId) {
|
||||
throw new Error("Audit nicht gefunden oder nicht berechtigt");
|
||||
}
|
||||
return audit;
|
||||
}
|
||||
|
||||
export const approveAudit = mutation({
|
||||
args: { auditId: v.id("audits") },
|
||||
handler: async (ctx, { auditId }) => {
|
||||
await requireOwnedAudit(ctx, auditId);
|
||||
await ctx.db.patch(auditId, { status: "approved" });
|
||||
},
|
||||
});
|
||||
|
||||
export const publishAudit = mutation({
|
||||
args: { auditId: v.id("audits") },
|
||||
handler: async (ctx, { auditId }) => {
|
||||
const audit = await requireOwnedAudit(ctx, auditId);
|
||||
if (audit.status !== "approved") throw new Error("Erst freigeben");
|
||||
await ctx.db.patch(auditId, { status: "published", publishedAt: Date.now() });
|
||||
},
|
||||
});
|
||||
|
||||
// ---- Öffentliche Audit-Seite -------------------------------------------------
|
||||
|
||||
export const getBySlug = query({
|
||||
args: { slug: v.string() },
|
||||
handler: async (ctx, { slug }) => {
|
||||
const audit = await ctx.db
|
||||
.query("audits")
|
||||
.withIndex("by_slug", (q) => q.eq("slug", slug))
|
||||
.first();
|
||||
if (!audit) return null;
|
||||
|
||||
// Nicht freigegeben → extern nur Hinweis, keine Inhalte.
|
||||
if (audit.status !== "published") {
|
||||
return { status: audit.status, notReady: true as const };
|
||||
}
|
||||
const lead = await ctx.db.get(audit.leadId);
|
||||
return {
|
||||
status: audit.status,
|
||||
notReady: false as const,
|
||||
companyName: lead?.companyName ?? "",
|
||||
publicAuditText: audit.publicAuditText ?? "",
|
||||
ctaType: audit.ctaType ?? "anruf",
|
||||
desktopUrl: audit.screenshots?.desktop
|
||||
? await ctx.storage.getUrl(audit.screenshots.desktop)
|
||||
: null,
|
||||
mobileUrl: audit.screenshots?.mobile
|
||||
? await ctx.storage.getUrl(audit.screenshots.mobile)
|
||||
: null,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
// ---- Lifecycle (von crons.ts, täglich) ---------------------------------------
|
||||
|
||||
const DAY = 24 * 60 * 60 * 1000;
|
||||
|
||||
export const processLifecycle = internalMutation({
|
||||
args: {},
|
||||
handler: async (ctx) => {
|
||||
const now = Date.now();
|
||||
// Hinweis: bei Volumen einen dedizierten by_status-Index ergänzen.
|
||||
const published = await ctx.db
|
||||
.query("audits")
|
||||
.filter((q) => q.eq(q.field("status"), "published"))
|
||||
.collect();
|
||||
|
||||
for (const a of published) {
|
||||
if (!a.publishedAt) continue;
|
||||
const age = now - a.publishedAt;
|
||||
if (age >= 60 * DAY) {
|
||||
await ctx.db.patch(a._id, { status: "expired", deactivatedAt: now });
|
||||
} else if (age >= 30 * DAY && !a.noticeSentAt) {
|
||||
await ctx.db.patch(a._id, { noticeSentAt: now }); // Dashboard-Hinweis
|
||||
}
|
||||
}
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user