feat: add OpenRouter audit generation pipeline
This commit is contained in:
4
convex/_generated/api.d.ts
vendored
4
convex/_generated/api.d.ts
vendored
@@ -8,6 +8,8 @@
|
||||
* @module
|
||||
*/
|
||||
|
||||
import type * as auditGeneration from "../auditGeneration.js";
|
||||
import type * as auditGenerationAction from "../auditGenerationAction.js";
|
||||
import type * as auditInputs from "../auditInputs.js";
|
||||
import type * as audits from "../audits.js";
|
||||
import type * as blacklist from "../blacklist.js";
|
||||
@@ -32,6 +34,8 @@ import type {
|
||||
} from "convex/server";
|
||||
|
||||
declare const fullApi: ApiFromModules<{
|
||||
auditGeneration: typeof auditGeneration;
|
||||
auditGenerationAction: typeof auditGenerationAction;
|
||||
auditInputs: typeof auditInputs;
|
||||
audits: typeof audits;
|
||||
blacklist: typeof blacklist;
|
||||
|
||||
578
convex/auditGeneration.ts
Normal file
578
convex/auditGeneration.ts
Normal file
@@ -0,0 +1,578 @@
|
||||
import { internal } from "./_generated/api";
|
||||
import type { Doc, Id } from "./_generated/dataModel";
|
||||
import { internalMutation, internalQuery } from "./_generated/server";
|
||||
import {
|
||||
AUDIT_GENERATION_STAGES,
|
||||
AUDIT_GENERATION_STATUSES,
|
||||
RUN_STATUSES,
|
||||
} from "./domain";
|
||||
import { v } from "convex/values";
|
||||
import {
|
||||
type PageSpeedAuditErrorType,
|
||||
type PageSpeedMinimalAuditResult,
|
||||
} from "../lib/pagespeed-audit-input";
|
||||
|
||||
export const MAX_PROMPT_BYTES = 12_000;
|
||||
export const MAX_RAW_RESPONSE_BYTES = 12_000;
|
||||
export const MAX_PARSED_JSON_BYTES = 12_000;
|
||||
const TRUNCATION_MARKER = "\n\n[... abgeschnitten ...]";
|
||||
|
||||
const auditGenerationStage = v.union(
|
||||
...AUDIT_GENERATION_STAGES.map((stage) => v.literal(stage)),
|
||||
);
|
||||
const auditGenerationStatus = v.union(
|
||||
...AUDIT_GENERATION_STATUSES.map((status) => v.literal(status)),
|
||||
);
|
||||
const runStatus = v.union(...RUN_STATUSES.map((status) => v.literal(status)));
|
||||
|
||||
const auditGenerationParsedValue = v.union(
|
||||
v.string(),
|
||||
v.number(),
|
||||
v.boolean(),
|
||||
v.null(),
|
||||
v.array(v.any()),
|
||||
v.record(v.string(), v.any()),
|
||||
);
|
||||
|
||||
const auditGenerationParsedJson = v.union(
|
||||
v.string(),
|
||||
v.record(v.string(), auditGenerationParsedValue),
|
||||
);
|
||||
|
||||
type AuditGenerationLead = Pick<
|
||||
Doc<"leads">,
|
||||
| "_id"
|
||||
| "companyName"
|
||||
| "niche"
|
||||
| "city"
|
||||
| "address"
|
||||
| "websiteUrl"
|
||||
| "websiteDomain"
|
||||
| "phone"
|
||||
| "contactPerson"
|
||||
>;
|
||||
type AuditGenerationEvidenceCrawlPage = Pick<
|
||||
Doc<"websiteCrawlPages">,
|
||||
| "sourceUrl"
|
||||
| "finalUrl"
|
||||
| "title"
|
||||
| "metaDescription"
|
||||
| "pageKind"
|
||||
| "hasContactFormSignal"
|
||||
| "hasContactCtaSignal"
|
||||
| "visibleTextExcerpt"
|
||||
>;
|
||||
type AuditGenerationEvidenceTechnicalCheck = Pick<
|
||||
Doc<"websiteTechnicalChecks">,
|
||||
| "sourceUrl"
|
||||
| "finalUrl"
|
||||
| "usesHttps"
|
||||
| "missingTitle"
|
||||
| "missingMetaDescription"
|
||||
| "hasVisibleContactPath"
|
||||
| "brokenInternalLinkCount"
|
||||
>;
|
||||
type AuditGenerationEvidenceScreenshot = Pick<
|
||||
Doc<"websiteCrawlScreenshots">,
|
||||
| "storageId"
|
||||
| "viewport"
|
||||
| "sourceUrl"
|
||||
| "capturedAt"
|
||||
| "width"
|
||||
| "height"
|
||||
| "mimeType"
|
||||
>;
|
||||
|
||||
type AuditGenerationEvidence = {
|
||||
lead: AuditGenerationLead;
|
||||
crawlPages: AuditGenerationEvidenceCrawlPage[];
|
||||
technicalChecks: AuditGenerationEvidenceTechnicalCheck[];
|
||||
screenshots: AuditGenerationEvidenceScreenshot[];
|
||||
pageSpeedInputs: PageSpeedMinimalAuditResult[];
|
||||
};
|
||||
|
||||
function byteLength(value: string) {
|
||||
return new TextEncoder().encode(value).byteLength;
|
||||
}
|
||||
|
||||
function truncateToByteLimit(value: string, maxBytes: number) {
|
||||
if (maxBytes <= 0) {
|
||||
return "";
|
||||
}
|
||||
|
||||
let usedBytes = 0;
|
||||
let endIndex = 0;
|
||||
|
||||
for (const char of value) {
|
||||
const charBytes = byteLength(char);
|
||||
if (usedBytes + charBytes > maxBytes) {
|
||||
break;
|
||||
}
|
||||
usedBytes += charBytes;
|
||||
endIndex += char.length;
|
||||
}
|
||||
|
||||
return value.slice(0, endIndex);
|
||||
}
|
||||
|
||||
function truncateWithMarker(value: string, maxBytes: number) {
|
||||
if (byteLength(value) <= maxBytes) {
|
||||
return value;
|
||||
}
|
||||
|
||||
const markerBytes = byteLength(TRUNCATION_MARKER);
|
||||
if (markerBytes >= maxBytes) {
|
||||
const markerBytesBuffer = new TextEncoder().encode(TRUNCATION_MARKER);
|
||||
return new TextDecoder().decode(markerBytesBuffer.slice(0, maxBytes));
|
||||
}
|
||||
|
||||
const byteBudget = Math.max(0, maxBytes - markerBytes);
|
||||
const trimmed = truncateToByteLimit(value, byteBudget);
|
||||
|
||||
return `${trimmed}${TRUNCATION_MARKER}`;
|
||||
}
|
||||
|
||||
function sanitizeAndCapString(value: string | undefined, maxBytes: number) {
|
||||
if (!value) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const safe = (sanitizeSecretCandidates(value) ?? "").trim();
|
||||
return byteLength(safe) > maxBytes ? truncateWithMarker(safe, maxBytes) : safe;
|
||||
}
|
||||
|
||||
function safeStringify(value: unknown): string {
|
||||
try {
|
||||
return JSON.stringify(value);
|
||||
} catch {
|
||||
return "[unserializable payload]";
|
||||
}
|
||||
}
|
||||
|
||||
function sanitizeAndCapParsedJson(parsedJson: unknown) {
|
||||
if (parsedJson === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (typeof parsedJson === "string") {
|
||||
return sanitizeAndCapString(parsedJson, MAX_PARSED_JSON_BYTES);
|
||||
}
|
||||
|
||||
const serialized = safeStringify(parsedJson);
|
||||
const safeSerialized = sanitizeSecretCandidates(serialized) ?? "";
|
||||
if (byteLength(safeSerialized) <= MAX_PARSED_JSON_BYTES) {
|
||||
return safeSerialized;
|
||||
}
|
||||
|
||||
return truncateWithMarker(safeSerialized, MAX_PARSED_JSON_BYTES);
|
||||
}
|
||||
|
||||
function normalizePageSpeedResultRow(
|
||||
row: Doc<"pageSpeedResults">,
|
||||
): PageSpeedMinimalAuditResult {
|
||||
return {
|
||||
strategy: row.strategy,
|
||||
status: row.status,
|
||||
sourceUrl: row.sourceUrl,
|
||||
...(row.finalUrl ? { finalUrl: row.finalUrl } : {}),
|
||||
...(row.normalized ? { normalized: row.normalized } : {}),
|
||||
...(row.errorType ? { errorType: row.errorType as PageSpeedAuditErrorType } : {}),
|
||||
...(row.errorSummary ? { errorSummary: row.errorSummary } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
const auditGenerationUsage = v.object({
|
||||
promptTokens: v.optional(v.number()),
|
||||
completionTokens: v.optional(v.number()),
|
||||
totalTokens: v.optional(v.number()),
|
||||
cacheReadTokens: v.optional(v.number()),
|
||||
totalCostUsd: v.optional(v.number()),
|
||||
});
|
||||
|
||||
const secretHints = [
|
||||
"OPENROUTER_API_KEY",
|
||||
"GOOGLE_PLACES_API_KEY",
|
||||
"GOOGLE_GEOCODING_API_KEY",
|
||||
"PAGESPEED_API_KEY",
|
||||
"SMTP_PASSWORD",
|
||||
"SMTP_HOST",
|
||||
"SMTP_USER",
|
||||
"BETTER_AUTH_SECRET",
|
||||
"RYBBIT_API_KEY",
|
||||
];
|
||||
|
||||
function sanitizeSecretCandidates(value: string | undefined): string | undefined {
|
||||
if (!value) {
|
||||
return value;
|
||||
}
|
||||
|
||||
let sanitized = value;
|
||||
|
||||
for (const key of secretHints) {
|
||||
const secret = process.env[key];
|
||||
if (!secret) {
|
||||
continue;
|
||||
}
|
||||
|
||||
sanitized = sanitized.replace(
|
||||
new RegExp(escapeRegExp(secret), "g"),
|
||||
"[REDACTED]",
|
||||
);
|
||||
}
|
||||
|
||||
return sanitized
|
||||
.replace(/\b(?:api[_-]?key|token|secret|password)\s*[:=]\s*[^\s\"']+/gi, "[REDACTED]")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function escapeRegExp(value: string) {
|
||||
return value.replace(/[.*+?^${}()|[\\]\\]/g, "\\$&");
|
||||
}
|
||||
|
||||
type StartLeadSnapshot = Pick<
|
||||
Doc<"leads">,
|
||||
"_id" | "websiteUrl" | "websiteDomain" | "contactStatus"
|
||||
>;
|
||||
|
||||
export const getAuditGenerationEvidence = internalQuery({
|
||||
args: {
|
||||
runId: v.id("agentRuns"),
|
||||
},
|
||||
handler: async (ctx, args): Promise<AuditGenerationEvidence | null> => {
|
||||
const run = await ctx.db.get(args.runId);
|
||||
if (!run || !run.leadId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const lead = await ctx.db.get(run.leadId);
|
||||
if (!lead) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const runIdFilter = {
|
||||
table: "by_runId" as const,
|
||||
value: args.runId,
|
||||
};
|
||||
const leadIdFilter = {
|
||||
table: "by_leadId" as const,
|
||||
value: lead._id,
|
||||
};
|
||||
|
||||
const crawlPagesByRun = await ctx.db
|
||||
.query("websiteCrawlPages")
|
||||
.withIndex("by_runId", (q) => q.eq("runId", runIdFilter.value))
|
||||
.order("desc")
|
||||
.take(40);
|
||||
|
||||
const technicalChecksByRun = await ctx.db
|
||||
.query("websiteTechnicalChecks")
|
||||
.withIndex("by_runId", (q) => q.eq("runId", runIdFilter.value))
|
||||
.order("desc")
|
||||
.take(80);
|
||||
|
||||
const screenshotsByRun = await ctx.db
|
||||
.query("websiteCrawlScreenshots")
|
||||
.withIndex("by_runId", (q) => q.eq("runId", runIdFilter.value))
|
||||
.order("desc")
|
||||
.take(20);
|
||||
|
||||
const pageSpeedByRun = run.auditId
|
||||
? await ctx.db
|
||||
.query("pageSpeedResults")
|
||||
.withIndex("by_auditId", (q) => q.eq("auditId", run.auditId as Id<"audits">))
|
||||
.order("desc")
|
||||
.take(20)
|
||||
: await ctx.db
|
||||
.query("pageSpeedResults")
|
||||
.withIndex("by_leadId", (q) => q.eq("leadId", leadIdFilter.value))
|
||||
.order("desc")
|
||||
.take(20);
|
||||
|
||||
const crawlPages = crawlPagesByRun;
|
||||
const technicalChecks = technicalChecksByRun;
|
||||
const screenshots = screenshotsByRun;
|
||||
|
||||
return {
|
||||
lead: {
|
||||
_id: lead._id,
|
||||
companyName: lead.companyName,
|
||||
niche: lead.niche,
|
||||
city: lead.city,
|
||||
address: lead.address,
|
||||
websiteUrl: lead.websiteUrl,
|
||||
websiteDomain: lead.websiteDomain,
|
||||
phone: lead.phone,
|
||||
contactPerson: lead.contactPerson,
|
||||
},
|
||||
crawlPages,
|
||||
technicalChecks,
|
||||
screenshots,
|
||||
pageSpeedInputs: pageSpeedByRun.map(normalizePageSpeedResultRow),
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const queueLeadAuditGeneration = internalMutation({
|
||||
args: {
|
||||
leadId: v.id("leads"),
|
||||
auditId: v.optional(v.id("audits")),
|
||||
parentRunId: v.optional(v.id("agentRuns")),
|
||||
},
|
||||
returns: v.union(v.id("agentRuns"), v.null()),
|
||||
handler: async (ctx, args): Promise<Id<"agentRuns"> | null> => {
|
||||
const now = Date.now();
|
||||
const lead = await ctx.db.get(args.leadId);
|
||||
|
||||
if (!lead) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const existingPending = await ctx.db
|
||||
.query("agentRuns")
|
||||
.withIndex("by_type_and_status_and_leadId", (q) =>
|
||||
q
|
||||
.eq("type", "audit_generation")
|
||||
.eq("status", "pending")
|
||||
.eq("leadId", args.leadId),
|
||||
)
|
||||
.take(1);
|
||||
|
||||
const existingRunning = await ctx.db
|
||||
.query("agentRuns")
|
||||
.withIndex("by_type_and_status_and_leadId", (q) =>
|
||||
q
|
||||
.eq("type", "audit_generation")
|
||||
.eq("status", "running")
|
||||
.eq("leadId", args.leadId),
|
||||
)
|
||||
.take(1);
|
||||
|
||||
if (existingPending.length > 0) {
|
||||
return existingPending[0]._id;
|
||||
}
|
||||
|
||||
if (existingRunning.length > 0) {
|
||||
return existingRunning[0]._id;
|
||||
}
|
||||
|
||||
const runId = await ctx.db.insert("agentRuns", {
|
||||
type: "audit_generation",
|
||||
leadId: args.leadId,
|
||||
...(args.auditId ? { auditId: args.auditId } : {}),
|
||||
status: "pending",
|
||||
currentStep: "audit_generation",
|
||||
counters: {
|
||||
leadsFound: 0,
|
||||
leadsCreated: 0,
|
||||
auditsCreated: 0,
|
||||
outreachPrepared: 0,
|
||||
errors: 0,
|
||||
},
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
});
|
||||
|
||||
await ctx.db.insert("agentRunEvents", {
|
||||
runId,
|
||||
level: "info",
|
||||
message: "Audit-Generierung wurde in die Warteschlange gesetzt.",
|
||||
details: [
|
||||
{ label: "Lead", value: args.leadId },
|
||||
...(args.parentRunId
|
||||
? [{ label: "Parent-Run", value: args.parentRunId }]
|
||||
: []),
|
||||
],
|
||||
createdAt: now,
|
||||
});
|
||||
|
||||
await ctx.scheduler.runAfter(
|
||||
0,
|
||||
internal.auditGenerationAction.processAuditGeneration,
|
||||
{
|
||||
runId,
|
||||
},
|
||||
);
|
||||
|
||||
return runId;
|
||||
},
|
||||
});
|
||||
|
||||
export const startAuditGenerationRun = internalMutation({
|
||||
args: {
|
||||
runId: v.id("agentRuns"),
|
||||
},
|
||||
returns: v.union(
|
||||
v.object({
|
||||
lead: v.object({
|
||||
_id: v.id("leads"),
|
||||
websiteUrl: v.optional(v.string()),
|
||||
websiteDomain: v.optional(v.string()),
|
||||
contactStatus: v.union(
|
||||
v.literal("new"),
|
||||
v.literal("missing_contact"),
|
||||
v.literal("audit_ready"),
|
||||
v.literal("outreach_ready"),
|
||||
v.literal("contacted"),
|
||||
v.literal("replied"),
|
||||
v.literal("do_not_contact"),
|
||||
),
|
||||
}),
|
||||
auditId: v.optional(v.id("audits")),
|
||||
}),
|
||||
v.null(),
|
||||
),
|
||||
handler: async (ctx, args): Promise<
|
||||
{ lead: StartLeadSnapshot; auditId?: Id<"audits"> } | null
|
||||
> => {
|
||||
const now = Date.now();
|
||||
const run = await ctx.db.get(args.runId);
|
||||
|
||||
if (!run || run.type !== "audit_generation" || run.status !== "pending") {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!run.leadId) {
|
||||
await ctx.db.patch(args.runId, {
|
||||
status: "failed",
|
||||
currentStep: "audit_generation",
|
||||
errorSummary: "Der Lauf hat keine Lead-ID.",
|
||||
updatedAt: now,
|
||||
finishedAt: now,
|
||||
});
|
||||
|
||||
await ctx.db.insert("agentRunEvents", {
|
||||
runId: args.runId,
|
||||
level: "error",
|
||||
message:
|
||||
"Audit-Generierung konnte nicht gestartet werden: Keine Lead-ID.",
|
||||
details: [{ label: "Lead-ID", value: "unbekannt" }],
|
||||
createdAt: now,
|
||||
});
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
const lead = await ctx.db.get(run.leadId);
|
||||
if (!lead) {
|
||||
await ctx.db.patch(args.runId, {
|
||||
status: "failed",
|
||||
currentStep: "audit_generation",
|
||||
errorSummary: "Lead wurde nicht gefunden.",
|
||||
updatedAt: now,
|
||||
finishedAt: now,
|
||||
});
|
||||
|
||||
await ctx.db.insert("agentRunEvents", {
|
||||
runId: args.runId,
|
||||
level: "error",
|
||||
message:
|
||||
"Audit-Generierung konnte nicht gestartet werden: Kein Lead mit dieser ID.",
|
||||
details: [{ label: "Lead-ID", value: run.leadId }],
|
||||
createdAt: now,
|
||||
});
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
await ctx.db.patch(args.runId, {
|
||||
status: "running",
|
||||
currentStep: "audit_generation",
|
||||
startedAt: now,
|
||||
updatedAt: now,
|
||||
errorSummary: undefined,
|
||||
});
|
||||
|
||||
await ctx.db.insert("agentRunEvents", {
|
||||
runId: args.runId,
|
||||
level: "info",
|
||||
message: "Audit-Generierung gestartet.",
|
||||
details: [{ label: "Lead-ID", value: lead._id }],
|
||||
createdAt: now,
|
||||
});
|
||||
|
||||
return {
|
||||
lead: {
|
||||
_id: lead._id,
|
||||
websiteUrl: lead.websiteUrl,
|
||||
websiteDomain: lead.websiteDomain,
|
||||
contactStatus: lead.contactStatus,
|
||||
},
|
||||
...(run.auditId ? { auditId: run.auditId } : {}),
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const persistAuditGenerationResult = internalMutation({
|
||||
args: {
|
||||
leadId: v.id("leads"),
|
||||
auditId: v.optional(v.id("audits")),
|
||||
runId: v.id("agentRuns"),
|
||||
stage: auditGenerationStage,
|
||||
modelProfile: v.string(),
|
||||
modelId: v.string(),
|
||||
prompt: v.string(),
|
||||
systemPrompt: v.optional(v.string()),
|
||||
rawResponse: v.optional(v.string()),
|
||||
parsedJson: v.optional(auditGenerationParsedJson),
|
||||
usage: v.optional(auditGenerationUsage),
|
||||
finishReason: v.optional(v.string()),
|
||||
status: auditGenerationStatus,
|
||||
errorSummary: v.optional(v.string()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const now = Date.now();
|
||||
|
||||
return await ctx.db.insert("auditGenerations", {
|
||||
leadId: args.leadId,
|
||||
auditId: args.auditId,
|
||||
runId: args.runId,
|
||||
stage: args.stage,
|
||||
modelProfile: args.modelProfile,
|
||||
modelId: args.modelId,
|
||||
prompt: sanitizeAndCapString(args.prompt, MAX_PROMPT_BYTES) ?? "",
|
||||
...(args.systemPrompt
|
||||
? { systemPrompt: sanitizeAndCapString(args.systemPrompt, MAX_PROMPT_BYTES) }
|
||||
: {}),
|
||||
...(args.rawResponse
|
||||
? { rawResponse: sanitizeAndCapString(args.rawResponse, MAX_RAW_RESPONSE_BYTES) }
|
||||
: {}),
|
||||
...(args.parsedJson ? { parsedJson: sanitizeAndCapParsedJson(args.parsedJson) } : {}),
|
||||
...(args.usage ? { usage: args.usage } : {}),
|
||||
...(args.finishReason ? { finishReason: args.finishReason } : {}),
|
||||
status: args.status,
|
||||
...(args.errorSummary
|
||||
? { errorSummary: sanitizeAndCapString(args.errorSummary, MAX_RAW_RESPONSE_BYTES) }
|
||||
: {}),
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
export const finishAuditGenerationRun = internalMutation({
|
||||
args: {
|
||||
runId: v.id("agentRuns"),
|
||||
status: runStatus,
|
||||
currentStep: v.optional(v.string()),
|
||||
errorSummary: v.optional(v.string()),
|
||||
errors: v.optional(v.number()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const now = Date.now();
|
||||
|
||||
await ctx.db.patch(args.runId, {
|
||||
status: args.status,
|
||||
updatedAt: now,
|
||||
finishedAt: now,
|
||||
currentStep: args.currentStep ?? "audit_generation",
|
||||
errorSummary: args.errorSummary,
|
||||
counters: {
|
||||
leadsFound: 0,
|
||||
leadsCreated: 0,
|
||||
auditsCreated: 0,
|
||||
outreachPrepared: 0,
|
||||
errors: args.errors ?? 0,
|
||||
},
|
||||
});
|
||||
},
|
||||
});
|
||||
1197
convex/auditGenerationAction.ts
Normal file
1197
convex/auditGenerationAction.ts
Normal file
File diff suppressed because it is too large
Load Diff
@@ -1,7 +1,7 @@
|
||||
import { v } from "convex/values";
|
||||
|
||||
import { normalizeListLimit } from "./domain";
|
||||
import { mutation, query } from "./_generated/server";
|
||||
import { internalMutation, mutation, query } from "./_generated/server";
|
||||
|
||||
const auditStatus = v.union(
|
||||
v.literal("draft"),
|
||||
@@ -17,6 +17,13 @@ const usedSkillsValidator = v.array(
|
||||
source: v.optional(v.string()),
|
||||
}),
|
||||
);
|
||||
const skillSummaryValidator = v.array(
|
||||
v.object({
|
||||
name: v.string(),
|
||||
purpose: v.string(),
|
||||
summary: v.string(),
|
||||
}),
|
||||
);
|
||||
|
||||
export const create = mutation({
|
||||
args: {
|
||||
@@ -71,6 +78,81 @@ export const get = query({
|
||||
},
|
||||
});
|
||||
|
||||
export const upsertFromAuditGeneration = internalMutation({
|
||||
args: {
|
||||
leadId: v.id("leads"),
|
||||
runId: v.id("agentRuns"),
|
||||
auditId: v.optional(v.id("audits")),
|
||||
checkedDomain: v.string(),
|
||||
checkedPages: v.array(v.string()),
|
||||
internalSummary: v.optional(v.string()),
|
||||
multimodalSummary: v.optional(v.string()),
|
||||
publicSummary: v.optional(v.string()),
|
||||
publicBody: v.optional(v.string()),
|
||||
usedSkills: v.optional(usedSkillsValidator),
|
||||
skillSummaries: v.optional(skillSummaryValidator),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const now = Date.now();
|
||||
const lead = await ctx.db.get(args.leadId);
|
||||
if (!lead) {
|
||||
throw new Error("Lead wurde nicht gefunden.");
|
||||
}
|
||||
|
||||
if (args.auditId) {
|
||||
const existing = await ctx.db.get(args.auditId);
|
||||
if (!existing) {
|
||||
throw new Error("Audit wurde nicht gefunden.");
|
||||
}
|
||||
|
||||
await ctx.db.patch(args.auditId, {
|
||||
checkedDomain: args.checkedDomain,
|
||||
checkedPages: args.checkedPages,
|
||||
internalSummary: args.internalSummary,
|
||||
multimodalSummary: args.multimodalSummary,
|
||||
publicSummary: args.publicSummary,
|
||||
publicBody: args.publicBody,
|
||||
usedSkills: args.usedSkills,
|
||||
skillSummaries: args.skillSummaries,
|
||||
updatedAt: now,
|
||||
});
|
||||
|
||||
return args.auditId;
|
||||
}
|
||||
|
||||
const safeCheckedDomain = args.checkedDomain.trim().toLowerCase();
|
||||
const domainTag = safeCheckedDomain.length > 0
|
||||
? safeCheckedDomain.replace(/[^a-z0-9]+/g, "-").slice(0, 50)
|
||||
: "lead";
|
||||
let slug = `audit-${domainTag}-${args.leadId}-${now}`;
|
||||
|
||||
const slugCandidates = await ctx.db
|
||||
.query("audits")
|
||||
.withIndex("by_slug", (q) => q.eq("slug", slug))
|
||||
.take(1);
|
||||
|
||||
if (slugCandidates.length > 0) {
|
||||
slug = `${slug}-${Math.floor(now / 1_000)}`;
|
||||
}
|
||||
|
||||
return await ctx.db.insert("audits", {
|
||||
leadId: args.leadId,
|
||||
status: "draft",
|
||||
slug,
|
||||
checkedDomain: args.checkedDomain,
|
||||
checkedPages: args.checkedPages,
|
||||
internalSummary: args.internalSummary,
|
||||
multimodalSummary: args.multimodalSummary,
|
||||
publicSummary: args.publicSummary,
|
||||
publicBody: args.publicBody,
|
||||
usedSkills: args.usedSkills,
|
||||
skillSummaries: args.skillSummaries,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
export const getBySlug = query({
|
||||
args: { slug: v.string() },
|
||||
handler: async (ctx, args) => {
|
||||
|
||||
@@ -82,6 +82,7 @@ export const RUN_TYPES = [
|
||||
"campaign",
|
||||
"lead_discovery",
|
||||
"audit",
|
||||
"audit_generation",
|
||||
"outreach",
|
||||
"lifecycle",
|
||||
"website_enrichment",
|
||||
@@ -93,6 +94,19 @@ export const RUN_STATUSES = [
|
||||
"failed",
|
||||
"canceled",
|
||||
] as const;
|
||||
export const AUDIT_GENERATION_STAGES = [
|
||||
"classification",
|
||||
"multimodalAudit",
|
||||
"germanCopy",
|
||||
"qualityReview",
|
||||
] as const;
|
||||
export const AUDIT_GENERATION_STATUSES = [
|
||||
"pending",
|
||||
"running",
|
||||
"succeeded",
|
||||
"failed",
|
||||
"canceled",
|
||||
] as const;
|
||||
export const RUN_EVENT_LEVELS = ["info", "warning", "error"] as const;
|
||||
export const SCREENSHOT_VIEWPORTS = ["desktop", "mobile"] as const;
|
||||
export const PAGE_SPEED_STRATEGIES = ["mobile", "desktop"] as const;
|
||||
@@ -122,6 +136,8 @@ export type OutreachSalesStatus = (typeof OUTREACH_SALES_STATUSES)[number];
|
||||
export type BlacklistType = (typeof BLACKLIST_TYPES)[number];
|
||||
export type RunType = (typeof RUN_TYPES)[number];
|
||||
export type RunStatus = (typeof RUN_STATUSES)[number];
|
||||
export type AuditGenerationStage = (typeof AUDIT_GENERATION_STAGES)[number];
|
||||
export type AuditGenerationStatus = (typeof AUDIT_GENERATION_STATUSES)[number];
|
||||
export type RunEventLevel = (typeof RUN_EVENT_LEVELS)[number];
|
||||
export type ScreenshotViewport = (typeof SCREENSHOT_VIEWPORTS)[number];
|
||||
export type PageSpeedStrategy = (typeof PAGE_SPEED_STRATEGIES)[number];
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { v } from "convex/values";
|
||||
|
||||
import { normalizeListLimit } from "./domain";
|
||||
import { mutation, query } from "./_generated/server";
|
||||
import { internalMutation, mutation, query } from "./_generated/server";
|
||||
|
||||
const strategy = v.union(
|
||||
v.literal("call_first"),
|
||||
@@ -35,6 +35,59 @@ export const create = mutation({
|
||||
},
|
||||
});
|
||||
|
||||
export const upsertFromAuditGeneration = internalMutation({
|
||||
args: {
|
||||
leadId: v.id("leads"),
|
||||
auditId: v.optional(v.id("audits")),
|
||||
strategy: strategy,
|
||||
phoneScript: v.optional(v.string()),
|
||||
emailSubject: v.optional(v.string()),
|
||||
emailBody: v.optional(v.string()),
|
||||
followUpDraft: v.optional(v.string()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const now = Date.now();
|
||||
|
||||
const existing = await ctx.db
|
||||
.query("outreachRecords")
|
||||
.withIndex("by_leadId", (q) => q.eq("leadId", args.leadId))
|
||||
.order("desc")
|
||||
.take(1);
|
||||
|
||||
if (existing.length > 0) {
|
||||
const current = existing[0]!;
|
||||
if (args.auditId) {
|
||||
await ctx.db.patch(current._id, { auditId: args.auditId });
|
||||
}
|
||||
|
||||
await ctx.db.patch(current._id, {
|
||||
strategy: args.strategy,
|
||||
...(args.phoneScript !== undefined ? { phoneScript: args.phoneScript } : {}),
|
||||
...(args.emailSubject !== undefined
|
||||
? { emailSubject: args.emailSubject }
|
||||
: {}),
|
||||
...(args.emailBody !== undefined ? { emailBody: args.emailBody } : {}),
|
||||
...(args.followUpDraft !== undefined
|
||||
? { followUpDraft: args.followUpDraft }
|
||||
: {}),
|
||||
updatedAt: now,
|
||||
});
|
||||
|
||||
return current._id;
|
||||
}
|
||||
|
||||
return await ctx.db.insert("outreachRecords", {
|
||||
...args,
|
||||
approvalStatus: "draft",
|
||||
sendStatus: "not_sent",
|
||||
responseStatus: "none",
|
||||
salesStatus: "follow_up_planned",
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
export const list = query({
|
||||
args: {
|
||||
leadId: v.optional(v.id("leads")),
|
||||
|
||||
@@ -2,6 +2,8 @@ import { defineSchema, defineTable } from "convex/server";
|
||||
import { v } from "convex/values";
|
||||
import { tables as authTables } from "./betterAuth/schema";
|
||||
import {
|
||||
AUDIT_GENERATION_STAGES,
|
||||
AUDIT_GENERATION_STATUSES,
|
||||
RUN_EVENT_LEVELS,
|
||||
RUN_STATUSES,
|
||||
RUN_TYPES,
|
||||
@@ -91,6 +93,35 @@ const websiteEnrichmentPageKind = v.union(
|
||||
);
|
||||
const runType = v.union(...RUN_TYPES.map((type) => v.literal(type)));
|
||||
const runStatus = v.union(...RUN_STATUSES.map((status) => v.literal(status)));
|
||||
const auditGenerationStatus = v.union(
|
||||
...AUDIT_GENERATION_STATUSES.map((status) => v.literal(status)),
|
||||
);
|
||||
const auditGenerationStage = v.union(
|
||||
...AUDIT_GENERATION_STAGES.map((stage) => v.literal(stage)),
|
||||
);
|
||||
const auditGenerationUsage = v.object({
|
||||
promptTokens: v.optional(v.number()),
|
||||
completionTokens: v.optional(v.number()),
|
||||
totalTokens: v.optional(v.number()),
|
||||
cacheReadTokens: v.optional(v.number()),
|
||||
totalCostUsd: v.optional(v.number()),
|
||||
});
|
||||
const auditGenerationParsedJson = v.union(
|
||||
v.string(),
|
||||
v.record(
|
||||
v.string(),
|
||||
v.union(
|
||||
v.string(),
|
||||
v.number(),
|
||||
v.boolean(),
|
||||
v.null(),
|
||||
v.array(v.string()),
|
||||
v.array(v.number()),
|
||||
v.array(v.boolean()),
|
||||
v.object({}),
|
||||
),
|
||||
),
|
||||
);
|
||||
const runEventLevel = v.union(
|
||||
...RUN_EVENT_LEVELS.map((level) => v.literal(level)),
|
||||
);
|
||||
@@ -323,6 +354,30 @@ export default defineSchema({
|
||||
.index("by_auditId", ["auditId"])
|
||||
.index("by_leadId_and_strategy", ["leadId", "strategy"]),
|
||||
|
||||
auditGenerations: defineTable({
|
||||
leadId: v.id("leads"),
|
||||
auditId: v.optional(v.id("audits")),
|
||||
runId: v.id("agentRuns"),
|
||||
stage: auditGenerationStage,
|
||||
modelProfile: v.string(),
|
||||
modelId: v.string(),
|
||||
prompt: v.string(),
|
||||
systemPrompt: v.optional(v.string()),
|
||||
rawResponse: v.optional(v.string()),
|
||||
parsedJson: v.optional(auditGenerationParsedJson),
|
||||
usage: v.optional(auditGenerationUsage),
|
||||
finishReason: v.optional(v.string()),
|
||||
status: auditGenerationStatus,
|
||||
errorSummary: v.optional(v.string()),
|
||||
createdAt: v.number(),
|
||||
updatedAt: v.number(),
|
||||
})
|
||||
.index("by_leadId", ["leadId"])
|
||||
.index("by_auditId", ["auditId"])
|
||||
.index("by_runId", ["runId"])
|
||||
.index("by_stage", ["stage"])
|
||||
.index("by_leadId_and_stage", ["leadId", "stage"]),
|
||||
|
||||
websiteCrawlPages: defineTable({
|
||||
leadId: v.id("leads"),
|
||||
runId: v.optional(v.id("agentRuns")),
|
||||
|
||||
@@ -23,6 +23,10 @@ import { internalAction, type ActionCtx } from "./_generated/server";
|
||||
|
||||
const DEFAULT_CRAWL_TIMEOUT_MS = 60_000;
|
||||
const DEFAULT_CRAWL_MAX_PAGES = 5;
|
||||
const DEFAULT_ACTION_BUDGET_MS = 120_000;
|
||||
const MIN_ACTION_BUDGET_MS = 30_000;
|
||||
const MAX_ACTION_BUDGET_MS = 140_000;
|
||||
const ACTION_TIMEOUT_BUFFER_MS = 5_000;
|
||||
const MAX_PERSISTED_LINKS = 120;
|
||||
const MAX_PERSISTED_EMAIL_CANDIDATES = 40;
|
||||
const SCREENSHOT_MIME_TYPE = "image/png";
|
||||
@@ -140,6 +144,47 @@ function crawlMaxPages() {
|
||||
);
|
||||
}
|
||||
|
||||
function actionBudgetMs() {
|
||||
return Math.max(
|
||||
MIN_ACTION_BUDGET_MS,
|
||||
Math.min(
|
||||
MAX_ACTION_BUDGET_MS,
|
||||
readPositiveIntEnv("TASK8_ACTION_BUDGET_MS", DEFAULT_ACTION_BUDGET_MS),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function remainingActionBudgetMs(startedAt: number, budgetMs: number) {
|
||||
const elapsed = Date.now() - startedAt;
|
||||
return Math.max(1_000, budgetMs - elapsed - ACTION_TIMEOUT_BUFFER_MS);
|
||||
}
|
||||
|
||||
async function withActionTimeout<T>(
|
||||
promise: Promise<T>,
|
||||
timeoutMs: number,
|
||||
label: string,
|
||||
): Promise<T> {
|
||||
let timeout: ReturnType<typeof setTimeout> | null = null;
|
||||
try {
|
||||
return await Promise.race([
|
||||
promise,
|
||||
new Promise<T>((_, reject) => {
|
||||
timeout = setTimeout(() => {
|
||||
reject(
|
||||
new Error(
|
||||
`Website-Enrichment Zeitbudget ueberschritten: ${label}.`,
|
||||
),
|
||||
);
|
||||
}, Math.max(1, timeoutMs));
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
if (timeout) {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function makePageKind(url: string, rootUrl: string): EnrichmentPageKind {
|
||||
const normalizedRoot = normalizeCrawlUrl(rootUrl);
|
||||
if (!normalizedRoot) {
|
||||
@@ -418,6 +463,8 @@ export const processLeadEnrichment = internalAction({
|
||||
handler: async (ctx, args) => {
|
||||
let started: StartedLead | null = null;
|
||||
const runId = args.runId;
|
||||
const actionStartedAt = Date.now();
|
||||
const actionBudget = actionBudgetMs();
|
||||
let browser: Browser | null = null;
|
||||
let desktopContext: BrowserContext | null = null;
|
||||
let mobileContext: BrowserContext | null = null;
|
||||
@@ -480,9 +527,15 @@ export const processLeadEnrichment = internalAction({
|
||||
const maxPages = crawlMaxPages();
|
||||
|
||||
const { playwrightCore, serverlessChromium } =
|
||||
await loadPlaywrightModules();
|
||||
const executablePath = await resolveChromiumExecutablePath(
|
||||
serverlessChromium,
|
||||
await withActionTimeout(
|
||||
loadPlaywrightModules(),
|
||||
remainingActionBudgetMs(actionStartedAt, actionBudget),
|
||||
"Playwright-Module laden",
|
||||
);
|
||||
const executablePath = await withActionTimeout(
|
||||
resolveChromiumExecutablePath(serverlessChromium),
|
||||
remainingActionBudgetMs(actionStartedAt, actionBudget),
|
||||
"Chromium executable vorbereiten",
|
||||
);
|
||||
|
||||
const prepareChromiumSharedLibraries = async (
|
||||
@@ -502,21 +555,50 @@ export const processLeadEnrichment = internalAction({
|
||||
chromiumRuntime.setupLambdaEnvironment(path.join(tmpdir(), "al2023", "lib"));
|
||||
};
|
||||
|
||||
await prepareChromiumSharedLibraries(serverlessChromium);
|
||||
browser = await playwrightCore.chromium.launch({
|
||||
headless: true,
|
||||
executablePath,
|
||||
args: serverlessChromium.args,
|
||||
});
|
||||
await withActionTimeout(
|
||||
prepareChromiumSharedLibraries(serverlessChromium),
|
||||
remainingActionBudgetMs(actionStartedAt, actionBudget),
|
||||
"Chromium-Bibliotheken vorbereiten",
|
||||
);
|
||||
browser = await withActionTimeout(
|
||||
playwrightCore.chromium.launch({
|
||||
headless: true,
|
||||
executablePath,
|
||||
args: serverlessChromium.args,
|
||||
timeout: remainingActionBudgetMs(actionStartedAt, actionBudget),
|
||||
}),
|
||||
remainingActionBudgetMs(actionStartedAt, actionBudget),
|
||||
"Chromium starten",
|
||||
);
|
||||
const { devices } = playwrightCore;
|
||||
desktopContext = await browser.newContext({
|
||||
...devices["Desktop Chrome"],
|
||||
});
|
||||
mobileContext = await browser.newContext({
|
||||
...devices["iPhone 11"],
|
||||
});
|
||||
desktopContext = await withActionTimeout(
|
||||
browser.newContext({
|
||||
...devices["Desktop Chrome"],
|
||||
}),
|
||||
remainingActionBudgetMs(actionStartedAt, actionBudget),
|
||||
"Desktop-Kontext erstellen",
|
||||
);
|
||||
mobileContext = await withActionTimeout(
|
||||
browser.newContext({
|
||||
...devices["iPhone 11"],
|
||||
}),
|
||||
remainingActionBudgetMs(actionStartedAt, actionBudget),
|
||||
"Mobile-Kontext erstellen",
|
||||
);
|
||||
|
||||
const homepage = await crawlPage(desktopContext, rootUrl, rootUrl, timeoutMs);
|
||||
const homepage = await withActionTimeout(
|
||||
crawlPage(
|
||||
desktopContext,
|
||||
rootUrl,
|
||||
rootUrl,
|
||||
Math.min(
|
||||
timeoutMs,
|
||||
remainingActionBudgetMs(actionStartedAt, actionBudget),
|
||||
),
|
||||
),
|
||||
remainingActionBudgetMs(actionStartedAt, actionBudget),
|
||||
"Homepage crawlen",
|
||||
);
|
||||
if (!homepage) {
|
||||
throw new Error("Homepage konnte nicht geladen werden.");
|
||||
}
|
||||
@@ -529,7 +611,19 @@ export const processLeadEnrichment = internalAction({
|
||||
const crawledPages: PageResult[] = [homepage];
|
||||
|
||||
for (const pageUrl of crawlTargets.slice(1)) {
|
||||
const crawled = await crawlPage(desktopContext, pageUrl, rootUrl, timeoutMs);
|
||||
const crawled = await withActionTimeout(
|
||||
crawlPage(
|
||||
desktopContext,
|
||||
pageUrl,
|
||||
rootUrl,
|
||||
Math.min(
|
||||
timeoutMs,
|
||||
remainingActionBudgetMs(actionStartedAt, actionBudget),
|
||||
),
|
||||
),
|
||||
remainingActionBudgetMs(actionStartedAt, actionBudget),
|
||||
`Unterseite crawlen: ${pageUrl}`,
|
||||
);
|
||||
if (crawled) {
|
||||
crawledPages.push(crawled);
|
||||
}
|
||||
@@ -552,7 +646,10 @@ export const processLeadEnrichment = internalAction({
|
||||
for (const href of uniqueInternalLinks.slice(0, 30)) {
|
||||
try {
|
||||
const response = await desktopContext.request.get(href, {
|
||||
timeout: Math.max(1_000, timeoutMs - 1_000),
|
||||
timeout: Math.min(
|
||||
Math.max(1_000, timeoutMs - 1_000),
|
||||
remainingActionBudgetMs(actionStartedAt, actionBudget),
|
||||
),
|
||||
});
|
||||
const status = response.status();
|
||||
checkMap.set(href, {
|
||||
@@ -567,19 +664,33 @@ export const processLeadEnrichment = internalAction({
|
||||
}
|
||||
}
|
||||
|
||||
const desktopScreenshot = await captureHomepageScreenshot(
|
||||
ctx,
|
||||
desktopContext,
|
||||
homepage.finalUrl,
|
||||
"desktop",
|
||||
timeoutMs,
|
||||
const desktopScreenshot = await withActionTimeout(
|
||||
captureHomepageScreenshot(
|
||||
ctx,
|
||||
desktopContext,
|
||||
homepage.finalUrl,
|
||||
"desktop",
|
||||
Math.min(
|
||||
timeoutMs,
|
||||
remainingActionBudgetMs(actionStartedAt, actionBudget),
|
||||
),
|
||||
),
|
||||
remainingActionBudgetMs(actionStartedAt, actionBudget),
|
||||
"Desktop-Screenshot erfassen",
|
||||
);
|
||||
const mobileScreenshot = await captureHomepageScreenshot(
|
||||
ctx,
|
||||
mobileContext,
|
||||
homepage.finalUrl,
|
||||
"mobile",
|
||||
timeoutMs,
|
||||
const mobileScreenshot = await withActionTimeout(
|
||||
captureHomepageScreenshot(
|
||||
ctx,
|
||||
mobileContext,
|
||||
homepage.finalUrl,
|
||||
"mobile",
|
||||
Math.min(
|
||||
timeoutMs,
|
||||
remainingActionBudgetMs(actionStartedAt, actionBudget),
|
||||
),
|
||||
),
|
||||
remainingActionBudgetMs(actionStartedAt, actionBudget),
|
||||
"Mobile-Screenshot erfassen",
|
||||
);
|
||||
|
||||
const technicalInput = buildTechnicalChecks({
|
||||
|
||||
Reference in New Issue
Block a user