Externalize audit pipeline services
This commit is contained in:
356
tests/usage-events-source.test.ts
Normal file
356
tests/usage-events-source.test.ts
Normal file
@@ -0,0 +1,356 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import ts from "typescript";
|
||||
|
||||
const schemaPath = join(process.cwd(), "convex", "schema.ts");
|
||||
const domainPath = join(process.cwd(), "convex", "domain.ts");
|
||||
const usageEventsPath = join(process.cwd(), "convex", "usageEvents.ts");
|
||||
|
||||
const schemaSource = readFileSync(schemaPath, "utf8");
|
||||
const domainSource = readFileSync(domainPath, "utf8");
|
||||
const usageEventsSource = existsSync(usageEventsPath)
|
||||
? readFileSync(usageEventsPath, "utf8")
|
||||
: "";
|
||||
|
||||
const usageEventsSourceFile = ts.createSourceFile(
|
||||
"usageEvents.ts",
|
||||
usageEventsSource,
|
||||
ts.ScriptTarget.ES2022,
|
||||
true,
|
||||
ts.ScriptKind.TS,
|
||||
);
|
||||
|
||||
function getExportedConstNames(file: ts.SourceFile) {
|
||||
const names = new Set<string>();
|
||||
|
||||
const visit = (node: ts.Node) => {
|
||||
if (ts.isVariableStatement(node)) {
|
||||
const isExported = node.modifiers?.some(
|
||||
(mod) => mod.kind === ts.SyntaxKind.ExportKeyword,
|
||||
);
|
||||
const isConst = node.declarationList.flags & ts.NodeFlags.Const;
|
||||
|
||||
if (isExported && isConst) {
|
||||
for (const declaration of node.declarationList.declarations) {
|
||||
if (ts.isIdentifier(declaration.name)) {
|
||||
names.add(declaration.name.text);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ts.forEachChild(node, visit);
|
||||
};
|
||||
|
||||
ts.forEachChild(file, visit);
|
||||
return names;
|
||||
}
|
||||
|
||||
function extractTableSection(tableName: string) {
|
||||
const marker = `${tableName}: defineTable({`;
|
||||
const markerIndex = schemaSource.indexOf(marker);
|
||||
assert.notEqual(
|
||||
markerIndex,
|
||||
-1,
|
||||
`Expected schema table definition for ${tableName}.`,
|
||||
);
|
||||
|
||||
const objectStart = schemaSource.indexOf("{", markerIndex);
|
||||
let depth = 0;
|
||||
let objectEnd = -1;
|
||||
|
||||
for (let index = objectStart; index < schemaSource.length; index += 1) {
|
||||
if (schemaSource[index] === "{") {
|
||||
depth += 1;
|
||||
} else if (schemaSource[index] === "}") {
|
||||
depth -= 1;
|
||||
if (depth === 0) {
|
||||
objectEnd = index;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert.notEqual(objectEnd, -1, `Could not parse schema object for ${tableName}.`);
|
||||
|
||||
const remainder = schemaSource.slice(objectEnd + 1);
|
||||
const nextTableMatch = remainder.match(/^\s*[a-zA-Z_][\w]*:\s*defineTable\(/m);
|
||||
const sectionEnd =
|
||||
nextTableMatch === null ? schemaSource.length : objectEnd + 1 + nextTableMatch.index!;
|
||||
|
||||
return {
|
||||
objectBlock: schemaSource.slice(markerIndex, objectEnd + 1),
|
||||
section: schemaSource.slice(markerIndex, sectionEnd),
|
||||
};
|
||||
}
|
||||
|
||||
function extractExportSource(name: string) {
|
||||
const marker = `export const ${name} = `;
|
||||
const declarationIndex = usageEventsSource.indexOf(marker);
|
||||
assert.notEqual(declarationIndex, -1, `Expected declaration for ${name}`);
|
||||
|
||||
const openBraceIndex = usageEventsSource.indexOf("{", declarationIndex);
|
||||
let depth = 0;
|
||||
let end = -1;
|
||||
|
||||
for (let index = openBraceIndex; index < usageEventsSource.length; index += 1) {
|
||||
const char = usageEventsSource[index];
|
||||
if (char === "{") {
|
||||
depth += 1;
|
||||
} else if (char === "}") {
|
||||
depth -= 1;
|
||||
if (depth === 0) {
|
||||
end = index;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert.notEqual(end, -1, `Expected balanced braces for ${name}`);
|
||||
return usageEventsSource.slice(openBraceIndex, end + 1);
|
||||
}
|
||||
|
||||
function assertHas(pattern: RegExp, source: string, message: string) {
|
||||
assert.equal(pattern.test(source), true, message);
|
||||
}
|
||||
|
||||
const usageReadQueries = [
|
||||
{
|
||||
name: "listLatestUsageEvents",
|
||||
indexAssertion:
|
||||
/withIndex\("by_createdAt"\)/,
|
||||
message: "latest query should use by_createdAt.",
|
||||
},
|
||||
{
|
||||
name: "listUsageEventsByRun",
|
||||
indexAssertion:
|
||||
/withIndex\("by_runId_and_createdAt"[\s\S]*?eq\("runId",\s*args\.runId\)/,
|
||||
message: "run query should use by_runId_and_createdAt with runId equality.",
|
||||
},
|
||||
{
|
||||
name: "listUsageEventsByLead",
|
||||
indexAssertion:
|
||||
/withIndex\("by_leadId_and_createdAt"[\s\S]*?eq\("leadId",\s*args\.leadId\)/,
|
||||
message: "lead query should use by_leadId_and_createdAt with leadId equality.",
|
||||
},
|
||||
{
|
||||
name: "listUsageEventsByAudit",
|
||||
indexAssertion:
|
||||
/withIndex\("by_auditId_and_createdAt"[\s\S]*?eq\("auditId",\s*args\.auditId\)/,
|
||||
message: "audit query should use by_auditId_and_createdAt with auditId equality.",
|
||||
},
|
||||
{
|
||||
name: "listUsageEventsByProvider",
|
||||
indexAssertion:
|
||||
/withIndex\("by_provider_and_createdAt"[\s\S]*?eq\("provider",\s*args\.provider\)/,
|
||||
message: "provider query should use by_provider_and_createdAt with provider equality.",
|
||||
},
|
||||
] as const;
|
||||
|
||||
test("usage domain constants declare supported providers and operations", () => {
|
||||
assertHas(
|
||||
/USAGE_EVENT_PROVIDERS\s*=\s*\[[\s\S]*"openrouter"[\s\S]*"screenshotone"[\s\S]*"jina"[\s\S]*"pagespeed"[\s\S]*"google_places"[\s\S]*\]\s*as const/,
|
||||
domainSource,
|
||||
"Domain should declare usage providers for all managed external services.",
|
||||
);
|
||||
assertHas(
|
||||
/USAGE_EVENT_OPERATIONS\s*=\s*\[[\s\S]*"audit_capture"[\s\S]*"audit_generation"[\s\S]*"lead_lookup"[\s\S]*\]\s*as const/,
|
||||
domainSource,
|
||||
"Domain should declare usage operations for capture, generation, and lookup.",
|
||||
);
|
||||
});
|
||||
|
||||
test("usageEvents schema stores cost and usage dimensions with bounded indexes", () => {
|
||||
const { objectBlock, section } = extractTableSection("usageEvents");
|
||||
|
||||
assertHas(/provider:\s*usageEventProvider/, objectBlock, "provider should use the provider validator.");
|
||||
assertHas(/operation:\s*usageEventOperation/, objectBlock, "operation should use the operation validator.");
|
||||
assertHas(
|
||||
/runId:\s*v\.optional\(\s*v\.id\(["']agentRuns["']\)\s*\)/,
|
||||
objectBlock,
|
||||
"runId should be optional for SaaS-ready attribution.",
|
||||
);
|
||||
assertHas(
|
||||
/leadId:\s*v\.optional\(\s*v\.id\(["']leads["']\)\s*\)/,
|
||||
objectBlock,
|
||||
"leadId should be optional for lead-level attribution.",
|
||||
);
|
||||
assertHas(
|
||||
/auditId:\s*v\.optional\(\s*v\.id\(["']audits["']\)\s*\)/,
|
||||
objectBlock,
|
||||
"auditId should be optional for audit-level attribution.",
|
||||
);
|
||||
assertHas(
|
||||
/estimatedCostUsd:\s*v\.number\(\)/,
|
||||
objectBlock,
|
||||
"estimatedCostUsd should be a required normalized number.",
|
||||
);
|
||||
assertHas(
|
||||
/tokens:\s*v\.optional\(\s*v\.object\([\s\S]*?inputTokens:\s*v\.optional\(v\.number\(\)\)[\s\S]*?outputTokens:\s*v\.optional\(v\.number\(\)\)[\s\S]*?promptTokens:\s*v\.optional\(v\.number\(\)\)[\s\S]*?completionTokens:\s*v\.optional\(v\.number\(\)\)[\s\S]*?totalTokens:\s*v\.optional\(v\.number\(\)\)[\s\S]*?\)\s*\)/,
|
||||
objectBlock,
|
||||
"tokens should capture OpenRouter-compatible token dimensions.",
|
||||
);
|
||||
assertHas(
|
||||
/callCounts:\s*v\.optional\(\s*v\.object\(/,
|
||||
objectBlock,
|
||||
"callCounts should be an optional object.",
|
||||
);
|
||||
for (const countName of ["requests", "pages", "screenshots", "lookups"]) {
|
||||
assertHas(
|
||||
new RegExp(`${countName}:\\s*v\\.optional\\(v\\.number\\(\\)\\)`),
|
||||
objectBlock,
|
||||
`callCounts.${countName} should be optional number.`,
|
||||
);
|
||||
}
|
||||
assertHas(/createdAt:\s*v\.number\(\)/, objectBlock, "createdAt should be required.");
|
||||
|
||||
for (const [indexName, fields] of [
|
||||
["by_runId_and_createdAt", '"runId",\\s*"createdAt"'],
|
||||
["by_leadId_and_createdAt", '"leadId",\\s*"createdAt"'],
|
||||
["by_auditId_and_createdAt", '"auditId",\\s*"createdAt"'],
|
||||
["by_provider_and_createdAt", '"provider",\\s*"createdAt"'],
|
||||
["by_createdAt", '"createdAt"'],
|
||||
] as const) {
|
||||
assertHas(
|
||||
new RegExp(`index\\("${indexName}",\\s*\\[${fields}\\]\\)`),
|
||||
section,
|
||||
`usageEvents should define ${indexName}.`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("usageEvents module exposes internal recorder and authenticated bounded read queries", () => {
|
||||
assert.equal(existsSync(usageEventsPath), true, "usageEvents.ts should be present.");
|
||||
|
||||
const exports = getExportedConstNames(usageEventsSourceFile);
|
||||
for (const exportName of [
|
||||
"recordUsageEvent",
|
||||
...usageReadQueries.map((readQuery) => readQuery.name),
|
||||
]) {
|
||||
assert.equal(exports.has(exportName), true, `Expected export: ${exportName}`);
|
||||
}
|
||||
|
||||
assertHas(
|
||||
/export const recordUsageEvent = internalMutation\s*\(/,
|
||||
usageEventsSource,
|
||||
"recordUsageEvent should be an internalMutation.",
|
||||
);
|
||||
for (const { name } of usageReadQueries) {
|
||||
assertHas(
|
||||
new RegExp(`export const ${name} = query\\s*\\(`),
|
||||
usageEventsSource,
|
||||
`${name} should remain a public authenticated bounded query.`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("usageEvents queries use indexes and bounded take without filters or collect", () => {
|
||||
const querySources = usageReadQueries.map((readQuery) => ({
|
||||
...readQuery,
|
||||
source: extractExportSource(readQuery.name),
|
||||
}));
|
||||
|
||||
for (const { source } of querySources) {
|
||||
assertHas(/limit:\s*v\.optional\(v\.number\(\)\)/, source, "read query should validate limit.");
|
||||
}
|
||||
for (const { source, indexAssertion, message } of querySources) {
|
||||
assertHas(indexAssertion, source, message);
|
||||
}
|
||||
|
||||
for (const source of [usageEventsSource, ...querySources.map((querySource) => querySource.source)]) {
|
||||
assert.doesNotMatch(source, /\.filter\s*\(/, "usageEvents should not use query filters.");
|
||||
assert.doesNotMatch(source, /\.collect\s*\(/, "usageEvents should not use unbounded collect.");
|
||||
}
|
||||
for (const { source } of querySources) {
|
||||
assertHas(
|
||||
/\.take\(\s*normalizeListLimit\(args\.limit\)\s*\)/,
|
||||
source,
|
||||
"read query should be bounded.",
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("usageEvents read queries require operator auth before reading telemetry", () => {
|
||||
assertHas(
|
||||
/const\s+requireOperator\s*=\s*async\s*\(\s*ctx:\s*QueryCtx\s*\)[\s\S]*ctx\.auth\.getUserIdentity\(\)[\s\S]*throw new Error\(["']Nicht autorisiert\.["']\)/,
|
||||
usageEventsSource,
|
||||
"usageEvents should define the local requireOperator auth guard.",
|
||||
);
|
||||
|
||||
for (const { name } of usageReadQueries) {
|
||||
const source = extractExportSource(name);
|
||||
const authIndex = source.indexOf("await requireOperator(ctx)");
|
||||
const readIndex = source.indexOf("ctx.db");
|
||||
|
||||
assert.notEqual(authIndex, -1, `${name} should require operator auth.`);
|
||||
assert.notEqual(readIndex, -1, `${name} should read from ctx.db.`);
|
||||
assert.equal(
|
||||
authIndex < readIndex,
|
||||
true,
|
||||
`${name} should require auth before reading usage telemetry.`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("recordUsageEvent guards finite non-negative usage numbers before insert", () => {
|
||||
const recordSource = extractExportSource("recordUsageEvent");
|
||||
const guardCallIndex = recordSource.indexOf("assertValidUsageEventNumbers(args)");
|
||||
const insertIndex = recordSource.indexOf('ctx.db.insert("usageEvents"');
|
||||
|
||||
assert.notEqual(
|
||||
guardCallIndex,
|
||||
-1,
|
||||
"recordUsageEvent should call assertValidUsageEventNumbers(args).",
|
||||
);
|
||||
assert.notEqual(insertIndex, -1, "recordUsageEvent should insert usageEvents.");
|
||||
assert.equal(
|
||||
guardCallIndex < insertIndex,
|
||||
true,
|
||||
"recordUsageEvent should validate usage numbers before inserting.",
|
||||
);
|
||||
|
||||
assertHas(
|
||||
/function\s+assertFiniteNonNegativeNumber[\s\S]*Number\.isFinite\(value\)[\s\S]*value\s*<\s*0/,
|
||||
usageEventsSource,
|
||||
"Cost guard should reject NaN, Infinity, and negative numbers.",
|
||||
);
|
||||
assertHas(
|
||||
/function\s+assertFiniteNonNegativeInteger[\s\S]*Number\.isFinite\(value\)[\s\S]*value\s*<\s*0[\s\S]*Number\.isInteger\(value\)/,
|
||||
usageEventsSource,
|
||||
"Token/count guard should require finite non-negative integers.",
|
||||
);
|
||||
assertHas(
|
||||
/assertFiniteNonNegativeNumber\(args\.estimatedCostUsd,\s*["']estimatedCostUsd["']\)/,
|
||||
usageEventsSource,
|
||||
"estimatedCostUsd should use the finite non-negative number guard.",
|
||||
);
|
||||
|
||||
for (const tokenName of [
|
||||
"inputTokens",
|
||||
"outputTokens",
|
||||
"promptTokens",
|
||||
"completionTokens",
|
||||
"totalTokens",
|
||||
"cacheReadTokens",
|
||||
]) {
|
||||
assertHas(
|
||||
new RegExp(
|
||||
`assertFiniteNonNegativeInteger\\(args\\.tokens\\?\\.${tokenName},\\s*["']tokens\\.${tokenName}["']\\)`,
|
||||
),
|
||||
usageEventsSource,
|
||||
`tokens.${tokenName} should use the finite non-negative integer guard.`,
|
||||
);
|
||||
}
|
||||
|
||||
for (const countName of ["requests", "pages", "screenshots", "lookups"]) {
|
||||
assertHas(
|
||||
new RegExp(
|
||||
`assertFiniteNonNegativeInteger\\(args\\.callCounts\\?\\.${countName},\\s*["']callCounts\\.${countName}["']\\)`,
|
||||
),
|
||||
usageEventsSource,
|
||||
`callCounts.${countName} should use the finite non-negative integer guard.`,
|
||||
);
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user