357 lines
12 KiB
TypeScript
357 lines
12 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { existsSync, readFileSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import test from "node:test";
|
|
import ts from "typescript";
|
|
|
|
const schemaPath = join(process.cwd(), "convex", "schema.ts");
|
|
const domainPath = join(process.cwd(), "convex", "domain.ts");
|
|
const usageEventsPath = join(process.cwd(), "convex", "usageEvents.ts");
|
|
|
|
const schemaSource = readFileSync(schemaPath, "utf8");
|
|
const domainSource = readFileSync(domainPath, "utf8");
|
|
const usageEventsSource = existsSync(usageEventsPath)
|
|
? readFileSync(usageEventsPath, "utf8")
|
|
: "";
|
|
|
|
const usageEventsSourceFile = ts.createSourceFile(
|
|
"usageEvents.ts",
|
|
usageEventsSource,
|
|
ts.ScriptTarget.ES2022,
|
|
true,
|
|
ts.ScriptKind.TS,
|
|
);
|
|
|
|
function getExportedConstNames(file: ts.SourceFile) {
|
|
const names = new Set<string>();
|
|
|
|
const visit = (node: ts.Node) => {
|
|
if (ts.isVariableStatement(node)) {
|
|
const isExported = node.modifiers?.some(
|
|
(mod) => mod.kind === ts.SyntaxKind.ExportKeyword,
|
|
);
|
|
const isConst = node.declarationList.flags & ts.NodeFlags.Const;
|
|
|
|
if (isExported && isConst) {
|
|
for (const declaration of node.declarationList.declarations) {
|
|
if (ts.isIdentifier(declaration.name)) {
|
|
names.add(declaration.name.text);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
ts.forEachChild(node, visit);
|
|
};
|
|
|
|
ts.forEachChild(file, visit);
|
|
return names;
|
|
}
|
|
|
|
function extractTableSection(tableName: string) {
|
|
const marker = `${tableName}: defineTable({`;
|
|
const markerIndex = schemaSource.indexOf(marker);
|
|
assert.notEqual(
|
|
markerIndex,
|
|
-1,
|
|
`Expected schema table definition for ${tableName}.`,
|
|
);
|
|
|
|
const objectStart = schemaSource.indexOf("{", markerIndex);
|
|
let depth = 0;
|
|
let objectEnd = -1;
|
|
|
|
for (let index = objectStart; index < schemaSource.length; index += 1) {
|
|
if (schemaSource[index] === "{") {
|
|
depth += 1;
|
|
} else if (schemaSource[index] === "}") {
|
|
depth -= 1;
|
|
if (depth === 0) {
|
|
objectEnd = index;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
assert.notEqual(objectEnd, -1, `Could not parse schema object for ${tableName}.`);
|
|
|
|
const remainder = schemaSource.slice(objectEnd + 1);
|
|
const nextTableMatch = remainder.match(/^\s*[a-zA-Z_][\w]*:\s*defineTable\(/m);
|
|
const sectionEnd =
|
|
nextTableMatch === null ? schemaSource.length : objectEnd + 1 + nextTableMatch.index!;
|
|
|
|
return {
|
|
objectBlock: schemaSource.slice(markerIndex, objectEnd + 1),
|
|
section: schemaSource.slice(markerIndex, sectionEnd),
|
|
};
|
|
}
|
|
|
|
function extractExportSource(name: string) {
|
|
const marker = `export const ${name} = `;
|
|
const declarationIndex = usageEventsSource.indexOf(marker);
|
|
assert.notEqual(declarationIndex, -1, `Expected declaration for ${name}`);
|
|
|
|
const openBraceIndex = usageEventsSource.indexOf("{", declarationIndex);
|
|
let depth = 0;
|
|
let end = -1;
|
|
|
|
for (let index = openBraceIndex; index < usageEventsSource.length; index += 1) {
|
|
const char = usageEventsSource[index];
|
|
if (char === "{") {
|
|
depth += 1;
|
|
} else if (char === "}") {
|
|
depth -= 1;
|
|
if (depth === 0) {
|
|
end = index;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
assert.notEqual(end, -1, `Expected balanced braces for ${name}`);
|
|
return usageEventsSource.slice(openBraceIndex, end + 1);
|
|
}
|
|
|
|
function assertHas(pattern: RegExp, source: string, message: string) {
|
|
assert.equal(pattern.test(source), true, message);
|
|
}
|
|
|
|
const usageReadQueries = [
|
|
{
|
|
name: "listLatestUsageEvents",
|
|
indexAssertion:
|
|
/withIndex\("by_createdAt"\)/,
|
|
message: "latest query should use by_createdAt.",
|
|
},
|
|
{
|
|
name: "listUsageEventsByRun",
|
|
indexAssertion:
|
|
/withIndex\("by_runId_and_createdAt"[\s\S]*?eq\("runId",\s*args\.runId\)/,
|
|
message: "run query should use by_runId_and_createdAt with runId equality.",
|
|
},
|
|
{
|
|
name: "listUsageEventsByLead",
|
|
indexAssertion:
|
|
/withIndex\("by_leadId_and_createdAt"[\s\S]*?eq\("leadId",\s*args\.leadId\)/,
|
|
message: "lead query should use by_leadId_and_createdAt with leadId equality.",
|
|
},
|
|
{
|
|
name: "listUsageEventsByAudit",
|
|
indexAssertion:
|
|
/withIndex\("by_auditId_and_createdAt"[\s\S]*?eq\("auditId",\s*args\.auditId\)/,
|
|
message: "audit query should use by_auditId_and_createdAt with auditId equality.",
|
|
},
|
|
{
|
|
name: "listUsageEventsByProvider",
|
|
indexAssertion:
|
|
/withIndex\("by_provider_and_createdAt"[\s\S]*?eq\("provider",\s*args\.provider\)/,
|
|
message: "provider query should use by_provider_and_createdAt with provider equality.",
|
|
},
|
|
] as const;
|
|
|
|
test("usage domain constants declare supported providers and operations", () => {
|
|
assertHas(
|
|
/USAGE_EVENT_PROVIDERS\s*=\s*\[[\s\S]*"openrouter"[\s\S]*"screenshotone"[\s\S]*"jina"[\s\S]*"pagespeed"[\s\S]*"google_places"[\s\S]*\]\s*as const/,
|
|
domainSource,
|
|
"Domain should declare usage providers for all managed external services.",
|
|
);
|
|
assertHas(
|
|
/USAGE_EVENT_OPERATIONS\s*=\s*\[[\s\S]*"audit_capture"[\s\S]*"audit_generation"[\s\S]*"lead_lookup"[\s\S]*\]\s*as const/,
|
|
domainSource,
|
|
"Domain should declare usage operations for capture, generation, and lookup.",
|
|
);
|
|
});
|
|
|
|
test("usageEvents schema stores cost and usage dimensions with bounded indexes", () => {
|
|
const { objectBlock, section } = extractTableSection("usageEvents");
|
|
|
|
assertHas(/provider:\s*usageEventProvider/, objectBlock, "provider should use the provider validator.");
|
|
assertHas(/operation:\s*usageEventOperation/, objectBlock, "operation should use the operation validator.");
|
|
assertHas(
|
|
/runId:\s*v\.optional\(\s*v\.id\(["']agentRuns["']\)\s*\)/,
|
|
objectBlock,
|
|
"runId should be optional for SaaS-ready attribution.",
|
|
);
|
|
assertHas(
|
|
/leadId:\s*v\.optional\(\s*v\.id\(["']leads["']\)\s*\)/,
|
|
objectBlock,
|
|
"leadId should be optional for lead-level attribution.",
|
|
);
|
|
assertHas(
|
|
/auditId:\s*v\.optional\(\s*v\.id\(["']audits["']\)\s*\)/,
|
|
objectBlock,
|
|
"auditId should be optional for audit-level attribution.",
|
|
);
|
|
assertHas(
|
|
/estimatedCostUsd:\s*v\.number\(\)/,
|
|
objectBlock,
|
|
"estimatedCostUsd should be a required normalized number.",
|
|
);
|
|
assertHas(
|
|
/tokens:\s*v\.optional\(\s*v\.object\([\s\S]*?inputTokens:\s*v\.optional\(v\.number\(\)\)[\s\S]*?outputTokens:\s*v\.optional\(v\.number\(\)\)[\s\S]*?promptTokens:\s*v\.optional\(v\.number\(\)\)[\s\S]*?completionTokens:\s*v\.optional\(v\.number\(\)\)[\s\S]*?totalTokens:\s*v\.optional\(v\.number\(\)\)[\s\S]*?\)\s*\)/,
|
|
objectBlock,
|
|
"tokens should capture OpenRouter-compatible token dimensions.",
|
|
);
|
|
assertHas(
|
|
/callCounts:\s*v\.optional\(\s*v\.object\(/,
|
|
objectBlock,
|
|
"callCounts should be an optional object.",
|
|
);
|
|
for (const countName of ["requests", "pages", "screenshots", "lookups"]) {
|
|
assertHas(
|
|
new RegExp(`${countName}:\\s*v\\.optional\\(v\\.number\\(\\)\\)`),
|
|
objectBlock,
|
|
`callCounts.${countName} should be optional number.`,
|
|
);
|
|
}
|
|
assertHas(/createdAt:\s*v\.number\(\)/, objectBlock, "createdAt should be required.");
|
|
|
|
for (const [indexName, fields] of [
|
|
["by_runId_and_createdAt", '"runId",\\s*"createdAt"'],
|
|
["by_leadId_and_createdAt", '"leadId",\\s*"createdAt"'],
|
|
["by_auditId_and_createdAt", '"auditId",\\s*"createdAt"'],
|
|
["by_provider_and_createdAt", '"provider",\\s*"createdAt"'],
|
|
["by_createdAt", '"createdAt"'],
|
|
] as const) {
|
|
assertHas(
|
|
new RegExp(`index\\("${indexName}",\\s*\\[${fields}\\]\\)`),
|
|
section,
|
|
`usageEvents should define ${indexName}.`,
|
|
);
|
|
}
|
|
});
|
|
|
|
test("usageEvents module exposes internal recorder and authenticated bounded read queries", () => {
|
|
assert.equal(existsSync(usageEventsPath), true, "usageEvents.ts should be present.");
|
|
|
|
const exports = getExportedConstNames(usageEventsSourceFile);
|
|
for (const exportName of [
|
|
"recordUsageEvent",
|
|
...usageReadQueries.map((readQuery) => readQuery.name),
|
|
]) {
|
|
assert.equal(exports.has(exportName), true, `Expected export: ${exportName}`);
|
|
}
|
|
|
|
assertHas(
|
|
/export const recordUsageEvent = internalMutation\s*\(/,
|
|
usageEventsSource,
|
|
"recordUsageEvent should be an internalMutation.",
|
|
);
|
|
for (const { name } of usageReadQueries) {
|
|
assertHas(
|
|
new RegExp(`export const ${name} = query\\s*\\(`),
|
|
usageEventsSource,
|
|
`${name} should remain a public authenticated bounded query.`,
|
|
);
|
|
}
|
|
});
|
|
|
|
test("usageEvents queries use indexes and bounded take without filters or collect", () => {
|
|
const querySources = usageReadQueries.map((readQuery) => ({
|
|
...readQuery,
|
|
source: extractExportSource(readQuery.name),
|
|
}));
|
|
|
|
for (const { source } of querySources) {
|
|
assertHas(/limit:\s*v\.optional\(v\.number\(\)\)/, source, "read query should validate limit.");
|
|
}
|
|
for (const { source, indexAssertion, message } of querySources) {
|
|
assertHas(indexAssertion, source, message);
|
|
}
|
|
|
|
for (const source of [usageEventsSource, ...querySources.map((querySource) => querySource.source)]) {
|
|
assert.doesNotMatch(source, /\.filter\s*\(/, "usageEvents should not use query filters.");
|
|
assert.doesNotMatch(source, /\.collect\s*\(/, "usageEvents should not use unbounded collect.");
|
|
}
|
|
for (const { source } of querySources) {
|
|
assertHas(
|
|
/\.take\(\s*normalizeListLimit\(args\.limit\)\s*\)/,
|
|
source,
|
|
"read query should be bounded.",
|
|
);
|
|
}
|
|
});
|
|
|
|
test("usageEvents read queries require operator auth before reading telemetry", () => {
|
|
assertHas(
|
|
/const\s+requireOperator\s*=\s*async\s*\(\s*ctx:\s*QueryCtx\s*\)[\s\S]*ctx\.auth\.getUserIdentity\(\)[\s\S]*throw new Error\(["']Nicht autorisiert\.["']\)/,
|
|
usageEventsSource,
|
|
"usageEvents should define the local requireOperator auth guard.",
|
|
);
|
|
|
|
for (const { name } of usageReadQueries) {
|
|
const source = extractExportSource(name);
|
|
const authIndex = source.indexOf("await requireOperator(ctx)");
|
|
const readIndex = source.indexOf("ctx.db");
|
|
|
|
assert.notEqual(authIndex, -1, `${name} should require operator auth.`);
|
|
assert.notEqual(readIndex, -1, `${name} should read from ctx.db.`);
|
|
assert.equal(
|
|
authIndex < readIndex,
|
|
true,
|
|
`${name} should require auth before reading usage telemetry.`,
|
|
);
|
|
}
|
|
});
|
|
|
|
test("recordUsageEvent guards finite non-negative usage numbers before insert", () => {
|
|
const recordSource = extractExportSource("recordUsageEvent");
|
|
const guardCallIndex = recordSource.indexOf("assertValidUsageEventNumbers(args)");
|
|
const insertIndex = recordSource.indexOf('ctx.db.insert("usageEvents"');
|
|
|
|
assert.notEqual(
|
|
guardCallIndex,
|
|
-1,
|
|
"recordUsageEvent should call assertValidUsageEventNumbers(args).",
|
|
);
|
|
assert.notEqual(insertIndex, -1, "recordUsageEvent should insert usageEvents.");
|
|
assert.equal(
|
|
guardCallIndex < insertIndex,
|
|
true,
|
|
"recordUsageEvent should validate usage numbers before inserting.",
|
|
);
|
|
|
|
assertHas(
|
|
/function\s+assertFiniteNonNegativeNumber[\s\S]*Number\.isFinite\(value\)[\s\S]*value\s*<\s*0/,
|
|
usageEventsSource,
|
|
"Cost guard should reject NaN, Infinity, and negative numbers.",
|
|
);
|
|
assertHas(
|
|
/function\s+assertFiniteNonNegativeInteger[\s\S]*Number\.isFinite\(value\)[\s\S]*value\s*<\s*0[\s\S]*Number\.isInteger\(value\)/,
|
|
usageEventsSource,
|
|
"Token/count guard should require finite non-negative integers.",
|
|
);
|
|
assertHas(
|
|
/assertFiniteNonNegativeNumber\(args\.estimatedCostUsd,\s*["']estimatedCostUsd["']\)/,
|
|
usageEventsSource,
|
|
"estimatedCostUsd should use the finite non-negative number guard.",
|
|
);
|
|
|
|
for (const tokenName of [
|
|
"inputTokens",
|
|
"outputTokens",
|
|
"promptTokens",
|
|
"completionTokens",
|
|
"totalTokens",
|
|
"cacheReadTokens",
|
|
]) {
|
|
assertHas(
|
|
new RegExp(
|
|
`assertFiniteNonNegativeInteger\\(args\\.tokens\\?\\.${tokenName},\\s*["']tokens\\.${tokenName}["']\\)`,
|
|
),
|
|
usageEventsSource,
|
|
`tokens.${tokenName} should use the finite non-negative integer guard.`,
|
|
);
|
|
}
|
|
|
|
for (const countName of ["requests", "pages", "screenshots", "lookups"]) {
|
|
assertHas(
|
|
new RegExp(
|
|
`assertFiniteNonNegativeInteger\\(args\\.callCounts\\?\\.${countName},\\s*["']callCounts\\.${countName}["']\\)`,
|
|
),
|
|
usageEventsSource,
|
|
`callCounts.${countName} should use the finite non-negative integer guard.`,
|
|
);
|
|
}
|
|
});
|