matthias/finanzen
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix FinTS transaction deduplication

Browse Source
This commit is contained in:
Matthias
2026-06-23 12:37:41 +02:00
parent 993616c2f9
commit 6e687dc9b5
8 changed files with 451 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
backlog/tasks/task-15 - Fix-FinTS-transaction-deduplication.md Normal file
View File

@@ -0,0 +1,52 @@
---
id: TASK-15
title: Fix FinTS transaction deduplication
status: Done
assignee: []
created_date: '2026-06-23 10:22'
updated_date: '2026-06-23 10:36'
labels: []
dependencies: []
priority: high
ordinal: 15000
---
## Description
<!-- SECTION:DESCRIPTION:BEGIN -->
Repair FinTS import deduplication so placeholder MT940/CAMT references do not cause valid comdirect Girokonto transactions to be skipped.
<!-- SECTION:DESCRIPTION:END -->
## Acceptance Criteria
<!-- AC:BEGIN -->
- [x] #1 FinTS placeholder references such as NONREF and NOTPROVIDED are not stored as externalRef
- [x] #2 Multiple FinTS rows with placeholder references but distinct transaction content import successfully
- [x] #3 Real duplicate externalRef values are still skipped
- [x] #4 Repeated import of the same rows skips via dedupHash
- [x] #5 Focused tests cover FinTS mapping and import commit behavior
<!-- AC:END -->
## Implementation Plan
<!-- SECTION:PLAN:BEGIN -->
1. Add failing regression tests for FinTS placeholder references and import skip behavior
2. Normalize FinTS external references so placeholders are discarded
3. Add import skip diagnostics for externalRef vs dedupHash
4. Run focused tests and project verification
<!-- SECTION:PLAN:END -->
## Implementation Notes
<!-- SECTION:NOTES:BEGIN -->
Implemented FinTS externalRef normalization for MT940/CAMT placeholder values and added import commit diagnostics split by externalRef and dedupHash skips. Verified with focused Vitest, broader Convex tests, focused ESLint, diff whitespace check, and production build.
Follow-up from live Convex logs: all 42 rows were skipped by externalRef. Read-only Convex data inspection showed FinTS externalRef values like POS 13, which are MT940 positional markers and collide across statement periods. Extended normalization to discard POS-style references and added regression coverage for legacy POS externalRef records not blocking new rows.
Follow-up from visible UI gaps: imported rows existed, but transactions.list used by_user_effmonth ordering for assignment-month mode, which sorts by effectiveMonth and creation time rather than bookingDate. Added by_user_effmonth_booking index and switched effective-basis listing to it so current month rows are ordered by booking date descending. Added regression coverage.
<!-- SECTION:NOTES:END -->
## Final Summary
<!-- SECTION:FINAL_SUMMARY:BEGIN -->
Fixed FinTS transaction deduplication by discarding placeholder and POS-style external references, added import skip diagnostics, and corrected effective-month transaction listing to order by booking date. Verified with focused and related Vitest suites, focused ESLint, diff check, and production build.
<!-- SECTION:FINAL_SUMMARY:END -->

65
convex/bank/fintsMap.test.ts Normal file
View File

@@ -0,0 +1,65 @@
import { describe, expect, test } from "vitest";
import { mapFinTsTransaction } from "./fintsMap";
const salaryShift = {
enabled: true,
categoryNames: ["Gehalt & Besoldung"],
dayThreshold: 25,
};
function baseTransaction(overrides: Partial<Parameters<typeof mapFinTsTransaction>[0]> = {}) {
return {
valueDate: new Date(2026, 5, 22),
entryDate: new Date(2026, 5, 22),
amount: -12.34,
transactionType: "NMSC",
bankReference: "",
customerReference: "",
bookingText: "Kartenzahlung",
purpose: "REWE SAGT DANKE",
remoteName: "REWE",
...overrides,
};
}
describe("mapFinTsTransaction", () => {
test("does not expose MT940 placeholder references as external references", () => {
const placeholders = [
"NONREF",
"NOTPROVIDED",
"NOT PROVIDED",
"N/A",
"POS 13",
"POS-42",
"POS_1155",
"-",
"",
];
for (const placeholder of placeholders) {
const mapped = mapFinTsTransaction(
baseTransaction({
bankReference: placeholder,
customerReference: placeholder,
}),
[],
salaryShift,
);
expect(mapped.externalRef).toBeUndefined();
}
});
test("keeps real FinTS references for provider-level duplicate detection", () => {
const mapped = mapFinTsTransaction(
baseTransaction({
bankReference: "5J2C21XL0470L56V/39761",
customerReference: "NONREF",
}),
[],
salaryShift,
);
expect(mapped.externalRef).toBe("5J2C21XL0470L56V/39761");
});
});

50
convex/bank/fintsMap.ts
View File

@@ -14,6 +14,54 @@ export type FintsStatementTransaction = {
customerReference?: string;
};
const FINTS_EXTERNAL_REF_PLACEHOLDERS = new Set([
"0",
"00",
"000",
"0000",
"00000",
"000000",
"0000000",
"00000000",
"000000000",
"0000000000",
"KEINE",
"N/A",
"NA",
"NONE",
"NONREF",
"NOREF",
"NOTPROVIDED",
"NULL",
"UNBEKANNT",
]);
function normalizeFinTsExternalRefValue(value?: string): string | undefined {
const trimmed = value?.trim();
if (!trimmed) return undefined;
const compact = trimmed.replace(/[\s_-]+/g, "").toUpperCase();
if (
!compact ||
FINTS_EXTERNAL_REF_PLACEHOLDERS.has(compact) ||
/^POS\d+$/.test(compact)
) {
return undefined;
}
return trimmed;
}
export function normalizeFinTsExternalRef(
bankReference?: string,
customerReference?: string,
): string | undefined {
return (
normalizeFinTsExternalRefValue(bankReference) ??
normalizeFinTsExternalRefValue(customerReference)
);
}
export function formatFinTsDate(date: Date): string {
const y = date.getFullYear();
const m = String(date.getMonth() + 1).padStart(2, "0");
@@ -55,7 +103,7 @@ export function mapFinTsTransaction(
vorgang,
isPending: false,
rawText: rawText || undefined,
externalRef: tx.bankReference || tx.customerReference || undefined,
externalRef: normalizeFinTsExternalRef(tx.bankReference, tx.customerReference),
categoryName,
assignedMonth,
effectiveMonth,

198
convex/imports.test.ts Normal file
View File

@@ -0,0 +1,198 @@
/// <reference types="vite/client" />
import { convexTest } from "convex-test";
import { makeFunctionReference } from "convex/server";
import { describe, expect, test } from "vitest";
import type { Id } from "./_generated/dataModel";
import schema from "./schema";
const modules = import.meta.glob("./**/*.ts");
delete modules["./imports.test.ts"];
const commitRowsInternal = makeFunctionReference<"mutation">(
"imports:commitRowsInternal",
);
async function seedImportFixture() {
const t = convexTest(schema, modules);
const seeded = await t.run(async (ctx) => {
const userId = await ctx.db.insert("users", {
name: "Import User",
email: "import@example.com",
});
const accountId = await ctx.db.insert("accounts", {
userId,
name: "Girokonto",
type: "giro",
iban: "DE40200411110592825400",
openingBalance: 0,
currency: "EUR",
isArchived: false,
externalId: "592825400",
});
return { userId, accountId };
});
return { t, seeded };
}
describe("imports.commitRowsInternal", () => {
test("imports multiple FinTS rows without external references when transaction content differs", async () => {
const { t, seeded } = await seedImportFixture();
const result = await t.mutation(commitRowsInternal, {
userId: seeded.userId,
filename: "fints-sync-2026-06-15-2026-06-23",
source: "fints",
accountId: seeded.accountId as Id<"accounts">,
rows: [
{
accountId: seeded.accountId as Id<"accounts">,
bookingDate: "2026-06-22",
valueDate: "2026-06-22",
description: "REWE",
amount: -12.34,
vorgang: "NMSC",
isPending: false,
rawText: "REWE SAGT DANKE",
},
{
accountId: seeded.accountId as Id<"accounts">,
bookingDate: "2026-06-22",
valueDate: "2026-06-22",
description: "ALDI",
amount: -23.45,
vorgang: "NMSC",
isPending: false,
rawText: "ALDI SAGT DANKE",
},
],
});
const rows = await t.run(async (ctx) =>
ctx.db
.query("transactions")
.withIndex("by_user_account", (q) =>
q.eq("userId", seeded.userId).eq("accountId", seeded.accountId),
)
.collect(),
);
expect(result.importedCount).toBe(2);
expect(result.skippedCount).toBe(0);
expect(rows.map((row) => row.description).sort()).toEqual(["ALDI", "REWE"]);
expect(rows.every((row) => row.externalRef === undefined)).toBe(true);
});
test("does not let a legacy positional external reference block new FinTS rows without external references", async () => {
const { t, seeded } = await seedImportFixture();
await t.run(async (ctx) => {
await ctx.db.insert("transactions", {
userId: seeded.userId,
accountId: seeded.accountId,
bookingDate: "2024-09-30",
valueDate: "2024-09-30",
description: "Old positional transaction",
amount: -9.99,
isPending: false,
effectiveMonth: "2024-09",
dedupHash: "legacy-dedup",
externalRef: "592825400:POS 13",
});
});
const result = await t.mutation(commitRowsInternal, {
userId: seeded.userId,
filename: "fints-sync-2026-06-15-2026-06-23",
source: "fints",
accountId: seeded.accountId as Id<"accounts">,
rows: [
{
accountId: seeded.accountId as Id<"accounts">,
bookingDate: "2026-06-22",
valueDate: "2026-06-22",
description: "HEM TANKSTELLE",
amount: -62.01,
vorgang: "NMSC",
isPending: false,
rawText: "HEM TANKSTELLE",
},
],
});
expect(result.importedCount).toBe(1);
expect(result.skippedCount).toBe(0);
});
test("still skips real duplicate external references", async () => {
const { t, seeded } = await seedImportFixture();
const row = {
accountId: seeded.accountId as Id<"accounts">,
bookingDate: "2026-06-22",
valueDate: "2026-06-22",
description: "Kartenzahlung",
amount: -12.34,
vorgang: "NMSC",
isPending: false,
rawText: "Kartenzahlung",
externalRef: "592825400:5J2C21XL0470L56V/39761",
};
const first = await t.mutation(commitRowsInternal, {
userId: seeded.userId,
filename: "fints-sync-first",
source: "fints",
accountId: seeded.accountId as Id<"accounts">,
rows: [row],
});
const second = await t.mutation(commitRowsInternal, {
userId: seeded.userId,
filename: "fints-sync-second",
source: "fints",
accountId: seeded.accountId as Id<"accounts">,
rows: [{ ...row, description: "Different text same bank ref", amount: -99 }],
});
expect(first.importedCount).toBe(1);
expect(first.skippedCount).toBe(0);
expect(second.importedCount).toBe(0);
expect(second.skippedCount).toBe(1);
});
test("skips repeated rows without external references via dedup hash", async () => {
const { t, seeded } = await seedImportFixture();
const row = {
accountId: seeded.accountId as Id<"accounts">,
bookingDate: "2026-06-22",
valueDate: "2026-06-22",
description: "REWE",
amount: -12.34,
vorgang: "NMSC",
isPending: false,
rawText: "REWE SAGT DANKE",
};
const first = await t.mutation(commitRowsInternal, {
userId: seeded.userId,
filename: "fints-sync-first",
source: "fints",
accountId: seeded.accountId as Id<"accounts">,
rows: [row],
});
const second = await t.mutation(commitRowsInternal, {
userId: seeded.userId,
filename: "fints-sync-second",
source: "fints",
accountId: seeded.accountId as Id<"accounts">,
rows: [row],
});
expect(first.importedCount).toBe(1);
expect(first.skippedCount).toBe(0);
expect(second.importedCount).toBe(0);
expect(second.skippedCount).toBe(1);
});
});

13
convex/imports.ts
View File

@@ -132,6 +132,8 @@ async function commitRowsHandler(
let importedCount = 0;
let skippedCount = 0;
let skippedByExternalRef = 0;
let skippedByDedupHash = 0;
for (const row of args.rows) {
if (row.externalRef) {
@@ -143,6 +145,7 @@ async function commitRowsHandler(
.unique();
if (existingRef) {
skippedCount++;
skippedByExternalRef++;
continue;
}
}
@@ -162,6 +165,7 @@ async function commitRowsHandler(
.unique();
if (existingDedup) {
skippedCount++;
skippedByDedupHash++;
continue;
}
@@ -204,5 +208,14 @@ async function commitRowsHandler(
status: "completed",
});
console.info("[imports] Commit-Ergebnis", {
source: args.source,
rowCount: args.rows.length,
importedCount,
skippedCount,
skippedByExternalRef,
skippedByDedupHash,
});
return { importId, importedCount, skippedCount };
}

1
convex/schema.ts
View File

@@ -77,6 +77,7 @@ export default defineSchema({
.index("by_user", ["userId"])
.index("by_user_booking", ["userId", "bookingDate"])
.index("by_user_effmonth", ["userId", "effectiveMonth"])
.index("by_user_effmonth_booking", ["userId", "effectiveMonth", "bookingDate"])
.index("by_user_category", ["userId", "categoryId"])
.index("by_user_account", ["userId", "accountId"])
.index("by_user_account_booking", ["userId", "accountId", "bookingDate"])

72
convex/transactions.test.ts
View File

@@ -186,4 +186,76 @@ describe("transactions.list", () => {
expect(effectiveResult.page.map((tx) => tx._id)).toEqual([seeded.shiftedId]);
expect(bookingResult.page).toEqual([]);
});
test("orders effective month results by booking date instead of import creation time", async () => {
const t = convexTest(schema, modules);
const seeded = await t.run(async (ctx) => {
const userId = await ctx.db.insert("users", {
name: "Ordering User",
email: "ordering@example.com",
});
const accountId = await ctx.db.insert("accounts", {
userId,
name: "Girokonto",
type: "checking",
openingBalance: 0,
currency: "EUR",
isArchived: false,
});
const olderCreatedNewestBookingId = await ctx.db.insert("transactions", {
userId,
accountId,
bookingDate: "2026-06-23",
description: "Newest booking created first",
amount: -10,
isPending: false,
effectiveMonth: "2026-06",
});
const newerCreatedOldestBookingId = await ctx.db.insert("transactions", {
userId,
accountId,
bookingDate: "2026-06-15",
description: "Oldest booking created later",
amount: -20,
isPending: false,
effectiveMonth: "2026-06",
});
const middleBookingId = await ctx.db.insert("transactions", {
userId,
accountId,
bookingDate: "2026-06-22",
description: "Middle booking created last",
amount: -30,
isPending: false,
effectiveMonth: "2026-06",
});
return {
userId,
olderCreatedNewestBookingId,
newerCreatedOldestBookingId,
middleBookingId,
};
});
const asUser = t.withIdentity({
subject: `${seeded.userId}|test-session`,
tokenIdentifier: `test:${seeded.userId}`,
});
const result = await asUser.query(api.transactions.list, {
paginationOpts: { cursor: null, numItems: 20 },
from: "2026-06-01",
to: "2026-06-30",
basis: "effective",
});
expect(result.page.map((tx) => tx._id)).toEqual([
seeded.olderCreatedNewestBookingId,
seeded.middleBookingId,
seeded.newerCreatedOldestBookingId,
]);
});
});

2
convex/transactions.ts
View File

@@ -67,7 +67,7 @@ export const list = query({
if (basis === "effective") {
q = ctx.db
.query("transactions")
.withIndex("by_user_effmonth", (iq) => {
.withIndex("by_user_effmonth_booking", (iq) => {
if (fromMonth && toMonth) {
return iq.eq("userId", userId).gte("effectiveMonth", fromMonth).lte("effectiveMonth", toMonth);
}