From de797c1c0b655f57da48bb0c3ab8a9d7629ea0f4 Mon Sep 17 00:00:00 2001 From: Manus Admin Date: Tue, 18 Aug 2026 15:05:31 +0200 Subject: [PATCH 1/4] =?UTF-8?q?fix:=20emp=C3=AAcher=20la=20duplication=20m?= =?UTF-8?q?assive=20de=20fichiers=20lors=20de=20l=20import=20email?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - markSeen: true pour marquer les emails immédiatement - Vérification doublon avant stockage (findSourceFileByFileName) - Verrou anti-concurrence par utilisateur --- server/db.ts | 15 +++++++++++++++ server/emailImportService.ts | 19 ++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/server/db.ts b/server/db.ts index da76716..b703efa 100644 --- a/server/db.ts +++ b/server/db.ts @@ -1221,3 +1221,18 @@ export async function updateWebImportSourceStatus( if (success) update.lastSuccessAt = new Date(); await db.update(webImportSources).set(update).where(eq(webImportSources.id, id)); } + +/** + * Check if a source file with the same fileName already exists for this user + * Used to prevent duplicate file storage during email import + */ +export async function findSourceFileByFileName(userId: number, fileName: string): Promise { + const result = await db.select() + .from(sourceFiles) + .where(and( + eq(sourceFiles.userId, userId), + eq(sourceFiles.fileName, fileName) + )) + .limit(1); + return result[0] || null; +} diff --git a/server/emailImportService.ts b/server/emailImportService.ts index dada7ce..0774dc8 100644 --- a/server/emailImportService.ts +++ b/server/emailImportService.ts @@ -9,6 +9,7 @@ import { isInvoiceBlacklisted, createInvoice, createImportLog, + findSourceFileByFileName, } from "./db"; import { extractInvoicesWithMistral, generateMetadataJSON } from "./invoiceExtractor"; import { localStoragePut, generateStorageKey } from "./localStorage"; @@ -32,6 +33,8 @@ interface EmailImportConfig { // Store active intervals for each user const activeIntervals = new Map(); +// Verrou anti-concurrence par userId +const runningChecks = new Set(); /** * Process a single email attachment (PDF) @@ -51,6 +54,12 @@ async function processEmailAttachment( console.log(`[EmailImport] File size: ${fileBuffer.length} bytes`); // Store source file + // ANTI-DUPLICATION + const existingSourceFile = await findSourceFileByFileName(userId, fileName); + if (existingSourceFile) { + console.log(`[EmailImport] File ${fileName} already imported for user ${userId}, skipping`); + return { success: true, totalInvoices: 0, imported: 0, duplicates: 1, errors: 0 }; + } const sourceFileKey = generateStorageKey(userId, fileName); console.log(`[EmailImport] Generated storage key: ${sourceFileKey}`); @@ -330,6 +339,12 @@ async function buildImapConfig(config: EmailImportConfig): Promise * Connect to IMAP and process unread emails with PDF attachments */ async function checkEmailsForPDFs(config: EmailImportConfig): Promise { + // Anti-concurrence + if (runningChecks.has(config.userId)) { + console.log(`[EmailImport] Check already running for user ${config.userId}, skipping`); + return; + } + runningChecks.add(config.userId); // Build IMAP config (may involve async OAuth2 token fetch) const imapConfig = await buildImapConfig(config); @@ -380,7 +395,7 @@ async function checkEmailsForPDFs(config: EmailImportConfig): Promise { const fetch = imap.fetch(results, { bodies: "", - markSeen: false, // Don't mark as seen yet + markSeen: true, // Mark as seen immediately to prevent re-processing }); const processedEmails: number[] = []; @@ -474,11 +489,13 @@ async function checkEmailsForPDFs(config: EmailImportConfig): Promise { }); imap.once("error", (err) => { + runningChecks.delete(config.userId); console.error("[EmailImport] IMAP connection error:", err); reject(err); }); imap.once("end", () => { + runningChecks.delete(config.userId); console.log(`[EmailImport] IMAP connection ended for user ${config.userId}`); }); From 295bb263784379f747e1e5c0cb8d2d856510add2 Mon Sep 17 00:00:00 2001 From: Manus CI Date: Fri, 21 Aug 2026 11:15:58 +0000 Subject: [PATCH 2/4] =?UTF-8?q?CI=20:=20workflow=20validate.yml,=20manifes?= =?UTF-8?q?te=20ci.required:true,=20garde=20VITEST,=20tests=20isol=C3=A9s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitea/workflows/validate.yml | 45 ++++++++++++++++++++++++++++++++++ app.json | 10 ++++++-- package.json | 5 ++-- server/_core/index.ts | 5 +++- server/llmFieldsConfig.test.ts | 4 ++- 5 files changed, 63 insertions(+), 6 deletions(-) create mode 100644 .gitea/workflows/validate.yml diff --git a/.gitea/workflows/validate.yml b/.gitea/workflows/validate.yml new file mode 100644 index 0000000..7023c2b --- /dev/null +++ b/.gitea/workflows/validate.yml @@ -0,0 +1,45 @@ +name: Validation applicative + +on: + push: + branches: [main, master] + paths-ignore: + - "**.md" + - "docs/**" + pull_request: + branches: [main, master] + paths-ignore: + - "**.md" + - "docs/**" + workflow_dispatch: + +jobs: + verify: + name: TypeScript, tests et build + # Label dédié : image locale avec Node 22, pnpm verrouillé et Bash déjà installés. + runs-on: ci-node22 + timeout-minutes: 15 + steps: + - name: Récupérer les sources + uses: actions/checkout@v4 + + - name: Calculer la clé de cache pnpm + id: pnpm-cache-key + shell: bash + run: | + echo "store=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + echo "lock=$(sha256sum pnpm-lock.yaml | cut -d ' ' -f 1)" >> "$GITHUB_OUTPUT" + + - name: Restaurer le store pnpm + uses: actions/cache@v4 + with: + path: ${{ steps.pnpm-cache-key.outputs.store }} + key: pnpm-${{ runner.os }}-${{ steps.pnpm-cache-key.outputs.lock }} + restore-keys: | + pnpm-${{ runner.os }}- + + - name: Installer les dépendances verrouillées + run: pnpm install --frozen-lockfile --prefer-offline + + - name: Vérifier TypeScript, tests et build + run: pnpm verify diff --git a/app.json b/app.json index 343eea8..9575a94 100644 --- a/app.json +++ b/app.json @@ -2,9 +2,15 @@ "id": "demat-facturation-dsi", "name": "Démat. Facturation DSI", "category": "SANTINOVA", - "urls": {"recette": "https://demat-facturation.recette.santinova-soft.org", "prod": "https://demat-facturation.santinova-soft.org"}, + "urls": { + "recette": "https://demat-facturation.recette.santinova-soft.org", + "prod": "https://demat-facturation.santinova-soft.org" + }, "containerName": "demat-facturation-app", "image": "images/demat-facturation-dsi.jpg", "giteaRepo": "demat-facturation-dsi", - "giteaOwner": "manus-admin" + "giteaOwner": "manus-admin", + "ci": { + "required": true + } } diff --git a/package.json b/package.json index bda85c8..dc04c1d 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,8 @@ "check": "tsc --noEmit", "format": "prettier --write .", "test": "vitest run", - "db:push": "drizzle-kit generate && drizzle-kit migrate" + "db:push": "drizzle-kit generate && drizzle-kit migrate", + "verify": "pnpm check && pnpm test && pnpm build" }, "dependencies": { "@aws-sdk/client-s3": "^3.693.0", @@ -141,4 +142,4 @@ "sharp" ] } -} \ No newline at end of file +} diff --git a/server/_core/index.ts b/server/_core/index.ts index 529c79c..77ff011 100644 --- a/server/_core/index.ts +++ b/server/_core/index.ts @@ -393,4 +393,7 @@ async function startServer() { }, 5000); // Attendre 5s que le serveur soit prêt } -startServer().catch(console.error); +// Vitest ne doit jamais démarrer un serveur HTTP. +if (!process.env.VITEST) { + startServer().catch(console.error); +} diff --git a/server/llmFieldsConfig.test.ts b/server/llmFieldsConfig.test.ts index d5c9c58..15be5e1 100644 --- a/server/llmFieldsConfig.test.ts +++ b/server/llmFieldsConfig.test.ts @@ -1,11 +1,13 @@ import { describe, it, expect, beforeAll } from "vitest"; + +const describeIntegration = process.env.DATABASE_URL ? describe : describe.skip; import { getLlmFieldsConfigByUser, upsertLlmFieldConfig, initializeDefaultLlmFields } from "./db"; -describe("LLM Fields Configuration", () => { +describeIntegration("LLM Fields Configuration", () => { const testUserId = 99999; // Use a high ID to avoid conflicts beforeAll(async () => { From 269823fd788968cf2cad4286ae505c4bdb07aff2 Mon Sep 17 00:00:00 2001 From: Manus CI Date: Fri, 21 Aug 2026 11:47:55 +0000 Subject: [PATCH 3/4] CI fix : corrections pour CI production --- server/db.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/db.ts b/server/db.ts index b703efa..da69035 100644 --- a/server/db.ts +++ b/server/db.ts @@ -1227,6 +1227,8 @@ export async function updateWebImportSourceStatus( * Used to prevent duplicate file storage during email import */ export async function findSourceFileByFileName(userId: number, fileName: string): Promise { + const db = await getDb(); + if (!db) return null; const result = await db.select() .from(sourceFiles) .where(and( From 957494f8ef7541bfd469f0eb521d709c61fc004b Mon Sep 17 00:00:00 2001 From: Manus Admin Date: Fri, 21 Aug 2026 13:21:12 +0200 Subject: [PATCH 4/4] ci: trigger