Checkpoint: Correction robuste des doublons de factures : empreinte SHA-256 unique des PDF sources, détection globale avant import manuel/email/dossier/web, verrou contre les vérifications IMAP concurrentes, attente réelle de fin de traitement avant marquage lu, migration et tests de non-régression.

drizzle/0037_goofy_quentin_quire.sql

@@ -0,0 +1,2 @@
+ALTER TABLE `sourceFiles` ADD `contentHash` varchar(64);--> statement-breakpoint
+ALTER TABLE `sourceFiles` ADD CONSTRAINT `source_file_content_hash_unique` UNIQUE(`contentHash`);

drizzle/meta/_journal.json

@@ -260,6 +260,13 @@
       "when": 1785419093588,
       "tag": "0036_broken_rattler",
       "breakpoints": true
+    },
+    {
+      "idx": 37,
+      "version": "5",
+      "when": 1787394418464,
+      "tag": "0037_goofy_quentin_quire",
+      "breakpoints": true
     }
   ]
 }

drizzle/schema.ts

@@ -36,12 +36,16 @@ export const sourceFiles = mysqlTable("sourceFiles", {
   fileName: varchar("fileName", { length: 255 }).notNull(),
   fileKey: text("fileKey").notNull(), // Local storage key with YYYY-MM prefix
   fileUrl: text("fileUrl").notNull(), // Public URL
+  /** Empreinte du PDF source, globale à l'application pour bloquer tout réimport identique. */
+  contentHash: varchar("contentHash", { length: 64 }),
   totalInvoicesDetected: int("totalInvoicesDetected").default(0).notNull(),
   processingStatus: mysqlEnum("processingStatus", ["processing", "completed", "error"]).default("processing").notNull(),
   processingProgress: varchar("processingProgress", { length: 255 }), // Progress message (e.g., "Extraction 3/9 factures...")
   createdAt: timestamp("createdAt").defaultNow().notNull(),
   updatedAt: timestamp("updatedAt").defaultNow().onUpdateNow().notNull(),
-});
+}, (table) => ({
+  contentHashIdx: uniqueIndex("source_file_content_hash_unique").on(table.contentHash),
+}));
 
 export type SourceFile = typeof sourceFiles.$inferSelect;
 export type InsertSourceFile = typeof sourceFiles.$inferInsert;

server/_core/index.ts

@@ -17,7 +17,8 @@ import { startEmailImportService } from "../emailImportService";
 import { startFolderImportService } from "../folderImportService";
 import { handleAzureCallback, isAzureAdConfigured, generateToken, verifyToken } from "../auth";
 import { createDatabaseBackup } from "../databaseBackup";
-import { generateStorageKey, localStoragePut } from "../localStorage";
+import { generateStorageKey, localStorageDelete, localStoragePut } from "../localStorage";
+import { calculateFileSha256 } from "../fileFingerprint";
 
 const MAX_WEB_IMPORT_BYTES = 20 * 1024 * 1024;
 
@@ -305,7 +306,7 @@ async function startServer() {
         return;
       }
 
-      const { getWebImportSourceByToken, createInvoice, findDuplicateInvoice, isInvoiceBlacklisted, updateWebImportSourceStatus, createSourceFile } = await import('../db');
+      const { getWebImportSourceByToken, createInvoice, findDuplicateInvoice, isInvoiceBlacklisted, updateWebImportSourceStatus, createSourceFile, getSourceFileByContentHash } = await import('../db');
       const source = await getWebImportSourceByToken(apiToken);
       if (!source) {
         res.status(401).json({ error: "Token invalide" });
@@ -317,15 +318,35 @@ async function startServer() {
         return;
       }
 
+      const contentHash = calculateFileSha256(pdfBuffer);
+      const existingSource = await getSourceFileByContentHash(contentHash);
+      if (existingSource) {
+        await updateWebImportSourceStatus(source.id, "success", 0, true);
+        res.json({ success: true, imported: 0, duplicates: 1, total: 1 });
+        return;
+      }
+
       // Stocker d'abord le PDF de façon persistante, comme les autres sources d'import.
       const storageKey = generateStorageKey(source.userId, safeFileName);
       const { url: fileUrl } = await localStoragePut(storageKey, pdfBuffer, "application/pdf");
-      const sourceFile = await createSourceFile({
+      let sourceFile;
+      try {
+        sourceFile = await createSourceFile({
           userId: source.userId,
           fileName: safeFileName,
           fileKey: storageKey,
           fileUrl,
+          contentHash,
         });
+      } catch (error: any) {
+        if (error?.code === "ER_DUP_ENTRY" || error?.errno === 1062) {
+          await localStorageDelete(storageKey).catch(() => undefined);
+          await updateWebImportSourceStatus(source.id, "success", 0, true);
+          res.json({ success: true, imported: 0, duplicates: 1, total: 1 });
+          return;
+        }
+        throw error;
+      }
       const userSettings = await getUserSettings(source.userId);
       const aiSettings = {
         aiProvider: userSettings?.aiProvider || "manus",

server/db.ts

@@ -204,6 +204,22 @@ export async function createSourceFile(data: InsertSourceFile): Promise<SourceFi
   return inserted[0]!;
 }
 
+/**
+ * Recherche un PDF déjà ingéré, quel que soit le compte utilisateur.
+ * L'import email est partagé entre plusieurs identités : la détection doit donc
+ * être globale pour éviter qu'une même pièce soit retraitée sous chaque compte.
+ */
+export async function getSourceFileByContentHash(contentHash: string): Promise<SourceFile | undefined> {
+  const db = await getDb();
+  if (!db) return undefined;
+  const result = await db
+    .select()
+    .from(sourceFiles)
+    .where(eq(sourceFiles.contentHash, contentHash))
+    .limit(1);
+  return result[0];
+}
+
 export async function getSourceFileById(id: number): Promise<SourceFile | undefined> {
   const db = await getDb();
   if (!db) return undefined;

server/emailImportService.ts

@@ -5,13 +5,15 @@ import {
   createSourceFile,
   updateSourceFile,
   getUserSettings,
+  getSourceFileByContentHash,
   findDuplicateInvoice,
   isInvoiceBlacklisted,
   createInvoice,
   createImportLog,
 } from "./db";
 import { extractInvoicesWithMistral, generateMetadataJSON } from "./invoiceExtractor";
-import { localStoragePut, generateStorageKey } from "./localStorage";
+import { localStorageDelete, localStoragePut, generateStorageKey } from "./localStorage";
+import { calculateFileSha256 } from "./fileFingerprint";
 import { sendImportNotification } from "./notificationService";
 import { getOffice365ImapToken, buildXOAuth2String } from "./office365OAuth";
 import { applyAutomationRules } from "./automationEngine";
@@ -32,6 +34,23 @@ interface EmailImportConfig {
 
 // Store active intervals for each user
 const activeIntervals = new Map<number, NodeJS.Timeout>();
+// Une extraction IA peut dépasser la fréquence configurée : ce verrou évite
+// qu'un second cycle IMAP traite les mêmes messages avant la fin du premier.
+const activeChecks = new Map<number, Promise<void>>();
+
+function runEmailCheckExclusive(config: EmailImportConfig): Promise<void> {
+  const runningCheck = activeChecks.get(config.userId);
+  if (runningCheck) {
+    console.log(`[EmailImport] Vérification déjà en cours pour user ${config.userId}, cycle ignoré`);
+    return runningCheck;
+  }
+
+  const check = checkEmailsForPDFs(config).finally(() => {
+    if (activeChecks.get(config.userId) === check) activeChecks.delete(config.userId);
+  });
+  activeChecks.set(config.userId, check);
+  return check;
+}
 
 /**
  * Process a single email attachment (PDF)
@@ -50,6 +69,21 @@ async function processEmailAttachment(
     const fileBuffer = attachment.content;
     console.log(`[EmailImport] File size: ${fileBuffer.length} bytes`);
 
+    const contentHash = calculateFileSha256(fileBuffer);
+    const existingSource = await getSourceFileByContentHash(contentHash);
+    if (existingSource) {
+      console.log(
+        `[EmailImport] PDF déjà importé, extraction ignorée: ${fileName} -> source ${existingSource.id}`,
+      );
+      return {
+        success: true,
+        totalInvoices: Math.max(existingSource.totalInvoicesDetected, 1),
+        imported: 0,
+        duplicates: Math.max(existingSource.totalInvoicesDetected, 1),
+        errors: 0,
+      };
+    }
+    
     // Store source file
     const sourceFileKey = generateStorageKey(userId, fileName);
     console.log(`[EmailImport] Generated storage key: ${sourceFileKey}`);
@@ -65,13 +99,25 @@ async function processEmailAttachment(
     }
     
     // Create source file record
-    const sourceFile = await createSourceFile({
+    let sourceFile;
+    try {
+      sourceFile = await createSourceFile({
         userId,
         fileName,
         fileKey: sourceFileKey,
         fileUrl: sourceFileUrl,
+        contentHash,
         processingStatus: "processing",
       });
+    } catch (error: any) {
+      // La contrainte unique protège également contre deux imports concurrents.
+      if (error?.code === "ER_DUP_ENTRY" || error?.errno === 1062) {
+        await localStorageDelete(sourceFileKey).catch(() => undefined);
+        console.log(`[EmailImport] PDF réservé par un autre traitement: ${fileName}`);
+        return { success: true, totalInvoices: 1, imported: 0, duplicates: 1, errors: 0 };
+      }
+      throw error;
+    }
     
     console.log(`[EmailImport] Source file record created with ID: ${sourceFile.id}`);
     
@@ -383,46 +429,40 @@ async function checkEmailsForPDFs(config: EmailImportConfig): Promise<void> {
             markSeen: false, // Don't mark as seen yet
           });
 
-          const processedEmails: number[] = [];
+          const processedEmailUids: number[] = [];
+          const messageTasks: Promise<void>[] = [];
 
           fetch.on("message", (msg, seqno) => {
-            msg.on("body", (stream) => {
+            const uidPromise = new Promise<number>((resolveUid) => {
-              simpleParser(stream as any, async (err, parsed: ParsedMail) => {
+              msg.once("attributes", (attributes) => resolveUid(attributes.uid));
-                if (err) {
+            });
-                  console.error("[EmailImport] Error parsing email:", err);
-                  return;
-                }
 
-                // Check if email has PDF attachments
+            msg.on("body", (stream) => {
+              const task = (async () => {
+                try {
+                  const parsed: ParsedMail = await simpleParser(stream as any);
                   const pdfAttachments = parsed.attachments.filter(
-                  (att) =>
+                    (attachment) =>
-                    att.contentType === "application/pdf" ||
+                      attachment.contentType === "application/pdf" ||
-                    att.filename?.toLowerCase().endsWith(".pdf")
+                      attachment.filename?.toLowerCase().endsWith(".pdf"),
                   );
 
-                if (pdfAttachments.length === 0) {
+                  if (pdfAttachments.length === 0) return;
-                  return;
-                }
 
                   console.log(
-                  `[EmailImport] Email ${seqno} has ${pdfAttachments.length} PDF attachment(s)`
+                    `[EmailImport] Email ${seqno} has ${pdfAttachments.length} PDF attachment(s)`,
                   );
 
-                // Process each PDF attachment
+                  let allAttachmentsSucceeded = true;
                   for (const attachment of pdfAttachments) {
                     try {
                       const result = await processEmailAttachment(
                         config.userId,
                         attachment,
-                      parsed.subject || "No subject"
+                        parsed.subject || "No subject",
                       );
+                      allAttachmentsSucceeded = allAttachmentsSucceeded && result.success;
 
-                    // Mark this email as successfully processed
-                    if (!processedEmails.includes(seqno)) {
-                      processedEmails.push(seqno);
-                    }
-                    
-                    // Send notification after successful processing
                       if (result.success) {
                         await sendImportNotification(config.userId, {
                           source: "email",
@@ -434,13 +474,24 @@ async function checkEmailsForPDFs(config: EmailImportConfig): Promise<void> {
                         });
                       }
                     } catch (error) {
+                      allAttachmentsSucceeded = false;
                       console.error(
                         `[EmailImport] Failed to process attachment from email ${seqno}:`,
-                      error
+                        error,
                       );
                     }
                   }
-              });
+
+                  if (allAttachmentsSucceeded) {
+                    const uid = await uidPromise;
+                    if (!processedEmailUids.includes(uid)) processedEmailUids.push(uid);
+                  }
+                } catch (error) {
+                  console.error(`[EmailImport] Error parsing email ${seqno}:`, error);
+                }
+              })();
+
+              messageTasks.push(task);
             });
           });
 
@@ -450,16 +501,19 @@ async function checkEmailsForPDFs(config: EmailImportConfig): Promise<void> {
             reject(err);
           });
 
-          fetch.once("end", () => {
+          fetch.once("end", async () => {
             console.log(`[EmailImport] Finished fetching emails for user ${config.userId}`);
 
-            // Mark successfully processed emails as seen
+            // Le flux IMAP peut se terminer avant les traitements IA asynchrones.
-            if (processedEmails.length > 0) {
+            // On attend explicitement chaque message avant de le marquer comme lu.
-              imap.addFlags(processedEmails, ["\\Seen"], (err) => {
+            await Promise.allSettled(messageTasks);
+
+            if (processedEmailUids.length > 0) {
+              imap.addFlags(processedEmailUids, ["\\Seen"], (err) => {
                 if (err) {
                   console.error("[EmailImport] Error marking emails as seen:", err);
                 } else {
-                  console.log(`[EmailImport] Marked ${processedEmails.length} emails as seen`);
+                  console.log(`[EmailImport] Marked ${processedEmailUids.length} emails as seen`);
                 }
                 imap.end();
                 resolve();
@@ -598,13 +652,13 @@ export async function startEmailImportService(userId: number): Promise<boolean>
     );
 
     // Run immediately on start
-    checkEmailsForPDFs(config).catch((error) => {
+    runEmailCheckExclusive(config).catch((error) => {
       console.error(`[EmailImport] Error checking emails for user ${userId}:`, error);
     });
 
     // Set up interval for periodic checks
     const interval = setInterval(() => {
-      checkEmailsForPDFs(config).catch((error) => {
+      runEmailCheckExclusive(config).catch((error) => {
         console.error(`[EmailImport] Error checking emails for user ${userId}:`, error);
       });
     }, frequencyMs);
@@ -670,7 +724,7 @@ export async function triggerEmailCheck(userId: number): Promise<{ success: bool
     };
 
     console.log(`[EmailImport] Manual check triggered for user ${userId}`);
-    await checkEmailsForPDFs(config);
+    await runEmailCheckExclusive(config);
     
     return { success: true, message: "Vérification terminée avec succès" };
   } catch (error: any) {

server/fileFingerprint.test.ts

@@ -0,0 +1,19 @@
+import { describe, expect, it } from "vitest";
+import { calculateFileSha256 } from "./fileFingerprint";
+
+describe("calculateFileSha256", () => {
+  it("retourne la même empreinte pour un contenu identique", () => {
+    const content = Buffer.from("facture-pdf");
+    expect(calculateFileSha256(content)).toBe(calculateFileSha256(Buffer.from(content)));
+  });
+
+  it("distingue deux contenus différents", () => {
+    expect(calculateFileSha256(Buffer.from("facture-a"))).not.toBe(
+      calculateFileSha256(Buffer.from("facture-b")),
+    );
+  });
+
+  it("produit une empreinte SHA-256 hexadécimale", () => {
+    expect(calculateFileSha256(Buffer.from("facture"))).toMatch(/^[a-f0-9]{64}$/);
+  });
+});

server/fileFingerprint.ts

@@ -0,0 +1,12 @@
+import { createHash } from "node:crypto";
+
+/**
+ * Calcule une empreinte déterministe sur les octets du document original.
+ *
+ * L'empreinte est calculée avant tout stockage ou traitement IA : deux imports
+ * du même PDF sont donc reconnus même si le nom du fichier ou l'utilisateur
+ * diffèrent.
+ */
+export function calculateFileSha256(buffer: Buffer): string {
+  return createHash("sha256").update(buffer).digest("hex");
+}

server/folderImportService.ts

@@ -4,6 +4,7 @@ import path from "path";
 import {
   getImportSettingsByUser,
   createSourceFile,
+  getSourceFileByContentHash,
   updateSourceFile,
   getUserSettings,
   findDuplicateInvoice,
@@ -11,7 +12,8 @@ import {
   createImportLog,
 } from "./db";
 import { extractInvoicesWithMistral, generateMetadataJSON } from "./invoiceExtractor";
-import { localStoragePut, generateStorageKey } from "./localStorage";
+import { localStorageDelete, localStoragePut, generateStorageKey } from "./localStorage";
+import { calculateFileSha256 } from "./fileFingerprint";
 import { sendImportNotification } from "./notificationService";
 
 interface FolderImportConfig {
@@ -42,6 +44,13 @@ async function processFolderFile(
     const fileBuffer = await fs.readFile(filePath);
     console.log(`[FolderImport] File size: ${fileBuffer.length} bytes`);
 
+    const contentHash = calculateFileSha256(fileBuffer);
+    const existingSource = await getSourceFileByContentHash(contentHash);
+    if (existingSource) {
+      console.log(`[FolderImport] PDF déjà importé, fichier ignoré: ${fileName}`);
+      return { success: true, imported: 0, duplicates: 1, errors: 0 };
+    }
+    
     // Store source file
     const sourceFileKey = generateStorageKey(userId, fileName);
     console.log(`[FolderImport] Generated storage key: ${sourceFileKey}`);
@@ -57,13 +66,23 @@ async function processFolderFile(
     }
     
     // Create source file record
-    const sourceFile = await createSourceFile({
+    let sourceFile;
+    try {
+      sourceFile = await createSourceFile({
         userId,
         fileName,
         fileKey: sourceFileKey,
         fileUrl: sourceFileUrl,
+        contentHash,
         processingStatus: "processing",
       });
+    } catch (error: any) {
+      if (error?.code === "ER_DUP_ENTRY" || error?.errno === 1062) {
+        await localStorageDelete(sourceFileKey).catch(() => undefined);
+        return { success: true, imported: 0, duplicates: 1, errors: 0 };
+      }
+      throw error;
+    }
     
     console.log(`[FolderImport] Source file record created with ID: ${sourceFile.id}`);
     

server/routers.ts

@@ -24,6 +24,7 @@ import {
   searchInvoices,
   getInvoiceStats,
   createSourceFile,
+  getSourceFileByContentHash,
   getSourceFileById,
   updateSourceFile,
   getUserSettings,
@@ -87,6 +88,7 @@ import { loginLocal, hashPassword, getAzureAuthUrl, isAzureAdConfigured } from "
 import fsSync from "fs";
 import pathSync from "path";
 import { extractInvoicesWithMistral, generateMetadataJSON } from "./invoiceExtractor";
+import { calculateFileSha256 } from "./fileFingerprint";
 import { localStoragePut, generateStorageKey } from "./localStorage";
 import { testSftpConnection, exportInvoiceToSftp, getUserSftpConfig } from "./sftpExport";
 import { drawBapCartouche } from "./bapCartouche";
@@ -182,6 +184,17 @@ export const appRouter = router({
         const fileBuffer = Buffer.from(input.fileData, "base64");
         console.log(`[Upload] Received file: ${input.fileName}, size: ${fileBuffer.length} bytes`);
 
+        // Le contrôle sur les octets du PDF intervient avant le stockage et l'appel IA.
+        // Il reste fiable même si le nom du fichier ou le compte utilisateur diffère.
+        const contentHash = calculateFileSha256(fileBuffer);
+        const existingSource = await getSourceFileByContentHash(contentHash);
+        if (existingSource) {
+          throw new TRPCError({
+            code: "CONFLICT",
+            message: `Ce PDF a déjà été importé (${existingSource.fileName}).`,
+          });
+        }
+        
         // Store source file
         const sourceFileKey = generateStorageKey(userId, input.fileName);
         console.log(`[Upload] Generated storage key: ${sourceFileKey}`);
@@ -202,6 +215,7 @@ export const appRouter = router({
           fileName: input.fileName,
           fileKey: sourceFileKey,
           fileUrl: sourceFileUrl,
+          contentHash,
           processingStatus: "processing",
         });
         

todo.md

@@ -702,3 +702,18 @@
 - [x] Documenter les modules métier, les invariants et les décisions techniques critiques
 - [x] Ajouter des tests de non-régression ciblés et vérifier build, types et tests
 - [x] Normaliser les valeurs OAuth de loginMethod avant écriture en base
+
+## Incident production — erreur HTTP 404
+- [x] Reproduire la 404 et contrôler le domaine, Traefik et les conteneurs
+- [x] Identifier et corriger la cause racine sans modifier les données
+- [x] Vérifier le retour HTTP 200 et la santé des conteneurs
+
+## Audit production — 674 factures affichées
+- [x] Compter les factures par utilisateur, source et statut
+- [x] Identifier les groupes de doublons selon plusieurs clés métier
+- [x] Vérifier les références de stockage et les effets de la fusion précédente
+- [x] Préparer une correction réversible sans suppression immédiate
+- [x] Sauvegarder la base et le volume puis suspendre les imports email
+- [x] Bloquer les réimports par empreinte PDF et fiabiliser le traitement IMAP
+- [ ] Déployer le correctif anti-réimport et migrer la base de production
+- [ ] Appliquer la correction confirmée et vérifier le comptage final