Checkpoint: Correction robuste des doublons de factures : empreinte SHA-256 unique des PDF sources, détection globale avant import manuel/email/dossier/web, verrou contre les vérifications IMAP concurrentes, attente réelle de fin de traitement avant marquage lu, migration et tests de non-régression.

2026-08-22 10:31:54 +00:00
parent d8b2a8fe6f
commit d729a94a96
12 changed files with 2637 additions and 82 deletions
--- a/server/_core/index.ts
+++ b/server/_core/index.ts
@@ -17,7 +17,8 @@ import { startEmailImportService } from "../emailImportService";
 import { startFolderImportService } from "../folderImportService";
 import { handleAzureCallback, isAzureAdConfigured, generateToken, verifyToken } from "../auth";
 import { createDatabaseBackup } from "../databaseBackup";
-import { generateStorageKey, localStoragePut } from "../localStorage";
+import { generateStorageKey, localStorageDelete, localStoragePut } from "../localStorage";
+import { calculateFileSha256 } from "../fileFingerprint";

 const MAX_WEB_IMPORT_BYTES = 20 * 1024 * 1024;

@@ -305,7 +306,7 @@ async function startServer() {
        return;
      }

-      const { getWebImportSourceByToken, createInvoice, findDuplicateInvoice, isInvoiceBlacklisted, updateWebImportSourceStatus, createSourceFile } = await import('../db');
+      const { getWebImportSourceByToken, createInvoice, findDuplicateInvoice, isInvoiceBlacklisted, updateWebImportSourceStatus, createSourceFile, getSourceFileByContentHash } = await import('../db');
      const source = await getWebImportSourceByToken(apiToken);
      if (!source) {
        res.status(401).json({ error: "Token invalide" });
@@ -317,15 +318,35 @@ async function startServer() {
        return;
      }

+      const contentHash = calculateFileSha256(pdfBuffer);
+      const existingSource = await getSourceFileByContentHash(contentHash);
+      if (existingSource) {
+        await updateWebImportSourceStatus(source.id, "success", 0, true);
+        res.json({ success: true, imported: 0, duplicates: 1, total: 1 });
+        return;
+      }
+
      // Stocker d'abord le PDF de façon persistante, comme les autres sources d'import.
      const storageKey = generateStorageKey(source.userId, safeFileName);
      const { url: fileUrl } = await localStoragePut(storageKey, pdfBuffer, "application/pdf");
-      const sourceFile = await createSourceFile({
-        userId: source.userId,
-        fileName: safeFileName,
-        fileKey: storageKey,
-        fileUrl,
-      });
+      let sourceFile;
+      try {
+        sourceFile = await createSourceFile({
+          userId: source.userId,
+          fileName: safeFileName,
+          fileKey: storageKey,
+          fileUrl,
+          contentHash,
+        });
+      } catch (error: any) {
+        if (error?.code === "ER_DUP_ENTRY" || error?.errno === 1062) {
+          await localStorageDelete(storageKey).catch(() => undefined);
+          await updateWebImportSourceStatus(source.id, "success", 0, true);
+          res.json({ success: true, imported: 0, duplicates: 1, total: 1 });
+          return;
+        }
+        throw error;
+      }
      const userSettings = await getUserSettings(source.userId);
      const aiSettings = {
        aiProvider: userSettings?.aiProvider || "manus",
--- a/server/db.ts
+++ b/server/db.ts
@@ -204,6 +204,22 @@ export async function createSourceFile(data: InsertSourceFile): Promise<SourceFi
  return inserted[0]!;
 }

+/**
+ * Recherche un PDF déjà ingéré, quel que soit le compte utilisateur.
+ * L'import email est partagé entre plusieurs identités : la détection doit donc
+ * être globale pour éviter qu'une même pièce soit retraitée sous chaque compte.
+ */
+export async function getSourceFileByContentHash(contentHash: string): Promise<SourceFile | undefined> {
+  const db = await getDb();
+  if (!db) return undefined;
+  const result = await db
+    .select()
+    .from(sourceFiles)
+    .where(eq(sourceFiles.contentHash, contentHash))
+    .limit(1);
+  return result[0];
+}
+
 export async function getSourceFileById(id: number): Promise<SourceFile | undefined> {
  const db = await getDb();
  if (!db) return undefined;
--- a/server/emailImportService.ts
+++ b/server/emailImportService.ts
@@ -5,13 +5,15 @@ import {
  createSourceFile,
  updateSourceFile,
  getUserSettings,
+  getSourceFileByContentHash,
  findDuplicateInvoice,
  isInvoiceBlacklisted,
  createInvoice,
  createImportLog,
 } from "./db";
 import { extractInvoicesWithMistral, generateMetadataJSON } from "./invoiceExtractor";
-import { localStoragePut, generateStorageKey } from "./localStorage";
+import { localStorageDelete, localStoragePut, generateStorageKey } from "./localStorage";
+import { calculateFileSha256 } from "./fileFingerprint";
 import { sendImportNotification } from "./notificationService";
 import { getOffice365ImapToken, buildXOAuth2String } from "./office365OAuth";
 import { applyAutomationRules } from "./automationEngine";
@@ -32,6 +34,23 @@ interface EmailImportConfig {

 // Store active intervals for each user
 const activeIntervals = new Map<number, NodeJS.Timeout>();
+// Une extraction IA peut dépasser la fréquence configurée : ce verrou évite
+// qu'un second cycle IMAP traite les mêmes messages avant la fin du premier.
+const activeChecks = new Map<number, Promise<void>>();
+
+function runEmailCheckExclusive(config: EmailImportConfig): Promise<void> {
+  const runningCheck = activeChecks.get(config.userId);
+  if (runningCheck) {
+    console.log(`[EmailImport] Vérification déjà en cours pour user ${config.userId}, cycle ignoré`);
+    return runningCheck;
+  }
+
+  const check = checkEmailsForPDFs(config).finally(() => {
+    if (activeChecks.get(config.userId) === check) activeChecks.delete(config.userId);
+  });
+  activeChecks.set(config.userId, check);
+  return check;
+}

 /**
 * Process a single email attachment (PDF)
@@ -49,6 +68,21 @@ async function processEmailAttachment(
    // Convert attachment content to Buffer
    const fileBuffer = attachment.content;
    console.log(`[EmailImport] File size: ${fileBuffer.length} bytes`);
+
+    const contentHash = calculateFileSha256(fileBuffer);
+    const existingSource = await getSourceFileByContentHash(contentHash);
+    if (existingSource) {
+      console.log(
+        `[EmailImport] PDF déjà importé, extraction ignorée: ${fileName} -> source ${existingSource.id}`,
+      );
+      return {
+        success: true,
+        totalInvoices: Math.max(existingSource.totalInvoicesDetected, 1),
+        imported: 0,
+        duplicates: Math.max(existingSource.totalInvoicesDetected, 1),
+        errors: 0,
+      };
+    }
    
    // Store source file
    const sourceFileKey = generateStorageKey(userId, fileName);
@@ -65,13 +99,25 @@ async function processEmailAttachment(
    }
    
    // Create source file record
-    const sourceFile = await createSourceFile({
-      userId,
-      fileName,
-      fileKey: sourceFileKey,
-      fileUrl: sourceFileUrl,
-      processingStatus: "processing",
-    });
+    let sourceFile;
+    try {
+      sourceFile = await createSourceFile({
+        userId,
+        fileName,
+        fileKey: sourceFileKey,
+        fileUrl: sourceFileUrl,
+        contentHash,
+        processingStatus: "processing",
+      });
+    } catch (error: any) {
+      // La contrainte unique protège également contre deux imports concurrents.
+      if (error?.code === "ER_DUP_ENTRY" || error?.errno === 1062) {
+        await localStorageDelete(sourceFileKey).catch(() => undefined);
+        console.log(`[EmailImport] PDF réservé par un autre traitement: ${fileName}`);
+        return { success: true, totalInvoices: 1, imported: 0, duplicates: 1, errors: 0 };
+      }
+      throw error;
+    }
    
    console.log(`[EmailImport] Source file record created with ID: ${sourceFile.id}`);
    
@@ -383,64 +429,69 @@ async function checkEmailsForPDFs(config: EmailImportConfig): Promise<void> {
            markSeen: false, // Don't mark as seen yet
          });

-          const processedEmails: number[] = [];
+          const processedEmailUids: number[] = [];
+          const messageTasks: Promise<void>[] = [];

          fetch.on("message", (msg, seqno) => {
+            const uidPromise = new Promise<number>((resolveUid) => {
+              msg.once("attributes", (attributes) => resolveUid(attributes.uid));
+            });
+
            msg.on("body", (stream) => {
-              simpleParser(stream as any, async (err, parsed: ParsedMail) => {
-                if (err) {
-                  console.error("[EmailImport] Error parsing email:", err);
-                  return;
-                }
+              const task = (async () => {
+                try {
+                  const parsed: ParsedMail = await simpleParser(stream as any);
+                  const pdfAttachments = parsed.attachments.filter(
+                    (attachment) =>
+                      attachment.contentType === "application/pdf" ||
+                      attachment.filename?.toLowerCase().endsWith(".pdf"),
+                  );

-                // Check if email has PDF attachments
-                const pdfAttachments = parsed.attachments.filter(
-                  (att) =>
-                    att.contentType === "application/pdf" ||
-                    att.filename?.toLowerCase().endsWith(".pdf")
-                );
+                  if (pdfAttachments.length === 0) return;

-                if (pdfAttachments.length === 0) {
-                  return;
-                }
+                  console.log(
+                    `[EmailImport] Email ${seqno} has ${pdfAttachments.length} PDF attachment(s)`,
+                  );

-                console.log(
-                  `[EmailImport] Email ${seqno} has ${pdfAttachments.length} PDF attachment(s)`
-                );
+                  let allAttachmentsSucceeded = true;
+                  for (const attachment of pdfAttachments) {
+                    try {
+                      const result = await processEmailAttachment(
+                        config.userId,
+                        attachment,
+                        parsed.subject || "No subject",
+                      );
+                      allAttachmentsSucceeded = allAttachmentsSucceeded && result.success;

-                // Process each PDF attachment
-                for (const attachment of pdfAttachments) {
-                  try {
-                    const result = await processEmailAttachment(
-                      config.userId,
-                      attachment,
-                      parsed.subject || "No subject"
-                    );
-                    
-                    // Mark this email as successfully processed
-                    if (!processedEmails.includes(seqno)) {
-                      processedEmails.push(seqno);
+                      if (result.success) {
+                        await sendImportNotification(config.userId, {
+                          source: "email",
+                          fileName: attachment.filename || "email-attachment.pdf",
+                          totalInvoices: result.totalInvoices,
+                          imported: result.imported,
+                          duplicates: result.duplicates,
+                          errors: result.errors,
+                        });
+                      }
+                    } catch (error) {
+                      allAttachmentsSucceeded = false;
+                      console.error(
+                        `[EmailImport] Failed to process attachment from email ${seqno}:`,
+                        error,
+                      );
                    }
-                    
-                    // Send notification after successful processing
-                    if (result.success) {
-                      await sendImportNotification(config.userId, {
-                        source: "email",
-                        fileName: attachment.filename || "email-attachment.pdf",
-                        totalInvoices: result.totalInvoices,
-                        imported: result.imported,
-                        duplicates: result.duplicates,
-                        errors: result.errors,
-                      });
-                    }
-                  } catch (error) {
-                    console.error(
-                      `[EmailImport] Failed to process attachment from email ${seqno}:`,
-                      error
-                    );
                  }
+
+                  if (allAttachmentsSucceeded) {
+                    const uid = await uidPromise;
+                    if (!processedEmailUids.includes(uid)) processedEmailUids.push(uid);
+                  }
+                } catch (error) {
+                  console.error(`[EmailImport] Error parsing email ${seqno}:`, error);
                }
-              });
+              })();
+
+              messageTasks.push(task);
            });
          });

@@ -450,16 +501,19 @@ async function checkEmailsForPDFs(config: EmailImportConfig): Promise<void> {
            reject(err);
          });

-          fetch.once("end", () => {
+          fetch.once("end", async () => {
            console.log(`[EmailImport] Finished fetching emails for user ${config.userId}`);
-            
-            // Mark successfully processed emails as seen
-            if (processedEmails.length > 0) {
-              imap.addFlags(processedEmails, ["\\Seen"], (err) => {
+
+            // Le flux IMAP peut se terminer avant les traitements IA asynchrones.
+            // On attend explicitement chaque message avant de le marquer comme lu.
+            await Promise.allSettled(messageTasks);
+
+            if (processedEmailUids.length > 0) {
+              imap.addFlags(processedEmailUids, ["\\Seen"], (err) => {
                if (err) {
                  console.error("[EmailImport] Error marking emails as seen:", err);
                } else {
-                  console.log(`[EmailImport] Marked ${processedEmails.length} emails as seen`);
+                  console.log(`[EmailImport] Marked ${processedEmailUids.length} emails as seen`);
                }
                imap.end();
                resolve();
@@ -598,13 +652,13 @@ export async function startEmailImportService(userId: number): Promise<boolean>
    );

    // Run immediately on start
-    checkEmailsForPDFs(config).catch((error) => {
+    runEmailCheckExclusive(config).catch((error) => {
      console.error(`[EmailImport] Error checking emails for user ${userId}:`, error);
    });

    // Set up interval for periodic checks
    const interval = setInterval(() => {
-      checkEmailsForPDFs(config).catch((error) => {
+      runEmailCheckExclusive(config).catch((error) => {
        console.error(`[EmailImport] Error checking emails for user ${userId}:`, error);
      });
    }, frequencyMs);
@@ -670,7 +724,7 @@ export async function triggerEmailCheck(userId: number): Promise<{ success: bool
    };

    console.log(`[EmailImport] Manual check triggered for user ${userId}`);
-    await checkEmailsForPDFs(config);
+    await runEmailCheckExclusive(config);
    
    return { success: true, message: "Vérification terminée avec succès" };
  } catch (error: any) {
--- a/server/fileFingerprint.test.ts
+++ b/server/fileFingerprint.test.ts
@@ -0,0 +1,19 @@
+import { describe, expect, it } from "vitest";
+import { calculateFileSha256 } from "./fileFingerprint";
+
+describe("calculateFileSha256", () => {
+  it("retourne la même empreinte pour un contenu identique", () => {
+    const content = Buffer.from("facture-pdf");
+    expect(calculateFileSha256(content)).toBe(calculateFileSha256(Buffer.from(content)));
+  });
+
+  it("distingue deux contenus différents", () => {
+    expect(calculateFileSha256(Buffer.from("facture-a"))).not.toBe(
+      calculateFileSha256(Buffer.from("facture-b")),
+    );
+  });
+
+  it("produit une empreinte SHA-256 hexadécimale", () => {
+    expect(calculateFileSha256(Buffer.from("facture"))).toMatch(/^[a-f0-9]{64}$/);
+  });
+});
--- a/server/fileFingerprint.ts
+++ b/server/fileFingerprint.ts
@@ -0,0 +1,12 @@
+import { createHash } from "node:crypto";
+
+/**
+ * Calcule une empreinte déterministe sur les octets du document original.
+ *
+ * L'empreinte est calculée avant tout stockage ou traitement IA : deux imports
+ * du même PDF sont donc reconnus même si le nom du fichier ou l'utilisateur
+ * diffèrent.
+ */
+export function calculateFileSha256(buffer: Buffer): string {
+  return createHash("sha256").update(buffer).digest("hex");
+}
--- a/server/folderImportService.ts
+++ b/server/folderImportService.ts
@@ -4,6 +4,7 @@ import path from "path";
 import {
  getImportSettingsByUser,
  createSourceFile,
+  getSourceFileByContentHash,
  updateSourceFile,
  getUserSettings,
  findDuplicateInvoice,
@@ -11,7 +12,8 @@ import {
  createImportLog,
 } from "./db";
 import { extractInvoicesWithMistral, generateMetadataJSON } from "./invoiceExtractor";
-import { localStoragePut, generateStorageKey } from "./localStorage";
+import { localStorageDelete, localStoragePut, generateStorageKey } from "./localStorage";
+import { calculateFileSha256 } from "./fileFingerprint";
 import { sendImportNotification } from "./notificationService";

 interface FolderImportConfig {
@@ -41,6 +43,13 @@ async function processFolderFile(
    // Read the file
    const fileBuffer = await fs.readFile(filePath);
    console.log(`[FolderImport] File size: ${fileBuffer.length} bytes`);
+
+    const contentHash = calculateFileSha256(fileBuffer);
+    const existingSource = await getSourceFileByContentHash(contentHash);
+    if (existingSource) {
+      console.log(`[FolderImport] PDF déjà importé, fichier ignoré: ${fileName}`);
+      return { success: true, imported: 0, duplicates: 1, errors: 0 };
+    }
    
    // Store source file
    const sourceFileKey = generateStorageKey(userId, fileName);
@@ -57,13 +66,23 @@ async function processFolderFile(
    }
    
    // Create source file record
-    const sourceFile = await createSourceFile({
-      userId,
-      fileName,
-      fileKey: sourceFileKey,
-      fileUrl: sourceFileUrl,
-      processingStatus: "processing",
-    });
+    let sourceFile;
+    try {
+      sourceFile = await createSourceFile({
+        userId,
+        fileName,
+        fileKey: sourceFileKey,
+        fileUrl: sourceFileUrl,
+        contentHash,
+        processingStatus: "processing",
+      });
+    } catch (error: any) {
+      if (error?.code === "ER_DUP_ENTRY" || error?.errno === 1062) {
+        await localStorageDelete(sourceFileKey).catch(() => undefined);
+        return { success: true, imported: 0, duplicates: 1, errors: 0 };
+      }
+      throw error;
+    }
    
    console.log(`[FolderImport] Source file record created with ID: ${sourceFile.id}`);
    
--- a/server/routers.ts
+++ b/server/routers.ts
@@ -24,6 +24,7 @@ import {
  searchInvoices,
  getInvoiceStats,
  createSourceFile,
+  getSourceFileByContentHash,
  getSourceFileById,
  updateSourceFile,
  getUserSettings,
@@ -87,6 +88,7 @@ import { loginLocal, hashPassword, getAzureAuthUrl, isAzureAdConfigured } from "
 import fsSync from "fs";
 import pathSync from "path";
 import { extractInvoicesWithMistral, generateMetadataJSON } from "./invoiceExtractor";
+import { calculateFileSha256 } from "./fileFingerprint";
 import { localStoragePut, generateStorageKey } from "./localStorage";
 import { testSftpConnection, exportInvoiceToSftp, getUserSftpConfig } from "./sftpExport";
 import { drawBapCartouche } from "./bapCartouche";
@@ -181,6 +183,17 @@ export const appRouter = router({
        // Decode base64 file data
        const fileBuffer = Buffer.from(input.fileData, "base64");
        console.log(`[Upload] Received file: ${input.fileName}, size: ${fileBuffer.length} bytes`);
+
+        // Le contrôle sur les octets du PDF intervient avant le stockage et l'appel IA.
+        // Il reste fiable même si le nom du fichier ou le compte utilisateur diffère.
+        const contentHash = calculateFileSha256(fileBuffer);
+        const existingSource = await getSourceFileByContentHash(contentHash);
+        if (existingSource) {
+          throw new TRPCError({
+            code: "CONFLICT",
+            message: `Ce PDF a déjà été importé (${existingSource.fileName}).`,
+          });
+        }
        
        // Store source file
        const sourceFileKey = generateStorageKey(userId, input.fileName);
@@ -202,6 +215,7 @@ export const appRouter = router({
          fileName: input.fileName,
          fileKey: sourceFileKey,
          fileUrl: sourceFileUrl,
+          contentHash,
          processingStatus: "processing",
        });