Checkpoint: Correction robuste des doublons de factures : empreinte SHA-256 unique des PDF sources, détection globale avant import manuel/email/dossier/web, verrou contre les vérifications IMAP concurrentes, attente réelle de fin de traitement avant marquage lu, migration et tests de non-régression.

server/emailImportService.ts

@@ -5,13 +5,15 @@ import {
   createSourceFile,
   updateSourceFile,
   getUserSettings,
+  getSourceFileByContentHash,
   findDuplicateInvoice,
   isInvoiceBlacklisted,
   createInvoice,
   createImportLog,
 } from "./db";
 import { extractInvoicesWithMistral, generateMetadataJSON } from "./invoiceExtractor";
-import { localStoragePut, generateStorageKey } from "./localStorage";
+import { localStorageDelete, localStoragePut, generateStorageKey } from "./localStorage";
+import { calculateFileSha256 } from "./fileFingerprint";
 import { sendImportNotification } from "./notificationService";
 import { getOffice365ImapToken, buildXOAuth2String } from "./office365OAuth";
 import { applyAutomationRules } from "./automationEngine";
@@ -32,6 +34,23 @@ interface EmailImportConfig {
 
 // Store active intervals for each user
 const activeIntervals = new Map<number, NodeJS.Timeout>();
+// Une extraction IA peut dépasser la fréquence configurée : ce verrou évite
+// qu'un second cycle IMAP traite les mêmes messages avant la fin du premier.
+const activeChecks = new Map<number, Promise<void>>();
+
+function runEmailCheckExclusive(config: EmailImportConfig): Promise<void> {
+  const runningCheck = activeChecks.get(config.userId);
+  if (runningCheck) {
+    console.log(`[EmailImport] Vérification déjà en cours pour user ${config.userId}, cycle ignoré`);
+    return runningCheck;
+  }
+
+  const check = checkEmailsForPDFs(config).finally(() => {
+    if (activeChecks.get(config.userId) === check) activeChecks.delete(config.userId);
+  });
+  activeChecks.set(config.userId, check);
+  return check;
+}
 
 /**
  * Process a single email attachment (PDF)
@@ -49,6 +68,21 @@ async function processEmailAttachment(
     // Convert attachment content to Buffer
     const fileBuffer = attachment.content;
     console.log(`[EmailImport] File size: ${fileBuffer.length} bytes`);
+
+    const contentHash = calculateFileSha256(fileBuffer);
+    const existingSource = await getSourceFileByContentHash(contentHash);
+    if (existingSource) {
+      console.log(
+        `[EmailImport] PDF déjà importé, extraction ignorée: ${fileName} -> source ${existingSource.id}`,
+      );
+      return {
+        success: true,
+        totalInvoices: Math.max(existingSource.totalInvoicesDetected, 1),
+        imported: 0,
+        duplicates: Math.max(existingSource.totalInvoicesDetected, 1),
+        errors: 0,
+      };
+    }
     
     // Store source file
     const sourceFileKey = generateStorageKey(userId, fileName);
@@ -65,13 +99,25 @@ async function processEmailAttachment(
     }
     
     // Create source file record
-    const sourceFile = await createSourceFile({
-      userId,
-      fileName,
-      fileKey: sourceFileKey,
-      fileUrl: sourceFileUrl,
-      processingStatus: "processing",
-    });
+    let sourceFile;
+    try {
+      sourceFile = await createSourceFile({
+        userId,
+        fileName,
+        fileKey: sourceFileKey,
+        fileUrl: sourceFileUrl,
+        contentHash,
+        processingStatus: "processing",
+      });
+    } catch (error: any) {
+      // La contrainte unique protège également contre deux imports concurrents.
+      if (error?.code === "ER_DUP_ENTRY" || error?.errno === 1062) {
+        await localStorageDelete(sourceFileKey).catch(() => undefined);
+        console.log(`[EmailImport] PDF réservé par un autre traitement: ${fileName}`);
+        return { success: true, totalInvoices: 1, imported: 0, duplicates: 1, errors: 0 };
+      }
+      throw error;
+    }
     
     console.log(`[EmailImport] Source file record created with ID: ${sourceFile.id}`);
     
@@ -383,64 +429,69 @@ async function checkEmailsForPDFs(config: EmailImportConfig): Promise<void> {
             markSeen: false, // Don't mark as seen yet
           });
 
-          const processedEmails: number[] = [];
+          const processedEmailUids: number[] = [];
+          const messageTasks: Promise<void>[] = [];
 
           fetch.on("message", (msg, seqno) => {
+            const uidPromise = new Promise<number>((resolveUid) => {
+              msg.once("attributes", (attributes) => resolveUid(attributes.uid));
+            });
+
             msg.on("body", (stream) => {
-              simpleParser(stream as any, async (err, parsed: ParsedMail) => {
-                if (err) {
-                  console.error("[EmailImport] Error parsing email:", err);
-                  return;
-                }
+              const task = (async () => {
+                try {
+                  const parsed: ParsedMail = await simpleParser(stream as any);
+                  const pdfAttachments = parsed.attachments.filter(
+                    (attachment) =>
+                      attachment.contentType === "application/pdf" ||
+                      attachment.filename?.toLowerCase().endsWith(".pdf"),
+                  );
 
-                // Check if email has PDF attachments
-                const pdfAttachments = parsed.attachments.filter(
-                  (att) =>
-                    att.contentType === "application/pdf" ||
-                    att.filename?.toLowerCase().endsWith(".pdf")
-                );
+                  if (pdfAttachments.length === 0) return;
 
-                if (pdfAttachments.length === 0) {
-                  return;
-                }
+                  console.log(
+                    `[EmailImport] Email ${seqno} has ${pdfAttachments.length} PDF attachment(s)`,
+                  );
 
-                console.log(
-                  `[EmailImport] Email ${seqno} has ${pdfAttachments.length} PDF attachment(s)`
-                );
+                  let allAttachmentsSucceeded = true;
+                  for (const attachment of pdfAttachments) {
+                    try {
+                      const result = await processEmailAttachment(
+                        config.userId,
+                        attachment,
+                        parsed.subject || "No subject",
+                      );
+                      allAttachmentsSucceeded = allAttachmentsSucceeded && result.success;
 
-                // Process each PDF attachment
-                for (const attachment of pdfAttachments) {
-                  try {
-                    const result = await processEmailAttachment(
-                      config.userId,
-                      attachment,
-                      parsed.subject || "No subject"
-                    );
-                    
-                    // Mark this email as successfully processed
-                    if (!processedEmails.includes(seqno)) {
-                      processedEmails.push(seqno);
+                      if (result.success) {
+                        await sendImportNotification(config.userId, {
+                          source: "email",
+                          fileName: attachment.filename || "email-attachment.pdf",
+                          totalInvoices: result.totalInvoices,
+                          imported: result.imported,
+                          duplicates: result.duplicates,
+                          errors: result.errors,
+                        });
+                      }
+                    } catch (error) {
+                      allAttachmentsSucceeded = false;
+                      console.error(
+                        `[EmailImport] Failed to process attachment from email ${seqno}:`,
+                        error,
+                      );
                     }
-                    
-                    // Send notification after successful processing
-                    if (result.success) {
-                      await sendImportNotification(config.userId, {
-                        source: "email",
-                        fileName: attachment.filename || "email-attachment.pdf",
-                        totalInvoices: result.totalInvoices,
-                        imported: result.imported,
-                        duplicates: result.duplicates,
-                        errors: result.errors,
-                      });
-                    }
-                  } catch (error) {
-                    console.error(
-                      `[EmailImport] Failed to process attachment from email ${seqno}:`,
-                      error
-                    );
                   }
+
+                  if (allAttachmentsSucceeded) {
+                    const uid = await uidPromise;
+                    if (!processedEmailUids.includes(uid)) processedEmailUids.push(uid);
+                  }
+                } catch (error) {
+                  console.error(`[EmailImport] Error parsing email ${seqno}:`, error);
                 }
-              });
+              })();
+
+              messageTasks.push(task);
             });
           });
 
@@ -450,16 +501,19 @@ async function checkEmailsForPDFs(config: EmailImportConfig): Promise<void> {
             reject(err);
           });
 
-          fetch.once("end", () => {
+          fetch.once("end", async () => {
             console.log(`[EmailImport] Finished fetching emails for user ${config.userId}`);
-            
-            // Mark successfully processed emails as seen
-            if (processedEmails.length > 0) {
-              imap.addFlags(processedEmails, ["\\Seen"], (err) => {
+
+            // Le flux IMAP peut se terminer avant les traitements IA asynchrones.
+            // On attend explicitement chaque message avant de le marquer comme lu.
+            await Promise.allSettled(messageTasks);
+
+            if (processedEmailUids.length > 0) {
+              imap.addFlags(processedEmailUids, ["\\Seen"], (err) => {
                 if (err) {
                   console.error("[EmailImport] Error marking emails as seen:", err);
                 } else {
-                  console.log(`[EmailImport] Marked ${processedEmails.length} emails as seen`);
+                  console.log(`[EmailImport] Marked ${processedEmailUids.length} emails as seen`);
                 }
                 imap.end();
                 resolve();
@@ -598,13 +652,13 @@ export async function startEmailImportService(userId: number): Promise<boolean>
     );
 
     // Run immediately on start
-    checkEmailsForPDFs(config).catch((error) => {
+    runEmailCheckExclusive(config).catch((error) => {
       console.error(`[EmailImport] Error checking emails for user ${userId}:`, error);
     });
 
     // Set up interval for periodic checks
     const interval = setInterval(() => {
-      checkEmailsForPDFs(config).catch((error) => {
+      runEmailCheckExclusive(config).catch((error) => {
         console.error(`[EmailImport] Error checking emails for user ${userId}:`, error);
       });
     }, frequencyMs);
@@ -670,7 +724,7 @@ export async function triggerEmailCheck(userId: number): Promise<{ success: bool
     };
 
     console.log(`[EmailImport] Manual check triggered for user ${userId}`);
-    await checkEmailsForPDFs(config);
+    await runEmailCheckExclusive(config);
     
     return { success: true, message: "Vérification terminée avec succès" };
   } catch (error: any) {