fix: empêcher la duplication massive de fichiers lors de l'import email

- Marquer les emails comme lus immédiatement (markSeen: true)
- Vérifier si le fichier existe déjà avant de le stocker (findSourceFileByFileName)
- Ajouter un verrou anti-concurrence par utilisateur (runningChecks)
- Empêche la création de ~570x doublons par facture
This commit is contained in:
Manus Admin
2026-08-18 14:50:31 +02:00
parent 8759d85f3d
commit de76a761a3
2 changed files with 33 additions and 1 deletions

View File

@@ -1218,3 +1218,18 @@ export async function updateWebImportSourceStatus(
if (success) update.lastSuccessAt = new Date(); if (success) update.lastSuccessAt = new Date();
await db.update(webImportSources).set(update).where(eq(webImportSources.id, id)); await db.update(webImportSources).set(update).where(eq(webImportSources.id, id));
} }
/**
* Check if a source file with the same fileName already exists for this user
* Used to prevent duplicate file storage during email import
*/
export async function findSourceFileByFileName(userId: number, fileName: string): Promise<any | null> {
const result = await db.select()
.from(sourceFiles)
.where(and(
eq(sourceFiles.userId, userId),
eq(sourceFiles.fileName, fileName)
))
.limit(1);
return result[0] || null;
}

View File

@@ -9,6 +9,7 @@ import {
isInvoiceBlacklisted, isInvoiceBlacklisted,
createInvoice, createInvoice,
createImportLog, createImportLog,
findSourceFileByFileName,
} from "./db"; } from "./db";
import { extractInvoicesWithMistral, generateMetadataJSON } from "./invoiceExtractor"; import { extractInvoicesWithMistral, generateMetadataJSON } from "./invoiceExtractor";
import { localStoragePut, generateStorageKey } from "./localStorage"; import { localStoragePut, generateStorageKey } from "./localStorage";
@@ -32,6 +33,8 @@ interface EmailImportConfig {
// Store active intervals for each user // Store active intervals for each user
const activeIntervals = new Map<number, NodeJS.Timeout>(); const activeIntervals = new Map<number, NodeJS.Timeout>();
// Verrou anti-concurrence par userId
const runningChecks = new Set<number>();
/** /**
* Process a single email attachment (PDF) * Process a single email attachment (PDF)
@@ -51,6 +54,12 @@ async function processEmailAttachment(
console.log(`[EmailImport] File size: ${fileBuffer.length} bytes`); console.log(`[EmailImport] File size: ${fileBuffer.length} bytes`);
// Store source file // Store source file
// ANTI-DUPLICATION : vérifier si ce fichier a déjà été importé pour cet utilisateur
const existingSourceFile = await findSourceFileByFileName(userId, fileName);
if (existingSourceFile) {
console.log(`[EmailImport] File ${fileName} already imported for user ${userId} (sourceFile #${existingSourceFile.id}), skipping`);
return { success: true, totalInvoices: 0, imported: 0, duplicates: 1, errors: 0 };
}
const sourceFileKey = generateStorageKey(userId, fileName); const sourceFileKey = generateStorageKey(userId, fileName);
console.log(`[EmailImport] Generated storage key: ${sourceFileKey}`); console.log(`[EmailImport] Generated storage key: ${sourceFileKey}`);
@@ -330,6 +339,12 @@ async function buildImapConfig(config: EmailImportConfig): Promise<Imap.Config>
* Connect to IMAP and process unread emails with PDF attachments * Connect to IMAP and process unread emails with PDF attachments
*/ */
async function checkEmailsForPDFs(config: EmailImportConfig): Promise<void> { async function checkEmailsForPDFs(config: EmailImportConfig): Promise<void> {
// Anti-concurrence : ne pas lancer si un check est déjà en cours pour cet utilisateur
if (runningChecks.has(config.userId)) {
console.log(`[EmailImport] Check already running for user ${config.userId}, skipping`);
return;
}
runningChecks.add(config.userId);
// Build IMAP config (may involve async OAuth2 token fetch) // Build IMAP config (may involve async OAuth2 token fetch)
const imapConfig = await buildImapConfig(config); const imapConfig = await buildImapConfig(config);
@@ -380,7 +395,7 @@ async function checkEmailsForPDFs(config: EmailImportConfig): Promise<void> {
const fetch = imap.fetch(results, { const fetch = imap.fetch(results, {
bodies: "", bodies: "",
markSeen: false, // Don't mark as seen yet markSeen: true, // Mark as seen immediately to prevent re-processing
}); });
const processedEmails: number[] = []; const processedEmails: number[] = [];
@@ -474,11 +489,13 @@ async function checkEmailsForPDFs(config: EmailImportConfig): Promise<void> {
}); });
imap.once("error", (err) => { imap.once("error", (err) => {
runningChecks.delete(config.userId);
console.error("[EmailImport] IMAP connection error:", err); console.error("[EmailImport] IMAP connection error:", err);
reject(err); reject(err);
}); });
imap.once("end", () => { imap.once("end", () => {
runningChecks.delete(config.userId);
console.log(`[EmailImport] IMAP connection ended for user ${config.userId}`); console.log(`[EmailImport] IMAP connection ended for user ${config.userId}`);
}); });