fix: empêcher la duplication massive de fichiers lors de l'import email
- Marquer les emails comme lus immédiatement (markSeen: true) - Vérifier si le fichier existe déjà avant de le stocker (findSourceFileByFileName) - Ajouter un verrou anti-concurrence par utilisateur (runningChecks) - Empêche la création de ~570x doublons par facture
This commit is contained in:
@@ -9,6 +9,7 @@ import {
|
||||
isInvoiceBlacklisted,
|
||||
createInvoice,
|
||||
createImportLog,
|
||||
findSourceFileByFileName,
|
||||
} from "./db";
|
||||
import { extractInvoicesWithMistral, generateMetadataJSON } from "./invoiceExtractor";
|
||||
import { localStoragePut, generateStorageKey } from "./localStorage";
|
||||
@@ -32,6 +33,8 @@ interface EmailImportConfig {
|
||||
|
||||
// Store active intervals for each user
|
||||
const activeIntervals = new Map<number, NodeJS.Timeout>();
|
||||
// Verrou anti-concurrence par userId
|
||||
const runningChecks = new Set<number>();
|
||||
|
||||
/**
|
||||
* Process a single email attachment (PDF)
|
||||
@@ -51,6 +54,12 @@ async function processEmailAttachment(
|
||||
console.log(`[EmailImport] File size: ${fileBuffer.length} bytes`);
|
||||
|
||||
// Store source file
|
||||
// ANTI-DUPLICATION : vérifier si ce fichier a déjà été importé pour cet utilisateur
|
||||
const existingSourceFile = await findSourceFileByFileName(userId, fileName);
|
||||
if (existingSourceFile) {
|
||||
console.log(`[EmailImport] File ${fileName} already imported for user ${userId} (sourceFile #${existingSourceFile.id}), skipping`);
|
||||
return { success: true, totalInvoices: 0, imported: 0, duplicates: 1, errors: 0 };
|
||||
}
|
||||
const sourceFileKey = generateStorageKey(userId, fileName);
|
||||
console.log(`[EmailImport] Generated storage key: ${sourceFileKey}`);
|
||||
|
||||
@@ -330,6 +339,12 @@ async function buildImapConfig(config: EmailImportConfig): Promise<Imap.Config>
|
||||
* Connect to IMAP and process unread emails with PDF attachments
|
||||
*/
|
||||
async function checkEmailsForPDFs(config: EmailImportConfig): Promise<void> {
|
||||
// Anti-concurrence : ne pas lancer si un check est déjà en cours pour cet utilisateur
|
||||
if (runningChecks.has(config.userId)) {
|
||||
console.log(`[EmailImport] Check already running for user ${config.userId}, skipping`);
|
||||
return;
|
||||
}
|
||||
runningChecks.add(config.userId);
|
||||
// Build IMAP config (may involve async OAuth2 token fetch)
|
||||
const imapConfig = await buildImapConfig(config);
|
||||
|
||||
@@ -380,7 +395,7 @@ async function checkEmailsForPDFs(config: EmailImportConfig): Promise<void> {
|
||||
|
||||
const fetch = imap.fetch(results, {
|
||||
bodies: "",
|
||||
markSeen: false, // Don't mark as seen yet
|
||||
markSeen: true, // Mark as seen immediately to prevent re-processing
|
||||
});
|
||||
|
||||
const processedEmails: number[] = [];
|
||||
@@ -474,11 +489,13 @@ async function checkEmailsForPDFs(config: EmailImportConfig): Promise<void> {
|
||||
});
|
||||
|
||||
imap.once("error", (err) => {
|
||||
runningChecks.delete(config.userId);
|
||||
console.error("[EmailImport] IMAP connection error:", err);
|
||||
reject(err);
|
||||
});
|
||||
|
||||
imap.once("end", () => {
|
||||
runningChecks.delete(config.userId);
|
||||
console.log(`[EmailImport] IMAP connection ended for user ${config.userId}`);
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user