fix: empêcher la duplication massive de fichiers lors de l'import email
- Marquer les emails comme lus immédiatement (markSeen: true) - Vérifier si le fichier existe déjà avant de le stocker (findSourceFileByFileName) - Ajouter un verrou anti-concurrence par utilisateur (runningChecks) - Empêche la création de ~570x doublons par facture
This commit is contained in:
15
server/db.ts
15
server/db.ts
@@ -1218,3 +1218,18 @@ export async function updateWebImportSourceStatus(
|
|||||||
if (success) update.lastSuccessAt = new Date();
|
if (success) update.lastSuccessAt = new Date();
|
||||||
await db.update(webImportSources).set(update).where(eq(webImportSources.id, id));
|
await db.update(webImportSources).set(update).where(eq(webImportSources.id, id));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if a source file with the same fileName already exists for this user
|
||||||
|
* Used to prevent duplicate file storage during email import
|
||||||
|
*/
|
||||||
|
export async function findSourceFileByFileName(userId: number, fileName: string): Promise<any | null> {
|
||||||
|
const result = await db.select()
|
||||||
|
.from(sourceFiles)
|
||||||
|
.where(and(
|
||||||
|
eq(sourceFiles.userId, userId),
|
||||||
|
eq(sourceFiles.fileName, fileName)
|
||||||
|
))
|
||||||
|
.limit(1);
|
||||||
|
return result[0] || null;
|
||||||
|
}
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import {
|
|||||||
isInvoiceBlacklisted,
|
isInvoiceBlacklisted,
|
||||||
createInvoice,
|
createInvoice,
|
||||||
createImportLog,
|
createImportLog,
|
||||||
|
findSourceFileByFileName,
|
||||||
} from "./db";
|
} from "./db";
|
||||||
import { extractInvoicesWithMistral, generateMetadataJSON } from "./invoiceExtractor";
|
import { extractInvoicesWithMistral, generateMetadataJSON } from "./invoiceExtractor";
|
||||||
import { localStoragePut, generateStorageKey } from "./localStorage";
|
import { localStoragePut, generateStorageKey } from "./localStorage";
|
||||||
@@ -32,6 +33,8 @@ interface EmailImportConfig {
|
|||||||
|
|
||||||
// Store active intervals for each user
|
// Store active intervals for each user
|
||||||
const activeIntervals = new Map<number, NodeJS.Timeout>();
|
const activeIntervals = new Map<number, NodeJS.Timeout>();
|
||||||
|
// Verrou anti-concurrence par userId
|
||||||
|
const runningChecks = new Set<number>();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Process a single email attachment (PDF)
|
* Process a single email attachment (PDF)
|
||||||
@@ -51,6 +54,12 @@ async function processEmailAttachment(
|
|||||||
console.log(`[EmailImport] File size: ${fileBuffer.length} bytes`);
|
console.log(`[EmailImport] File size: ${fileBuffer.length} bytes`);
|
||||||
|
|
||||||
// Store source file
|
// Store source file
|
||||||
|
// ANTI-DUPLICATION : vérifier si ce fichier a déjà été importé pour cet utilisateur
|
||||||
|
const existingSourceFile = await findSourceFileByFileName(userId, fileName);
|
||||||
|
if (existingSourceFile) {
|
||||||
|
console.log(`[EmailImport] File ${fileName} already imported for user ${userId} (sourceFile #${existingSourceFile.id}), skipping`);
|
||||||
|
return { success: true, totalInvoices: 0, imported: 0, duplicates: 1, errors: 0 };
|
||||||
|
}
|
||||||
const sourceFileKey = generateStorageKey(userId, fileName);
|
const sourceFileKey = generateStorageKey(userId, fileName);
|
||||||
console.log(`[EmailImport] Generated storage key: ${sourceFileKey}`);
|
console.log(`[EmailImport] Generated storage key: ${sourceFileKey}`);
|
||||||
|
|
||||||
@@ -330,6 +339,12 @@ async function buildImapConfig(config: EmailImportConfig): Promise<Imap.Config>
|
|||||||
* Connect to IMAP and process unread emails with PDF attachments
|
* Connect to IMAP and process unread emails with PDF attachments
|
||||||
*/
|
*/
|
||||||
async function checkEmailsForPDFs(config: EmailImportConfig): Promise<void> {
|
async function checkEmailsForPDFs(config: EmailImportConfig): Promise<void> {
|
||||||
|
// Anti-concurrence : ne pas lancer si un check est déjà en cours pour cet utilisateur
|
||||||
|
if (runningChecks.has(config.userId)) {
|
||||||
|
console.log(`[EmailImport] Check already running for user ${config.userId}, skipping`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
runningChecks.add(config.userId);
|
||||||
// Build IMAP config (may involve async OAuth2 token fetch)
|
// Build IMAP config (may involve async OAuth2 token fetch)
|
||||||
const imapConfig = await buildImapConfig(config);
|
const imapConfig = await buildImapConfig(config);
|
||||||
|
|
||||||
@@ -380,7 +395,7 @@ async function checkEmailsForPDFs(config: EmailImportConfig): Promise<void> {
|
|||||||
|
|
||||||
const fetch = imap.fetch(results, {
|
const fetch = imap.fetch(results, {
|
||||||
bodies: "",
|
bodies: "",
|
||||||
markSeen: false, // Don't mark as seen yet
|
markSeen: true, // Mark as seen immediately to prevent re-processing
|
||||||
});
|
});
|
||||||
|
|
||||||
const processedEmails: number[] = [];
|
const processedEmails: number[] = [];
|
||||||
@@ -474,11 +489,13 @@ async function checkEmailsForPDFs(config: EmailImportConfig): Promise<void> {
|
|||||||
});
|
});
|
||||||
|
|
||||||
imap.once("error", (err) => {
|
imap.once("error", (err) => {
|
||||||
|
runningChecks.delete(config.userId);
|
||||||
console.error("[EmailImport] IMAP connection error:", err);
|
console.error("[EmailImport] IMAP connection error:", err);
|
||||||
reject(err);
|
reject(err);
|
||||||
});
|
});
|
||||||
|
|
||||||
imap.once("end", () => {
|
imap.once("end", () => {
|
||||||
|
runningChecks.delete(config.userId);
|
||||||
console.log(`[EmailImport] IMAP connection ended for user ${config.userId}`);
|
console.log(`[EmailImport] IMAP connection ended for user ${config.userId}`);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user