Files
demat-facturation/server/emailImportService.ts

669 lines
24 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { ImapFlow, type ImapFlowOptions, type SearchObject } from "imapflow";
import { simpleParser, ParsedMail, Attachment } from "mailparser";
import {
getImportSettingsByUser,
createSourceFile,
updateSourceFile,
getUserSettings,
getSourceFileByContentHash,
findDuplicateInvoice,
isInvoiceBlacklisted,
createInvoice,
createImportLog,
} from "./db";
import { extractInvoicesWithMistral, generateMetadataJSON } from "./invoiceExtractor";
import { localStorageDelete, localStoragePut, generateStorageKey } from "./localStorage";
import { calculateFileSha256 } from "./fileFingerprint";
import { sendImportNotification } from "./notificationService";
import { getOffice365ImapToken } from "./office365OAuth";
import { applyAutomationRules } from "./automationEngine";
export interface EmailImportConfig {
userId: number;
emailAddress: string;
password: string;
host: string;
port: number;
sinceDate?: number; // Unix timestamp (s) — ne pas lire les emails antérieurs à cette date
authMode?: "basic" | "oauth2"; // Mode d'authentification IMAP
// Credentials Azure AD pour OAuth2
azureTenantId?: string;
azureClientId?: string;
azureClientSecret?: string;
}
/**
* Construit les options ImapFlow sans effectuer d'appel réseau.
* ImapFlow reçoit le jeton brut et construit lui-même SASL XOAUTH2.
*/
export function createImapFlowOptions(
config: EmailImportConfig,
accessToken?: string,
): ImapFlowOptions {
const auth = config.authMode === "oauth2"
? { user: config.emailAddress, accessToken }
: { user: config.emailAddress, pass: config.password };
return {
host: config.host,
port: config.port,
secure: true,
auth,
tls: {
servername: config.host,
rejectUnauthorized: true,
},
logger: false,
disableAutoIdle: true,
connectionTimeout: 30_000,
greetingTimeout: 20_000,
socketTimeout: 120_000,
};
}
// Store active intervals for each user
const activeIntervals = new Map<number, NodeJS.Timeout>();
// Une extraction IA peut dépasser la fréquence configurée : ce verrou évite
// qu'un second cycle IMAP traite les mêmes messages avant la fin du premier.
const activeChecks = new Map<number, Promise<void>>();
function runEmailCheckExclusive(config: EmailImportConfig): Promise<void> {
const runningCheck = activeChecks.get(config.userId);
if (runningCheck) {
console.log(`[EmailImport] Vérification déjà en cours pour user ${config.userId}, cycle ignoré`);
return runningCheck;
}
const check = checkEmailsForPDFs(config).finally(() => {
if (activeChecks.get(config.userId) === check) activeChecks.delete(config.userId);
});
activeChecks.set(config.userId, check);
return check;
}
/**
* Process a single email attachment (PDF)
* Replicates the same logic as manual upload
*/
async function processEmailAttachment(
userId: number,
attachment: Attachment,
emailSubject: string
): Promise<{ success: boolean; totalInvoices: number; imported: number; duplicates: number; errors: number; quotaError?: boolean }> {
const fileName = attachment.filename || `email-attachment-${Date.now()}.pdf`;
console.log(`[EmailImport] Processing attachment: ${fileName} from email: ${emailSubject}`);
try {
// Convert attachment content to Buffer
const fileBuffer = attachment.content;
console.log(`[EmailImport] File size: ${fileBuffer.length} bytes`);
const contentHash = calculateFileSha256(fileBuffer);
const existingSource = await getSourceFileByContentHash(contentHash);
if (existingSource) {
console.log(
`[EmailImport] PDF déjà importé, extraction ignorée: ${fileName} -> source ${existingSource.id}`,
);
return {
success: true,
totalInvoices: Math.max(existingSource.totalInvoicesDetected, 1),
imported: 0,
duplicates: Math.max(existingSource.totalInvoicesDetected, 1),
errors: 0,
};
}
// Store source file
const sourceFileKey = generateStorageKey(userId, fileName);
console.log(`[EmailImport] Generated storage key: ${sourceFileKey}`);
let sourceFileUrl: string;
try {
const result = await localStoragePut(sourceFileKey, fileBuffer, "application/pdf");
sourceFileUrl = result.url;
console.log(`[EmailImport] File stored successfully at: ${sourceFileUrl}`);
} catch (error) {
console.error(`[EmailImport] FAILED to store file:`, error);
throw new Error("Failed to store PDF file");
}
// Create source file record
let sourceFile;
try {
sourceFile = await createSourceFile({
userId,
fileName,
fileKey: sourceFileKey,
fileUrl: sourceFileUrl,
contentHash,
processingStatus: "processing",
});
} catch (error: any) {
// La contrainte unique protège également contre deux imports concurrents.
if (error?.code === "ER_DUP_ENTRY" || error?.errno === 1062) {
await localStorageDelete(sourceFileKey).catch(() => undefined);
console.log(`[EmailImport] PDF réservé par un autre traitement: ${fileName}`);
return { success: true, totalInvoices: 1, imported: 0, duplicates: 1, errors: 0 };
}
throw error;
}
console.log(`[EmailImport] Source file record created with ID: ${sourceFile.id}`);
// Get user settings for custom keywords
const settings = await getUserSettings(userId);
const customKeywords = settings ? {
invoiceNumber: settings.invoiceNumberKeywords,
deliveryNote: settings.deliveryNoteKeywords,
orderNumber: settings.orderNumberKeywords,
supplier: settings.supplierKeywords,
totalAmount: settings.totalAmountKeywords,
subscription: settings.subscriptionKeywords,
recipient: settings.recipientKeywords,
} : undefined;
const model = settings?.llmModel || "mistral-large-latest";
const aiSettings = settings ? {
aiProvider: settings.aiProvider,
mistralApiKey: settings.mistralApiKey,
manusForgeApiKey: settings.manusForgeApiKey,
manusForgeApiUrl: settings.manusForgeApiUrl,
geminiApiKey: (settings as any).geminiApiKey || null,
} : undefined;
// Extract invoices
console.log(`[EmailImport] Starting invoice extraction (provider: ${aiSettings?.aiProvider || 'mistral'})...`);
const result = await extractInvoicesWithMistral(
fileBuffer,
userId,
sourceFile.id,
model,
customKeywords,
aiSettings
);
console.log(`[EmailImport] Extraction complete: ${result.invoiceCount} invoice(s) detected`);
// Update source file with total count
await updateSourceFile(sourceFile.id, {
totalInvoicesDetected: result.invoiceCount,
processingProgress: `Extraction ${result.invoiceCount} facture(s) détectée(s)`,
});
// Process each invoice
let importedCount = 0;
let duplicatesCount = 0;
let errorsCount = 0;
const duplicateDetails: any[] = [];
const errorDetails: any[] = [];
for (let i = 0; i < result.invoices.length; i++) {
const invoiceData = result.invoices[i]!;
try {
// Update progress
await updateSourceFile(sourceFile.id, {
processingProgress: `Extraction ${i + 1}/${result.invoiceCount} factures...`,
});
// Vérifier la blacklist (factures supprimées manuellement)
const blacklisted = await isInvoiceBlacklisted(invoiceData.invoiceNumber, userId);
if (blacklisted) {
duplicatesCount++;
duplicateDetails.push({
supplierName: invoiceData.supplierName,
invoiceNumber: invoiceData.invoiceNumber,
totalAmount: invoiceData.totalAmount,
reason: 'blacklisted',
});
continue;
}
// Check for duplicates (numéro de facture + montant)
const duplicate = await findDuplicateInvoice(
invoiceData.invoiceNumber,
invoiceData.totalAmount?.toString() ?? null,
userId
);
if (duplicate) {
duplicatesCount++;
duplicateDetails.push({
supplierName: invoiceData.supplierName,
invoiceNumber: invoiceData.invoiceNumber,
totalAmount: invoiceData.totalAmount,
});
continue;
}
// Generate metadata JSON
const metadataJson = generateMetadataJSON(invoiceData);
const metadataKey = generateStorageKey(userId, `${fileName}-${i + 1}-metadata.json`);
const { url: metadataUrl } = await localStoragePut(
metadataKey,
Buffer.from(metadataJson),
"application/json"
);
// Create invoice record
const newInvoice = await createInvoice({
userId,
sourceFileId: sourceFile.id,
invoiceIndexInFile: i + 1,
fileName: `${fileName} - Facture ${i + 1}`,
fileKey: sourceFileKey,
fileUrl: sourceFileUrl,
supplierName: invoiceData.supplierName,
invoiceNumber: invoiceData.invoiceNumber,
invoiceDate: invoiceData.invoiceDate,
deliveryNoteNumber: invoiceData.deliveryNoteNumber,
orderNumber: invoiceData.orderNumber,
totalAmount: invoiceData.totalAmount?.toString(),
recipientName: invoiceData.recipientName,
pageRange: invoiceData.pageRange,
qualityScore: invoiceData.qualityScore,
extractedText: invoiceData.extractedText,
metadataFileKey: metadataKey,
metadataFileUrl: metadataUrl,
status: "completed",
});
// Apply automation rules (ventilation, typeAchat, serviceConcerne)
try {
const { updateInvoice } = await import("./db");
const automationUpdates = await applyAutomationRules(userId, newInvoice);
if (Object.keys(automationUpdates).length > 0) {
await updateInvoice(newInvoice.id, automationUpdates);
console.log(`[EmailImport] Automation rules applied to invoice ${newInvoice.id}: ${Object.keys(automationUpdates).join(', ')}`);
}
} catch (autoError: any) {
console.error(`[EmailImport] Error applying automation rules:`, autoError.message);
}
importedCount++;
} catch (error: any) {
errorsCount++;
errorDetails.push({
invoiceIndex: i + 1,
error: error.message,
});
}
}
// Update source file status
await updateSourceFile(sourceFile.id, {
processingStatus: "completed",
processingProgress: `Terminé: ${importedCount} importée(s), ${duplicatesCount} doublon(s)`,
});
// Détecter si des erreurs de quota ont eu lieu
const quotaErrors = errorDetails.filter(e =>
e.error && (
e.error.includes('usage exhausted') ||
e.error.includes('quota') ||
e.error.includes('rate limit') ||
e.error.includes('Precondition Failed') ||
e.error.includes('429')
)
);
const warningMessage = quotaErrors.length > 0
? `Quota IA épuisé : ${quotaErrors.length} facture(s) non extraite(s). Vérifiez votre quota Manus ou configurez une clé API externe dans Paramétrage → IA.`
: null;
// Create import log
await createImportLog({
userId,
sourceFileId: sourceFile.id,
fileName,
totalInvoicesDetected: result.invoiceCount,
invoicesImported: importedCount,
duplicatesIgnored: duplicatesCount,
errors: errorsCount,
duplicateDetails: duplicateDetails.length > 0 ? JSON.stringify(duplicateDetails) : null,
errorDetails: errorDetails.length > 0 ? JSON.stringify(errorDetails) : null,
warningMessage,
importSource: "email",
});
console.log(`[EmailImport] Successfully processed attachment: ${fileName}`);
console.log(`[EmailImport] Results: ${importedCount} imported, ${duplicatesCount} duplicates, ${errorsCount} errors`);
return {
success: true,
totalInvoices: result.invoiceCount,
imported: importedCount,
duplicates: duplicatesCount,
errors: errorsCount,
};
} catch (error: any) {
console.error(`[EmailImport] Error processing attachment ${attachment.filename}:`, error);
// Détecter l'erreur de quota IA
const isQuotaError = error?.message && (
error.message.includes('usage exhausted') ||
error.message.includes('quota') ||
error.message.includes('429') ||
error.message.includes('rate limit') ||
error.message.includes('Precondition Failed')
);
return {
success: false,
totalInvoices: 0,
imported: 0,
duplicates: 0,
errors: 1,
quotaError: isQuotaError as boolean,
};
}
}
/**
* Construit la configuration IMAP en fonction du mode d'authentification.
* - basic : login/password classique
* - oauth2 : obtient un token Azure AD et utilise XOAUTH2
*/
async function buildImapConfig(config: EmailImportConfig): Promise<ImapFlowOptions> {
if (config.authMode === "oauth2") {
if (!config.azureTenantId || !config.azureClientId || !config.azureClientSecret) {
throw new Error(
"OAuth2 IMAP : credentials Azure AD incomplets (Tenant ID, Client ID, Client Secret requis)"
);
}
console.log(`[EmailImport] Obtaining OAuth2 token for ${config.emailAddress}...`);
const accessToken = await getOffice365ImapToken(
config.azureTenantId,
config.azureClientId,
config.azureClientSecret
);
console.log(`[EmailImport] OAuth2 token obtained successfully`);
return createImapFlowOptions(config, accessToken);
}
return createImapFlowOptions(config);
}
/**
* Connect to IMAP and process unread emails with PDF attachments
*/
async function checkEmailsForPDFs(config: EmailImportConfig): Promise<void> {
const imapConfig = await buildImapConfig(config);
const client = new ImapFlow(imapConfig);
client.on("error", (error) => {
console.error(`[EmailImport] IMAP connection error for user ${config.userId}:`, error);
});
let lock: Awaited<ReturnType<ImapFlow["getMailboxLock"]>> | undefined;
try {
await client.connect();
console.log(
`[EmailImport] Connected to IMAP server for user ${config.userId} (mode: ${config.authMode || "basic"})`,
);
lock = await client.getMailboxLock("INBOX", {
readOnly: false,
acquireTimeout: 30_000,
description: `invoice-import-user-${config.userId}`,
});
const searchCriteria: SearchObject = { seen: false };
if (config.sinceDate) {
searchCriteria.since = new Date(config.sinceDate * 1000);
console.log(
`[EmailImport] Filtering emails since ${searchCriteria.since.toISOString()} for user ${config.userId}`,
);
}
const unreadUids = await client.search(searchCriteria, { uid: true });
if (!unreadUids || unreadUids.length === 0) {
console.log(`[EmailImport] No unread emails found for user ${config.userId}`);
return;
}
console.log(`[EmailImport] Found ${unreadUids.length} unread emails for user ${config.userId}`);
// Le traitement reste séquentiel afin d'éviter plusieurs extractions IA
// concurrentes sur les mêmes pièces jointes.
for (const uid of unreadUids) {
const message = await client.fetchOne(uid, { source: true }, { uid: true });
if (!message || !message.source) {
console.warn(`[EmailImport] Message UID ${uid} without source, skipped`);
continue;
}
try {
const parsed: ParsedMail = await simpleParser(message.source);
const pdfAttachments = parsed.attachments.filter(
(attachment) =>
attachment.contentType === "application/pdf" ||
attachment.filename?.toLowerCase().endsWith(".pdf"),
);
if (pdfAttachments.length === 0) continue;
console.log(`[EmailImport] Email UID ${uid} has ${pdfAttachments.length} PDF attachment(s)`);
let allAttachmentsSucceeded = true;
for (const attachment of pdfAttachments) {
try {
const result = await processEmailAttachment(
config.userId,
attachment,
parsed.subject || "No subject",
);
allAttachmentsSucceeded = allAttachmentsSucceeded && result.success;
if (result.success) {
await sendImportNotification(config.userId, {
source: "email",
fileName: attachment.filename || "email-attachment.pdf",
totalInvoices: result.totalInvoices,
imported: result.imported,
duplicates: result.duplicates,
errors: result.errors,
});
}
} catch (error) {
allAttachmentsSucceeded = false;
console.error(`[EmailImport] Failed to process attachment from UID ${uid}:`, error);
}
}
if (allAttachmentsSucceeded) {
await client.messageFlagsAdd(uid, ["\\Seen"], { uid: true, silent: true });
console.log(`[EmailImport] Marked email UID ${uid} as seen`);
}
} catch (error) {
console.error(`[EmailImport] Error parsing email UID ${uid}:`, error);
}
}
console.log(`[EmailImport] Finished processing emails for user ${config.userId}`);
} finally {
lock?.release();
if (client.usable) await client.logout().catch(() => client.close());
else client.close();
}
}
/**
* Test IMAP connection for a user (without processing emails)
* Returns detailed error message if connection fails
*/
export async function testImapConnection(config: EmailImportConfig): Promise<{ success: boolean; message: string }> {
let client: ImapFlow | undefined;
try {
const imapConfig = await buildImapConfig(config);
client = new ImapFlow({ ...imapConfig, verifyOnly: true });
await client.connect();
console.log(`[EmailImport] Test connection successful for ${config.emailAddress}`);
return { success: true, message: `Connexion IMAP OAuth2 réussie pour ${config.emailAddress}` };
} catch (error: any) {
console.error(`[EmailImport] Test connection failed:`, error);
const rawMessage = error?.response || error?.message || String(error);
let message = `Erreur de connexion IMAP : ${rawMessage}`;
if (/AUTHENTICATE|authentication|invalid credentials/i.test(rawMessage)) {
message = config.authMode === "oauth2"
? "Authentification OAuth2 refusée. Vérifiez IMAP.AccessAsApp, le consentement administrateur, le service principal Exchange et lautorisation de la boîte."
: "Identifiants invalides. Pour Microsoft 365, utilisez OAuth2 au lieu de lauthentification basique.";
} else if (/ECONNREFUSED|ENOTFOUND/i.test(rawMessage)) {
message = `Impossible de joindre ${config.host}:${config.port}. Vérifiez ladresse et le port IMAP.`;
} else if (/certificate|TLS|SSL/i.test(rawMessage)) {
message = `Erreur TLS lors de la connexion à ${config.host}. Vérifiez le certificat et le port 993.`;
} else if (/timeout/i.test(rawMessage)) {
message = `Timeout lors de la connexion à ${config.host}:${config.port}.`;
}
return { success: false, message };
} finally {
if (client?.usable) await client.logout().catch(() => client?.close());
else client?.close();
}
}
/**
* Start email import service for a user
*/
export async function startEmailImportService(userId: number): Promise<boolean> {
try {
// Get user's import settings
const settings = await getImportSettingsByUser(userId);
if (!settings || settings.emailImportEnabled !== 1) {
console.log(`[EmailImport] Email import not enabled for user ${userId}`);
return false;
}
if (!settings.emailImportAddress || !settings.emailImportHost) {
console.log(`[EmailImport] Email import configuration incomplete for user ${userId}`);
return false;
}
const authMode = (settings as any).emailImportAuthMode as "basic" | "oauth2" || "basic";
// Vérification selon le mode d'auth
if (authMode === "basic" && !settings.emailImportPassword) {
console.log(`[EmailImport] Email import password missing for user ${userId}`);
return false;
}
if (authMode === "oauth2" && (!settings.azureTenantId || !settings.azureClientId || !settings.azureClientSecret)) {
console.log(`[EmailImport] OAuth2 credentials incomplete for user ${userId}`);
return false;
}
// Stop existing service if running
stopEmailImportService(userId);
const config: EmailImportConfig = {
userId,
emailAddress: settings.emailImportAddress,
password: settings.emailImportPassword || "",
host: settings.emailImportHost,
port: settings.emailImportPort || 993,
sinceDate: settings.emailImportSinceDate ?? undefined,
authMode,
azureTenantId: settings.azureTenantId || undefined,
azureClientId: settings.azureClientId || undefined,
azureClientSecret: settings.azureClientSecret || undefined,
};
const frequencyMs = (settings.emailImportFrequency || 30) * 60 * 1000; // Convert minutes to milliseconds
console.log(
`[EmailImport] Starting email import service for user ${userId} with frequency ${settings.emailImportFrequency} minutes (auth: ${authMode})`
);
// Run immediately on start
runEmailCheckExclusive(config).catch((error) => {
console.error(`[EmailImport] Error checking emails for user ${userId}:`, error);
});
// Set up interval for periodic checks
const interval = setInterval(() => {
runEmailCheckExclusive(config).catch((error) => {
console.error(`[EmailImport] Error checking emails for user ${userId}:`, error);
});
}, frequencyMs);
activeIntervals.set(userId, interval);
console.log(`[EmailImport] Email import service started for user ${userId}`);
return true;
} catch (error) {
console.error(`[EmailImport] Error starting email import service for user ${userId}:`, error);
return false;
}
}
/**
* Stop email import service for a user
*/
export function stopEmailImportService(userId: number): void {
const interval = activeIntervals.get(userId);
if (interval) {
clearInterval(interval);
activeIntervals.delete(userId);
console.log(`[EmailImport] Email import service stopped for user ${userId}`);
}
}
/**
* Check if email import service is running for a user
*/
export function isEmailImportServiceRunning(userId: number): boolean {
return activeIntervals.has(userId);
}
/**
* Manually trigger an immediate email check for a user
*/
export async function triggerEmailCheck(userId: number): Promise<{ success: boolean; message: string }> {
try {
// Get user's import settings
const settings = await getImportSettingsByUser(userId);
if (!settings || settings.emailImportEnabled !== 1) {
return { success: false, message: "Import par email non activé" };
}
if (!settings.emailImportAddress || !settings.emailImportHost) {
return { success: false, message: "Configuration IMAP incomplète" };
}
const authMode = (settings as any).emailImportAuthMode as "basic" | "oauth2" || "basic";
const config: EmailImportConfig = {
userId,
emailAddress: settings.emailImportAddress,
password: settings.emailImportPassword || "",
host: settings.emailImportHost,
port: settings.emailImportPort || 993,
sinceDate: settings.emailImportSinceDate ?? undefined,
authMode,
azureTenantId: settings.azureTenantId || undefined,
azureClientId: settings.azureClientId || undefined,
azureClientSecret: settings.azureClientSecret || undefined,
};
console.log(`[EmailImport] Manual check triggered for user ${userId}`);
await runEmailCheckExclusive(config);
return { success: true, message: "Vérification terminée avec succès" };
} catch (error: any) {
console.error(`[EmailImport] Error during manual check for user ${userId}:`, error);
return { success: false, message: `Erreur: ${error.message}` };
}
}
/**
* Stop all email import services
*/
export function stopAllEmailImportServices(): void {
activeIntervals.forEach((interval, userId) => {
clearInterval(interval);
console.log(`[EmailImport] Stopped email import service for user ${userId}`);
});
activeIntervals.clear();
}