Checkpoint: Remplacement complet de imap 0.8 par ImapFlow : OAuth2 moderne avec jeton brut, TLS strict, traitement séquentiel des messages non lus, verrou anti-concurrence conservé, marquage Seen uniquement après succès, test de connexion adapté, diagnostic OAuth2 et tests unitaires ajoutés. Validation : 31 tests, TypeScript, build et authentification réelle Microsoft 365 réussis.
This commit is contained in:
Manus
38
server/emailImportService.test.ts
Normal file
38
server/emailImportService.test.ts
Normal file
@@ -0,0 +1,38 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createImapFlowOptions, type EmailImportConfig } from "./emailImportService";
|
||||
|
||||
const baseConfig: EmailImportConfig = {
|
||||
userId: 2,
|
||||
emailAddress: "compta@example.org",
|
||||
password: "secret",
|
||||
host: "outlook.office365.com",
|
||||
port: 993,
|
||||
};
|
||||
|
||||
describe("createImapFlowOptions", () => {
|
||||
it("transmet le jeton brut à ImapFlow pour une authentification OAuth2", () => {
|
||||
const options = createImapFlowOptions(
|
||||
{ ...baseConfig, authMode: "oauth2" },
|
||||
"access-token-value",
|
||||
);
|
||||
|
||||
expect(options.secure).toBe(true);
|
||||
expect(options.auth).toEqual({
|
||||
user: "compta@example.org",
|
||||
accessToken: "access-token-value",
|
||||
});
|
||||
expect(options.auth).not.toHaveProperty("pass");
|
||||
expect(options.tls?.rejectUnauthorized).toBe(true);
|
||||
expect(options.disableAutoIdle).toBe(true);
|
||||
});
|
||||
|
||||
it("conserve le mot de passe uniquement pour le mode basique", () => {
|
||||
const options = createImapFlowOptions({ ...baseConfig, authMode: "basic" });
|
||||
|
||||
expect(options.auth).toEqual({
|
||||
user: "compta@example.org",
|
||||
pass: "secret",
|
||||
});
|
||||
expect(options.auth).not.toHaveProperty("accessToken");
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
import Imap from "imap";
|
||||
import { ImapFlow, type ImapFlowOptions, type SearchObject } from "imapflow";
|
||||
import { simpleParser, ParsedMail, Attachment } from "mailparser";
|
||||
import {
|
||||
getImportSettingsByUser,
|
||||
@@ -15,10 +15,10 @@ import { extractInvoicesWithMistral, generateMetadataJSON } from "./invoiceExtra
|
||||
import { localStorageDelete, localStoragePut, generateStorageKey } from "./localStorage";
|
||||
import { calculateFileSha256 } from "./fileFingerprint";
|
||||
import { sendImportNotification } from "./notificationService";
|
||||
import { getOffice365ImapToken, buildXOAuth2String } from "./office365OAuth";
|
||||
import { getOffice365ImapToken } from "./office365OAuth";
|
||||
import { applyAutomationRules } from "./automationEngine";
|
||||
|
||||
interface EmailImportConfig {
|
||||
export interface EmailImportConfig {
|
||||
userId: number;
|
||||
emailAddress: string;
|
||||
password: string;
|
||||
@@ -32,6 +32,35 @@ interface EmailImportConfig {
|
||||
azureClientSecret?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Construit les options ImapFlow sans effectuer d'appel réseau.
|
||||
* ImapFlow reçoit le jeton brut et construit lui-même SASL XOAUTH2.
|
||||
*/
|
||||
export function createImapFlowOptions(
|
||||
config: EmailImportConfig,
|
||||
accessToken?: string,
|
||||
): ImapFlowOptions {
|
||||
const auth = config.authMode === "oauth2"
|
||||
? { user: config.emailAddress, accessToken }
|
||||
: { user: config.emailAddress, pass: config.password };
|
||||
|
||||
return {
|
||||
host: config.host,
|
||||
port: config.port,
|
||||
secure: true,
|
||||
auth,
|
||||
tls: {
|
||||
servername: config.host,
|
||||
rejectUnauthorized: true,
|
||||
},
|
||||
logger: false,
|
||||
disableAutoIdle: true,
|
||||
connectionTimeout: 30_000,
|
||||
greetingTimeout: 20_000,
|
||||
socketTimeout: 120_000,
|
||||
};
|
||||
}
|
||||
|
||||
// Store active intervals for each user
|
||||
const activeIntervals = new Map<number, NodeJS.Timeout>();
|
||||
// Une extraction IA peut dépasser la fréquence configurée : ce verrou évite
|
||||
@@ -332,7 +361,7 @@ async function processEmailAttachment(
|
||||
* - basic : login/password classique
|
||||
* - oauth2 : obtient un token Azure AD et utilise XOAUTH2
|
||||
*/
|
||||
async function buildImapConfig(config: EmailImportConfig): Promise<Imap.Config> {
|
||||
async function buildImapConfig(config: EmailImportConfig): Promise<ImapFlowOptions> {
|
||||
if (config.authMode === "oauth2") {
|
||||
if (!config.azureTenantId || !config.azureClientId || !config.azureClientSecret) {
|
||||
throw new Error(
|
||||
@@ -346,198 +375,115 @@ async function buildImapConfig(config: EmailImportConfig): Promise<Imap.Config>
|
||||
config.azureClientId,
|
||||
config.azureClientSecret
|
||||
);
|
||||
const xoauth2 = buildXOAuth2String(config.emailAddress, accessToken);
|
||||
console.log(`[EmailImport] OAuth2 token obtained successfully`);
|
||||
|
||||
return {
|
||||
user: config.emailAddress,
|
||||
xoauth2,
|
||||
host: config.host,
|
||||
port: config.port,
|
||||
tls: true,
|
||||
tlsOptions: { rejectUnauthorized: false },
|
||||
authTimeout: 30000,
|
||||
} as any;
|
||||
return createImapFlowOptions(config, accessToken);
|
||||
}
|
||||
|
||||
// Basic auth (par défaut)
|
||||
return {
|
||||
user: config.emailAddress,
|
||||
password: config.password,
|
||||
host: config.host,
|
||||
port: config.port,
|
||||
tls: true,
|
||||
tlsOptions: { rejectUnauthorized: false },
|
||||
authTimeout: 30000,
|
||||
};
|
||||
return createImapFlowOptions(config);
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect to IMAP and process unread emails with PDF attachments
|
||||
*/
|
||||
async function checkEmailsForPDFs(config: EmailImportConfig): Promise<void> {
|
||||
// Build IMAP config (may involve async OAuth2 token fetch)
|
||||
const imapConfig = await buildImapConfig(config);
|
||||
const client = new ImapFlow(imapConfig);
|
||||
client.on("error", (error) => {
|
||||
console.error(`[EmailImport] IMAP connection error for user ${config.userId}:`, error);
|
||||
});
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const imap = new Imap(imapConfig);
|
||||
let lock: Awaited<ReturnType<ImapFlow["getMailboxLock"]>> | undefined;
|
||||
try {
|
||||
await client.connect();
|
||||
console.log(
|
||||
`[EmailImport] Connected to IMAP server for user ${config.userId} (mode: ${config.authMode || "basic"})`,
|
||||
);
|
||||
|
||||
function openInbox(cb: (err: Error | null, box?: any) => void) {
|
||||
imap.openBox("INBOX", false, cb);
|
||||
lock = await client.getMailboxLock("INBOX", {
|
||||
readOnly: false,
|
||||
acquireTimeout: 30_000,
|
||||
description: `invoice-import-user-${config.userId}`,
|
||||
});
|
||||
|
||||
const searchCriteria: SearchObject = { seen: false };
|
||||
if (config.sinceDate) {
|
||||
searchCriteria.since = new Date(config.sinceDate * 1000);
|
||||
console.log(
|
||||
`[EmailImport] Filtering emails since ${searchCriteria.since.toISOString()} for user ${config.userId}`,
|
||||
);
|
||||
}
|
||||
|
||||
imap.once("ready", () => {
|
||||
console.log(`[EmailImport] Connected to IMAP server for user ${config.userId} (mode: ${config.authMode || "basic"})`);
|
||||
|
||||
openInbox((err) => {
|
||||
if (err) {
|
||||
console.error("[EmailImport] Error opening inbox:", err);
|
||||
imap.end();
|
||||
reject(err);
|
||||
return;
|
||||
}
|
||||
const unreadUids = await client.search(searchCriteria, { uid: true });
|
||||
if (!unreadUids || unreadUids.length === 0) {
|
||||
console.log(`[EmailImport] No unread emails found for user ${config.userId}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Build search criteria: unread emails, optionally filtered by date
|
||||
const searchCriteria: any[] = ["UNSEEN"];
|
||||
if (config.sinceDate) {
|
||||
// IMAP SINCE expects a date string like "1-Jan-2026"
|
||||
const since = new Date(config.sinceDate * 1000);
|
||||
const months = ["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"];
|
||||
const sinceStr = `${since.getDate()}-${months[since.getMonth()]}-${since.getFullYear()}`;
|
||||
searchCriteria.push(["SINCE", sinceStr]);
|
||||
console.log(`[EmailImport] Filtering emails since ${sinceStr} for user ${config.userId}`);
|
||||
}
|
||||
imap.search(searchCriteria, (err, results) => {
|
||||
if (err) {
|
||||
console.error("[EmailImport] Error searching emails:", err);
|
||||
imap.end();
|
||||
reject(err);
|
||||
return;
|
||||
}
|
||||
console.log(`[EmailImport] Found ${unreadUids.length} unread emails for user ${config.userId}`);
|
||||
|
||||
if (!results || results.length === 0) {
|
||||
console.log(`[EmailImport] No unread emails found for user ${config.userId}`);
|
||||
imap.end();
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
// Le traitement reste séquentiel afin d'éviter plusieurs extractions IA
|
||||
// concurrentes sur les mêmes pièces jointes.
|
||||
for (const uid of unreadUids) {
|
||||
const message = await client.fetchOne(uid, { source: true }, { uid: true });
|
||||
if (!message || !message.source) {
|
||||
console.warn(`[EmailImport] Message UID ${uid} without source, skipped`);
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log(`[EmailImport] Found ${results.length} unread emails for user ${config.userId}`);
|
||||
try {
|
||||
const parsed: ParsedMail = await simpleParser(message.source);
|
||||
const pdfAttachments = parsed.attachments.filter(
|
||||
(attachment) =>
|
||||
attachment.contentType === "application/pdf" ||
|
||||
attachment.filename?.toLowerCase().endsWith(".pdf"),
|
||||
);
|
||||
|
||||
const fetch = imap.fetch(results, {
|
||||
bodies: "",
|
||||
markSeen: false, // Don't mark as seen yet
|
||||
});
|
||||
if (pdfAttachments.length === 0) continue;
|
||||
|
||||
const processedEmailUids: number[] = [];
|
||||
const messageTasks: Promise<void>[] = [];
|
||||
console.log(`[EmailImport] Email UID ${uid} has ${pdfAttachments.length} PDF attachment(s)`);
|
||||
let allAttachmentsSucceeded = true;
|
||||
|
||||
fetch.on("message", (msg, seqno) => {
|
||||
const uidPromise = new Promise<number>((resolveUid) => {
|
||||
msg.once("attributes", (attributes) => resolveUid(attributes.uid));
|
||||
});
|
||||
for (const attachment of pdfAttachments) {
|
||||
try {
|
||||
const result = await processEmailAttachment(
|
||||
config.userId,
|
||||
attachment,
|
||||
parsed.subject || "No subject",
|
||||
);
|
||||
allAttachmentsSucceeded = allAttachmentsSucceeded && result.success;
|
||||
|
||||
msg.on("body", (stream) => {
|
||||
const task = (async () => {
|
||||
try {
|
||||
const parsed: ParsedMail = await simpleParser(stream as any);
|
||||
const pdfAttachments = parsed.attachments.filter(
|
||||
(attachment) =>
|
||||
attachment.contentType === "application/pdf" ||
|
||||
attachment.filename?.toLowerCase().endsWith(".pdf"),
|
||||
);
|
||||
|
||||
if (pdfAttachments.length === 0) return;
|
||||
|
||||
console.log(
|
||||
`[EmailImport] Email ${seqno} has ${pdfAttachments.length} PDF attachment(s)`,
|
||||
);
|
||||
|
||||
let allAttachmentsSucceeded = true;
|
||||
for (const attachment of pdfAttachments) {
|
||||
try {
|
||||
const result = await processEmailAttachment(
|
||||
config.userId,
|
||||
attachment,
|
||||
parsed.subject || "No subject",
|
||||
);
|
||||
allAttachmentsSucceeded = allAttachmentsSucceeded && result.success;
|
||||
|
||||
if (result.success) {
|
||||
await sendImportNotification(config.userId, {
|
||||
source: "email",
|
||||
fileName: attachment.filename || "email-attachment.pdf",
|
||||
totalInvoices: result.totalInvoices,
|
||||
imported: result.imported,
|
||||
duplicates: result.duplicates,
|
||||
errors: result.errors,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
allAttachmentsSucceeded = false;
|
||||
console.error(
|
||||
`[EmailImport] Failed to process attachment from email ${seqno}:`,
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (allAttachmentsSucceeded) {
|
||||
const uid = await uidPromise;
|
||||
if (!processedEmailUids.includes(uid)) processedEmailUids.push(uid);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`[EmailImport] Error parsing email ${seqno}:`, error);
|
||||
}
|
||||
})();
|
||||
|
||||
messageTasks.push(task);
|
||||
});
|
||||
});
|
||||
|
||||
fetch.once("error", (err) => {
|
||||
console.error("[EmailImport] Fetch error:", err);
|
||||
imap.end();
|
||||
reject(err);
|
||||
});
|
||||
|
||||
fetch.once("end", async () => {
|
||||
console.log(`[EmailImport] Finished fetching emails for user ${config.userId}`);
|
||||
|
||||
// Le flux IMAP peut se terminer avant les traitements IA asynchrones.
|
||||
// On attend explicitement chaque message avant de le marquer comme lu.
|
||||
await Promise.allSettled(messageTasks);
|
||||
|
||||
if (processedEmailUids.length > 0) {
|
||||
imap.addFlags(processedEmailUids, ["\\Seen"], (err) => {
|
||||
if (err) {
|
||||
console.error("[EmailImport] Error marking emails as seen:", err);
|
||||
} else {
|
||||
console.log(`[EmailImport] Marked ${processedEmailUids.length} emails as seen`);
|
||||
}
|
||||
imap.end();
|
||||
resolve();
|
||||
if (result.success) {
|
||||
await sendImportNotification(config.userId, {
|
||||
source: "email",
|
||||
fileName: attachment.filename || "email-attachment.pdf",
|
||||
totalInvoices: result.totalInvoices,
|
||||
imported: result.imported,
|
||||
duplicates: result.duplicates,
|
||||
errors: result.errors,
|
||||
});
|
||||
} else {
|
||||
imap.end();
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
} catch (error) {
|
||||
allAttachmentsSucceeded = false;
|
||||
console.error(`[EmailImport] Failed to process attachment from UID ${uid}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
imap.once("error", (err) => {
|
||||
console.error("[EmailImport] IMAP connection error:", err);
|
||||
reject(err);
|
||||
});
|
||||
if (allAttachmentsSucceeded) {
|
||||
await client.messageFlagsAdd(uid, ["\\Seen"], { uid: true, silent: true });
|
||||
console.log(`[EmailImport] Marked email UID ${uid} as seen`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`[EmailImport] Error parsing email UID ${uid}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
imap.once("end", () => {
|
||||
console.log(`[EmailImport] IMAP connection ended for user ${config.userId}`);
|
||||
});
|
||||
|
||||
imap.connect();
|
||||
});
|
||||
console.log(`[EmailImport] Finished processing emails for user ${config.userId}`);
|
||||
} finally {
|
||||
lock?.release();
|
||||
if (client.usable) await client.logout().catch(() => client.close());
|
||||
else client.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -545,57 +491,34 @@ async function checkEmailsForPDFs(config: EmailImportConfig): Promise<void> {
|
||||
* Returns detailed error message if connection fails
|
||||
*/
|
||||
export async function testImapConnection(config: EmailImportConfig): Promise<{ success: boolean; message: string }> {
|
||||
let client: ImapFlow | undefined;
|
||||
try {
|
||||
const imapConfig = await buildImapConfig(config);
|
||||
|
||||
return new Promise((resolve) => {
|
||||
const imap = new Imap(imapConfig);
|
||||
let resolved = false;
|
||||
|
||||
const done = (result: { success: boolean; message: string }) => {
|
||||
if (!resolved) {
|
||||
resolved = true;
|
||||
try { imap.destroy(); } catch {}
|
||||
resolve(result);
|
||||
}
|
||||
};
|
||||
|
||||
imap.once("ready", () => {
|
||||
console.log(`[EmailImport] Test connection successful for ${config.emailAddress}`);
|
||||
done({ success: true, message: `Connexion IMAP réussie pour ${config.emailAddress}` });
|
||||
});
|
||||
|
||||
imap.once("error", (err: any) => {
|
||||
console.error(`[EmailImport] Test connection failed:`, err);
|
||||
let message = `Erreur de connexion IMAP : ${err.message || err}`;
|
||||
|
||||
// Messages d'erreur plus clairs
|
||||
if (err.message?.includes("Invalid credentials") || err.message?.includes("AUTHENTICATE")) {
|
||||
if (config.authMode === "oauth2") {
|
||||
message = "Authentification OAuth2 refusée. Vérifiez que l'application Azure AD a bien la permission IMAP.AccessAsApp et que le consentement admin a été accordé.";
|
||||
} else {
|
||||
message = "Identifiants invalides. Pour Office 365, l'authentification basique est désactivée. Activez le mode OAuth2 et configurez les credentials Azure AD.";
|
||||
}
|
||||
} else if (err.message?.includes("ECONNREFUSED") || err.message?.includes("ENOTFOUND")) {
|
||||
message = `Impossible de se connecter au serveur ${config.host}:${config.port}. Vérifiez l'adresse et le port IMAP.`;
|
||||
} else if (err.message?.includes("certificate") || err.message?.includes("SSL")) {
|
||||
message = `Erreur SSL/TLS lors de la connexion à ${config.host}. Vérifiez le port (993 pour SSL).`;
|
||||
} else if (err.message?.includes("timeout") || err.message?.includes("Timeout")) {
|
||||
message = `Timeout de connexion à ${config.host}:${config.port}. Vérifiez l'adresse du serveur IMAP.`;
|
||||
}
|
||||
|
||||
done({ success: false, message });
|
||||
});
|
||||
|
||||
// Timeout de sécurité
|
||||
setTimeout(() => {
|
||||
done({ success: false, message: `Timeout : impossible de se connecter à ${config.host}:${config.port} dans les 15 secondes.` });
|
||||
}, 15000);
|
||||
|
||||
imap.connect();
|
||||
});
|
||||
client = new ImapFlow({ ...imapConfig, verifyOnly: true });
|
||||
await client.connect();
|
||||
console.log(`[EmailImport] Test connection successful for ${config.emailAddress}`);
|
||||
return { success: true, message: `Connexion IMAP OAuth2 réussie pour ${config.emailAddress}` };
|
||||
} catch (error: any) {
|
||||
return { success: false, message: `Erreur : ${error.message || error}` };
|
||||
console.error(`[EmailImport] Test connection failed:`, error);
|
||||
const rawMessage = error?.response || error?.message || String(error);
|
||||
let message = `Erreur de connexion IMAP : ${rawMessage}`;
|
||||
|
||||
if (/AUTHENTICATE|authentication|invalid credentials/i.test(rawMessage)) {
|
||||
message = config.authMode === "oauth2"
|
||||
? "Authentification OAuth2 refusée. Vérifiez IMAP.AccessAsApp, le consentement administrateur, le service principal Exchange et l’autorisation de la boîte."
|
||||
: "Identifiants invalides. Pour Microsoft 365, utilisez OAuth2 au lieu de l’authentification basique.";
|
||||
} else if (/ECONNREFUSED|ENOTFOUND/i.test(rawMessage)) {
|
||||
message = `Impossible de joindre ${config.host}:${config.port}. Vérifiez l’adresse et le port IMAP.`;
|
||||
} else if (/certificate|TLS|SSL/i.test(rawMessage)) {
|
||||
message = `Erreur TLS lors de la connexion à ${config.host}. Vérifiez le certificat et le port 993.`;
|
||||
} else if (/timeout/i.test(rawMessage)) {
|
||||
message = `Timeout lors de la connexion à ${config.host}:${config.port}.`;
|
||||
}
|
||||
|
||||
return { success: false, message };
|
||||
} finally {
|
||||
if (client?.usable) await client.logout().catch(() => client?.close());
|
||||
else client?.close();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user