Compare commits

...

15 Commits

Author SHA1 Message Date
Manus
52b5e49082 Merge remote-tracking branch 'recette/server-state'
All checks were successful
Validation applicative / TypeScript, tests et build (push) Successful in 10m21s
# Conflicts:
#	app.json
#	server/emailImportService.ts
2026-08-22 16:09:35 +00:00
Manus
d27703f878 Merge remote-tracking branch 'recette/main'
All checks were successful
Validation applicative / TypeScript, tests et build (push) Successful in 8m30s
# Conflicts:
#	app.json
#	server/emailImportService.ts
#	todo.md
2026-08-22 15:57:13 +00:00
Manus
098d707289 Checkpoint: Remplacement complet de imap 0.8 par ImapFlow : OAuth2 moderne avec jeton brut, TLS strict, traitement séquentiel des messages non lus, verrou anti-concurrence conservé, marquage Seen uniquement après succès, test de connexion adapté, diagnostic OAuth2 et tests unitaires ajoutés. Validation : 31 tests, TypeScript, build et authentification réelle Microsoft 365 réussis. 2026-08-22 15:51:06 +00:00
Manus
66c4940aba Checkpoint: Version finale après fusion du dépôt Gitea de production et nettoyage contrôlé : empreinte SHA-256 unique des PDF, import IMAP séquencé et non concurrent, protections étendues à toutes les sources, migration validée et suivi de production clôturé. 2026-08-22 10:55:15 +00:00
Manus
147dd6e5a0 Merge remote-tracking branch 'prod-dsi/main'
# Conflicts:
#	server/emailImportService.ts
2026-08-22 10:39:08 +00:00
Manus
b7525ab51e Merge remote-tracking branch 'prod/main' 2026-08-22 10:33:05 +00:00
Manus
d729a94a96 Checkpoint: Correction robuste des doublons de factures : empreinte SHA-256 unique des PDF sources, détection globale avant import manuel/email/dossier/web, verrou contre les vérifications IMAP concurrentes, attente réelle de fin de traitement avant marquage lu, migration et tests de non-régression. 2026-08-22 10:31:54 +00:00
Manus Admin
957494f8ef ci: trigger 2026-08-21 13:48:09 +02:00
Manus CI
269823fd78 CI fix : corrections pour CI production 2026-08-21 11:47:55 +00:00
Manus CI
295bb26378 CI : workflow validate.yml, manifeste ci.required:true, garde VITEST, tests isolés 2026-08-21 11:15:58 +00:00
Manus Admin
de797c1c0b fix: empêcher la duplication massive de fichiers lors de l import email
- markSeen: true pour marquer les emails immédiatement
- Vérification doublon avant stockage (findSourceFileByFileName)
- Verrou anti-concurrence par utilisateur
2026-08-18 15:05:31 +02:00
Manus Admin
de76a761a3 fix: empêcher la duplication massive de fichiers lors de l'import email
- Marquer les emails comme lus immédiatement (markSeen: true)
- Vérifier si le fichier existe déjà avant de le stocker (findSourceFileByFileName)
- Ajouter un verrou anti-concurrence par utilisateur (runningChecks)
- Empêche la création de ~570x doublons par facture
2026-08-18 14:50:31 +02:00
Manus
8759d85f3d Merge remote-tracking branch 'recette/main' 2026-08-18 07:42:06 +00:00
Manus
89c65ca979 chore: versionner le manifeste de déploiement 2026-08-18 09:40:27 +02:00
Manus
eecbd07b5c chore: versionner le manifeste de déploiement 2026-08-17 22:35:40 +02:00
16 changed files with 2974 additions and 330 deletions

View File

@@ -0,0 +1,2 @@
ALTER TABLE `sourceFiles` ADD `contentHash` varchar(64);--> statement-breakpoint
ALTER TABLE `sourceFiles` ADD CONSTRAINT `source_file_content_hash_unique` UNIQUE(`contentHash`);

File diff suppressed because it is too large Load Diff

View File

@@ -260,6 +260,13 @@
"when": 1785419093588,
"tag": "0036_broken_rattler",
"breakpoints": true
},
{
"idx": 37,
"version": "5",
"when": 1787394418464,
"tag": "0037_goofy_quentin_quire",
"breakpoints": true
}
]
}

View File

@@ -36,12 +36,16 @@ export const sourceFiles = mysqlTable("sourceFiles", {
fileName: varchar("fileName", { length: 255 }).notNull(),
fileKey: text("fileKey").notNull(), // Local storage key with YYYY-MM prefix
fileUrl: text("fileUrl").notNull(), // Public URL
/** Empreinte du PDF source, globale à l'application pour bloquer tout réimport identique. */
contentHash: varchar("contentHash", { length: 64 }),
totalInvoicesDetected: int("totalInvoicesDetected").default(0).notNull(),
processingStatus: mysqlEnum("processingStatus", ["processing", "completed", "error"]).default("processing").notNull(),
processingProgress: varchar("processingProgress", { length: 255 }), // Progress message (e.g., "Extraction 3/9 factures...")
createdAt: timestamp("createdAt").defaultNow().notNull(),
updatedAt: timestamp("updatedAt").defaultNow().onUpdateNow().notNull(),
});
}, (table) => ({
contentHashIdx: uniqueIndex("source_file_content_hash_unique").on(table.contentHash),
}));
export type SourceFile = typeof sourceFiles.$inferSelect;
export type InsertSourceFile = typeof sourceFiles.$inferInsert;

View File

@@ -53,7 +53,6 @@
"@types/archiver": "^7.0.0",
"@types/bcrypt": "^6.0.0",
"@types/chokidar": "^2.1.7",
"@types/imap": "^0.8.43",
"@types/jsonwebtoken": "^9.0.10",
"@types/mailparser": "^3.4.6",
"@types/ssh2-sftp-client": "^9.0.6",
@@ -71,7 +70,7 @@
"embla-carousel-react": "^8.6.0",
"express": "^4.21.2",
"framer-motion": "^12.23.22",
"imap": "^0.8.19",
"imapflow": "^1.7.2",
"input-otp": "^1.4.2",
"jose": "6.1.0",
"jsonwebtoken": "^9.0.3",

209
pnpm-lock.yaml generated
View File

@@ -133,9 +133,6 @@ importers:
'@types/chokidar':
specifier: ^2.1.7
version: 2.1.7
'@types/imap':
specifier: ^0.8.43
version: 0.8.43
'@types/jsonwebtoken':
specifier: ^9.0.10
version: 9.0.10
@@ -187,9 +184,9 @@ importers:
framer-motion:
specifier: ^12.23.22
version: 12.23.22(react-dom@19.2.1(react@19.2.1))(react@19.2.1)
imap:
specifier: ^0.8.19
version: 0.8.19
imapflow:
specifier: ^1.7.2
version: 1.7.2
input-otp:
specifier: ^1.4.2
version: 1.4.2(react-dom@19.2.1(react@19.2.1))(react@19.2.1)
@@ -1438,6 +1435,9 @@ packages:
'@pdf-lib/upng@1.0.1':
resolution: {integrity: sha512-dQK2FUMQtowVP00mtIksrlZhdFXQZPC+taih1q4CvPZ5vqdxR/LKBaFg0oAfzd1GlHZXXSPdQfzQnt+ViGvEIQ==}
'@pinojs/redact@0.4.0':
resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==}
'@pkgjs/parseargs@0.11.0':
resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==}
engines: {node: '>=14'}
@@ -2580,9 +2580,6 @@ packages:
'@types/http-errors@2.0.5':
resolution: {integrity: sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==}
'@types/imap@0.8.43':
resolution: {integrity: sha512-POPoqrDax9mxM2N4ITZYCWaFtg1ORVfzJe4S7xwSh9aHawdEb7FwWTJYiAhzIvWp7DM+6BajnzYOwZ1BUrqtow==}
'@types/jsonwebtoken@9.0.10':
resolution: {integrity: sha512-asx5hIG9Qmf/1oStypjanR7iKTv0gXQ1Ov/jfrX6kS/EO0OFni8orbmGCn0672NHR3kXHwpAwR+B368ZGN/2rA==}
@@ -2681,6 +2678,9 @@ packages:
'@vitest/utils@2.1.9':
resolution: {integrity: sha512-v0psaMSkNJ3A2NMrUEHFRzJtDPFn+/VWZ5WxImB21T9fjucJRmS7xCS3ppEnARb9y11OAzaD+P2Ps+b+BGX5iQ==}
'@zone-eu/mailsplit@5.4.15':
resolution: {integrity: sha512-c7ZpxauvF4AEkDJlKDYO7iMUtMuqJMBnDWNff1cyx+d7zaBVR3iFEmXhNHOVoMmVyVF3pTZLsLIJsEFKDldOAA==}
'@zone-eu/mailsplit@5.4.8':
resolution: {integrity: sha512-eEyACj4JZ7sjzRvy26QhLgKEMWwQbsw1+QZnlLX+/gihcNH07lVPOcnwf5U6UAL7gkc//J3jVd76o/WS+taUiA==}
@@ -2744,6 +2744,10 @@ packages:
asynckit@0.4.0:
resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==}
atomic-sleep@1.0.0:
resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==}
engines: {node: '>=8.0.0'}
autoprefixer@10.4.21:
resolution: {integrity: sha512-O+A6LWV5LDHSJD3LjHYoNi4VLsj/Whi7k6zG12xTYaU4cQ8oxQGckXNX8cRHK5yOZ/ppVHe0ZBXGzSV9jXdVbQ==}
engines: {node: ^10 || ^12 || >=14}
@@ -3542,12 +3546,15 @@ packages:
resolution: {integrity: sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==}
engines: {node: '>=0.10.0'}
iconv-lite@0.7.3:
resolution: {integrity: sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==}
engines: {node: '>=0.10.0'}
ieee754@1.2.1:
resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}
imap@0.8.19:
resolution: {integrity: sha512-z5DxEA1uRnZG73UcPA4ES5NSCGnPuuouUx43OPX7KZx1yzq3N8/vx2mtXEShT5inxB3pRgnfG1hijfu7XN2YMw==}
engines: {node: '>=0.8.0'}
imapflow@1.7.2:
resolution: {integrity: sha512-1pWZgWQ/M2Q7kPSW7Sp7QDn+ZPEqs/9IymYh34RY+3J7d3vfPayhSmRAl0tB7weblGU0SR/t7eYES3TW6vSiOQ==}
inherits@2.0.4:
resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==}
@@ -3565,6 +3572,10 @@ packages:
iobuffer@5.4.0:
resolution: {integrity: sha512-DRebOWuqDvxunfkNJAlc3IzWIPD5xVxwUNbHr7xKB8E6aLJxIPfNX3CoMJghcFjpv6RWQsrcJbghtEwSPoJqMA==}
ip-address@10.5.0:
resolution: {integrity: sha512-R5SnVLJmgYYvf2F2ZgwSBnelz5G4q5AxIC277GDfUaNbrZKNANcBC7RHqYYePlszf4kBolVkJauG0ZjHHFh55g==}
engines: {node: '>= 12'}
ipaddr.js@1.9.1:
resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==}
engines: {node: '>= 0.10'}
@@ -3598,9 +3609,6 @@ packages:
resolution: {integrity: sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw==}
engines: {node: '>=16'}
isarray@0.0.1:
resolution: {integrity: sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==}
isarray@1.0.0:
resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==}
@@ -3661,6 +3669,9 @@ packages:
libmime@5.3.7:
resolution: {integrity: sha512-FlDb3Wtha8P01kTL3P9M+ZDNDWPKPmKHWaU/cG/lg5pfuAwdflVpZE+wm9m7pKmC5ww6s+zTxBKS1p6yl3KpSw==}
libmime@5.4.2:
resolution: {integrity: sha512-+IQnCOdPiufGBkOii+Ze8F7iniyBzOwvWDbn1DyExBpc9pT2B3IEMQi7GUc/PpqhNUh/sr1SG9UXDITQoR0VIA==}
libqp@2.1.1:
resolution: {integrity: sha512-0Wd+GPz1O134cP62YU2GTOPNA7Qgl09XwCqM5zpBv87ERCXdfDtyKXvV7c9U22yWJh44QZqBocFnXN11K96qow==}
@@ -3931,6 +3942,10 @@ packages:
resolution: {integrity: sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==}
engines: {node: '>= 0.4'}
on-exit-leak-free@2.1.2:
resolution: {integrity: sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==}
engines: {node: '>=14.0.0'}
on-finished@2.4.1:
resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==}
engines: {node: '>= 0.8'}
@@ -4008,6 +4023,16 @@ packages:
resolution: {integrity: sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==}
engines: {node: '>=12'}
pino-abstract-transport@3.0.0:
resolution: {integrity: sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==}
pino-std-serializers@7.1.0:
resolution: {integrity: sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==}
pino@10.3.1:
resolution: {integrity: sha512-r34yH/GlQpKZbU1BvFFqOjhISRo1MNx1tWYsYvmj6KIRHSPMT2+yHOEb1SG6NMvRoHRF0a07kCOox/9yakl1vg==}
hasBin: true
pnpm@10.18.0:
resolution: {integrity: sha512-6AT4ifHOzEDVctsITuw+SIFzn43sacD/ENLRvv+aTjCTg7ontbdQBZ1/TBSVNbbNDSyx7Trrc5I5pChKaPQM+g==}
engines: {node: '>=18.12'}
@@ -4032,6 +4057,9 @@ packages:
process-nextick-args@2.0.1:
resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==}
process-warning@5.1.0:
resolution: {integrity: sha512-jQSaVHsPgtyw60e1rQ/A+/ArPEj/S8pS/vFnyGa/gYFXrKk/6RuDkoqVDQ5NI5MmS01698ltlAk0NoDBNLujRw==}
process@0.11.10:
resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==}
engines: {node: '>= 0.6.0'}
@@ -4054,6 +4082,9 @@ packages:
resolution: {integrity: sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==}
engines: {node: '>=0.6'}
quick-format-unescaped@4.0.4:
resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==}
raf@3.4.1:
resolution: {integrity: sha512-Sq4CW4QhwOHE8ucn6J34MqtZCeWFP2aQSmrlroYgqAV1PjStIhJXxYuTgUIfkEk7zTLjmIjLmU5q+fbD1NnOJA==}
@@ -4154,9 +4185,6 @@ packages:
resolution: {integrity: sha512-DGrYcCWK7tvYMnWh79yrPHt+vdx9tY+1gPZa7nJQtO/p8bLTDaHp4dzwEhQB7pZ4Xe3ok4XKuEPrVuc+wlpkmw==}
engines: {node: '>=0.10.0'}
readable-stream@1.1.14:
resolution: {integrity: sha512-+MeVjFf4L44XUkhM1eYbD8fyEsxcV81pqMSR5gblfcLCHfZvbrqy4/qYHE+/R5HoBUT11WV5O08Cr1n3YXkWVQ==}
readable-stream@2.3.8:
resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==}
@@ -4175,6 +4203,13 @@ packages:
resolution: {integrity: sha512-9u/XQ1pvrQtYyMpZe7DXKv2p5CNvyVwzUB6uhLAnQwHMSgKMBR62lc7AHljaeteeHXn11XTAaLLUVZYVZyuRBQ==}
engines: {node: '>= 20.19.0'}
real-require@0.2.0:
resolution: {integrity: sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==}
engines: {node: '>= 12.13.0'}
real-require@1.0.0:
resolution: {integrity: sha512-P4nbQYQfePJxRSmY+v/KINxVucm4NF3p3s7pJveMTtom52FR4YGltUQLB8idDXwDDWW+eYrWDFbuzUnjoWHF7g==}
recharts-scale@0.4.5:
resolution: {integrity: sha512-kivNFO+0OcUNu7jQquLXAxz1FIwZj8nrj+YkOKc5694NbjCvcT6aSZiIzNzd2Kul4o4rTto8QVR9lMNtxD4G1w==}
@@ -4214,6 +4249,10 @@ packages:
safe-buffer@5.2.1:
resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==}
safe-stable-stringify@2.5.0:
resolution: {integrity: sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==}
engines: {node: '>=10'}
safer-buffer@2.1.2:
resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==}
@@ -4223,10 +4262,6 @@ packages:
selderee@0.11.0:
resolution: {integrity: sha512-5TF+l7p4+OsnP8BCCvSyZiSPc4x4//p5uPwK8TCnVPJYRmU2aYKMpOXvw8zM5a5JvuuCGN1jmsMwuU2W02ukfA==}
semver@5.3.0:
resolution: {integrity: sha512-mfmm3/H9+67MCVix1h+IXTpDwL6710LyHuk7+cWC9T1mE0qz4iHhh6r4hU2wrIT9iTsAAC2XQRvfblL028cpLw==}
hasBin: true
semver@6.3.1:
resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==}
hasBin: true
@@ -4285,6 +4320,17 @@ packages:
resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==}
engines: {node: '>=14'}
smart-buffer@4.2.0:
resolution: {integrity: sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==}
engines: {node: '>= 6.0.0', npm: '>= 3.0.0'}
socks@2.8.9:
resolution: {integrity: sha512-LJhUYUvItdQ0LkJTmPeaEObWXAqFyfmP85x0tch/ez9cahmhlBBLbIqDFnvBnUJGagb0JbIQrkBs1wJ+yRYpEw==}
engines: {node: '>= 10.0.0', npm: '>= 3.0.0'}
sonic-boom@4.2.1:
resolution: {integrity: sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==}
sonner@2.0.7:
resolution: {integrity: sha512-W6ZN4p58k8aDKA4XPcx2hpIQXBRAgyiWVkYhT7CvK6D3iAu7xjvVyhQHg2/iaKJZ1XVJ4r7XuwGL+WGEK37i9w==}
peerDependencies:
@@ -4302,6 +4348,10 @@ packages:
resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==}
engines: {node: '>=0.10.0'}
split2@4.2.0:
resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==}
engines: {node: '>= 10.x'}
sqlstring@2.3.3:
resolution: {integrity: sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg==}
engines: {node: '>= 0.6'}
@@ -4343,9 +4393,6 @@ packages:
resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==}
engines: {node: '>=12'}
string_decoder@0.10.31:
resolution: {integrity: sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ==}
string_decoder@1.1.1:
resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==}
@@ -4402,6 +4449,10 @@ packages:
text-segmentation@1.0.3:
resolution: {integrity: sha512-iOiPUo/BGnZ6+54OsWxZidGCsdU8YbE4PSpdPinp7DeMtUJNJBoJ/ouUSTJjHkh1KntHaltHl/gDs2FC4i5+Nw==}
thread-stream@4.2.0:
resolution: {integrity: sha512-e2zZ96wSChazBsbENf/Pcm/4swHt2cEKQ92rhUjkL9GCKiTDJIaTBenjE/m9DXi0QBmTMDkFDdOomUy20A1tDQ==}
engines: {node: '>=20'}
tiny-invariant@1.3.3:
resolution: {integrity: sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==}
@@ -4508,9 +4559,6 @@ packages:
peerDependencies:
react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0
utf7@1.0.2:
resolution: {integrity: sha512-qQrPtYLLLl12NF4DrM9CvfkxkYI97xOb5dsnGZHE3teFr0tWiEZ9UdgMPczv24vl708cYMpe6mGXGHrotIp3Bw==}
util-deprecate@1.0.2:
resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==}
@@ -5883,6 +5931,8 @@ snapshots:
dependencies:
pako: 1.0.11
'@pinojs/redact@0.4.0': {}
'@pkgjs/parseargs@0.11.0':
optional: true
@@ -7136,10 +7186,6 @@ snapshots:
'@types/http-errors@2.0.5': {}
'@types/imap@0.8.43':
dependencies:
'@types/node': 24.7.0
'@types/jsonwebtoken@9.0.10':
dependencies:
'@types/ms': 2.1.0
@@ -7269,6 +7315,12 @@ snapshots:
loupe: 3.2.1
tinyrainbow: 1.2.0
'@zone-eu/mailsplit@5.4.15':
dependencies:
libbase64: 1.3.0
libmime: 5.4.2
libqp: 2.1.1
'@zone-eu/mailsplit@5.4.8':
dependencies:
libbase64: 1.3.0
@@ -7338,6 +7390,8 @@ snapshots:
asynckit@0.4.0: {}
atomic-sleep@1.0.0: {}
autoprefixer@10.4.21(postcss@8.5.6):
dependencies:
browserslist: 4.26.3
@@ -8118,12 +8172,22 @@ snapshots:
dependencies:
safer-buffer: 2.1.2
iconv-lite@0.7.3:
dependencies:
safer-buffer: 2.1.2
ieee754@1.2.1: {}
imap@0.8.19:
imapflow@1.7.2:
dependencies:
readable-stream: 1.1.14
utf7: 1.0.2
'@zone-eu/mailsplit': 5.4.15
encoding-japanese: 2.2.0
iconv-lite: 0.7.3
libbase64: 1.3.0
libmime: 5.4.2
libqp: 2.1.1
pino: 10.3.1
socks: 2.8.9
inherits@2.0.4: {}
@@ -8136,6 +8200,8 @@ snapshots:
iobuffer@5.4.0: {}
ip-address@10.5.0: {}
ipaddr.js@1.9.1: {}
is-docker@3.0.0: {}
@@ -8156,8 +8222,6 @@ snapshots:
dependencies:
is-inside-container: 1.0.0
isarray@0.0.1: {}
isarray@1.0.0: {}
isexe@2.0.0: {}
@@ -8232,6 +8296,13 @@ snapshots:
libbase64: 1.3.0
libqp: 2.1.1
libmime@5.4.2:
dependencies:
encoding-japanese: 2.2.0
iconv-lite: 0.7.3
libbase64: 1.3.0
libqp: 2.1.1
libqp@2.1.1: {}
lightningcss-darwin-arm64@1.30.1:
@@ -8439,6 +8510,8 @@ snapshots:
object-inspect@1.13.4: {}
on-exit-leak-free@2.1.2: {}
on-finished@2.4.1:
dependencies:
ee-first: 1.1.1
@@ -8508,6 +8581,26 @@ snapshots:
picomatch@4.0.3: {}
pino-abstract-transport@3.0.0:
dependencies:
split2: 4.2.0
pino-std-serializers@7.1.0: {}
pino@10.3.1:
dependencies:
'@pinojs/redact': 0.4.0
atomic-sleep: 1.0.0
on-exit-leak-free: 2.1.2
pino-abstract-transport: 3.0.0
pino-std-serializers: 7.1.0
process-warning: 5.1.0
quick-format-unescaped: 4.0.4
real-require: 0.2.0
safe-stable-stringify: 2.5.0
sonic-boom: 4.2.1
thread-stream: 4.2.0
pnpm@10.18.0: {}
postcss-selector-parser@6.0.10:
@@ -8527,6 +8620,8 @@ snapshots:
process-nextick-args@2.0.1: {}
process-warning@5.1.0: {}
process@0.11.10: {}
prop-types@15.8.1:
@@ -8548,6 +8643,8 @@ snapshots:
dependencies:
side-channel: 1.1.0
quick-format-unescaped@4.0.4: {}
raf@3.4.1:
dependencies:
performance-now: 2.1.0
@@ -8650,13 +8747,6 @@ snapshots:
react@19.2.1: {}
readable-stream@1.1.14:
dependencies:
core-util-is: 1.0.3
inherits: 2.0.4
isarray: 0.0.1
string_decoder: 0.10.31
readable-stream@2.3.8:
dependencies:
core-util-is: 1.0.3
@@ -8687,6 +8777,10 @@ snapshots:
readdirp@5.0.0: {}
real-require@0.2.0: {}
real-require@1.0.0: {}
recharts-scale@0.4.5:
dependencies:
decimal.js-light: 2.5.1
@@ -8748,6 +8842,8 @@ snapshots:
safe-buffer@5.2.1: {}
safe-stable-stringify@2.5.0: {}
safer-buffer@2.1.2: {}
scheduler@0.27.0: {}
@@ -8756,8 +8852,6 @@ snapshots:
dependencies:
parseley: 0.12.1
semver@5.3.0: {}
semver@6.3.1: {}
semver@7.7.3: {}
@@ -8862,6 +8956,17 @@ snapshots:
signal-exit@4.1.0: {}
smart-buffer@4.2.0: {}
socks@2.8.9:
dependencies:
ip-address: 10.5.0
smart-buffer: 4.2.0
sonic-boom@4.2.1:
dependencies:
atomic-sleep: 1.0.0
sonner@2.0.7(react-dom@19.2.1(react@19.2.1))(react@19.2.1):
dependencies:
react: 19.2.1
@@ -8876,6 +8981,8 @@ snapshots:
source-map@0.6.1: {}
split2@4.2.0: {}
sqlstring@2.3.3: {}
ssf@0.11.2:
@@ -8925,8 +9032,6 @@ snapshots:
emoji-regex: 9.2.2
strip-ansi: 7.2.0
string_decoder@0.10.31: {}
string_decoder@1.1.1:
dependencies:
safe-buffer: 5.1.2
@@ -8999,6 +9104,10 @@ snapshots:
utrie: 1.0.2
optional: true
thread-stream@4.2.0:
dependencies:
real-require: 1.0.0
tiny-invariant@1.3.3: {}
tinybench@2.9.0: {}
@@ -9077,10 +9186,6 @@ snapshots:
dependencies:
react: 19.2.1
utf7@1.0.2:
dependencies:
semver: 5.3.0
util-deprecate@1.0.2: {}
utils-merge@1.0.1: {}

View File

@@ -0,0 +1,49 @@
import { ImapFlow } from "imapflow";
const required = (name) => {
const value = process.env[name];
if (!value) throw new Error(`Variable ${name} manquante`);
return value;
};
const tenantId = required("AZURE_AD_TENANT_ID");
const clientId = required("AZURE_AD_CLIENT_ID");
const clientSecret = required("AZURE_AD_CLIENT_SECRET");
const email = required("IMAP_TEST_EMAIL");
const host = process.env.IMAP_TEST_HOST || "outlook.office365.com";
const response = await fetch(
`https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`,
{
method: "POST",
headers: { "Content-Type": "application/x-www-form-urlencoded" },
body: new URLSearchParams({
client_id: clientId,
client_secret: clientSecret,
scope: "https://outlook.office365.com/.default",
grant_type: "client_credentials",
}),
},
);
const tokenResponse = await response.json();
if (!response.ok || !tokenResponse.access_token) {
throw new Error(`Échec OAuth2 : ${tokenResponse.error || response.status}`);
}
const client = new ImapFlow({
host,
port: 993,
secure: true,
auth: { user: email, accessToken: tokenResponse.access_token },
tls: { servername: host, rejectUnauthorized: true },
verifyOnly: true,
logger: false,
});
try {
await client.connect();
console.log(`Authentification ImapFlow OAuth2 réussie pour ${email}`);
} finally {
if (client.usable) await client.logout().catch(() => client.close());
else client.close();
}

View File

@@ -17,7 +17,8 @@ import { startEmailImportService } from "../emailImportService";
import { startFolderImportService } from "../folderImportService";
import { handleAzureCallback, isAzureAdConfigured, generateToken, verifyToken } from "../auth";
import { createDatabaseBackup } from "../databaseBackup";
import { generateStorageKey, localStoragePut } from "../localStorage";
import { generateStorageKey, localStorageDelete, localStoragePut } from "../localStorage";
import { calculateFileSha256 } from "../fileFingerprint";
const MAX_WEB_IMPORT_BYTES = 20 * 1024 * 1024;
@@ -305,7 +306,7 @@ async function startServer() {
return;
}
const { getWebImportSourceByToken, createInvoice, findDuplicateInvoice, isInvoiceBlacklisted, updateWebImportSourceStatus, createSourceFile } = await import('../db');
const { getWebImportSourceByToken, createInvoice, findDuplicateInvoice, isInvoiceBlacklisted, updateWebImportSourceStatus, createSourceFile, getSourceFileByContentHash } = await import('../db');
const source = await getWebImportSourceByToken(apiToken);
if (!source) {
res.status(401).json({ error: "Token invalide" });
@@ -317,15 +318,35 @@ async function startServer() {
return;
}
const contentHash = calculateFileSha256(pdfBuffer);
const existingSource = await getSourceFileByContentHash(contentHash);
if (existingSource) {
await updateWebImportSourceStatus(source.id, "success", 0, true);
res.json({ success: true, imported: 0, duplicates: 1, total: 1 });
return;
}
// Stocker d'abord le PDF de façon persistante, comme les autres sources d'import.
const storageKey = generateStorageKey(source.userId, safeFileName);
const { url: fileUrl } = await localStoragePut(storageKey, pdfBuffer, "application/pdf");
const sourceFile = await createSourceFile({
userId: source.userId,
fileName: safeFileName,
fileKey: storageKey,
fileUrl,
});
let sourceFile;
try {
sourceFile = await createSourceFile({
userId: source.userId,
fileName: safeFileName,
fileKey: storageKey,
fileUrl,
contentHash,
});
} catch (error: any) {
if (error?.code === "ER_DUP_ENTRY" || error?.errno === 1062) {
await localStorageDelete(storageKey).catch(() => undefined);
await updateWebImportSourceStatus(source.id, "success", 0, true);
res.json({ success: true, imported: 0, duplicates: 1, total: 1 });
return;
}
throw error;
}
const userSettings = await getUserSettings(source.userId);
const aiSettings = {
aiProvider: userSettings?.aiProvider || "manus",

View File

@@ -204,6 +204,22 @@ export async function createSourceFile(data: InsertSourceFile): Promise<SourceFi
return inserted[0]!;
}
/**
* Recherche un PDF déjà ingéré, quel que soit le compte utilisateur.
* L'import email est partagé entre plusieurs identités : la détection doit donc
* être globale pour éviter qu'une même pièce soit retraitée sous chaque compte.
*/
export async function getSourceFileByContentHash(contentHash: string): Promise<SourceFile | undefined> {
const db = await getDb();
if (!db) return undefined;
const result = await db
.select()
.from(sourceFiles)
.where(eq(sourceFiles.contentHash, contentHash))
.limit(1);
return result[0];
}
export async function getSourceFileById(id: number): Promise<SourceFile | undefined> {
const db = await getDb();
if (!db) return undefined;
@@ -1218,20 +1234,3 @@ export async function updateWebImportSourceStatus(
if (success) update.lastSuccessAt = new Date();
await db.update(webImportSources).set(update).where(eq(webImportSources.id, id));
}
/**
* Check if a source file with the same fileName already exists for this user
* Used to prevent duplicate file storage during email import
*/
export async function findSourceFileByFileName(userId: number, fileName: string): Promise<any | null> {
const db = await getDb();
if (!db) return null;
const result = await db.select()
.from(sourceFiles)
.where(and(
eq(sourceFiles.userId, userId),
eq(sourceFiles.fileName, fileName)
))
.limit(1);
return result[0] || null;
}

View File

@@ -0,0 +1,38 @@
import { describe, expect, it } from "vitest";
import { createImapFlowOptions, type EmailImportConfig } from "./emailImportService";
const baseConfig: EmailImportConfig = {
userId: 2,
emailAddress: "compta@example.org",
password: "secret",
host: "outlook.office365.com",
port: 993,
};
describe("createImapFlowOptions", () => {
it("transmet le jeton brut à ImapFlow pour une authentification OAuth2", () => {
const options = createImapFlowOptions(
{ ...baseConfig, authMode: "oauth2" },
"access-token-value",
);
expect(options.secure).toBe(true);
expect(options.auth).toEqual({
user: "compta@example.org",
accessToken: "access-token-value",
});
expect(options.auth).not.toHaveProperty("pass");
expect(options.tls?.rejectUnauthorized).toBe(true);
expect(options.disableAutoIdle).toBe(true);
});
it("conserve le mot de passe uniquement pour le mode basique", () => {
const options = createImapFlowOptions({ ...baseConfig, authMode: "basic" });
expect(options.auth).toEqual({
user: "compta@example.org",
pass: "secret",
});
expect(options.auth).not.toHaveProperty("accessToken");
});
});

View File

@@ -1,23 +1,24 @@
import Imap from "imap";
import { ImapFlow, type ImapFlowOptions, type SearchObject } from "imapflow";
import { simpleParser, ParsedMail, Attachment } from "mailparser";
import {
getImportSettingsByUser,
createSourceFile,
updateSourceFile,
getUserSettings,
getSourceFileByContentHash,
findDuplicateInvoice,
isInvoiceBlacklisted,
createInvoice,
createImportLog,
findSourceFileByFileName,
} from "./db";
import { extractInvoicesWithMistral, generateMetadataJSON } from "./invoiceExtractor";
import { localStoragePut, generateStorageKey } from "./localStorage";
import { localStorageDelete, localStoragePut, generateStorageKey } from "./localStorage";
import { calculateFileSha256 } from "./fileFingerprint";
import { sendImportNotification } from "./notificationService";
import { getOffice365ImapToken, buildXOAuth2String } from "./office365OAuth";
import { getOffice365ImapToken } from "./office365OAuth";
import { applyAutomationRules } from "./automationEngine";
interface EmailImportConfig {
export interface EmailImportConfig {
userId: number;
emailAddress: string;
password: string;
@@ -31,10 +32,54 @@ interface EmailImportConfig {
azureClientSecret?: string;
}
/**
* Construit les options ImapFlow sans effectuer d'appel réseau.
* ImapFlow reçoit le jeton brut et construit lui-même SASL XOAUTH2.
*/
export function createImapFlowOptions(
config: EmailImportConfig,
accessToken?: string,
): ImapFlowOptions {
const auth = config.authMode === "oauth2"
? { user: config.emailAddress, accessToken }
: { user: config.emailAddress, pass: config.password };
return {
host: config.host,
port: config.port,
secure: true,
auth,
tls: {
servername: config.host,
rejectUnauthorized: true,
},
logger: false,
disableAutoIdle: true,
connectionTimeout: 30_000,
greetingTimeout: 20_000,
socketTimeout: 120_000,
};
}
// Store active intervals for each user
const activeIntervals = new Map<number, NodeJS.Timeout>();
// Verrou anti-concurrence par userId
const runningChecks = new Set<number>();
// Une extraction IA peut dépasser la fréquence configurée : ce verrou évite
// qu'un second cycle IMAP traite les mêmes messages avant la fin du premier.
const activeChecks = new Map<number, Promise<void>>();
function runEmailCheckExclusive(config: EmailImportConfig): Promise<void> {
const runningCheck = activeChecks.get(config.userId);
if (runningCheck) {
console.log(`[EmailImport] Vérification déjà en cours pour user ${config.userId}, cycle ignoré`);
return runningCheck;
}
const check = checkEmailsForPDFs(config).finally(() => {
if (activeChecks.get(config.userId) === check) activeChecks.delete(config.userId);
});
activeChecks.set(config.userId, check);
return check;
}
/**
* Process a single email attachment (PDF)
@@ -53,13 +98,22 @@ async function processEmailAttachment(
const fileBuffer = attachment.content;
console.log(`[EmailImport] File size: ${fileBuffer.length} bytes`);
// Store source file
// ANTI-DUPLICATION : vérifier si ce fichier a déjà été importé pour cet utilisateur
const existingSourceFile = await findSourceFileByFileName(userId, fileName);
if (existingSourceFile) {
console.log(`[EmailImport] File ${fileName} already imported for user ${userId} (sourceFile #${existingSourceFile.id}), skipping`);
return { success: true, totalInvoices: 0, imported: 0, duplicates: 1, errors: 0 };
const contentHash = calculateFileSha256(fileBuffer);
const existingSource = await getSourceFileByContentHash(contentHash);
if (existingSource) {
console.log(
`[EmailImport] PDF déjà importé, extraction ignorée: ${fileName} -> source ${existingSource.id}`,
);
return {
success: true,
totalInvoices: Math.max(existingSource.totalInvoicesDetected, 1),
imported: 0,
duplicates: Math.max(existingSource.totalInvoicesDetected, 1),
errors: 0,
};
}
// Store source file
const sourceFileKey = generateStorageKey(userId, fileName);
console.log(`[EmailImport] Generated storage key: ${sourceFileKey}`);
@@ -74,13 +128,25 @@ async function processEmailAttachment(
}
// Create source file record
const sourceFile = await createSourceFile({
userId,
fileName,
fileKey: sourceFileKey,
fileUrl: sourceFileUrl,
processingStatus: "processing",
});
let sourceFile;
try {
sourceFile = await createSourceFile({
userId,
fileName,
fileKey: sourceFileKey,
fileUrl: sourceFileUrl,
contentHash,
processingStatus: "processing",
});
} catch (error: any) {
// La contrainte unique protège également contre deux imports concurrents.
if (error?.code === "ER_DUP_ENTRY" || error?.errno === 1062) {
await localStorageDelete(sourceFileKey).catch(() => undefined);
console.log(`[EmailImport] PDF réservé par un autre traitement: ${fileName}`);
return { success: true, totalInvoices: 1, imported: 0, duplicates: 1, errors: 0 };
}
throw error;
}
console.log(`[EmailImport] Source file record created with ID: ${sourceFile.id}`);
@@ -295,7 +361,7 @@ async function processEmailAttachment(
* - basic : login/password classique
* - oauth2 : obtient un token Azure AD et utilise XOAUTH2
*/
async function buildImapConfig(config: EmailImportConfig): Promise<Imap.Config> {
async function buildImapConfig(config: EmailImportConfig): Promise<ImapFlowOptions> {
if (config.authMode === "oauth2") {
if (!config.azureTenantId || !config.azureClientId || !config.azureClientSecret) {
throw new Error(
@@ -309,198 +375,115 @@ async function buildImapConfig(config: EmailImportConfig): Promise<Imap.Config>
config.azureClientId,
config.azureClientSecret
);
const xoauth2 = buildXOAuth2String(config.emailAddress, accessToken);
console.log(`[EmailImport] OAuth2 token obtained successfully`);
return {
user: config.emailAddress,
xoauth2,
host: config.host,
port: config.port,
tls: true,
tlsOptions: { rejectUnauthorized: false },
authTimeout: 30000,
} as any;
return createImapFlowOptions(config, accessToken);
}
// Basic auth (par défaut)
return {
user: config.emailAddress,
password: config.password,
host: config.host,
port: config.port,
tls: true,
tlsOptions: { rejectUnauthorized: false },
authTimeout: 30000,
};
return createImapFlowOptions(config);
}
/**
* Connect to IMAP and process unread emails with PDF attachments
*/
async function checkEmailsForPDFs(config: EmailImportConfig): Promise<void> {
// Anti-concurrence : ne pas lancer si un check est déjà en cours pour cet utilisateur
if (runningChecks.has(config.userId)) {
console.log(`[EmailImport] Check already running for user ${config.userId}, skipping`);
return;
}
runningChecks.add(config.userId);
// Build IMAP config (may involve async OAuth2 token fetch)
const imapConfig = await buildImapConfig(config);
const client = new ImapFlow(imapConfig);
client.on("error", (error) => {
console.error(`[EmailImport] IMAP connection error for user ${config.userId}:`, error);
});
return new Promise((resolve, reject) => {
const imap = new Imap(imapConfig);
let lock: Awaited<ReturnType<ImapFlow["getMailboxLock"]>> | undefined;
try {
await client.connect();
console.log(
`[EmailImport] Connected to IMAP server for user ${config.userId} (mode: ${config.authMode || "basic"})`,
);
function openInbox(cb: (err: Error | null, box?: any) => void) {
imap.openBox("INBOX", false, cb);
lock = await client.getMailboxLock("INBOX", {
readOnly: false,
acquireTimeout: 30_000,
description: `invoice-import-user-${config.userId}`,
});
const searchCriteria: SearchObject = { seen: false };
if (config.sinceDate) {
searchCriteria.since = new Date(config.sinceDate * 1000);
console.log(
`[EmailImport] Filtering emails since ${searchCriteria.since.toISOString()} for user ${config.userId}`,
);
}
imap.once("ready", () => {
console.log(`[EmailImport] Connected to IMAP server for user ${config.userId} (mode: ${config.authMode || "basic"})`);
const unreadUids = await client.search(searchCriteria, { uid: true });
if (!unreadUids || unreadUids.length === 0) {
console.log(`[EmailImport] No unread emails found for user ${config.userId}`);
return;
}
openInbox((err) => {
if (err) {
console.error("[EmailImport] Error opening inbox:", err);
imap.end();
reject(err);
return;
}
console.log(`[EmailImport] Found ${unreadUids.length} unread emails for user ${config.userId}`);
// Build search criteria: unread emails, optionally filtered by date
const searchCriteria: any[] = ["UNSEEN"];
if (config.sinceDate) {
// IMAP SINCE expects a date string like "1-Jan-2026"
const since = new Date(config.sinceDate * 1000);
const months = ["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"];
const sinceStr = `${since.getDate()}-${months[since.getMonth()]}-${since.getFullYear()}`;
searchCriteria.push(["SINCE", sinceStr]);
console.log(`[EmailImport] Filtering emails since ${sinceStr} for user ${config.userId}`);
}
imap.search(searchCriteria, (err, results) => {
if (err) {
console.error("[EmailImport] Error searching emails:", err);
imap.end();
reject(err);
return;
}
// Le traitement reste séquentiel afin d'éviter plusieurs extractions IA
// concurrentes sur les mêmes pièces jointes.
for (const uid of unreadUids) {
const message = await client.fetchOne(uid, { source: true }, { uid: true });
if (!message || !message.source) {
console.warn(`[EmailImport] Message UID ${uid} without source, skipped`);
continue;
}
if (!results || results.length === 0) {
console.log(`[EmailImport] No unread emails found for user ${config.userId}`);
imap.end();
resolve();
return;
}
try {
const parsed: ParsedMail = await simpleParser(message.source);
const pdfAttachments = parsed.attachments.filter(
(attachment) =>
attachment.contentType === "application/pdf" ||
attachment.filename?.toLowerCase().endsWith(".pdf"),
);
console.log(`[EmailImport] Found ${results.length} unread emails for user ${config.userId}`);
if (pdfAttachments.length === 0) continue;
const fetch = imap.fetch(results, {
bodies: "",
markSeen: true, // Mark as seen immediately to prevent re-processing
});
console.log(`[EmailImport] Email UID ${uid} has ${pdfAttachments.length} PDF attachment(s)`);
let allAttachmentsSucceeded = true;
const processedEmails: number[] = [];
for (const attachment of pdfAttachments) {
try {
const result = await processEmailAttachment(
config.userId,
attachment,
parsed.subject || "No subject",
);
allAttachmentsSucceeded = allAttachmentsSucceeded && result.success;
fetch.on("message", (msg, seqno) => {
msg.on("body", (stream) => {
simpleParser(stream as any, async (err, parsed: ParsedMail) => {
if (err) {
console.error("[EmailImport] Error parsing email:", err);
return;
}
// Check if email has PDF attachments
const pdfAttachments = parsed.attachments.filter(
(att) =>
att.contentType === "application/pdf" ||
att.filename?.toLowerCase().endsWith(".pdf")
);
if (pdfAttachments.length === 0) {
return;
}
console.log(
`[EmailImport] Email ${seqno} has ${pdfAttachments.length} PDF attachment(s)`
);
// Process each PDF attachment
for (const attachment of pdfAttachments) {
try {
const result = await processEmailAttachment(
config.userId,
attachment,
parsed.subject || "No subject"
);
// Mark this email as successfully processed
if (!processedEmails.includes(seqno)) {
processedEmails.push(seqno);
}
// Send notification after successful processing
if (result.success) {
await sendImportNotification(config.userId, {
source: "email",
fileName: attachment.filename || "email-attachment.pdf",
totalInvoices: result.totalInvoices,
imported: result.imported,
duplicates: result.duplicates,
errors: result.errors,
});
}
} catch (error) {
console.error(
`[EmailImport] Failed to process attachment from email ${seqno}:`,
error
);
}
}
if (result.success) {
await sendImportNotification(config.userId, {
source: "email",
fileName: attachment.filename || "email-attachment.pdf",
totalInvoices: result.totalInvoices,
imported: result.imported,
duplicates: result.duplicates,
errors: result.errors,
});
});
});
fetch.once("error", (err) => {
console.error("[EmailImport] Fetch error:", err);
imap.end();
reject(err);
});
fetch.once("end", () => {
console.log(`[EmailImport] Finished fetching emails for user ${config.userId}`);
// Mark successfully processed emails as seen
if (processedEmails.length > 0) {
imap.addFlags(processedEmails, ["\\Seen"], (err) => {
if (err) {
console.error("[EmailImport] Error marking emails as seen:", err);
} else {
console.log(`[EmailImport] Marked ${processedEmails.length} emails as seen`);
}
imap.end();
resolve();
});
} else {
imap.end();
resolve();
}
});
});
});
});
} catch (error) {
allAttachmentsSucceeded = false;
console.error(`[EmailImport] Failed to process attachment from UID ${uid}:`, error);
}
}
imap.once("error", (err) => {
runningChecks.delete(config.userId);
console.error("[EmailImport] IMAP connection error:", err);
reject(err);
});
if (allAttachmentsSucceeded) {
await client.messageFlagsAdd(uid, ["\\Seen"], { uid: true, silent: true });
console.log(`[EmailImport] Marked email UID ${uid} as seen`);
}
} catch (error) {
console.error(`[EmailImport] Error parsing email UID ${uid}:`, error);
}
}
imap.once("end", () => {
runningChecks.delete(config.userId);
console.log(`[EmailImport] IMAP connection ended for user ${config.userId}`);
});
imap.connect();
});
console.log(`[EmailImport] Finished processing emails for user ${config.userId}`);
} finally {
lock?.release();
if (client.usable) await client.logout().catch(() => client.close());
else client.close();
}
}
/**
@@ -508,57 +491,34 @@ async function checkEmailsForPDFs(config: EmailImportConfig): Promise<void> {
* Returns detailed error message if connection fails
*/
export async function testImapConnection(config: EmailImportConfig): Promise<{ success: boolean; message: string }> {
let client: ImapFlow | undefined;
try {
const imapConfig = await buildImapConfig(config);
return new Promise((resolve) => {
const imap = new Imap(imapConfig);
let resolved = false;
const done = (result: { success: boolean; message: string }) => {
if (!resolved) {
resolved = true;
try { imap.destroy(); } catch {}
resolve(result);
}
};
imap.once("ready", () => {
console.log(`[EmailImport] Test connection successful for ${config.emailAddress}`);
done({ success: true, message: `Connexion IMAP réussie pour ${config.emailAddress}` });
});
imap.once("error", (err: any) => {
console.error(`[EmailImport] Test connection failed:`, err);
let message = `Erreur de connexion IMAP : ${err.message || err}`;
// Messages d'erreur plus clairs
if (err.message?.includes("Invalid credentials") || err.message?.includes("AUTHENTICATE")) {
if (config.authMode === "oauth2") {
message = "Authentification OAuth2 refusée. Vérifiez que l'application Azure AD a bien la permission IMAP.AccessAsApp et que le consentement admin a été accordé.";
} else {
message = "Identifiants invalides. Pour Office 365, l'authentification basique est désactivée. Activez le mode OAuth2 et configurez les credentials Azure AD.";
}
} else if (err.message?.includes("ECONNREFUSED") || err.message?.includes("ENOTFOUND")) {
message = `Impossible de se connecter au serveur ${config.host}:${config.port}. Vérifiez l'adresse et le port IMAP.`;
} else if (err.message?.includes("certificate") || err.message?.includes("SSL")) {
message = `Erreur SSL/TLS lors de la connexion à ${config.host}. Vérifiez le port (993 pour SSL).`;
} else if (err.message?.includes("timeout") || err.message?.includes("Timeout")) {
message = `Timeout de connexion à ${config.host}:${config.port}. Vérifiez l'adresse du serveur IMAP.`;
}
done({ success: false, message });
});
// Timeout de sécurité
setTimeout(() => {
done({ success: false, message: `Timeout : impossible de se connecter à ${config.host}:${config.port} dans les 15 secondes.` });
}, 15000);
imap.connect();
});
client = new ImapFlow({ ...imapConfig, verifyOnly: true });
await client.connect();
console.log(`[EmailImport] Test connection successful for ${config.emailAddress}`);
return { success: true, message: `Connexion IMAP OAuth2 réussie pour ${config.emailAddress}` };
} catch (error: any) {
return { success: false, message: `Erreur : ${error.message || error}` };
console.error(`[EmailImport] Test connection failed:`, error);
const rawMessage = error?.response || error?.message || String(error);
let message = `Erreur de connexion IMAP : ${rawMessage}`;
if (/AUTHENTICATE|authentication|invalid credentials/i.test(rawMessage)) {
message = config.authMode === "oauth2"
? "Authentification OAuth2 refusée. Vérifiez IMAP.AccessAsApp, le consentement administrateur, le service principal Exchange et lautorisation de la boîte."
: "Identifiants invalides. Pour Microsoft 365, utilisez OAuth2 au lieu de lauthentification basique.";
} else if (/ECONNREFUSED|ENOTFOUND/i.test(rawMessage)) {
message = `Impossible de joindre ${config.host}:${config.port}. Vérifiez ladresse et le port IMAP.`;
} else if (/certificate|TLS|SSL/i.test(rawMessage)) {
message = `Erreur TLS lors de la connexion à ${config.host}. Vérifiez le certificat et le port 993.`;
} else if (/timeout/i.test(rawMessage)) {
message = `Timeout lors de la connexion à ${config.host}:${config.port}.`;
}
return { success: false, message };
} finally {
if (client?.usable) await client.logout().catch(() => client?.close());
else client?.close();
}
}
@@ -615,13 +575,13 @@ export async function startEmailImportService(userId: number): Promise<boolean>
);
// Run immediately on start
checkEmailsForPDFs(config).catch((error) => {
runEmailCheckExclusive(config).catch((error) => {
console.error(`[EmailImport] Error checking emails for user ${userId}:`, error);
});
// Set up interval for periodic checks
const interval = setInterval(() => {
checkEmailsForPDFs(config).catch((error) => {
runEmailCheckExclusive(config).catch((error) => {
console.error(`[EmailImport] Error checking emails for user ${userId}:`, error);
});
}, frequencyMs);
@@ -687,7 +647,7 @@ export async function triggerEmailCheck(userId: number): Promise<{ success: bool
};
console.log(`[EmailImport] Manual check triggered for user ${userId}`);
await checkEmailsForPDFs(config);
await runEmailCheckExclusive(config);
return { success: true, message: "Vérification terminée avec succès" };
} catch (error: any) {

View File

@@ -0,0 +1,19 @@
import { describe, expect, it } from "vitest";
import { calculateFileSha256 } from "./fileFingerprint";
describe("calculateFileSha256", () => {
it("retourne la même empreinte pour un contenu identique", () => {
const content = Buffer.from("facture-pdf");
expect(calculateFileSha256(content)).toBe(calculateFileSha256(Buffer.from(content)));
});
it("distingue deux contenus différents", () => {
expect(calculateFileSha256(Buffer.from("facture-a"))).not.toBe(
calculateFileSha256(Buffer.from("facture-b")),
);
});
it("produit une empreinte SHA-256 hexadécimale", () => {
expect(calculateFileSha256(Buffer.from("facture"))).toMatch(/^[a-f0-9]{64}$/);
});
});

12
server/fileFingerprint.ts Normal file
View File

@@ -0,0 +1,12 @@
import { createHash } from "node:crypto";
/**
* Calcule une empreinte déterministe sur les octets du document original.
*
* L'empreinte est calculée avant tout stockage ou traitement IA : deux imports
* du même PDF sont donc reconnus même si le nom du fichier ou l'utilisateur
* diffèrent.
*/
export function calculateFileSha256(buffer: Buffer): string {
return createHash("sha256").update(buffer).digest("hex");
}

View File

@@ -4,6 +4,7 @@ import path from "path";
import {
getImportSettingsByUser,
createSourceFile,
getSourceFileByContentHash,
updateSourceFile,
getUserSettings,
findDuplicateInvoice,
@@ -11,7 +12,8 @@ import {
createImportLog,
} from "./db";
import { extractInvoicesWithMistral, generateMetadataJSON } from "./invoiceExtractor";
import { localStoragePut, generateStorageKey } from "./localStorage";
import { localStorageDelete, localStoragePut, generateStorageKey } from "./localStorage";
import { calculateFileSha256 } from "./fileFingerprint";
import { sendImportNotification } from "./notificationService";
interface FolderImportConfig {
@@ -42,6 +44,13 @@ async function processFolderFile(
const fileBuffer = await fs.readFile(filePath);
console.log(`[FolderImport] File size: ${fileBuffer.length} bytes`);
const contentHash = calculateFileSha256(fileBuffer);
const existingSource = await getSourceFileByContentHash(contentHash);
if (existingSource) {
console.log(`[FolderImport] PDF déjà importé, fichier ignoré: ${fileName}`);
return { success: true, imported: 0, duplicates: 1, errors: 0 };
}
// Store source file
const sourceFileKey = generateStorageKey(userId, fileName);
console.log(`[FolderImport] Generated storage key: ${sourceFileKey}`);
@@ -57,13 +66,23 @@ async function processFolderFile(
}
// Create source file record
const sourceFile = await createSourceFile({
userId,
fileName,
fileKey: sourceFileKey,
fileUrl: sourceFileUrl,
processingStatus: "processing",
});
let sourceFile;
try {
sourceFile = await createSourceFile({
userId,
fileName,
fileKey: sourceFileKey,
fileUrl: sourceFileUrl,
contentHash,
processingStatus: "processing",
});
} catch (error: any) {
if (error?.code === "ER_DUP_ENTRY" || error?.errno === 1062) {
await localStorageDelete(sourceFileKey).catch(() => undefined);
return { success: true, imported: 0, duplicates: 1, errors: 0 };
}
throw error;
}
console.log(`[FolderImport] Source file record created with ID: ${sourceFile.id}`);

View File

@@ -24,6 +24,7 @@ import {
searchInvoices,
getInvoiceStats,
createSourceFile,
getSourceFileByContentHash,
getSourceFileById,
updateSourceFile,
getUserSettings,
@@ -87,6 +88,7 @@ import { loginLocal, hashPassword, getAzureAuthUrl, isAzureAdConfigured } from "
import fsSync from "fs";
import pathSync from "path";
import { extractInvoicesWithMistral, generateMetadataJSON } from "./invoiceExtractor";
import { calculateFileSha256 } from "./fileFingerprint";
import { localStoragePut, generateStorageKey } from "./localStorage";
import { testSftpConnection, exportInvoiceToSftp, getUserSftpConfig } from "./sftpExport";
import { drawBapCartouche } from "./bapCartouche";
@@ -182,6 +184,17 @@ export const appRouter = router({
const fileBuffer = Buffer.from(input.fileData, "base64");
console.log(`[Upload] Received file: ${input.fileName}, size: ${fileBuffer.length} bytes`);
// Le contrôle sur les octets du PDF intervient avant le stockage et l'appel IA.
// Il reste fiable même si le nom du fichier ou le compte utilisateur diffère.
const contentHash = calculateFileSha256(fileBuffer);
const existingSource = await getSourceFileByContentHash(contentHash);
if (existingSource) {
throw new TRPCError({
code: "CONFLICT",
message: `Ce PDF a déjà été importé (${existingSource.fileName}).`,
});
}
// Store source file
const sourceFileKey = generateStorageKey(userId, input.fileName);
console.log(`[Upload] Generated storage key: ${sourceFileKey}`);
@@ -202,6 +215,7 @@ export const appRouter = router({
fileName: input.fileName,
fileKey: sourceFileKey,
fileUrl: sourceFileUrl,
contentHash,
processingStatus: "processing",
});

32
todo.md
View File

@@ -703,7 +703,31 @@
- [x] Ajouter des tests de non-régression ciblés et vérifier build, types et tests
- [x] Normaliser les valeurs OAuth de loginMethod avant écriture en base
## Déploiement recette — audit de robustesse
- [ ] Pousser le checkpoint daudit vers Gitea recette
- [ ] Reconstruire lapplication sur le serveur de recette
- [ ] Vérifier le commit, les conteneurs et la disponibilité HTTP en recette
## Incident production — erreur HTTP 404
- [x] Reproduire la 404 et contrôler le domaine, Traefik et les conteneurs
- [x] Identifier et corriger la cause racine sans modifier les données
- [x] Vérifier le retour HTTP 200 et la santé des conteneurs
## Audit production — 674 factures affichées
- [x] Compter les factures par utilisateur, source et statut
- [x] Identifier les groupes de doublons selon plusieurs clés métier
- [x] Vérifier les références de stockage et les effets de la fusion précédente
- [x] Préparer une correction réversible sans suppression immédiate
- [x] Sauvegarder la base et le volume puis suspendre les imports email
- [x] Bloquer les réimports par empreinte PDF et fiabiliser le traitement IMAP
- [x] Déployer le correctif anti-réimport et migrer la base de production
- [x] Appliquer la correction confirmée et vérifier le comptage final
## Audit authentification IMAP Microsoft 365
- [x] Vérifier la génération du jeton Azure et le format XOAUTH2 envoyé à IMAP
- [x] Comparer les scopes, permissions et méthode dauthentification aux exigences Microsoft 365
- [x] Tester la configuration active de production sans exposer les secrets
- [x] Documenter la cause du refus IMAP et le correctif requis
## Migration import email vers ImapFlow
- [x] Remplacer la dépendance `imap` par `imapflow`
- [x] Réécrire la connexion OAuth2, la recherche UNSEEN et la lecture des messages
- [x] Conserver le traitement séquentiel, le verrou anti-concurrence et le marquage Seen après succès
- [x] Adapter le test de connexion IMAP et les messages derreur
- [x] Ajouter des tests de non-régression du flux ImapFlow
- [x] Vérifier TypeScript, tests, build et authentification OAuth2 réelle