diff --git a/.manus/db/db-query-1773685654769.json b/.manus/db/db-query-1773685654769.json new file mode 100644 index 0000000..343849a --- /dev/null +++ b/.manus/db/db-query-1773685654769.json @@ -0,0 +1,95 @@ +{ + "query": "SELECT * FROM app_settings; SELECT * FROM import_logs ORDER BY startedAt DESC LIMIT 5;", + "command": "mysql --batch --raw --column-names --default-character-set=utf8mb4 --host gateway02.us-east-1.prod.aws.tidbcloud.com --port 4000 --user 4CrrYuB5tme73Qo.63b125a8f9ca --database VepzDyqR8YkJNcqpZ729Bw --execute SELECT * FROM app_settings; SELECT * FROM import_logs ORDER BY startedAt DESC LIMIT 5;", + "rows": [ + { + "id": "1", + "key": "source_type", + "value": "local", + "updatedAt": "2026-03-16 18:23:41" + }, + { + "id": "2", + "key": "veille_file_path", + "value": "D:\\OneDrive - ITINOVA\\@ITINOVA\\Projets\\@Logiciel veille réglementaire", + "updatedAt": "2026-03-16 18:23:41" + }, + { + "id": "3", + "key": "aap_file_path", + "value": "D:\\OneDrive - ITINOVA\\@ITINOVA\\Projets\\@Logiciel veille réglementaire", + "updatedAt": "2026-03-16 18:23:41" + }, + { + "id": "4", + "key": "ftp_host", + "value": "", + "updatedAt": "2026-03-16 18:23:41" + }, + { + "id": "5", + "key": "ftp_port", + "value": "21", + "updatedAt": "2026-03-16 18:23:41" + }, + { + "id": "6", + "key": "ftp_user", + "value": "", + "updatedAt": "2026-03-16 18:23:41" + }, + { + "id": "7", + "key": "ftp_password", + "value": "", + "updatedAt": "2026-03-16 18:23:41" + }, + { + "id": "8", + "key": "ftp_secure", + "value": "false", + "updatedAt": "2026-03-16 18:23:41" + }, + { + "id": "9", + "key": "onedrive_token", + "value": "", + "updatedAt": "2026-03-16 18:23:41" + }, + { + "id": "10", + "key": "sharepoint_site_url", + "value": "", + "updatedAt": "2026-03-16 18:23:41" + }, + { + "id": "11", + "key": "sharepoint_token", + "value": "", + "updatedAt": "2026-03-16 18:23:41" + }, + { + "id": "12", + "key": "auth_mode", + "value": "local", + "updatedAt": "2026-03-16 18:23:41" + }, + { + "id": "13", + "key": "import_time", + "value": "06:00", + "updatedAt": "2026-03-16 18:23:41" + } + ], + "messages": [ + "id\tfileType\tsource\tstatus\ttotalRows\tnewRows\tskippedRows\terrorMessage\tdetails\tstartedAt\tcompletedAt", + "8\tveille\tD:\\OneDrive - ITINOVA\\@ITINOVA\\Projets\\@Logiciel veille réglementaire\terror\t0\t0\t0\tFichier introuvable : D:\\OneDrive - ITINOVA\\@ITINOVA\\Projets\\@Logiciel veille réglementaire\tNULL\t2026-03-16 18:25:31\t2026-03-16 18:25:31", + "7\taap\tD:\\OneDrive - ITINOVA\\@ITINOVA\\Projets\\@Logiciel veille réglementaire\terror\t0\t0\t0\tFichier introuvable : D:\\OneDrive - ITINOVA\\@ITINOVA\\Projets\\@Logiciel veille réglementaire\tNULL\t2026-03-16 18:25:31\t2026-03-16 18:25:31", + "5\tveille\tD:\\OneDrive - ITINOVA\\@ITINOVA\\Projets\\@Logiciel veille réglementaire\terror\t0\t0\t0\tFichier introuvable : D:\\OneDrive - ITINOVA\\@ITINOVA\\Projets\\@Logiciel veille réglementaire\tNULL\t2026-03-16 18:24:39\t2026-03-16 18:24:39", + "6\taap\tD:\\OneDrive - ITINOVA\\@ITINOVA\\Projets\\@Logiciel veille réglementaire\terror\t0\t0\t0\tFichier introuvable : D:\\OneDrive - ITINOVA\\@ITINOVA\\Projets\\@Logiciel veille réglementaire\tNULL\t2026-03-16 18:24:39\t2026-03-16 18:24:39", + "4\taap\tD:\\OneDrive - ITINOVA\\@ITINOVA\\Projets\\@Logiciel veille réglementaire\terror\t0\t0\t0\tFichier introuvable : D:\\OneDrive - ITINOVA\\@ITINOVA\\Projets\\@Logiciel veille réglementaire\tNULL\t2026-03-16 18:24:15\t2026-03-16 18:24:15" + ], + "stdout": "id\tkey\tvalue\tupdatedAt\n1\tsource_type\tlocal\t2026-03-16 18:23:41\n2\tveille_file_path\tD:\\OneDrive - ITINOVA\\@ITINOVA\\Projets\\@Logiciel veille réglementaire\t2026-03-16 18:23:41\n3\taap_file_path\tD:\\OneDrive - ITINOVA\\@ITINOVA\\Projets\\@Logiciel veille réglementaire\t2026-03-16 18:23:41\n4\tftp_host\t\t2026-03-16 18:23:41\n5\tftp_port\t21\t2026-03-16 18:23:41\n6\tftp_user\t\t2026-03-16 18:23:41\n7\tftp_password\t\t2026-03-16 18:23:41\n8\tftp_secure\tfalse\t2026-03-16 18:23:41\n9\tonedrive_token\t\t2026-03-16 18:23:41\n10\tsharepoint_site_url\t\t2026-03-16 18:23:41\n11\tsharepoint_token\t\t2026-03-16 18:23:41\n12\tauth_mode\tlocal\t2026-03-16 18:23:41\n13\timport_time\t06:00\t2026-03-16 18:23:41\nid\tfileType\tsource\tstatus\ttotalRows\tnewRows\tskippedRows\terrorMessage\tdetails\tstartedAt\tcompletedAt\n8\tveille\tD:\\OneDrive - ITINOVA\\@ITINOVA\\Projets\\@Logiciel veille réglementaire\terror\t0\t0\t0\tFichier introuvable : D:\\OneDrive - ITINOVA\\@ITINOVA\\Projets\\@Logiciel veille réglementaire\tNULL\t2026-03-16 18:25:31\t2026-03-16 18:25:31\n7\taap\tD:\\OneDrive - ITINOVA\\@ITINOVA\\Projets\\@Logiciel veille réglementaire\terror\t0\t0\t0\tFichier introuvable : D:\\OneDrive - ITINOVA\\@ITINOVA\\Projets\\@Logiciel veille réglementaire\tNULL\t2026-03-16 18:25:31\t2026-03-16 18:25:31\n5\tveille\tD:\\OneDrive - ITINOVA\\@ITINOVA\\Projets\\@Logiciel veille réglementaire\terror\t0\t0\t0\tFichier introuvable : D:\\OneDrive - ITINOVA\\@ITINOVA\\Projets\\@Logiciel veille réglementaire\tNULL\t2026-03-16 18:24:39\t2026-03-16 18:24:39\n6\taap\tD:\\OneDrive - ITINOVA\\@ITINOVA\\Projets\\@Logiciel veille réglementaire\terror\t0\t0\t0\tFichier introuvable : D:\\OneDrive - ITINOVA\\@ITINOVA\\Projets\\@Logiciel veille réglementaire\tNULL\t2026-03-16 18:24:39\t2026-03-16 18:24:39\n4\taap\tD:\\OneDrive - ITINOVA\\@ITINOVA\\Projets\\@Logiciel veille réglementaire\terror\t0\t0\t0\tFichier introuvable : D:\\OneDrive - ITINOVA\\@ITINOVA\\Projets\\@Logiciel veille réglementaire\tNULL\t2026-03-16 18:24:15\t2026-03-16 18:24:15\n", + "stderr": "", + "execution_time_ms": 56 +} \ No newline at end of file diff --git a/.manus/db/db-query-1773686056762.json b/.manus/db/db-query-1773686056762.json new file mode 100644 index 0000000..cb5f12f --- /dev/null +++ b/.manus/db/db-query-1773686056762.json @@ -0,0 +1,49 @@ +{ + "query": "SELECT COUNT(*) as total_veille FROM veille_items; SELECT COUNT(*) as total_aap FROM aap_items; SELECT typeVeille, COUNT(*) as nb FROM veille_items GROUP BY typeVeille; SELECT categorie, COUNT(*) as nb FROM aap_items GROUP BY categorie;", + "command": "mysql --batch --raw --column-names --default-character-set=utf8mb4 --host gateway02.us-east-1.prod.aws.tidbcloud.com --port 4000 --user 4CrrYuB5tme73Qo.63b125a8f9ca --database VepzDyqR8YkJNcqpZ729Bw --execute SELECT COUNT(*) as total_veille FROM veille_items; SELECT COUNT(*) as total_aap FROM aap_items; SELECT typeVeille, COUNT(*) as nb FROM veille_items GROUP BY typeVeille; SELECT categorie, COUNT(*) as nb FROM aap_items GROUP BY categorie;", + "rows": [ + { + "total_veille": "38" + }, + { + "total_veille": "total_aap" + }, + { + "total_veille": "7" + }, + { + "total_veille": "typeVeille\tnb" + }, + { + "total_veille": "concurrentielle\t4" + }, + { + "total_veille": "technologique\t4" + }, + { + "total_veille": "reglementaire\t13" + }, + { + "total_veille": "generale\t17" + }, + { + "total_veille": "categorie\tnb" + }, + { + "total_veille": "PA\t1" + }, + { + "total_veille": "Sanitaire\t4" + }, + { + "total_veille": "Handicap\t1" + }, + { + "total_veille": "Autre\t1" + } + ], + "messages": [], + "stdout": "total_veille\n38\ntotal_aap\n7\ntypeVeille\tnb\nconcurrentielle\t4\ntechnologique\t4\nreglementaire\t13\ngenerale\t17\ncategorie\tnb\nPA\t1\nSanitaire\t4\nHandicap\t1\nAutre\t1\n", + "stderr": "", + "execution_time_ms": 66 +} \ No newline at end of file diff --git a/.manus/db/db-query-error-1773685646158.json b/.manus/db/db-query-error-1773685646158.json new file mode 100644 index 0000000..f3b85c0 --- /dev/null +++ b/.manus/db/db-query-error-1773685646158.json @@ -0,0 +1,22 @@ +{ + "query": "SELECT * FROM app_settings; SELECT * FROM import_logs ORDER BY started_at DESC LIMIT 5;", + "command": "mysql --batch --raw --column-names --default-character-set=utf8mb4 --host gateway02.us-east-1.prod.aws.tidbcloud.com --port 4000 --user 4CrrYuB5tme73Qo.63b125a8f9ca --database VepzDyqR8YkJNcqpZ729Bw --execute SELECT * FROM app_settings; SELECT * FROM import_logs ORDER BY started_at DESC LIMIT 5;", + "returncode": 1, + "logs": [ + "id\tkey\tvalue\tupdatedAt", + "1\tsource_type\tlocal\t2026-03-16 18:23:41", + "2\tveille_file_path\tD:\\OneDrive - ITINOVA\\@ITINOVA\\Projets\\@Logiciel veille réglementaire\t2026-03-16 18:23:41", + "3\taap_file_path\tD:\\OneDrive - ITINOVA\\@ITINOVA\\Projets\\@Logiciel veille réglementaire\t2026-03-16 18:23:41", + "4\tftp_host\t\t2026-03-16 18:23:41", + "5\tftp_port\t21\t2026-03-16 18:23:41", + "6\tftp_user\t\t2026-03-16 18:23:41", + "7\tftp_password\t\t2026-03-16 18:23:41", + "8\tftp_secure\tfalse\t2026-03-16 18:23:41", + "9\tonedrive_token\t\t2026-03-16 18:23:41", + "10\tsharepoint_site_url\t\t2026-03-16 18:23:41", + "11\tsharepoint_token\t\t2026-03-16 18:23:41", + "12\tauth_mode\tlocal\t2026-03-16 18:23:41", + "13\timport_time\t06:00\t2026-03-16 18:23:41", + "ERROR 1054 (42S22) at line 1: Unknown column 'started_at' in 'order clause'" + ] +} \ No newline at end of file diff --git a/client/src/pages/ImportLogs.tsx b/client/src/pages/ImportLogs.tsx index bc170ac..b72743b 100644 --- a/client/src/pages/ImportLogs.tsx +++ b/client/src/pages/ImportLogs.tsx @@ -1,8 +1,8 @@ -import { useState } from "react"; +import { useState, useRef, useCallback } from "react"; import { trpc } from "@/lib/trpc"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; -import { Card, CardContent } from "@/components/ui/card"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Activity, CheckCircle, @@ -13,10 +13,14 @@ import { ChevronRight, Clock, FileText, + Upload, + FileSpreadsheet, + AlertCircle, } from "lucide-react"; import { format } from "date-fns"; import { fr } from "date-fns/locale"; import { cn } from "@/lib/utils"; +import { toast } from "sonner"; interface ImportLog { id: number; @@ -31,6 +35,18 @@ interface ImportLog { source: string | null; } +interface UploadResult { + success: boolean; + fileType: "veille" | "aap"; + fileName: string; + totalRows: number; + newRows: number; + skippedRows: number; + errors: string[]; + status: "success" | "partial" | "error"; + error?: string; +} + const STATUS_CONFIG = { success: { label: "Succès", color: "bg-emerald-100 text-emerald-800 border-emerald-200", icon: }, error: { label: "Erreur", color: "bg-red-100 text-red-800 border-red-200", icon: }, @@ -44,6 +60,150 @@ const FILE_CONFIG = { const PAGE_SIZE = 20; +// ─── Composant UploadZone ───────────────────────────────────────────────────── + +function UploadZone({ + fileType, + label, + color, + onSuccess, +}: { + fileType: "veille" | "aap"; + label: string; + color: string; + onSuccess: () => void; +}) { + const [isDragging, setIsDragging] = useState(false); + const [isUploading, setIsUploading] = useState(false); + const [lastResult, setLastResult] = useState(null); + const inputRef = useRef(null); + + const handleFile = useCallback( + async (file: File) => { + if (!file) return; + if (!file.name.endsWith(".xlsx") && !file.name.endsWith(".xls")) { + toast.error("Seuls les fichiers Excel (.xlsx, .xls) sont acceptés"); + return; + } + + setIsUploading(true); + setLastResult(null); + + try { + const formData = new FormData(); + formData.append("file", file); + formData.append("fileType", fileType); + + const res = await fetch("/api/upload-excel", { + method: "POST", + body: formData, + }); + + const data: UploadResult = await res.json(); + + if (!res.ok || !data.success) { + throw new Error(data.error || "Erreur lors de l'import"); + } + + setLastResult(data); + onSuccess(); + + if (data.newRows > 0) { + toast.success(`Import réussi — ${data.newRows} nouvelle(s) entrée(s) ajoutée(s)`); + } else { + toast.info(`Import terminé — aucune nouvelle entrée (${data.skippedRows} déjà présentes)`); + } + } catch (e: unknown) { + const msg = e instanceof Error ? e.message : String(e); + toast.error(`Erreur : ${msg}`); + setLastResult({ success: false, fileType, fileName: file.name, totalRows: 0, newRows: 0, skippedRows: 0, errors: [msg], status: "error", error: msg }); + } finally { + setIsUploading(false); + } + }, + [fileType, onSuccess] + ); + + const onDrop = useCallback( + (e: React.DragEvent) => { + e.preventDefault(); + setIsDragging(false); + const file = e.dataTransfer.files[0]; + if (file) handleFile(file); + }, + [handleFile] + ); + + return ( +
+
{ e.preventDefault(); setIsDragging(true); }} + onDragLeave={() => setIsDragging(false)} + onDrop={onDrop} + onClick={() => inputRef.current?.click()} + > + { const f = e.target.files?.[0]; if (f) handleFile(f); e.target.value = ""; }} + /> + + {isUploading ? ( +
+ +

Import en cours…

+
+ ) : ( +
+
+ +
+
+

{label}

+

+ Glissez-déposez ou parcourez +

+

.xlsx ou .xls · max 50 MB

+
+
+ )} +
+ + {/* Résultat du dernier upload */} + {lastResult && ( +
+ {lastResult.status === "success" ? : + lastResult.status === "partial" ? : + } +
+

{lastResult.fileName}

+

+ {lastResult.newRows} nouvelle(s) · {lastResult.skippedRows} ignorée(s) · {lastResult.totalRows} total +

+ {lastResult.errors.length > 0 && ( +

{lastResult.errors[0]}

+ )} +
+
+ )} +
+ ); +} + +// ─── Page principale ────────────────────────────────────────────────────────── + export default function ImportLogs() { const [page, setPage] = useState(1); @@ -56,6 +216,10 @@ export default function ImportLogs() { const total = logsQuery.data?.total ?? 0; const totalPages = Math.ceil(total / PAGE_SIZE); + const handleUploadSuccess = () => { + logsQuery.refetch(); + }; + return (
{/* En-tête */} @@ -70,13 +234,43 @@ export default function ImportLogs() {
+ {/* Upload direct */} + + + + + Import direct depuis votre ordinateur + +

+ Déposez vos fichiers Excel directement — les nouvelles entrées seront ajoutées immédiatement +

+
+ +
+ + +
+
+
+ {/* Stats rapides */} {logsQuery.data?.stats && (
@@ -124,7 +318,7 @@ export default function ImportLogs() { Ignorées Total Durée - Message + Source @@ -151,19 +345,19 @@ export default function ImportLogs() { - +{log.newRows} + +{log.newRows ?? 0} - {log.skippedRows} - {log.totalRows} + {log.skippedRows ?? 0} + {log.totalRows ?? 0} {log.startedAt && log.completedAt ? `${((new Date(log.completedAt).getTime() - new Date(log.startedAt).getTime()) / 1000).toFixed(1)}s` : "—"} - - {log.errorMessage ? ( - {log.errorMessage} - ) : "—"} + + {log.source + ? log.source.split(/[\\/]/).pop() || log.source + : "—"} ); diff --git a/package.json b/package.json index 3b4320d..5a0dabd 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "@trpc/react-query": "^11.6.0", "@trpc/server": "^11.6.0", "@types/bcryptjs": "^3.0.0", + "@types/multer": "^2.1.0", "@types/node-cron": "^3.0.11", "axios": "^1.12.0", "basic-ftp": "^5.2.0", @@ -64,6 +65,7 @@ "input-otp": "^1.4.2", "jose": "6.1.0", "lucide-react": "^0.453.0", + "multer": "^2.1.1", "mysql2": "^3.15.0", "nanoid": "^5.1.5", "next-themes": "^0.4.6", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 08d8b53..a2eb3f7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -118,6 +118,9 @@ importers: '@types/bcryptjs': specifier: ^3.0.0 version: 3.0.0 + '@types/multer': + specifier: ^2.1.0 + version: 2.1.0 '@types/node-cron': specifier: ^3.0.11 version: 3.0.11 @@ -169,6 +172,9 @@ importers: lucide-react: specifier: ^0.453.0 version: 0.453.0(react@19.2.1) + multer: + specifier: ^2.1.1 + version: 2.1.1 mysql2: specifier: ^3.15.0 version: 3.15.1 @@ -2316,6 +2322,9 @@ packages: '@types/ms@2.1.0': resolution: {integrity: sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==} + '@types/multer@2.1.0': + resolution: {integrity: sha512-zYZb0+nJhOHtPpGDb3vqPjwpdeGlGC157VpkqNQL+UU2qwoacoQ7MpsAmUptI/0Oa127X32JzWDqQVEXp2RcIA==} + '@types/node-cron@3.0.11': resolution: {integrity: sha512-0ikrnug3/IyneSHqCBeslAhlK2aBfYek1fGo4bP4QnZPmiqSGRK+Oy7ZMisLWkesffJvQ1cqAcBnJC+8+nxIAg==} @@ -2408,6 +2417,9 @@ packages: resolution: {integrity: sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==} engines: {node: '>=0.8'} + append-field@1.0.0: + resolution: {integrity: sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==} + aria-hidden@1.2.6: resolution: {integrity: sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA==} engines: {node: '>=10'} @@ -2466,6 +2478,10 @@ packages: buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} + busboy@1.6.0: + resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==} + engines: {node: '>=10.16.0'} + bytes@3.1.2: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} @@ -2556,6 +2572,10 @@ packages: resolution: {integrity: sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==} engines: {node: '>= 12'} + concat-stream@2.0.0: + resolution: {integrity: sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==} + engines: {'0': node >= 6.0} + confbox@0.1.8: resolution: {integrity: sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==} @@ -3637,6 +3657,10 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + multer@2.1.1: + resolution: {integrity: sha512-mo+QTzKlx8R7E5ylSXxWzGoXoZbOsRMpyitcht8By2KHvMbf3tjwosZ/Mu/XYU6UuJ3VZnODIrak5ZrPiPyB6A==} + engines: {node: '>= 10.16.0'} + mysql2@3.15.1: resolution: {integrity: sha512-WZMIRZstT2MFfouEaDz/AGFnGi1A2GwaDe7XvKTdRJEYiAHbOrh4S3d8KFmQeh11U85G+BFjIvS1Di5alusZsw==} engines: {node: '>= 8.0'} @@ -3879,6 +3903,10 @@ packages: resolution: {integrity: sha512-DGrYcCWK7tvYMnWh79yrPHt+vdx9tY+1gPZa7nJQtO/p8bLTDaHp4dzwEhQB7pZ4Xe3ok4XKuEPrVuc+wlpkmw==} engines: {node: '>=0.10.0'} + readable-stream@3.6.2: + resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} + engines: {node: '>= 6'} + recharts-scale@0.4.5: resolution: {integrity: sha512-kivNFO+0OcUNu7jQquLXAxz1FIwZj8nrj+YkOKc5694NbjCvcT6aSZiIzNzd2Kul4o4rTto8QVR9lMNtxD4G1w==} @@ -4035,6 +4063,13 @@ packages: peerDependencies: react: ^18.0.0 || ^19.0.0 + streamsearch@1.1.0: + resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} + engines: {node: '>=10.0.0'} + + string_decoder@1.3.0: + resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} + stringify-entities@4.0.4: resolution: {integrity: sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg==} @@ -4130,6 +4165,9 @@ packages: resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==} engines: {node: '>= 0.6'} + typedarray@0.0.6: + resolution: {integrity: sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==} + typescript@5.9.3: resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} engines: {node: '>=14.17'} @@ -6717,6 +6755,10 @@ snapshots: '@types/ms@2.1.0': {} + '@types/multer@2.1.0': + dependencies: + '@types/express': 4.17.21 + '@types/node-cron@3.0.11': {} '@types/node@24.7.0': @@ -6822,6 +6864,8 @@ snapshots: adler-32@1.3.1: {} + append-field@1.0.0: {} + aria-hidden@1.2.6: dependencies: tslib: 2.8.1 @@ -6889,6 +6933,10 @@ snapshots: buffer-from@1.1.2: {} + busboy@1.6.0: + dependencies: + streamsearch: 1.1.0 + bytes@3.1.2: {} cac@6.7.14: {} @@ -6976,6 +7024,13 @@ snapshots: commander@8.3.0: {} + concat-stream@2.0.0: + dependencies: + buffer-from: 1.1.2 + inherits: 2.0.4 + readable-stream: 3.6.2 + typedarray: 0.0.6 + confbox@0.1.8: {} confbox@0.2.2: {} @@ -8292,6 +8347,13 @@ snapshots: ms@2.1.3: {} + multer@2.1.1: + dependencies: + append-field: 1.0.0 + busboy: 1.6.0 + concat-stream: 2.0.0 + type-is: 1.6.18 + mysql2@3.15.1: dependencies: aws-ssl-profiles: 1.1.2 @@ -8532,6 +8594,12 @@ snapshots: react@19.2.1: {} + readable-stream@3.6.2: + dependencies: + inherits: 2.0.4 + string_decoder: 1.3.0 + util-deprecate: 1.0.2 + recharts-scale@0.4.5: dependencies: decimal.js-light: 2.5.1 @@ -8791,6 +8859,12 @@ snapshots: - '@types/react' - supports-color + streamsearch@1.1.0: {} + + string_decoder@1.3.0: + dependencies: + safe-buffer: 5.2.1 + stringify-entities@4.0.4: dependencies: character-entities-html4: 2.1.0 @@ -8873,6 +8947,8 @@ snapshots: media-typer: 0.3.0 mime-types: 2.1.35 + typedarray@0.0.6: {} + typescript@5.9.3: {} ufo@1.6.1: {} diff --git a/server/_core/index.ts b/server/_core/index.ts index 756e58f..97a11c5 100644 --- a/server/_core/index.ts +++ b/server/_core/index.ts @@ -9,6 +9,7 @@ import { appRouter } from "../routers"; import { createContext } from "./context"; import { serveStatic, setupVite } from "./vite"; import { runFullImport } from "../importer"; +import uploadRoutes from "../uploadRoutes"; import { ensureAdminExists } from "../localAuth"; import { getSetting } from "../db"; @@ -63,6 +64,7 @@ async function startServer() { app.use(express.urlencoded({ limit: "50mb", extended: true })); registerOAuthRoutes(app); + app.use(uploadRoutes); app.use( "/api/trpc", diff --git a/server/uploadRoutes.ts b/server/uploadRoutes.ts new file mode 100644 index 0000000..8d18f1b --- /dev/null +++ b/server/uploadRoutes.ts @@ -0,0 +1,293 @@ +import express, { Router } from "express"; +import multer from "multer"; +import * as XLSX from "xlsx"; +import { getDb } from "./db"; +import { veilleItems, aapItems, importLogs, appSettings } from "../drizzle/schema"; +import { eq } from "drizzle-orm"; +import * as crypto from "crypto"; + +const router: Router = express.Router(); + +// Multer en mémoire — on traite le buffer directement +const upload = multer({ + storage: multer.memoryStorage(), + limits: { fileSize: 50 * 1024 * 1024 }, // 50 MB max + fileFilter: (_req, file, cb) => { + if ( + file.mimetype === "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" || + file.mimetype === "application/vnd.ms-excel" || + file.originalname.endsWith(".xlsx") || + file.originalname.endsWith(".xls") + ) { + cb(null, true); + } else { + cb(new Error("Seuls les fichiers Excel (.xlsx, .xls) sont acceptés")); + } + }, +}); + +// ─── Utilitaires ───────────────────────────────────────────────────────────── + +function makeDedupKey(titre: string, lien?: string | null): string { + const raw = `${(titre || "").trim().toLowerCase()}|${(lien || "").trim().toLowerCase()}`; + return crypto.createHash("md5").update(raw).digest("hex"); +} + +function parseDate(value: unknown): Date | null { + if (!value) return null; + if (value instanceof Date) return isNaN(value.getTime()) ? null : value; + if (typeof value === "string") { + const cleaned = value.replace("Z", "").trim(); + const d = new Date(cleaned); + return isNaN(d.getTime()) ? null : d; + } + if (typeof value === "number") { + const d = XLSX.SSF.parse_date_code(value); + if (d) return new Date(d.y, d.m - 1, d.d); + } + return null; +} + +function stripHtml(html: string): string { + return html + .replace(/<[^>]+>/g, ' ') + .replace(/ /g, ' ') + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/\s+/g, ' ') + .trim(); +} + +function normalizeStr(v: unknown): string | null { + if (v === null || v === undefined) return null; + const s = String(v).trim(); + return s === "" || s === "Non renseigné" ? null : s; +} + +// ─── Mapping des feuilles ───────────────────────────────────────────────────── + +const VEILLE_SHEETS: Record = { + "réglementaire": "reglementaire", + "reglementaire": "reglementaire", + "concurrentielle": "concurrentielle", + "technologique": "technologique", + "générale": "generale", + "generale": "generale", +}; + +const AAP_SHEETS: Record = { + "handicap": "Handicap", + "pa": "PA", + "enfance": "Enfance", + "précarité": "Précarité", + "precarite": "Précarité", + "sanitaire": "Sanitaire", + "autre": "Autre", +}; + +// ─── Import depuis buffer ───────────────────────────────────────────────────── + +async function importVeilleFromBuffer(buffer: Buffer, fileName: string) { + const startedAt = new Date(); + const errors: string[] = []; + let totalRows = 0, newRows = 0, skippedRows = 0; + + const db = await getDb(); + if (!db) throw new Error("Base de données indisponible"); + + const workbook = XLSX.read(buffer, { type: "buffer", cellDates: true }); + + for (const sheetName of workbook.SheetNames) { + const normalized = sheetName.toLowerCase().trim(); + if (normalized === "poubelle") continue; + const typeVeille = VEILLE_SHEETS[normalized]; + if (!typeVeille) continue; + + const sheet = workbook.Sheets[sheetName]; + const rows = XLSX.utils.sheet_to_json>(sheet, { defval: null }); + + for (const row of rows) { + totalRows++; + const titre = normalizeStr(row["Titre"]); + if (!titre) { skippedRows++; continue; } + + const lien = normalizeStr(row["Lien"]); + const dedupKey = makeDedupKey(titre, lien); + + const existing = await db + .select({ id: veilleItems.id }) + .from(veilleItems) + .where(eq(veilleItems.dedupKey, dedupKey)) + .limit(1); + + if (existing.length > 0) { skippedRows++; continue; } + + // La colonne "Source" contient une date ISO dans ce fichier + const sourceRaw = row["Source"]; + const datePublication = parseDate(sourceRaw); + const sourceStr = normalizeStr(sourceRaw instanceof Date ? null : sourceRaw); + + try { + await db.insert(veilleItems).values({ + dedupKey, + titre, + categorie: normalizeStr(row["Catégorie"]), + niveau: normalizeStr(row["Niveau"]), + territoire: normalizeStr(row["Territoire"]), + resume: (() => { + const raw = normalizeStr(row[" Résumé"] ?? row["Résumé"] ?? row["Resume"]); + return raw ? stripHtml(raw) : null; + })(), + source: sourceStr, + passage: normalizeStr(row["passage"] ?? row["Passage"]), + lien, + typeVeille, + datePublication, + }); + newRows++; + } catch (e: unknown) { + const msg = e instanceof Error ? e.message : String(e); + errors.push(`[${sheetName}] ${titre.substring(0, 50)}: ${msg}`); + skippedRows++; + } + } + } + + const status = errors.length === 0 ? "success" : newRows > 0 ? "partial" : "error"; + await db.insert(importLogs).values({ + fileType: "veille", + source: fileName, + status, + totalRows, + newRows, + skippedRows, + errorMessage: errors.length > 0 ? errors.join("\n") : null, + details: errors.length > 0 ? { errors } as Record : null, + startedAt, + completedAt: new Date(), + }); + + return { totalRows, newRows, skippedRows, errors, status }; +} + +async function importAAPFromBuffer(buffer: Buffer, fileName: string) { + const startedAt = new Date(); + const errors: string[] = []; + let totalRows = 0, newRows = 0, skippedRows = 0; + + const db = await getDb(); + if (!db) throw new Error("Base de données indisponible"); + + const workbook = XLSX.read(buffer, { type: "buffer", cellDates: true }); + + for (const sheetName of workbook.SheetNames) { + const normalized = sheetName.toLowerCase().trim(); + const normalizedAcc = normalized.replace(/é/g, "e").replace(/è/g, "e").replace(/ê/g, "e"); + const categorie = AAP_SHEETS[normalized] || AAP_SHEETS[normalizedAcc]; + if (!categorie) continue; + + const sheet = workbook.Sheets[sheetName]; + const rows = XLSX.utils.sheet_to_json>(sheet, { defval: null }); + + for (const row of rows) { + totalRows++; + const titre = normalizeStr(row["Titre"]); + if (!titre) { skippedRows++; continue; } + + const lien = normalizeStr(row["Lien"]); + const dedupKey = makeDedupKey(titre, lien); + + const existing = await db + .select({ id: aapItems.id }) + .from(aapItems) + .where(eq(aapItems.dedupKey, dedupKey)) + .limit(1); + + if (existing.length > 0) { skippedRows++; continue; } + + const datePublication = parseDate(row["Date publication"]); + const dateCloture = parseDate(row["Date clôture"]); + + try { + await db.insert(aapItems).values({ + dedupKey, + titre, + categorie, + region: normalizeStr(row["Région"]), + departement: normalizeStr(row["Département"]), + dateCloture, + datePublication, + lien, + }); + newRows++; + } catch (e: unknown) { + const msg = e instanceof Error ? e.message : String(e); + errors.push(`[${sheetName}] ${titre.substring(0, 50)}: ${msg}`); + skippedRows++; + } + } + } + + const status = errors.length === 0 ? "success" : newRows > 0 ? "partial" : "error"; + await db.insert(importLogs).values({ + fileType: "aap", + source: fileName, + status, + totalRows, + newRows, + skippedRows, + errorMessage: errors.length > 0 ? errors.join("\n") : null, + details: errors.length > 0 ? { errors } as Record : null, + startedAt, + completedAt: new Date(), + }); + + return { totalRows, newRows, skippedRows, errors, status }; +} + +// ─── Route POST /api/upload-excel ───────────────────────────────────────────── + +router.post( + "/api/upload-excel", + upload.single("file"), + async (req: express.Request, res: express.Response): Promise => { + try { + const fileType = req.body?.fileType as "veille" | "aap"; + if (!fileType || !["veille", "aap"].includes(fileType)) { + res.status(400).json({ error: "fileType doit être 'veille' ou 'aap'" }); + return; + } + + if (!req.file) { + res.status(400).json({ error: "Aucun fichier reçu" }); + return; + } + + const buffer = req.file.buffer; + const fileName = req.file.originalname; + + let result; + if (fileType === "veille") { + result = await importVeilleFromBuffer(buffer, fileName); + } else { + result = await importAAPFromBuffer(buffer, fileName); + } + + res.json({ + success: true, + fileType, + fileName, + ...result, + }); + } catch (e: unknown) { + const msg = e instanceof Error ? e.message : String(e); + console.error("[Upload] Erreur:", msg); + res.status(500).json({ error: msg }); + } + } +); + +export default router; diff --git a/todo.md b/todo.md index 3c5a722..c360bfb 100644 --- a/todo.md +++ b/todo.md @@ -25,3 +25,15 @@ - [x] Thème visuel Itinova (bleu marine, palette professionnelle) - [x] 13 tests Vitest passants (auth + veille) - [x] Compte admin par défaut créé au démarrage du serveur + +## Bugs à corriger + +- [x] Import Excel affiche 0 nouvelles entrées alors que les fichiers contiennent des données + +## Nouvelles fonctionnalités + +- [x] Backend : endpoint POST /api/upload-excel (multipart) pour recevoir les fichiers Excel +- [x] Backend : stocker les fichiers uploadés en S3 et déclencher l'import immédiatement +- [x] Frontend : bouton "Importer un fichier" dans la page Logs d'import avec sélecteur veille/AAP +- [x] Frontend : zone de dépôt (drag & drop) dans la page Paramètres pour les deux fichiers +- [x] Frontend : afficher le résultat de l'import (nouvelles entrées, erreurs) après upload