import express, { Router } from "express"; import multer from "multer"; import * as XLSX from "xlsx"; import { getDb } from "./db"; import { veilleItems, aapItems, importLogs, appSettings } from "../drizzle/schema"; import { eq } from "drizzle-orm"; import * as crypto from "crypto"; const router: Router = express.Router(); // Multer en mémoire — on traite le buffer directement const upload = multer({ storage: multer.memoryStorage(), limits: { fileSize: 50 * 1024 * 1024 }, // 50 MB max fileFilter: (_req, file, cb) => { if ( file.mimetype === "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" || file.mimetype === "application/vnd.ms-excel" || file.originalname.endsWith(".xlsx") || file.originalname.endsWith(".xls") ) { cb(null, true); } else { cb(new Error("Seuls les fichiers Excel (.xlsx, .xls) sont acceptés")); } }, }); // ─── Utilitaires ───────────────────────────────────────────────────────────── function makeDedupKey(titre: string, lien?: string | null): string { const raw = `${(titre || "").trim().toLowerCase()}|${(lien || "").trim().toLowerCase()}`; return crypto.createHash("md5").update(raw).digest("hex"); } function parseDate(value: unknown): Date | null { if (!value) return null; if (value instanceof Date) return isNaN(value.getTime()) ? null : value; if (typeof value === "string") { const cleaned = value.replace("Z", "").trim(); const d = new Date(cleaned); return isNaN(d.getTime()) ? null : d; } if (typeof value === "number") { const d = XLSX.SSF.parse_date_code(value); if (d) return new Date(d.y, d.m - 1, d.d); } return null; } function stripHtml(html: string): string { return html .replace(/<[^>]+>/g, ' ') .replace(/ /g, ' ') .replace(/&/g, '&') .replace(/</g, '<') .replace(/>/g, '>') .replace(/"/g, '"') .replace(/'/g, "'") .replace(/\s+/g, ' ') .trim(); } function normalizeStr(v: unknown): string | null { if (v === null || v === undefined) return null; const s = String(v).trim(); return s === "" || s === "Non renseigné" ? null : s; } // ─── Mapping des feuilles ───────────────────────────────────────────────────── const VEILLE_SHEETS: Record = { "réglementaire": "reglementaire", "reglementaire": "reglementaire", "concurrentielle": "concurrentielle", "technologique": "technologique", "générale": "generale", "generale": "generale", }; const AAP_SHEETS: Record = { "handicap": "Handicap", "pa": "PA", "enfance": "Enfance", "précarité": "Précarité", "precarite": "Précarité", "sanitaire": "Sanitaire", "autre": "Autre", }; // ─── Import depuis buffer ───────────────────────────────────────────────────── async function importVeilleFromBuffer(buffer: Buffer, fileName: string) { const startedAt = new Date(); const errors: string[] = []; let totalRows = 0, newRows = 0, skippedRows = 0; const db = await getDb(); if (!db) throw new Error("Base de données indisponible"); const workbook = XLSX.read(buffer, { type: "buffer", cellDates: true }); for (const sheetName of workbook.SheetNames) { const normalized = sheetName.toLowerCase().trim(); if (normalized === "poubelle") continue; const typeVeille = VEILLE_SHEETS[normalized]; if (!typeVeille) continue; const sheet = workbook.Sheets[sheetName]; const rows = XLSX.utils.sheet_to_json>(sheet, { defval: null }); for (const row of rows) { totalRows++; const titre = normalizeStr(row["Titre"]); if (!titre) { skippedRows++; continue; } const lien = normalizeStr(row["Lien"]); const dedupKey = makeDedupKey(titre, lien); const existing = await db .select({ id: veilleItems.id }) .from(veilleItems) .where(eq(veilleItems.dedupKey, dedupKey)) .limit(1); if (existing.length > 0) { skippedRows++; continue; } // La colonne "Source" contient une date ISO dans ce fichier const sourceRaw = row["Source"]; const datePublication = parseDate(sourceRaw); const sourceStr = normalizeStr(sourceRaw instanceof Date ? null : sourceRaw); try { await db.insert(veilleItems).values({ dedupKey, titre, categorie: normalizeStr(row["Catégorie"]), niveau: normalizeStr(row["Niveau"]), territoire: normalizeStr(row["Territoire"]), resume: (() => { const raw = normalizeStr(row[" Résumé"] ?? row["Résumé"] ?? row["Resume"]); return raw ? stripHtml(raw) : null; })(), source: sourceStr, passage: normalizeStr(row["passage"] ?? row["Passage"]), lien, typeVeille, datePublication, }); newRows++; } catch (e: unknown) { const msg = e instanceof Error ? e.message : String(e); errors.push(`[${sheetName}] ${titre.substring(0, 50)}: ${msg}`); skippedRows++; } } } const status = errors.length === 0 ? "success" : newRows > 0 ? "partial" : "error"; await db.insert(importLogs).values({ fileType: "veille", source: fileName, status, totalRows, newRows, skippedRows, errorMessage: errors.length > 0 ? errors.join("\n") : null, details: errors.length > 0 ? { errors } as Record : null, startedAt, completedAt: new Date(), }); return { totalRows, newRows, skippedRows, errors, status }; } async function importAAPFromBuffer(buffer: Buffer, fileName: string) { const startedAt = new Date(); const errors: string[] = []; let totalRows = 0, newRows = 0, skippedRows = 0; const db = await getDb(); if (!db) throw new Error("Base de données indisponible"); const workbook = XLSX.read(buffer, { type: "buffer", cellDates: true }); for (const sheetName of workbook.SheetNames) { const normalized = sheetName.toLowerCase().trim(); const normalizedAcc = normalized.replace(/é/g, "e").replace(/è/g, "e").replace(/ê/g, "e"); const categorie = AAP_SHEETS[normalized] || AAP_SHEETS[normalizedAcc]; if (!categorie) continue; const sheet = workbook.Sheets[sheetName]; const rows = XLSX.utils.sheet_to_json>(sheet, { defval: null }); for (const row of rows) { totalRows++; const titre = normalizeStr(row["Titre"]); if (!titre) { skippedRows++; continue; } const lien = normalizeStr(row["Lien"]); const dedupKey = makeDedupKey(titre, lien); const existing = await db .select({ id: aapItems.id }) .from(aapItems) .where(eq(aapItems.dedupKey, dedupKey)) .limit(1); if (existing.length > 0) { skippedRows++; continue; } const datePublication = parseDate(row["Date publication"]); const dateCloture = parseDate(row["Date clôture"]); try { await db.insert(aapItems).values({ dedupKey, titre, categorie, region: normalizeStr(row["Région"]), departement: normalizeStr(row["Département"]), dateCloture, datePublication, lien, }); newRows++; } catch (e: unknown) { const msg = e instanceof Error ? e.message : String(e); errors.push(`[${sheetName}] ${titre.substring(0, 50)}: ${msg}`); skippedRows++; } } } const status = errors.length === 0 ? "success" : newRows > 0 ? "partial" : "error"; await db.insert(importLogs).values({ fileType: "aap", source: fileName, status, totalRows, newRows, skippedRows, errorMessage: errors.length > 0 ? errors.join("\n") : null, details: errors.length > 0 ? { errors } as Record : null, startedAt, completedAt: new Date(), }); return { totalRows, newRows, skippedRows, errors, status }; } // ─── Route POST /api/upload-excel ───────────────────────────────────────────── router.post( "/api/upload-excel", upload.single("file"), async (req: express.Request, res: express.Response): Promise => { try { const fileType = req.body?.fileType as "veille" | "aap"; if (!fileType || !["veille", "aap"].includes(fileType)) { res.status(400).json({ error: "fileType doit être 'veille' ou 'aap'" }); return; } if (!req.file) { res.status(400).json({ error: "Aucun fichier reçu" }); return; } const buffer = req.file.buffer; const fileName = req.file.originalname; let result; if (fileType === "veille") { result = await importVeilleFromBuffer(buffer, fileName); } else { result = await importAAPFromBuffer(buffer, fileName); } res.json({ success: true, fileType, fileName, ...result, }); } catch (e: unknown) { const msg = e instanceof Error ? e.message : String(e); console.error("[Upload] Erreur:", msg); res.status(500).json({ error: msg }); } } ); export default router;