Checkpoint: Ajout de l'upload direct de fichiers Excel depuis le navigateur (drag & drop), nettoyage automatique des balises HTML dans les résumés, import fonctionnel avec 38 entrées Veille + 7 AAP
This commit is contained in:
Manus
293
server/uploadRoutes.ts
Normal file
293
server/uploadRoutes.ts
Normal file
@@ -0,0 +1,293 @@
|
||||
import express, { Router } from "express";
|
||||
import multer from "multer";
|
||||
import * as XLSX from "xlsx";
|
||||
import { getDb } from "./db";
|
||||
import { veilleItems, aapItems, importLogs, appSettings } from "../drizzle/schema";
|
||||
import { eq } from "drizzle-orm";
|
||||
import * as crypto from "crypto";
|
||||
|
||||
const router: Router = express.Router();
|
||||
|
||||
// Multer en mémoire — on traite le buffer directement
|
||||
const upload = multer({
|
||||
storage: multer.memoryStorage(),
|
||||
limits: { fileSize: 50 * 1024 * 1024 }, // 50 MB max
|
||||
fileFilter: (_req, file, cb) => {
|
||||
if (
|
||||
file.mimetype === "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" ||
|
||||
file.mimetype === "application/vnd.ms-excel" ||
|
||||
file.originalname.endsWith(".xlsx") ||
|
||||
file.originalname.endsWith(".xls")
|
||||
) {
|
||||
cb(null, true);
|
||||
} else {
|
||||
cb(new Error("Seuls les fichiers Excel (.xlsx, .xls) sont acceptés"));
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
// ─── Utilitaires ─────────────────────────────────────────────────────────────
|
||||
|
||||
function makeDedupKey(titre: string, lien?: string | null): string {
|
||||
const raw = `${(titre || "").trim().toLowerCase()}|${(lien || "").trim().toLowerCase()}`;
|
||||
return crypto.createHash("md5").update(raw).digest("hex");
|
||||
}
|
||||
|
||||
function parseDate(value: unknown): Date | null {
|
||||
if (!value) return null;
|
||||
if (value instanceof Date) return isNaN(value.getTime()) ? null : value;
|
||||
if (typeof value === "string") {
|
||||
const cleaned = value.replace("Z", "").trim();
|
||||
const d = new Date(cleaned);
|
||||
return isNaN(d.getTime()) ? null : d;
|
||||
}
|
||||
if (typeof value === "number") {
|
||||
const d = XLSX.SSF.parse_date_code(value);
|
||||
if (d) return new Date(d.y, d.m - 1, d.d);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function stripHtml(html: string): string {
|
||||
return html
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/ /g, ' ')
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function normalizeStr(v: unknown): string | null {
|
||||
if (v === null || v === undefined) return null;
|
||||
const s = String(v).trim();
|
||||
return s === "" || s === "Non renseigné" ? null : s;
|
||||
}
|
||||
|
||||
// ─── Mapping des feuilles ─────────────────────────────────────────────────────
|
||||
|
||||
const VEILLE_SHEETS: Record<string, "reglementaire" | "concurrentielle" | "technologique" | "generale"> = {
|
||||
"réglementaire": "reglementaire",
|
||||
"reglementaire": "reglementaire",
|
||||
"concurrentielle": "concurrentielle",
|
||||
"technologique": "technologique",
|
||||
"générale": "generale",
|
||||
"generale": "generale",
|
||||
};
|
||||
|
||||
const AAP_SHEETS: Record<string, "Handicap" | "PA" | "Enfance" | "Précarité" | "Sanitaire" | "Autre"> = {
|
||||
"handicap": "Handicap",
|
||||
"pa": "PA",
|
||||
"enfance": "Enfance",
|
||||
"précarité": "Précarité",
|
||||
"precarite": "Précarité",
|
||||
"sanitaire": "Sanitaire",
|
||||
"autre": "Autre",
|
||||
};
|
||||
|
||||
// ─── Import depuis buffer ─────────────────────────────────────────────────────
|
||||
|
||||
async function importVeilleFromBuffer(buffer: Buffer, fileName: string) {
|
||||
const startedAt = new Date();
|
||||
const errors: string[] = [];
|
||||
let totalRows = 0, newRows = 0, skippedRows = 0;
|
||||
|
||||
const db = await getDb();
|
||||
if (!db) throw new Error("Base de données indisponible");
|
||||
|
||||
const workbook = XLSX.read(buffer, { type: "buffer", cellDates: true });
|
||||
|
||||
for (const sheetName of workbook.SheetNames) {
|
||||
const normalized = sheetName.toLowerCase().trim();
|
||||
if (normalized === "poubelle") continue;
|
||||
const typeVeille = VEILLE_SHEETS[normalized];
|
||||
if (!typeVeille) continue;
|
||||
|
||||
const sheet = workbook.Sheets[sheetName];
|
||||
const rows = XLSX.utils.sheet_to_json<Record<string, unknown>>(sheet, { defval: null });
|
||||
|
||||
for (const row of rows) {
|
||||
totalRows++;
|
||||
const titre = normalizeStr(row["Titre"]);
|
||||
if (!titre) { skippedRows++; continue; }
|
||||
|
||||
const lien = normalizeStr(row["Lien"]);
|
||||
const dedupKey = makeDedupKey(titre, lien);
|
||||
|
||||
const existing = await db
|
||||
.select({ id: veilleItems.id })
|
||||
.from(veilleItems)
|
||||
.where(eq(veilleItems.dedupKey, dedupKey))
|
||||
.limit(1);
|
||||
|
||||
if (existing.length > 0) { skippedRows++; continue; }
|
||||
|
||||
// La colonne "Source" contient une date ISO dans ce fichier
|
||||
const sourceRaw = row["Source"];
|
||||
const datePublication = parseDate(sourceRaw);
|
||||
const sourceStr = normalizeStr(sourceRaw instanceof Date ? null : sourceRaw);
|
||||
|
||||
try {
|
||||
await db.insert(veilleItems).values({
|
||||
dedupKey,
|
||||
titre,
|
||||
categorie: normalizeStr(row["Catégorie"]),
|
||||
niveau: normalizeStr(row["Niveau"]),
|
||||
territoire: normalizeStr(row["Territoire"]),
|
||||
resume: (() => {
|
||||
const raw = normalizeStr(row[" Résumé"] ?? row["Résumé"] ?? row["Resume"]);
|
||||
return raw ? stripHtml(raw) : null;
|
||||
})(),
|
||||
source: sourceStr,
|
||||
passage: normalizeStr(row["passage"] ?? row["Passage"]),
|
||||
lien,
|
||||
typeVeille,
|
||||
datePublication,
|
||||
});
|
||||
newRows++;
|
||||
} catch (e: unknown) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
errors.push(`[${sheetName}] ${titre.substring(0, 50)}: ${msg}`);
|
||||
skippedRows++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const status = errors.length === 0 ? "success" : newRows > 0 ? "partial" : "error";
|
||||
await db.insert(importLogs).values({
|
||||
fileType: "veille",
|
||||
source: fileName,
|
||||
status,
|
||||
totalRows,
|
||||
newRows,
|
||||
skippedRows,
|
||||
errorMessage: errors.length > 0 ? errors.join("\n") : null,
|
||||
details: errors.length > 0 ? { errors } as Record<string, unknown> : null,
|
||||
startedAt,
|
||||
completedAt: new Date(),
|
||||
});
|
||||
|
||||
return { totalRows, newRows, skippedRows, errors, status };
|
||||
}
|
||||
|
||||
async function importAAPFromBuffer(buffer: Buffer, fileName: string) {
|
||||
const startedAt = new Date();
|
||||
const errors: string[] = [];
|
||||
let totalRows = 0, newRows = 0, skippedRows = 0;
|
||||
|
||||
const db = await getDb();
|
||||
if (!db) throw new Error("Base de données indisponible");
|
||||
|
||||
const workbook = XLSX.read(buffer, { type: "buffer", cellDates: true });
|
||||
|
||||
for (const sheetName of workbook.SheetNames) {
|
||||
const normalized = sheetName.toLowerCase().trim();
|
||||
const normalizedAcc = normalized.replace(/é/g, "e").replace(/è/g, "e").replace(/ê/g, "e");
|
||||
const categorie = AAP_SHEETS[normalized] || AAP_SHEETS[normalizedAcc];
|
||||
if (!categorie) continue;
|
||||
|
||||
const sheet = workbook.Sheets[sheetName];
|
||||
const rows = XLSX.utils.sheet_to_json<Record<string, unknown>>(sheet, { defval: null });
|
||||
|
||||
for (const row of rows) {
|
||||
totalRows++;
|
||||
const titre = normalizeStr(row["Titre"]);
|
||||
if (!titre) { skippedRows++; continue; }
|
||||
|
||||
const lien = normalizeStr(row["Lien"]);
|
||||
const dedupKey = makeDedupKey(titre, lien);
|
||||
|
||||
const existing = await db
|
||||
.select({ id: aapItems.id })
|
||||
.from(aapItems)
|
||||
.where(eq(aapItems.dedupKey, dedupKey))
|
||||
.limit(1);
|
||||
|
||||
if (existing.length > 0) { skippedRows++; continue; }
|
||||
|
||||
const datePublication = parseDate(row["Date publication"]);
|
||||
const dateCloture = parseDate(row["Date clôture"]);
|
||||
|
||||
try {
|
||||
await db.insert(aapItems).values({
|
||||
dedupKey,
|
||||
titre,
|
||||
categorie,
|
||||
region: normalizeStr(row["Région"]),
|
||||
departement: normalizeStr(row["Département"]),
|
||||
dateCloture,
|
||||
datePublication,
|
||||
lien,
|
||||
});
|
||||
newRows++;
|
||||
} catch (e: unknown) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
errors.push(`[${sheetName}] ${titre.substring(0, 50)}: ${msg}`);
|
||||
skippedRows++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const status = errors.length === 0 ? "success" : newRows > 0 ? "partial" : "error";
|
||||
await db.insert(importLogs).values({
|
||||
fileType: "aap",
|
||||
source: fileName,
|
||||
status,
|
||||
totalRows,
|
||||
newRows,
|
||||
skippedRows,
|
||||
errorMessage: errors.length > 0 ? errors.join("\n") : null,
|
||||
details: errors.length > 0 ? { errors } as Record<string, unknown> : null,
|
||||
startedAt,
|
||||
completedAt: new Date(),
|
||||
});
|
||||
|
||||
return { totalRows, newRows, skippedRows, errors, status };
|
||||
}
|
||||
|
||||
// ─── Route POST /api/upload-excel ─────────────────────────────────────────────
|
||||
|
||||
router.post(
|
||||
"/api/upload-excel",
|
||||
upload.single("file"),
|
||||
async (req: express.Request, res: express.Response): Promise<void> => {
|
||||
try {
|
||||
const fileType = req.body?.fileType as "veille" | "aap";
|
||||
if (!fileType || !["veille", "aap"].includes(fileType)) {
|
||||
res.status(400).json({ error: "fileType doit être 'veille' ou 'aap'" });
|
||||
return;
|
||||
}
|
||||
|
||||
if (!req.file) {
|
||||
res.status(400).json({ error: "Aucun fichier reçu" });
|
||||
return;
|
||||
}
|
||||
|
||||
const buffer = req.file.buffer;
|
||||
const fileName = req.file.originalname;
|
||||
|
||||
let result;
|
||||
if (fileType === "veille") {
|
||||
result = await importVeilleFromBuffer(buffer, fileName);
|
||||
} else {
|
||||
result = await importAAPFromBuffer(buffer, fileName);
|
||||
}
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
fileType,
|
||||
fileName,
|
||||
...result,
|
||||
});
|
||||
} catch (e: unknown) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
console.error("[Upload] Erreur:", msg);
|
||||
res.status(500).json({ error: msg });
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
export default router;
|
||||
Reference in New Issue
Block a user