veille-reglementaire/server/uploadRoutes.ts

import express, { Router } from "express";
import multer from "multer";
import * as XLSX from "xlsx";
import { getDb } from "./db";
import { veilleItems, aapItems, importLogs, appSettings } from "../drizzle/schema";
import { eq } from "drizzle-orm";
import * as crypto from "crypto";

const router: Router = express.Router();

// Multer en mémoire — on traite le buffer directement
const upload = multer({
  storage: multer.memoryStorage(),
  limits: { fileSize: 50 * 1024 * 1024 }, // 50 MB max
  fileFilter: (_req, file, cb) => {
    if (
      file.mimetype === "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" ||
      file.mimetype === "application/vnd.ms-excel" ||
      file.originalname.endsWith(".xlsx") ||
      file.originalname.endsWith(".xls")
    ) {
      cb(null, true);
    } else {
      cb(new Error("Seuls les fichiers Excel (.xlsx, .xls) sont acceptés"));
    }
  },
});

// ─── Utilitaires ─────────────────────────────────────────────────────────────

function makeDedupKey(titre: string, lien?: string | null): string {
  const raw = `${(titre || "").trim().toLowerCase()}|${(lien || "").trim().toLowerCase()}`;
  return crypto.createHash("md5").update(raw).digest("hex");
}

function parseDate(value: unknown): Date | null {
  if (!value) return null;
  if (value instanceof Date) return isNaN(value.getTime()) ? null : value;
  if (typeof value === "string") {
    const cleaned = value.replace("Z", "").trim();
    const d = new Date(cleaned);
    return isNaN(d.getTime()) ? null : d;
  }
  if (typeof value === "number") {
    const d = XLSX.SSF.parse_date_code(value);
    if (d) return new Date(d.y, d.m - 1, d.d);
  }
  return null;
}

function stripHtml(html: string): string {
  return html
    .replace(/<[^>]+>/g, ' ')
    .replace(/&nbsp;/g, ' ')
    .replace(/&amp;/g, '&')
    .replace(/&lt;/g, '<')
    .replace(/&gt;/g, '>')
    .replace(/&quot;/g, '"')
    .replace(/&#39;/g, "'")
    .replace(/\s+/g, ' ')
    .trim();
}

function normalizeStr(v: unknown): string | null {
  if (v === null || v === undefined) return null;
  const s = String(v).trim();
  return s === "" || s === "Non renseigné" ? null : s;
}

// ─── Mapping des feuilles ─────────────────────────────────────────────────────

const VEILLE_SHEETS: Record<string, "reglementaire" | "concurrentielle" | "technologique" | "generale"> = {
  "réglementaire": "reglementaire",
  "reglementaire": "reglementaire",
  "concurrentielle": "concurrentielle",
  "technologique": "technologique",
  "générale": "generale",
  "generale": "generale",
};

const AAP_SHEETS: Record<string, "Handicap" | "PA" | "Enfance" | "Précarité" | "Sanitaire" | "Autre"> = {
  "handicap": "Handicap",
  "pa": "PA",
  "enfance": "Enfance",
  "précarité": "Précarité",
  "precarite": "Précarité",
  "sanitaire": "Sanitaire",
  "autre": "Autre",
};

// ─── Import depuis buffer ─────────────────────────────────────────────────────

async function importVeilleFromBuffer(buffer: Buffer, fileName: string) {
  const startedAt = new Date();
  const errors: string[] = [];
  let totalRows = 0, newRows = 0, skippedRows = 0;

  const db = await getDb();
  if (!db) throw new Error("Base de données indisponible");

  const workbook = XLSX.read(buffer, { type: "buffer", cellDates: true });

  for (const sheetName of workbook.SheetNames) {
    const normalized = sheetName.toLowerCase().trim();
    if (normalized === "poubelle") continue;
    const typeVeille = VEILLE_SHEETS[normalized];
    if (!typeVeille) continue;

    const sheet = workbook.Sheets[sheetName];
    const rows = XLSX.utils.sheet_to_json<Record<string, unknown>>(sheet, { defval: null });

    for (const row of rows) {
      totalRows++;
      const titre = normalizeStr(row["Titre"]);
      if (!titre) { skippedRows++; continue; }

      const lien = normalizeStr(row["Lien"]);
      const dedupKey = makeDedupKey(titre, lien);

      const existing = await db
        .select({ id: veilleItems.id })
        .from(veilleItems)
        .where(eq(veilleItems.dedupKey, dedupKey))
        .limit(1);

      if (existing.length > 0) { skippedRows++; continue; }

      // La colonne "Source" contient une date ISO dans ce fichier
      const sourceRaw = row["Source"];
      const datePublication = parseDate(sourceRaw);
      const sourceStr = normalizeStr(sourceRaw instanceof Date ? null : sourceRaw);

      try {
        await db.insert(veilleItems).values({
          dedupKey,
          titre,
          categorie: normalizeStr(row["Catégorie"]),
          niveau: normalizeStr(row["Niveau"]),
          territoire: normalizeStr(row["Territoire"]),
          resume: (() => {
            const raw = normalizeStr(row[" Résumé"] ?? row["Résumé"] ?? row["Resume"]);
            return raw ? stripHtml(raw) : null;
          })(),
          source: sourceStr,
          passage: normalizeStr(row["passage"] ?? row["Passage"]),
          lien,
          typeVeille,
          datePublication,
        });
        newRows++;
      } catch (e: unknown) {
        const msg = e instanceof Error ? e.message : String(e);
        errors.push(`[${sheetName}] ${titre.substring(0, 50)}: ${msg}`);
        skippedRows++;
      }
    }
  }

  const status = errors.length === 0 ? "success" : newRows > 0 ? "partial" : "error";
  await db.insert(importLogs).values({
    fileType: "veille",
    source: fileName,
    status,
    totalRows,
    newRows,
    skippedRows,
    errorMessage: errors.length > 0 ? errors.join("\n") : null,
    details: errors.length > 0 ? { errors } as Record<string, unknown> : null,
    startedAt,
    completedAt: new Date(),
  });

  return { totalRows, newRows, skippedRows, errors, status };
}

async function importAAPFromBuffer(buffer: Buffer, fileName: string) {
  const startedAt = new Date();
  const errors: string[] = [];
  let totalRows = 0, newRows = 0, skippedRows = 0;

  const db = await getDb();
  if (!db) throw new Error("Base de données indisponible");

  const workbook = XLSX.read(buffer, { type: "buffer", cellDates: true });

  for (const sheetName of workbook.SheetNames) {
    const normalized = sheetName.toLowerCase().trim();
    const normalizedAcc = normalized.replace(/é/g, "e").replace(/è/g, "e").replace(/ê/g, "e");
    const categorie = AAP_SHEETS[normalized] || AAP_SHEETS[normalizedAcc];
    if (!categorie) continue;

    const sheet = workbook.Sheets[sheetName];
    const rows = XLSX.utils.sheet_to_json<Record<string, unknown>>(sheet, { defval: null });

    for (const row of rows) {
      totalRows++;
      const titre = normalizeStr(row["Titre"]);
      if (!titre) { skippedRows++; continue; }

      const lien = normalizeStr(row["Lien"]);
      const dedupKey = makeDedupKey(titre, lien);

      const existing = await db
        .select({ id: aapItems.id })
        .from(aapItems)
        .where(eq(aapItems.dedupKey, dedupKey))
        .limit(1);

      if (existing.length > 0) { skippedRows++; continue; }

      const datePublication = parseDate(row["Date publication"]);
      const dateCloture = parseDate(row["Date clôture"]);

      try {
        await db.insert(aapItems).values({
          dedupKey,
          titre,
          categorie,
          region: normalizeStr(row["Région"]),
          departement: normalizeStr(row["Département"]),
          dateCloture,
          datePublication,
          lien,
        });
        newRows++;
      } catch (e: unknown) {
        const msg = e instanceof Error ? e.message : String(e);
        errors.push(`[${sheetName}] ${titre.substring(0, 50)}: ${msg}`);
        skippedRows++;
      }
    }
  }

  const status = errors.length === 0 ? "success" : newRows > 0 ? "partial" : "error";
  await db.insert(importLogs).values({
    fileType: "aap",
    source: fileName,
    status,
    totalRows,
    newRows,
    skippedRows,
    errorMessage: errors.length > 0 ? errors.join("\n") : null,
    details: errors.length > 0 ? { errors } as Record<string, unknown> : null,
    startedAt,
    completedAt: new Date(),
  });

  return { totalRows, newRows, skippedRows, errors, status };
}

// ─── Route POST /api/upload-excel ─────────────────────────────────────────────

router.post(
  "/api/upload-excel",
  upload.single("file"),
  async (req: express.Request, res: express.Response): Promise<void> => {
    try {
      const fileType = req.body?.fileType as "veille" | "aap";
      if (!fileType || !["veille", "aap"].includes(fileType)) {
        res.status(400).json({ error: "fileType doit être 'veille' ou 'aap'" });
        return;
      }

      if (!req.file) {
        res.status(400).json({ error: "Aucun fichier reçu" });
        return;
      }

      const buffer = req.file.buffer;
      const fileName = req.file.originalname;

      let result;
      if (fileType === "veille") {
        result = await importVeilleFromBuffer(buffer, fileName);
      } else {
        result = await importAAPFromBuffer(buffer, fileName);
      }

      res.json({
        success: true,
        fileType,
        fileName,
        ...result,
      });
    } catch (e: unknown) {
      const msg = e instanceof Error ? e.message : String(e);
      console.error("[Upload] Erreur:", msg);
      res.status(500).json({ error: msg });
    }
  }
);

export default router;