Checkpoint: Application complète : deux tableaux de bord (Veille Stratégique + AAP), import Excel quotidien avec déduplication, sources multiples (local/OneDrive/FTP/SharePoint), affichage liste/vignettes, filtres multi-critères, gestion utilisateurs, logs d'import, page paramètres, authentification locale, tâche cron 06h00, 13 tests Vitest passants.

This commit is contained in:
Manus
2026-03-16 10:45:35 -04:00
parent 5000fc555d
commit 8fb71e8bda
27 changed files with 4525 additions and 184 deletions

View File

@@ -3,47 +3,72 @@ import express from "express";
import { createServer } from "http";
import net from "net";
import { createExpressMiddleware } from "@trpc/server/adapters/express";
import * as cron from "node-cron";
import { registerOAuthRoutes } from "./oauth";
import { appRouter } from "../routers";
import { createContext } from "./context";
import { serveStatic, setupVite } from "./vite";
import { runFullImport } from "../importer";
import { ensureAdminExists } from "../localAuth";
import { getSetting } from "../db";
function isPortAvailable(port: number): Promise<boolean> {
return new Promise(resolve => {
const server = net.createServer();
server.listen(port, () => {
server.close(() => resolve(true));
});
server.listen(port, () => { server.close(() => resolve(true)); });
server.on("error", () => resolve(false));
});
}
async function findAvailablePort(startPort: number = 3000): Promise<number> {
for (let port = startPort; port < startPort + 20; port++) {
if (await isPortAvailable(port)) {
return port;
}
if (await isPortAvailable(port)) return port;
}
throw new Error(`No available port found starting from ${startPort}`);
}
// ─── Tâche d'import quotidien ─────────────────────────────────────────────────
let cronJob: ReturnType<typeof cron.schedule> | null = null;
async function scheduleDailyImport() {
// Heure configurable, défaut 06:00
const importTime = (await getSetting("import_time")) || "06:00";
const [hour, minute] = importTime.split(":").map(Number);
const cronExpr = `0 ${minute ?? 0} ${hour ?? 6} * * *`;
if (cronJob) {
cronJob.stop();
cronJob = null;
}
cronJob = cron.schedule(cronExpr, async () => {
console.log(`[Cron] Import automatique démarré à ${new Date().toISOString()}`);
try {
const result = await runFullImport();
console.log(`[Cron] Import terminé — Veille: +${result.veille.newRows} | AAP: +${result.aap.newRows}`);
} catch (e) {
console.error("[Cron] Erreur lors de l'import:", e);
}
});
console.log(`[Cron] Import quotidien planifié à ${importTime} (${cronExpr})`);
}
async function startServer() {
const app = express();
const server = createServer(app);
// Configure body parser with larger size limit for file uploads
app.use(express.json({ limit: "50mb" }));
app.use(express.urlencoded({ limit: "50mb", extended: true }));
// OAuth callback under /api/oauth/callback
registerOAuthRoutes(app);
// tRPC API
app.use(
"/api/trpc",
createExpressMiddleware({
router: appRouter,
createContext,
})
createExpressMiddleware({ router: appRouter, createContext })
);
// development mode uses Vite, production mode uses static files
if (process.env.NODE_ENV === "development") {
await setupVite(app, server);
} else {
@@ -57,8 +82,16 @@ async function startServer() {
console.log(`Port ${preferredPort} is busy, using port ${port} instead`);
}
server.listen(port, () => {
server.listen(port, async () => {
console.log(`Server running on http://localhost:${port}/`);
// Initialisation post-démarrage
try {
await ensureAdminExists();
await scheduleDailyImport();
} catch (e) {
console.error("[Init] Erreur d'initialisation:", e);
}
});
}

View File

@@ -1,11 +1,19 @@
import { eq } from "drizzle-orm";
import { eq, desc, and, like, gte, lte, or, sql } from "drizzle-orm";
import { drizzle } from "drizzle-orm/mysql2";
import { InsertUser, users } from "../drizzle/schema";
import { ENV } from './_core/env';
import {
InsertUser,
users,
localUsers,
veilleItems,
aapItems,
appSettings,
importLogs,
InsertLocalUser,
} from "../drizzle/schema";
import { ENV } from "./_core/env";
let _db: ReturnType<typeof drizzle> | null = null;
// Lazily create the drizzle instance so local tooling can run without a DB.
export async function getDb() {
if (!_db && process.env.DATABASE_URL) {
try {
@@ -18,75 +26,292 @@ export async function getDb() {
return _db;
}
// ─── Users (Manus OAuth) ─────────────────────────────────────────────────────
export async function upsertUser(user: InsertUser): Promise<void> {
if (!user.openId) {
throw new Error("User openId is required for upsert");
}
if (!user.openId) throw new Error("User openId is required for upsert");
const db = await getDb();
if (!db) {
console.warn("[Database] Cannot upsert user: database not available");
return;
if (!db) { console.warn("[Database] Cannot upsert user: database not available"); return; }
const values: InsertUser = { openId: user.openId };
const updateSet: Record<string, unknown> = {};
const textFields = ["name", "email", "loginMethod"] as const;
for (const field of textFields) {
const value = user[field];
if (value === undefined) continue;
const normalized = value ?? null;
values[field] = normalized;
updateSet[field] = normalized;
}
try {
const values: InsertUser = {
openId: user.openId,
};
const updateSet: Record<string, unknown> = {};
if (user.lastSignedIn !== undefined) { values.lastSignedIn = user.lastSignedIn; updateSet.lastSignedIn = user.lastSignedIn; }
if (user.role !== undefined) { values.role = user.role; updateSet.role = user.role; }
else if (user.openId === ENV.ownerOpenId) { values.role = "admin"; updateSet.role = "admin"; }
if (!values.lastSignedIn) values.lastSignedIn = new Date();
if (Object.keys(updateSet).length === 0) updateSet.lastSignedIn = new Date();
const textFields = ["name", "email", "loginMethod"] as const;
type TextField = (typeof textFields)[number];
const assignNullable = (field: TextField) => {
const value = user[field];
if (value === undefined) return;
const normalized = value ?? null;
values[field] = normalized;
updateSet[field] = normalized;
};
textFields.forEach(assignNullable);
if (user.lastSignedIn !== undefined) {
values.lastSignedIn = user.lastSignedIn;
updateSet.lastSignedIn = user.lastSignedIn;
}
if (user.role !== undefined) {
values.role = user.role;
updateSet.role = user.role;
} else if (user.openId === ENV.ownerOpenId) {
values.role = 'admin';
updateSet.role = 'admin';
}
if (!values.lastSignedIn) {
values.lastSignedIn = new Date();
}
if (Object.keys(updateSet).length === 0) {
updateSet.lastSignedIn = new Date();
}
await db.insert(users).values(values).onDuplicateKeyUpdate({
set: updateSet,
});
} catch (error) {
console.error("[Database] Failed to upsert user:", error);
throw error;
}
await db.insert(users).values(values).onDuplicateKeyUpdate({ set: updateSet });
}
export async function getUserByOpenId(openId: string) {
const db = await getDb();
if (!db) {
console.warn("[Database] Cannot get user: database not available");
return undefined;
}
if (!db) return undefined;
const result = await db.select().from(users).where(eq(users.openId, openId)).limit(1);
return result.length > 0 ? result[0] : undefined;
return result[0];
}
// TODO: add feature queries here as your schema grows.
// ─── Local Users ─────────────────────────────────────────────────────────────
export async function getLocalUsers() {
const db = await getDb();
if (!db) return [];
return db
.select({
id: localUsers.id,
name: localUsers.name,
email: localUsers.email,
role: localUsers.role,
isActive: localUsers.isActive,
createdAt: localUsers.createdAt,
lastSignedIn: localUsers.lastSignedIn,
})
.from(localUsers)
.orderBy(desc(localUsers.createdAt));
}
export async function createLocalUser(data: Omit<InsertLocalUser, "id" | "createdAt" | "updatedAt">) {
const db = await getDb();
if (!db) throw new Error("DB unavailable");
await db.insert(localUsers).values(data);
}
export async function updateLocalUser(id: number, data: Partial<InsertLocalUser>) {
const db = await getDb();
if (!db) throw new Error("DB unavailable");
await db.update(localUsers).set(data).where(eq(localUsers.id, id));
}
export async function deleteLocalUser(id: number) {
const db = await getDb();
if (!db) throw new Error("DB unavailable");
await db.delete(localUsers).where(eq(localUsers.id, id));
}
// ─── Veille Items ─────────────────────────────────────────────────────────────
export interface VeilleFilters {
typeVeille?: string;
categorie?: string;
niveau?: string;
territoire?: string;
search?: string;
dateFrom?: Date;
dateTo?: Date;
page?: number;
pageSize?: number;
}
export async function getVeilleItems(filters: VeilleFilters = {}) {
const db = await getDb();
if (!db) return { items: [], total: 0 };
const { page = 1, pageSize = 50, ...f } = filters;
const offset = (page - 1) * pageSize;
const conditions = [];
if (f.typeVeille) conditions.push(eq(veilleItems.typeVeille, f.typeVeille as "reglementaire" | "concurrentielle" | "technologique" | "generale"));
if (f.categorie) conditions.push(like(veilleItems.categorie, `%${f.categorie}%`));
if (f.niveau) conditions.push(like(veilleItems.niveau, `%${f.niveau}%`));
if (f.territoire) conditions.push(like(veilleItems.territoire, `%${f.territoire}%`));
if (f.search) {
conditions.push(
or(
like(veilleItems.titre, `%${f.search}%`),
like(veilleItems.resume, `%${f.search}%`)
)
);
}
if (f.dateFrom) conditions.push(gte(veilleItems.datePublication, f.dateFrom));
if (f.dateTo) conditions.push(lte(veilleItems.datePublication, f.dateTo));
const where = conditions.length > 0 ? and(...conditions) : undefined;
const [items, countResult] = await Promise.all([
db
.select()
.from(veilleItems)
.where(where)
.orderBy(desc(veilleItems.datePublication), desc(veilleItems.importedAt))
.limit(pageSize)
.offset(offset),
db
.select({ count: sql<number>`count(*)` })
.from(veilleItems)
.where(where),
]);
return { items, total: Number(countResult[0]?.count ?? 0) };
}
export async function getVeilleDistinctValues() {
const db = await getDb();
if (!db) return { categories: [], niveaux: [], territoires: [] };
const [cats, niveaux, territoires] = await Promise.all([
db.selectDistinct({ value: veilleItems.categorie }).from(veilleItems).where(sql`${veilleItems.categorie} IS NOT NULL`),
db.selectDistinct({ value: veilleItems.niveau }).from(veilleItems).where(sql`${veilleItems.niveau} IS NOT NULL`),
db.selectDistinct({ value: veilleItems.territoire }).from(veilleItems).where(sql`${veilleItems.territoire} IS NOT NULL`),
]);
return {
categories: cats.map((r) => r.value!).filter(Boolean).sort(),
niveaux: niveaux.map((r) => r.value!).filter(Boolean).sort(),
territoires: territoires.map((r) => r.value!).filter(Boolean).sort(),
};
}
// ─── AAP Items ────────────────────────────────────────────────────────────────
export interface AapFilters {
categorie?: string;
region?: string;
departement?: string;
search?: string;
dateFrom?: Date;
dateTo?: Date;
clotureFrom?: Date;
clotureTo?: Date;
page?: number;
pageSize?: number;
}
export async function getAapItems(filters: AapFilters = {}) {
const db = await getDb();
if (!db) return { items: [], total: 0 };
const { page = 1, pageSize = 50, ...f } = filters;
const offset = (page - 1) * pageSize;
const conditions = [];
if (f.categorie) conditions.push(eq(aapItems.categorie, f.categorie as "Handicap" | "PA" | "Enfance" | "Précarité" | "Sanitaire" | "Autre"));
if (f.region) conditions.push(like(aapItems.region, `%${f.region}%`));
if (f.departement) conditions.push(like(aapItems.departement, `%${f.departement}%`));
if (f.search) conditions.push(like(aapItems.titre, `%${f.search}%`));
if (f.dateFrom) conditions.push(gte(aapItems.datePublication, f.dateFrom));
if (f.dateTo) conditions.push(lte(aapItems.datePublication, f.dateTo));
if (f.clotureFrom) conditions.push(gte(aapItems.dateCloture, f.clotureFrom));
if (f.clotureTo) conditions.push(lte(aapItems.dateCloture, f.clotureTo));
const where = conditions.length > 0 ? and(...conditions) : undefined;
const [items, countResult] = await Promise.all([
db
.select()
.from(aapItems)
.where(where)
.orderBy(desc(aapItems.datePublication), desc(aapItems.importedAt))
.limit(pageSize)
.offset(offset),
db
.select({ count: sql<number>`count(*)` })
.from(aapItems)
.where(where),
]);
return { items, total: Number(countResult[0]?.count ?? 0) };
}
export async function getAapDistinctValues() {
const db = await getDb();
if (!db) return { regions: [], departements: [] };
const [regions, departements] = await Promise.all([
db.selectDistinct({ value: aapItems.region }).from(aapItems).where(sql`${aapItems.region} IS NOT NULL`),
db.selectDistinct({ value: aapItems.departement }).from(aapItems).where(sql`${aapItems.departement} IS NOT NULL`),
]);
return {
regions: regions.map((r) => r.value!).filter(Boolean).sort(),
departements: departements.map((r) => r.value!).filter(Boolean).sort(),
};
}
// ─── App Settings ─────────────────────────────────────────────────────────────
export async function getSetting(key: string): Promise<string | null> {
const db = await getDb();
if (!db) return null;
const rows = await db.select().from(appSettings).where(eq(appSettings.key, key)).limit(1);
return rows[0]?.value ?? null;
}
export async function getAllSettings(): Promise<Record<string, string>> {
const db = await getDb();
if (!db) return {};
const rows = await db.select().from(appSettings);
const map: Record<string, string> = {};
for (const r of rows) {
if (r.key && r.value !== null && r.value !== undefined) map[r.key] = r.value;
}
return map;
}
export async function setSetting(key: string, value: string): Promise<void> {
const db = await getDb();
if (!db) throw new Error("DB unavailable");
await db
.insert(appSettings)
.values({ key, value })
.onDuplicateKeyUpdate({ set: { value } });
}
export async function setSettings(settings: Record<string, string>): Promise<void> {
const db = await getDb();
if (!db) throw new Error("DB unavailable");
for (const [key, value] of Object.entries(settings)) {
await db
.insert(appSettings)
.values({ key, value })
.onDuplicateKeyUpdate({ set: { value } });
}
}
// ─── Import Logs ──────────────────────────────────────────────────────────────
export async function getImportLogs(limit = 50) {
const db = await getDb();
if (!db) return [];
return db
.select()
.from(importLogs)
.orderBy(desc(importLogs.startedAt))
.limit(limit);
}
export async function getImportStats() {
const db = await getDb();
if (!db) return { totalVeille: 0, totalAap: 0, lastImport: null, total: 0, success: 0, errors: 0, totalNewRows: 0 };
const [veilleCount, aapCount, lastLog, allLogs] = await Promise.all([
db.select({ count: sql<number>`count(*)` }).from(veilleItems),
db.select({ count: sql<number>`count(*)` }).from(aapItems),
db.select().from(importLogs).orderBy(desc(importLogs.startedAt)).limit(1),
db.select().from(importLogs),
]);
const total = allLogs.length;
const success = allLogs.filter(l => l.status === 'success').length;
const errors = allLogs.filter(l => l.status === 'error').length;
const totalNewRows = allLogs.reduce((sum, l) => sum + (l.newRows ?? 0), 0);
return {
totalVeille: Number(veilleCount[0]?.count ?? 0),
totalAap: Number(aapCount[0]?.count ?? 0),
lastImport: lastLog[0] ?? null,
total,
success,
errors,
totalNewRows,
};
}

390
server/importer.ts Normal file
View File

@@ -0,0 +1,390 @@
import * as XLSX from "xlsx";
import * as crypto from "crypto";
import * as fs from "fs";
import * as path from "path";
import * as ftp from "basic-ftp";
import * as https from "https";
import * as http from "http";
import { getDb } from "./db";
import { veilleItems, aapItems, importLogs, appSettings } from "../drizzle/schema";
import { eq, inArray } from "drizzle-orm";
// ─── Types ───────────────────────────────────────────────────────────────────
export type SourceType = "local" | "onedrive" | "ftp" | "sharepoint";
export interface ImportConfig {
sourceType: SourceType;
veilleFilePath?: string;
aapFilePath?: string;
ftpHost?: string;
ftpPort?: number;
ftpUser?: string;
ftpPassword?: string;
ftpSecure?: boolean;
onedriveToken?: string;
sharepointSiteUrl?: string;
sharepointToken?: string;
}
export interface ImportResult {
fileType: "veille" | "aap";
totalRows: number;
newRows: number;
skippedRows: number;
errors: string[];
status: "success" | "partial" | "error";
}
// ─── Utilitaires ─────────────────────────────────────────────────────────────
function makeDedupKey(titre: string, lien?: string | null): string {
const raw = `${(titre || "").trim().toLowerCase()}|${(lien || "").trim().toLowerCase()}`;
return crypto.createHash("md5").update(raw).digest("hex");
}
function parseDate(value: unknown): Date | null {
if (!value) return null;
if (value instanceof Date) return isNaN(value.getTime()) ? null : value;
if (typeof value === "string") {
const cleaned = value.replace("Z", "").trim();
const d = new Date(cleaned);
return isNaN(d.getTime()) ? null : d;
}
if (typeof value === "number") {
// Excel serial date
const d = XLSX.SSF.parse_date_code(value);
if (d) return new Date(d.y, d.m - 1, d.d);
}
return null;
}
function normalizeStr(v: unknown): string | null {
if (v === null || v === undefined) return null;
const s = String(v).trim();
return s === "" || s === "Non renseigné" ? null : s;
}
// ─── Téléchargement des fichiers selon la source ─────────────────────────────
async function downloadFile(
filePath: string,
config: ImportConfig
): Promise<Buffer> {
switch (config.sourceType) {
case "local": {
if (!fs.existsSync(filePath)) {
throw new Error(`Fichier introuvable : ${filePath}`);
}
return fs.readFileSync(filePath);
}
case "ftp": {
const client = new ftp.Client();
client.ftp.verbose = false;
try {
await client.access({
host: config.ftpHost!,
port: config.ftpPort || 21,
user: config.ftpUser!,
password: config.ftpPassword!,
secure: config.ftpSecure || false,
});
const tmpPath = `/tmp/veille_import_${Date.now()}.xlsx`;
await client.downloadTo(tmpPath, filePath);
const buf = fs.readFileSync(tmpPath);
fs.unlinkSync(tmpPath);
return buf;
} finally {
client.close();
}
}
case "onedrive":
case "sharepoint": {
const token =
config.sourceType === "onedrive"
? config.onedriveToken
: config.sharepointToken;
if (!token) throw new Error("Token d'authentification manquant");
return new Promise((resolve, reject) => {
const url = new URL(filePath);
const options = {
hostname: url.hostname,
path: url.pathname + url.search,
headers: { Authorization: `Bearer ${token}` },
};
const protocol = url.protocol === "https:" ? https : http;
protocol
.get(options, (res) => {
const chunks: Buffer[] = [];
res.on("data", (c) => chunks.push(c));
res.on("end", () => resolve(Buffer.concat(chunks)));
res.on("error", reject);
})
.on("error", reject);
});
}
default:
throw new Error(`Source non supportée : ${config.sourceType}`);
}
}
// ─── Lecture des paramètres depuis la BDD ────────────────────────────────────
export async function getImportConfig(): Promise<ImportConfig> {
const db = await getDb();
if (!db) return { sourceType: "local" };
const rows = await db.select().from(appSettings);
const map: Record<string, string> = {};
for (const r of rows) {
if (r.key && r.value) map[r.key] = r.value;
}
return {
sourceType: (map["source_type"] as SourceType) || "local",
veilleFilePath: map["veille_file_path"] || "",
aapFilePath: map["aap_file_path"] || "",
ftpHost: map["ftp_host"],
ftpPort: map["ftp_port"] ? parseInt(map["ftp_port"]) : 21,
ftpUser: map["ftp_user"],
ftpPassword: map["ftp_password"],
ftpSecure: map["ftp_secure"] === "true",
onedriveToken: map["onedrive_token"],
sharepointSiteUrl: map["sharepoint_site_url"],
sharepointToken: map["sharepoint_token"],
};
}
// ─── Import Veille Stratégique ───────────────────────────────────────────────
const VEILLE_SHEETS: Record<string, "reglementaire" | "concurrentielle" | "technologique" | "generale"> = {
réglementaire: "reglementaire",
reglementaire: "reglementaire",
concurrentielle: "concurrentielle",
technologique: "technologique",
générale: "generale",
generale: "generale",
};
export async function importVeille(config: ImportConfig): Promise<ImportResult> {
const startedAt = new Date();
const errors: string[] = [];
let totalRows = 0;
let newRows = 0;
let skippedRows = 0;
const db = await getDb();
if (!db) throw new Error("Base de données indisponible");
const filePath = config.veilleFilePath;
if (!filePath) throw new Error("Chemin du fichier Veille non configuré");
let buffer: Buffer;
try {
buffer = await downloadFile(filePath, config);
} catch (e: unknown) {
const msg = e instanceof Error ? e.message : String(e);
await logImport(db, "veille", filePath, "error", 0, 0, 0, msg, null, startedAt);
return { fileType: "veille", totalRows: 0, newRows: 0, skippedRows: 0, errors: [msg], status: "error" };
}
const workbook = XLSX.read(buffer, { type: "buffer", cellDates: true });
for (const sheetName of workbook.SheetNames) {
const normalized = sheetName.toLowerCase().trim();
if (normalized === "poubelle") continue;
const typeVeille = VEILLE_SHEETS[normalized];
if (!typeVeille) continue;
const sheet = workbook.Sheets[sheetName];
const rows = XLSX.utils.sheet_to_json<Record<string, unknown>>(sheet, { defval: null });
for (const row of rows) {
totalRows++;
const titre = normalizeStr(row["Titre"]);
if (!titre) { skippedRows++; continue; }
const lien = normalizeStr(row["Lien"]);
const dedupKey = makeDedupKey(titre, lien);
// Vérifier si déjà présent
const existing = await db
.select({ id: veilleItems.id })
.from(veilleItems)
.where(eq(veilleItems.dedupKey, dedupKey))
.limit(1);
if (existing.length > 0) { skippedRows++; continue; }
// Extraire la date depuis la colonne Source (qui contient une date ISO)
const sourceRaw = row["Source"];
const datePublication = parseDate(sourceRaw);
// La vraie source (URL) semble être dans Lien pour certaines feuilles
const sourceStr = normalizeStr(sourceRaw instanceof Date ? null : sourceRaw);
try {
await db.insert(veilleItems).values({
dedupKey,
titre,
categorie: normalizeStr(row["Catégorie"]),
niveau: normalizeStr(row["Niveau"]),
territoire: normalizeStr(row["Territoire"]),
resume: normalizeStr(row[" Résumé"] ?? row["Résumé"] ?? row["Resume"]),
source: sourceStr,
passage: normalizeStr(row["passage"] ?? row["Passage"]),
lien,
typeVeille,
datePublication,
});
newRows++;
} catch (e: unknown) {
const msg = e instanceof Error ? e.message : String(e);
errors.push(`[${sheetName}] ${titre?.substring(0, 50)}: ${msg}`);
skippedRows++;
}
}
}
const status = errors.length === 0 ? "success" : newRows > 0 ? "partial" : "error";
await logImport(db, "veille", filePath, status, totalRows, newRows, skippedRows, errors.join("\n") || null, { errors }, startedAt);
return { fileType: "veille", totalRows, newRows, skippedRows, errors, status };
}
// ─── Import Appels à Projets ─────────────────────────────────────────────────
const AAP_SHEETS: Record<string, "Handicap" | "PA" | "Enfance" | "Précarité" | "Sanitaire" | "Autre"> = {
handicap: "Handicap",
pa: "PA",
enfance: "Enfance",
"précarité": "Précarité",
precarite: "Précarité",
sanitaire: "Sanitaire",
autre: "Autre",
};
export async function importAAP(config: ImportConfig): Promise<ImportResult> {
const startedAt = new Date();
const errors: string[] = [];
let totalRows = 0;
let newRows = 0;
let skippedRows = 0;
const db = await getDb();
if (!db) throw new Error("Base de données indisponible");
const filePath = config.aapFilePath;
if (!filePath) throw new Error("Chemin du fichier AAP non configuré");
let buffer: Buffer;
try {
buffer = await downloadFile(filePath, config);
} catch (e: unknown) {
const msg = e instanceof Error ? e.message : String(e);
await logImport(db, "aap", filePath, "error", 0, 0, 0, msg, null, startedAt);
return { fileType: "aap", totalRows: 0, newRows: 0, skippedRows: 0, errors: [msg], status: "error" };
}
const workbook = XLSX.read(buffer, { type: "buffer", cellDates: true });
for (const sheetName of workbook.SheetNames) {
const normalized = sheetName.toLowerCase().trim().replace(/é/g, "e").replace(/è/g, "e");
const categorie = AAP_SHEETS[sheetName.toLowerCase().trim()] || AAP_SHEETS[normalized];
if (!categorie) continue;
const sheet = workbook.Sheets[sheetName];
const rows = XLSX.utils.sheet_to_json<Record<string, unknown>>(sheet, { defval: null });
for (const row of rows) {
totalRows++;
const titre = normalizeStr(row["Titre"]);
if (!titre) { skippedRows++; continue; }
const lien = normalizeStr(row["Lien"]);
const dedupKey = makeDedupKey(titre, lien);
const existing = await db
.select({ id: aapItems.id })
.from(aapItems)
.where(eq(aapItems.dedupKey, dedupKey))
.limit(1);
if (existing.length > 0) { skippedRows++; continue; }
const datePublication = parseDate(row["Date publication"]);
const dateCloture = parseDate(row["Date clôture"]);
try {
await db.insert(aapItems).values({
dedupKey,
titre,
categorie,
region: normalizeStr(row["Région"]),
departement: normalizeStr(row["Département"]),
dateCloture,
datePublication,
lien,
});
newRows++;
} catch (e: unknown) {
const msg = e instanceof Error ? e.message : String(e);
errors.push(`[${sheetName}] ${titre?.substring(0, 50)}: ${msg}`);
skippedRows++;
}
}
}
const status = errors.length === 0 ? "success" : newRows > 0 ? "partial" : "error";
await logImport(db, "aap", filePath, status, totalRows, newRows, skippedRows, errors.join("\n") || null, { errors }, startedAt);
return { fileType: "aap", totalRows, newRows, skippedRows, errors, status };
}
// ─── Import complet (veille + AAP) ───────────────────────────────────────────
export async function runFullImport(): Promise<{ veille: ImportResult; aap: ImportResult }> {
const config = await getImportConfig();
const [veille, aap] = await Promise.all([
importVeille(config),
importAAP(config),
]);
return { veille, aap };
}
// ─── Enregistrement des logs ─────────────────────────────────────────────────
async function logImport(
db: Awaited<ReturnType<typeof getDb>>,
fileType: "veille" | "aap",
source: string,
status: "success" | "partial" | "error",
totalRows: number,
newRows: number,
skippedRows: number,
errorMessage: string | null,
details: unknown,
startedAt: Date
) {
if (!db) return;
try {
await db.insert(importLogs).values({
fileType,
source,
status,
totalRows,
newRows,
skippedRows,
errorMessage,
details: details as Record<string, unknown> | null,
startedAt,
completedAt: new Date(),
});
} catch (e) {
console.error("[Import] Erreur lors de l'enregistrement du log:", e);
}
}

96
server/localAuth.ts Normal file
View File

@@ -0,0 +1,96 @@
import bcrypt from "bcryptjs";
import { getDb } from "./db";
import { localUsers } from "../drizzle/schema";
import { eq } from "drizzle-orm";
import { SignJWT, jwtVerify } from "jose";
import { ENV } from "./_core/env";
const SALT_ROUNDS = 12;
const JWT_EXPIRY = "7d";
const LOCAL_AUTH_COOKIE = "veille_local_auth";
export async function hashPassword(password: string): Promise<string> {
return bcrypt.hash(password, SALT_ROUNDS);
}
export async function verifyPassword(password: string, hash: string): Promise<boolean> {
return bcrypt.compare(password, hash);
}
export async function generateLocalToken(userId: number, role: string): Promise<string> {
const secret = new TextEncoder().encode(ENV.cookieSecret);
return new SignJWT({ sub: String(userId), role, type: "local" })
.setProtectedHeader({ alg: "HS256" })
.setIssuedAt()
.setExpirationTime(JWT_EXPIRY)
.sign(secret);
}
export async function verifyLocalToken(token: string): Promise<{ userId: number; role: string } | null> {
try {
const secret = new TextEncoder().encode(ENV.cookieSecret);
const { payload } = await jwtVerify(token, secret);
if (payload.type !== "local" || !payload.sub) return null;
return { userId: parseInt(payload.sub), role: payload.role as string };
} catch {
return null;
}
}
export async function loginLocalUser(email: string, password: string) {
const db = await getDb();
if (!db) throw new Error("Base de données indisponible");
const users = await db
.select()
.from(localUsers)
.where(eq(localUsers.email, email.toLowerCase().trim()))
.limit(1);
const user = users[0];
if (!user || !user.isActive) {
throw new Error("Identifiants incorrects ou compte désactivé");
}
const valid = await verifyPassword(password, user.passwordHash);
if (!valid) throw new Error("Identifiants incorrects ou compte désactivé");
// Mise à jour lastSignedIn
await db
.update(localUsers)
.set({ lastSignedIn: new Date() })
.where(eq(localUsers.id, user.id));
const token = await generateLocalToken(user.id, user.role);
return { token, user: { id: user.id, name: user.name, email: user.email, role: user.role } };
}
export async function getLocalUserById(id: number) {
const db = await getDb();
if (!db) return null;
const users = await db.select().from(localUsers).where(eq(localUsers.id, id)).limit(1);
return users[0] ?? null;
}
export async function ensureAdminExists() {
const db = await getDb();
if (!db) return;
const admins = await db
.select({ id: localUsers.id })
.from(localUsers)
.where(eq(localUsers.role, "admin"))
.limit(1);
if (admins.length === 0) {
const hash = await hashPassword("Admin@Itinova2024!");
await db.insert(localUsers).values({
name: "Administrateur",
email: "admin@itinova.fr",
passwordHash: hash,
role: "admin",
isActive: true,
});
console.log("[LocalAuth] Compte admin par défaut créé : admin@itinova.fr / Admin@Itinova2024!");
}
}

View File

@@ -1,28 +1,243 @@
import { z } from "zod";
import { TRPCError } from "@trpc/server";
import { COOKIE_NAME } from "@shared/const";
import { getSessionCookieOptions } from "./_core/cookies";
import { systemRouter } from "./_core/systemRouter";
import { publicProcedure, router } from "./_core/trpc";
import { publicProcedure, protectedProcedure, router } from "./_core/trpc";
import {
getVeilleItems,
getVeilleDistinctValues,
getAapItems,
getAapDistinctValues,
getAllSettings,
setSettings,
getImportLogs,
getImportStats,
getLocalUsers,
createLocalUser,
updateLocalUser,
deleteLocalUser,
} from "./db";
import { importVeille, importAAP, runFullImport, getImportConfig } from "./importer";
import { loginLocalUser, hashPassword, ensureAdminExists } from "./localAuth";
// ─── Middleware admin ─────────────────────────────────────────────────────────
const adminProcedure = protectedProcedure.use(({ ctx, next }) => {
if (ctx.user.role !== "admin") {
throw new TRPCError({ code: "FORBIDDEN", message: "Accès réservé aux administrateurs" });
}
return next({ ctx });
});
// ─── Router principal ─────────────────────────────────────────────────────────
export const appRouter = router({
// if you need to use socket.io, read and register route in server/_core/index.ts, all api should start with '/api/' so that the gateway can route correctly
system: systemRouter,
// ─── Auth ───────────────────────────────────────────────────────────────────
auth: router({
me: publicProcedure.query(opts => opts.ctx.user),
me: publicProcedure.query((opts) => opts.ctx.user),
logout: publicProcedure.mutation(({ ctx }) => {
const cookieOptions = getSessionCookieOptions(ctx.req);
ctx.res.clearCookie(COOKIE_NAME, { ...cookieOptions, maxAge: -1 });
return {
success: true,
} as const;
return { success: true } as const;
}),
// Connexion locale
localLogin: publicProcedure
.input(z.object({ email: z.string().email(), password: z.string().min(1) }))
.mutation(async ({ input, ctx }) => {
const result = await loginLocalUser(input.email, input.password);
// Stocker le token dans un cookie
const cookieOptions = getSessionCookieOptions(ctx.req);
ctx.res.cookie("veille_local_auth", result.token, {
...cookieOptions,
maxAge: 7 * 24 * 60 * 60 * 1000,
});
return { success: true, user: result.user };
}),
localLogout: publicProcedure.mutation(({ ctx }) => {
const cookieOptions = getSessionCookieOptions(ctx.req);
ctx.res.clearCookie("veille_local_auth", { ...cookieOptions, maxAge: -1 });
return { success: true };
}),
}),
// TODO: add feature routers here, e.g.
// todo: router({
// list: protectedProcedure.query(({ ctx }) =>
// db.getUserTodos(ctx.user.id)
// ),
// }),
// ─── Veille ─────────────────────────────────────────────────────────────────
veille: router({
list: publicProcedure
.input(
z.object({
typeVeille: z.enum(["reglementaire", "concurrentielle", "technologique", "generale"]).optional(),
categorie: z.string().optional(),
niveau: z.string().optional(),
territoire: z.string().optional(),
search: z.string().optional(),
dateFrom: z.date().optional(),
dateTo: z.date().optional(),
page: z.number().int().positive().default(1),
pageSize: z.number().int().positive().max(200).default(50),
})
)
.query(async ({ input }) => {
return getVeilleItems(input);
}),
filters: publicProcedure.query(async () => {
return getVeilleDistinctValues();
}),
}),
// ─── AAP ────────────────────────────────────────────────────────────────────
aap: router({
list: publicProcedure
.input(
z.object({
categorie: z.enum(["Handicap", "PA", "Enfance", "Précarité", "Sanitaire", "Autre"]).optional(),
region: z.string().optional(),
departement: z.string().optional(),
search: z.string().optional(),
dateFrom: z.date().optional(),
dateTo: z.date().optional(),
clotureFrom: z.date().optional(),
clotureTo: z.date().optional(),
page: z.number().int().positive().default(1),
pageSize: z.number().int().positive().max(200).default(50),
})
)
.query(async ({ input }) => {
return getAapItems(input);
}),
filters: publicProcedure.query(async () => {
return getAapDistinctValues();
}),
}),
// ─── Import ─────────────────────────────────────────────────────────────────
import: router({
run: adminProcedure
.input(z.object({ type: z.enum(["veille", "aap", "all"]).default("all") }))
.mutation(async ({ input }) => {
const config = await getImportConfig();
if (input.type === "all") return runFullImport();
if (input.type === "veille") return { veille: await importVeille(config) };
return { aap: await importAAP(config) };
}),
logs: adminProcedure
.input(z.object({ page: z.number().int().positive().default(1), pageSize: z.number().int().positive().max(100).default(20) }))
.query(async ({ input }) => {
const allLogs = await getImportLogs(500);
const start = (input.page - 1) * input.pageSize;
const logs = allLogs.slice(start, start + input.pageSize);
const stats = await getImportStats();
return { logs, total: allLogs.length, stats };
}),
stats: publicProcedure.query(async () => {
return getImportStats();
}),
}),
// ─── Paramètres ─────────────────────────────────────────────────────────────
settings: router({
get: adminProcedure.query(async () => {
const all = await getAllSettings();
// Masquer les mots de passe
const safe = { ...all };
if (safe.ftp_password) safe.ftp_password = "••••••••";
if (safe.onedrive_token) safe.onedrive_token = "••••••••";
if (safe.sharepoint_token) safe.sharepoint_token = "••••••••";
return safe;
}),
save: adminProcedure
.input(
z.object({
source_type: z.enum(["local", "onedrive", "ftp", "sharepoint"]),
veille_file_path: z.string().optional(),
aap_file_path: z.string().optional(),
ftp_host: z.string().optional(),
ftp_port: z.string().optional(),
ftp_user: z.string().optional(),
ftp_password: z.string().optional(),
ftp_secure: z.string().optional(),
onedrive_token: z.string().optional(),
sharepoint_site_url: z.string().optional(),
sharepoint_token: z.string().optional(),
auth_mode: z.enum(["local", "free"]).optional(),
import_time: z.string().optional(),
})
)
.mutation(async ({ input }) => {
const toSave: Record<string, string> = {};
for (const [k, v] of Object.entries(input)) {
if (v !== undefined && v !== "••••••••") toSave[k] = v;
}
await setSettings(toSave);
return { success: true };
}),
}),
// ─── Utilisateurs locaux ─────────────────────────────────────────────────────
users: router({
list: adminProcedure.query(async () => {
return getLocalUsers();
}),
create: adminProcedure
.input(
z.object({
name: z.string().min(2).max(255),
email: z.string().email(),
password: z.string().min(8),
role: z.enum(["admin", "user", "readonly"]).default("user"),
})
)
.mutation(async ({ input }) => {
const passwordHash = await hashPassword(input.password);
await createLocalUser({
name: input.name,
email: input.email.toLowerCase(),
passwordHash,
role: input.role,
isActive: true,
});
return { success: true };
}),
update: adminProcedure
.input(
z.object({
id: z.number().int().positive(),
name: z.string().min(2).max(255).optional(),
email: z.string().email().optional(),
password: z.string().min(8).optional(),
role: z.enum(["admin", "user", "readonly"]).optional(),
isActive: z.boolean().optional(),
})
)
.mutation(async ({ input }) => {
const { id, password, ...rest } = input;
const data: Record<string, unknown> = { ...rest };
if (password) data.passwordHash = await hashPassword(password);
await updateLocalUser(id, data as Parameters<typeof updateLocalUser>[1]);
return { success: true };
}),
delete: adminProcedure
.input(z.object({ id: z.number().int().positive() }))
.mutation(async ({ input }) => {
await deleteLocalUser(input.id);
return { success: true };
}),
ensureAdmin: publicProcedure.mutation(async () => {
await ensureAdminExists();
return { success: true };
}),
}),
});
export type AppRouter = typeof appRouter;

165
server/veille.test.ts Normal file
View File

@@ -0,0 +1,165 @@
import { describe, expect, it, vi, beforeEach } from "vitest";
import { appRouter } from "./routers";
import type { TrpcContext } from "./_core/context";
// ─── Helpers ─────────────────────────────────────────────────────────────────
function makeAdminCtx(): TrpcContext {
return {
user: {
id: 1,
openId: "admin-test",
email: "admin@itinova.fr",
name: "Admin Test",
loginMethod: "local",
role: "admin",
createdAt: new Date(),
updatedAt: new Date(),
lastSignedIn: new Date(),
},
req: { protocol: "https", headers: {} } as TrpcContext["req"],
res: {
clearCookie: vi.fn(),
cookie: vi.fn(),
} as unknown as TrpcContext["res"],
};
}
function makeUserCtx(): TrpcContext {
return {
user: {
id: 2,
openId: "user-test",
email: "user@itinova.fr",
name: "User Test",
loginMethod: "local",
role: "user",
createdAt: new Date(),
updatedAt: new Date(),
lastSignedIn: new Date(),
},
req: { protocol: "https", headers: {} } as TrpcContext["req"],
res: {
clearCookie: vi.fn(),
cookie: vi.fn(),
} as unknown as TrpcContext["res"],
};
}
function makeAnonCtx(): TrpcContext {
return {
user: null,
req: { protocol: "https", headers: {} } as TrpcContext["req"],
res: {
clearCookie: vi.fn(),
cookie: vi.fn(),
} as unknown as TrpcContext["res"],
};
}
// ─── Tests Auth ───────────────────────────────────────────────────────────────
describe("auth.logout", () => {
it("efface le cookie de session et retourne success", async () => {
const { ctx } = { ctx: makeAdminCtx() };
const clearedCookies: string[] = [];
ctx.res.clearCookie = (name: string) => { clearedCookies.push(name); };
const caller = appRouter.createCaller(ctx);
const result = await caller.auth.logout();
expect(result.success).toBe(true);
expect(clearedCookies.length).toBeGreaterThan(0);
});
it("retourne l'utilisateur connecté via auth.me", async () => {
const ctx = makeAdminCtx();
const caller = appRouter.createCaller(ctx);
const user = await caller.auth.me();
expect(user).not.toBeNull();
expect(user?.email).toBe("admin@itinova.fr");
});
it("retourne null pour un utilisateur non connecté", async () => {
const ctx = makeAnonCtx();
const caller = appRouter.createCaller(ctx);
const user = await caller.auth.me();
expect(user).toBeNull();
});
});
// ─── Tests protection admin ───────────────────────────────────────────────────
describe("protection admin", () => {
it("refuse l'accès aux logs pour un utilisateur non admin", async () => {
const ctx = makeUserCtx();
const caller = appRouter.createCaller(ctx);
await expect(caller.import.logs({ page: 1, pageSize: 10 })).rejects.toThrow();
});
it("refuse l'accès aux paramètres pour un utilisateur non admin", async () => {
const ctx = makeUserCtx();
const caller = appRouter.createCaller(ctx);
await expect(caller.settings.get()).rejects.toThrow();
});
it("refuse la gestion des utilisateurs pour un non admin", async () => {
const ctx = makeUserCtx();
const caller = appRouter.createCaller(ctx);
await expect(caller.users.list()).rejects.toThrow();
});
});
// ─── Tests accès public ───────────────────────────────────────────────────────
describe("accès public", () => {
it("veille.list est accessible sans authentification", async () => {
const ctx = makeAnonCtx();
const caller = appRouter.createCaller(ctx);
// Ne devrait pas lever d'erreur UNAUTHORIZED
const result = await caller.veille.list({ page: 1, pageSize: 10 });
expect(result).toHaveProperty("items");
expect(result).toHaveProperty("total");
});
it("aap.list est accessible sans authentification", async () => {
const ctx = makeAnonCtx();
const caller = appRouter.createCaller(ctx);
const result = await caller.aap.list({ page: 1, pageSize: 10 });
expect(result).toHaveProperty("items");
expect(result).toHaveProperty("total");
});
it("veille.filters est accessible sans authentification", async () => {
const ctx = makeAnonCtx();
const caller = appRouter.createCaller(ctx);
const result = await caller.veille.filters();
expect(result).toHaveProperty("categories");
expect(result).toHaveProperty("niveaux");
expect(result).toHaveProperty("territoires");
});
it("aap.filters est accessible sans authentification", async () => {
const ctx = makeAnonCtx();
const caller = appRouter.createCaller(ctx);
const result = await caller.aap.filters();
expect(result).toHaveProperty("regions");
expect(result).toHaveProperty("departements");
});
});
// ─── Tests import (admin seulement) ──────────────────────────────────────────
describe("import.run protection", () => {
it("refuse l'import pour un utilisateur non authentifié", async () => {
const ctx = makeAnonCtx();
const caller = appRouter.createCaller(ctx);
await expect(caller.import.run({ type: "all" })).rejects.toThrow();
});
it("refuse l'import pour un utilisateur standard", async () => {
const ctx = makeUserCtx();
const caller = appRouter.createCaller(ctx);
await expect(caller.import.run({ type: "all" })).rejects.toThrow();
});
});