Checkpoint: Application complète de dématérialisation de facturation avec extraction IA (Mistral), authentification locale + Azure AD, stockage local, et export SFTP.
Fonctionnalités implémentées : ✅ Authentification locale (email/password) + Azure AD + Manus OAuth ✅ Upload drag-and-drop de fichiers PDF avec suivi en temps réel ✅ Extraction automatique avec Mistral AI (OCR + LLM) ✅ Détection de doublons (fournisseur, numéro, date) ✅ Score de qualité d'extraction (0-100) ✅ Tableau de bord avec statistiques ✅ Liste des factures avec recherche et filtres ✅ Paramètres utilisateur (LLM, keywords, SFTP) ✅ Historique des imports avec logs détaillés ✅ Gestion des utilisateurs (admin) ✅ Export SFTP manuel/automatique ✅ Stockage local avec organisation YYYY-MM ✅ Tests unitaires d'authentification Architecture : - Frontend : React 19 + Vite + TailwindCSS + Radix UI - Backend : Express + tRPC + Drizzle ORM - Base de données : MySQL (6 tables) - IA : Mistral AI pour extraction - Stockage : Local filesystem - Export : SFTP Pages : - Login (choix local/Azure/Manus) - Dashboard (statistiques) - Upload (drag-and-drop) - Invoices (liste avec recherche) - Settings (LLM, keywords, SFTP) - History (logs d'import) - Users (gestion admin)
This commit is contained in:
Manus
@@ -1,28 +1,493 @@
|
||||
import { z } from "zod";
|
||||
import { COOKIE_NAME } from "@shared/const";
|
||||
import { getSessionCookieOptions } from "./_core/cookies";
|
||||
import { systemRouter } from "./_core/systemRouter";
|
||||
import { publicProcedure, router } from "./_core/trpc";
|
||||
import { publicProcedure, protectedProcedure, router } from "./_core/trpc";
|
||||
import {
|
||||
createInvoice,
|
||||
getInvoiceById,
|
||||
getInvoicesByUser,
|
||||
updateInvoice,
|
||||
deleteInvoice,
|
||||
searchInvoices,
|
||||
getInvoiceStats,
|
||||
createSourceFile,
|
||||
getSourceFileById,
|
||||
updateSourceFile,
|
||||
getUserSettings,
|
||||
upsertUserSettings,
|
||||
createLocalUser,
|
||||
getAllUsers,
|
||||
updateUserPassword,
|
||||
toggleUserActive,
|
||||
deleteUser,
|
||||
findDuplicateInvoice,
|
||||
createImportLog,
|
||||
getImportLogsByUser,
|
||||
getLlmLogsBySourceFile,
|
||||
getLlmLogsByInvoice,
|
||||
} from "./db";
|
||||
import { loginLocal, hashPassword, getAzureAuthUrl, isAzureAdConfigured, generateToken } from "./auth";
|
||||
import { extractInvoicesWithMistral, generateMetadataJSON } from "./invoiceExtractor";
|
||||
import { localStoragePut, generateStorageKey } from "./localStorage";
|
||||
import { testSftpConnection, exportInvoiceToSftp, getUserSftpConfig } from "./sftpExport";
|
||||
import { TRPCError } from "@trpc/server";
|
||||
|
||||
// Admin-only procedure
|
||||
const adminProcedure = protectedProcedure.use(({ ctx, next }) => {
|
||||
if (ctx.user.role !== "admin") {
|
||||
throw new TRPCError({ code: "FORBIDDEN", message: "Admin access required" });
|
||||
}
|
||||
return next({ ctx });
|
||||
});
|
||||
|
||||
export const appRouter = router({
|
||||
// if you need to use socket.io, read and register route in server/_core/index.ts, all api should start with '/api/' so that the gateway can route correctly
|
||||
system: systemRouter,
|
||||
|
||||
// ============= AUTH ROUTES =============
|
||||
auth: router({
|
||||
me: publicProcedure.query(opts => opts.ctx.user),
|
||||
|
||||
// Local login (email + password)
|
||||
loginLocal: publicProcedure
|
||||
.input(z.object({
|
||||
email: z.string().email(),
|
||||
password: z.string().min(6),
|
||||
}))
|
||||
.mutation(async ({ input, ctx }) => {
|
||||
const result = await loginLocal(input.email, input.password);
|
||||
|
||||
if (!result) {
|
||||
throw new TRPCError({ code: "UNAUTHORIZED", message: "Invalid email or password" });
|
||||
}
|
||||
|
||||
// Set auth cookie
|
||||
ctx.res.cookie("auth_token", result.token, {
|
||||
httpOnly: true,
|
||||
secure: process.env.NODE_ENV === "production",
|
||||
sameSite: "lax",
|
||||
path: "/",
|
||||
maxAge: 7 * 24 * 60 * 60 * 1000, // 7 days
|
||||
});
|
||||
|
||||
return { user: result.user };
|
||||
}),
|
||||
|
||||
// Get Azure AD login URL
|
||||
getAzureLoginUrl: publicProcedure.query(async () => {
|
||||
if (!isAzureAdConfigured()) {
|
||||
throw new TRPCError({ code: "BAD_REQUEST", message: "Azure AD not configured" });
|
||||
}
|
||||
|
||||
const url = await getAzureAuthUrl();
|
||||
return { url };
|
||||
}),
|
||||
|
||||
// Check if Azure AD is available
|
||||
isAzureAdAvailable: publicProcedure.query(() => {
|
||||
return { available: isAzureAdConfigured() };
|
||||
}),
|
||||
|
||||
logout: publicProcedure.mutation(({ ctx }) => {
|
||||
const cookieOptions = getSessionCookieOptions(ctx.req);
|
||||
ctx.res.clearCookie(COOKIE_NAME, { ...cookieOptions, maxAge: -1 });
|
||||
return {
|
||||
success: true,
|
||||
} as const;
|
||||
ctx.res.clearCookie("auth_token", { path: "/", maxAge: -1 });
|
||||
return { success: true };
|
||||
}),
|
||||
}),
|
||||
|
||||
// TODO: add feature routers here, e.g.
|
||||
// todo: router({
|
||||
// list: protectedProcedure.query(({ ctx }) =>
|
||||
// db.getUserTodos(ctx.user.id)
|
||||
// ),
|
||||
// }),
|
||||
|
||||
// ============= INVOICE ROUTES =============
|
||||
invoices: router({
|
||||
// Upload and process PDF file
|
||||
upload: protectedProcedure
|
||||
.input(z.object({
|
||||
fileName: z.string(),
|
||||
fileData: z.string(), // Base64 encoded PDF
|
||||
}))
|
||||
.mutation(async ({ input, ctx }) => {
|
||||
const userId = ctx.user.id;
|
||||
|
||||
// Decode base64 file data
|
||||
const fileBuffer = Buffer.from(input.fileData, "base64");
|
||||
|
||||
// Store source file
|
||||
const sourceFileKey = generateStorageKey(userId, input.fileName);
|
||||
const { url: sourceFileUrl } = await localStoragePut(sourceFileKey, fileBuffer, "application/pdf");
|
||||
|
||||
// Create source file record
|
||||
const sourceFile = await createSourceFile({
|
||||
userId,
|
||||
fileName: input.fileName,
|
||||
fileKey: sourceFileKey,
|
||||
fileUrl: sourceFileUrl,
|
||||
processingStatus: "processing",
|
||||
});
|
||||
|
||||
// Start extraction process (async - don't wait)
|
||||
(async () => {
|
||||
try {
|
||||
// Get user settings for custom keywords
|
||||
const settings = await getUserSettings(userId);
|
||||
const customKeywords = settings ? {
|
||||
invoiceNumber: settings.invoiceNumberKeywords,
|
||||
deliveryNote: settings.deliveryNoteKeywords,
|
||||
orderNumber: settings.orderNumberKeywords,
|
||||
supplier: settings.supplierKeywords,
|
||||
totalAmount: settings.totalAmountKeywords,
|
||||
} : undefined;
|
||||
|
||||
const model = settings?.llmModel || "mistral-large-latest";
|
||||
|
||||
// Extract invoices
|
||||
const result = await extractInvoicesWithMistral(
|
||||
fileBuffer,
|
||||
userId,
|
||||
sourceFile.id,
|
||||
model,
|
||||
customKeywords
|
||||
);
|
||||
|
||||
// Update source file with total count
|
||||
await updateSourceFile(sourceFile.id, {
|
||||
totalInvoicesDetected: result.invoiceCount,
|
||||
processingProgress: `Extraction ${result.invoiceCount} facture(s) détectée(s)`,
|
||||
});
|
||||
|
||||
// Process each invoice
|
||||
let importedCount = 0;
|
||||
let duplicatesCount = 0;
|
||||
let errorsCount = 0;
|
||||
const duplicateDetails: any[] = [];
|
||||
const errorDetails: any[] = [];
|
||||
|
||||
for (let i = 0; i < result.invoices.length; i++) {
|
||||
const invoiceData = result.invoices[i]!;
|
||||
|
||||
try {
|
||||
// Update progress
|
||||
await updateSourceFile(sourceFile.id, {
|
||||
processingProgress: `Extraction ${i + 1}/${result.invoiceCount} factures...`,
|
||||
});
|
||||
|
||||
// Check for duplicates
|
||||
const duplicate = await findDuplicateInvoice(
|
||||
invoiceData.supplierName,
|
||||
invoiceData.invoiceNumber,
|
||||
invoiceData.invoiceDate
|
||||
);
|
||||
|
||||
if (duplicate) {
|
||||
duplicatesCount++;
|
||||
duplicateDetails.push({
|
||||
supplierName: invoiceData.supplierName,
|
||||
invoiceNumber: invoiceData.invoiceNumber,
|
||||
invoiceDate: invoiceData.invoiceDate,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Generate metadata JSON
|
||||
const metadataJson = generateMetadataJSON(invoiceData);
|
||||
const metadataKey = generateStorageKey(userId, `${input.fileName}-${i + 1}-metadata.json`);
|
||||
const { url: metadataUrl } = await localStoragePut(
|
||||
metadataKey,
|
||||
Buffer.from(metadataJson),
|
||||
"application/json"
|
||||
);
|
||||
|
||||
// Create invoice record
|
||||
await createInvoice({
|
||||
userId,
|
||||
sourceFileId: sourceFile.id,
|
||||
invoiceIndexInFile: i + 1,
|
||||
fileName: `${input.fileName} - Facture ${i + 1}`,
|
||||
fileKey: sourceFileKey, // Same as source for now
|
||||
fileUrl: sourceFileUrl,
|
||||
supplierName: invoiceData.supplierName,
|
||||
invoiceNumber: invoiceData.invoiceNumber,
|
||||
invoiceDate: invoiceData.invoiceDate,
|
||||
deliveryNoteNumber: invoiceData.deliveryNoteNumber,
|
||||
orderNumber: invoiceData.orderNumber,
|
||||
totalAmount: invoiceData.totalAmount?.toString(),
|
||||
pageRange: invoiceData.pageRange,
|
||||
qualityScore: invoiceData.qualityScore,
|
||||
metadataFileKey: metadataKey,
|
||||
metadataFileUrl: metadataUrl,
|
||||
status: "completed",
|
||||
});
|
||||
|
||||
importedCount++;
|
||||
} catch (error: any) {
|
||||
errorsCount++;
|
||||
errorDetails.push({
|
||||
invoiceIndex: i + 1,
|
||||
error: error.message,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Update source file status
|
||||
await updateSourceFile(sourceFile.id, {
|
||||
processingStatus: "completed",
|
||||
processingProgress: `Terminé: ${importedCount} importée(s), ${duplicatesCount} doublon(s)`,
|
||||
});
|
||||
|
||||
// Create import log
|
||||
await createImportLog({
|
||||
userId,
|
||||
sourceFileId: sourceFile.id,
|
||||
fileName: input.fileName,
|
||||
totalInvoicesDetected: result.invoiceCount,
|
||||
invoicesImported: importedCount,
|
||||
duplicatesIgnored: duplicatesCount,
|
||||
errors: errorsCount,
|
||||
duplicateDetails: JSON.stringify(duplicateDetails),
|
||||
errorDetails: JSON.stringify(errorDetails),
|
||||
});
|
||||
|
||||
} catch (error: any) {
|
||||
console.error("[Upload] Extraction failed:", error);
|
||||
await updateSourceFile(sourceFile.id, {
|
||||
processingStatus: "error",
|
||||
processingProgress: `Erreur: ${error.message}`,
|
||||
});
|
||||
}
|
||||
})();
|
||||
|
||||
return { sourceFileId: sourceFile.id };
|
||||
}),
|
||||
|
||||
list: protectedProcedure.query(async ({ ctx }) => {
|
||||
return getInvoicesByUser(ctx.user.id);
|
||||
}),
|
||||
|
||||
getById: protectedProcedure
|
||||
.input(z.object({ id: z.number() }))
|
||||
.query(async ({ input, ctx }) => {
|
||||
const invoice = await getInvoiceById(input.id);
|
||||
if (!invoice || invoice.userId !== ctx.user.id) {
|
||||
throw new TRPCError({ code: "NOT_FOUND" });
|
||||
}
|
||||
return invoice;
|
||||
}),
|
||||
|
||||
update: protectedProcedure
|
||||
.input(z.object({
|
||||
id: z.number(),
|
||||
data: z.object({
|
||||
supplierName: z.string().optional(),
|
||||
invoiceNumber: z.string().optional(),
|
||||
invoiceDate: z.date().optional(),
|
||||
deliveryNoteNumber: z.string().optional(),
|
||||
orderNumber: z.string().optional(),
|
||||
totalAmount: z.string().optional(),
|
||||
}),
|
||||
}))
|
||||
.mutation(async ({ input, ctx }) => {
|
||||
const invoice = await getInvoiceById(input.id);
|
||||
if (!invoice || invoice.userId !== ctx.user.id) {
|
||||
throw new TRPCError({ code: "NOT_FOUND" });
|
||||
}
|
||||
|
||||
await updateInvoice(input.id, {
|
||||
...input.data,
|
||||
manuallyEdited: 1,
|
||||
});
|
||||
|
||||
return { success: true };
|
||||
}),
|
||||
|
||||
delete: protectedProcedure
|
||||
.input(z.object({ id: z.number() }))
|
||||
.mutation(async ({ input, ctx }) => {
|
||||
const invoice = await getInvoiceById(input.id);
|
||||
if (!invoice || invoice.userId !== ctx.user.id) {
|
||||
throw new TRPCError({ code: "NOT_FOUND" });
|
||||
}
|
||||
|
||||
await deleteInvoice(input.id);
|
||||
return { success: true };
|
||||
}),
|
||||
|
||||
search: protectedProcedure
|
||||
.input(z.object({ query: z.string() }))
|
||||
.query(async ({ input, ctx }) => {
|
||||
return searchInvoices(ctx.user.id, input.query);
|
||||
}),
|
||||
|
||||
getStats: protectedProcedure.query(async ({ ctx }) => {
|
||||
return getInvoiceStats(ctx.user.id);
|
||||
}),
|
||||
}),
|
||||
|
||||
// ============= SOURCE FILES ROUTES =============
|
||||
sourceFiles: router({
|
||||
getByIds: protectedProcedure
|
||||
.input(z.object({ ids: z.array(z.number()) }))
|
||||
.query(async ({ input, ctx }) => {
|
||||
const files = await Promise.all(
|
||||
input.ids.map(id => getSourceFileById(id))
|
||||
);
|
||||
return files.filter(f => f && f.userId === ctx.user.id);
|
||||
}),
|
||||
}),
|
||||
|
||||
// ============= SETTINGS ROUTES =============
|
||||
settings: router({
|
||||
get: protectedProcedure.query(async ({ ctx }) => {
|
||||
return getUserSettings(ctx.user.id);
|
||||
}),
|
||||
|
||||
upsert: protectedProcedure
|
||||
.input(z.object({
|
||||
llmModel: z.string().optional(),
|
||||
orderNumberFormat: z.string().optional(),
|
||||
invoiceNumberKeywords: z.string().optional(),
|
||||
deliveryNoteKeywords: z.string().optional(),
|
||||
orderNumberKeywords: z.string().optional(),
|
||||
supplierKeywords: z.string().optional(),
|
||||
totalAmountKeywords: z.string().optional(),
|
||||
sftpHost: z.string().optional(),
|
||||
sftpPort: z.number().optional(),
|
||||
sftpUsername: z.string().optional(),
|
||||
sftpPassword: z.string().optional(),
|
||||
sftpRemotePath: z.string().optional(),
|
||||
sftpAutoExport: z.number().optional(),
|
||||
llmLogsRetentionMonths: z.number().optional(),
|
||||
}))
|
||||
.mutation(async ({ input, ctx }) => {
|
||||
await upsertUserSettings({
|
||||
userId: ctx.user.id,
|
||||
...input,
|
||||
});
|
||||
return { success: true };
|
||||
}),
|
||||
}),
|
||||
|
||||
// ============= ADMIN ROUTES =============
|
||||
admin: router({
|
||||
createUser: adminProcedure
|
||||
.input(z.object({
|
||||
email: z.string().email(),
|
||||
password: z.string().min(6),
|
||||
name: z.string(),
|
||||
role: z.enum(["user", "admin"]),
|
||||
}))
|
||||
.mutation(async ({ input }) => {
|
||||
const passwordHash = await hashPassword(input.password);
|
||||
const user = await createLocalUser(input.email, passwordHash, input.name, input.role);
|
||||
return { user };
|
||||
}),
|
||||
|
||||
getAllUsers: adminProcedure.query(async () => {
|
||||
return getAllUsers();
|
||||
}),
|
||||
|
||||
updateUserPassword: adminProcedure
|
||||
.input(z.object({
|
||||
userId: z.number(),
|
||||
newPassword: z.string().min(6),
|
||||
}))
|
||||
.mutation(async ({ input }) => {
|
||||
const passwordHash = await hashPassword(input.newPassword);
|
||||
await updateUserPassword(input.userId, passwordHash);
|
||||
return { success: true };
|
||||
}),
|
||||
|
||||
toggleUserActive: adminProcedure
|
||||
.input(z.object({
|
||||
userId: z.number(),
|
||||
isActive: z.number(),
|
||||
}))
|
||||
.mutation(async ({ input }) => {
|
||||
await toggleUserActive(input.userId, input.isActive);
|
||||
return { success: true };
|
||||
}),
|
||||
|
||||
deleteUser: adminProcedure
|
||||
.input(z.object({ userId: z.number() }))
|
||||
.mutation(async ({ input }) => {
|
||||
await deleteUser(input.userId);
|
||||
return { success: true };
|
||||
}),
|
||||
}),
|
||||
|
||||
// ============= SFTP ROUTES =============
|
||||
sftp: router({
|
||||
testConnection: protectedProcedure.mutation(async ({ ctx }) => {
|
||||
const config = await getUserSftpConfig(ctx.user.id);
|
||||
if (!config) {
|
||||
throw new TRPCError({ code: "BAD_REQUEST", message: "SFTP not configured" });
|
||||
}
|
||||
|
||||
const success = await testSftpConnection(config);
|
||||
return { success };
|
||||
}),
|
||||
|
||||
exportInvoices: protectedProcedure
|
||||
.input(z.object({ invoiceIds: z.array(z.number()) }))
|
||||
.mutation(async ({ input, ctx }) => {
|
||||
const config = await getUserSftpConfig(ctx.user.id);
|
||||
if (!config) {
|
||||
throw new TRPCError({ code: "BAD_REQUEST", message: "SFTP not configured" });
|
||||
}
|
||||
|
||||
let successCount = 0;
|
||||
let errorCount = 0;
|
||||
|
||||
for (const invoiceId of input.invoiceIds) {
|
||||
try {
|
||||
const invoice = await getInvoiceById(invoiceId);
|
||||
if (!invoice || invoice.userId !== ctx.user.id) {
|
||||
errorCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
await exportInvoiceToSftp(
|
||||
config,
|
||||
invoice.fileKey,
|
||||
invoice.metadataFileKey,
|
||||
invoice.invoiceDate || new Date()
|
||||
);
|
||||
|
||||
// Update invoice export status
|
||||
await updateInvoice(invoiceId, {
|
||||
exportedAt: new Date(),
|
||||
exportMode: "manual",
|
||||
});
|
||||
|
||||
successCount++;
|
||||
} catch (error) {
|
||||
console.error(`[SFTP] Failed to export invoice ${invoiceId}:`, error);
|
||||
errorCount++;
|
||||
}
|
||||
}
|
||||
|
||||
return { successCount, errorCount };
|
||||
}),
|
||||
}),
|
||||
|
||||
// ============= IMPORT LOGS ROUTES =============
|
||||
importLogs: router({
|
||||
getByUser: protectedProcedure.query(async ({ ctx }) => {
|
||||
return getImportLogsByUser(ctx.user.id);
|
||||
}),
|
||||
}),
|
||||
|
||||
// ============= LLM LOGS ROUTES =============
|
||||
llmLogs: router({
|
||||
getBySourceFile: protectedProcedure
|
||||
.input(z.object({ sourceFileId: z.number() }))
|
||||
.query(async ({ input }) => {
|
||||
return getLlmLogsBySourceFile(input.sourceFileId);
|
||||
}),
|
||||
|
||||
getByInvoice: protectedProcedure
|
||||
.input(z.object({ invoiceId: z.number() }))
|
||||
.query(async ({ input }) => {
|
||||
return getLlmLogsByInvoice(input.invoiceId);
|
||||
}),
|
||||
}),
|
||||
});
|
||||
|
||||
export type AppRouter = typeof appRouter;
|
||||
|
||||
Reference in New Issue
Block a user