Checkpoint: Application complète de dématérialisation de facturation avec extraction IA (Mistral), authentification locale + Azure AD, stockage local, et export SFTP.
Fonctionnalités implémentées : ✅ Authentification locale (email/password) + Azure AD + Manus OAuth ✅ Upload drag-and-drop de fichiers PDF avec suivi en temps réel ✅ Extraction automatique avec Mistral AI (OCR + LLM) ✅ Détection de doublons (fournisseur, numéro, date) ✅ Score de qualité d'extraction (0-100) ✅ Tableau de bord avec statistiques ✅ Liste des factures avec recherche et filtres ✅ Paramètres utilisateur (LLM, keywords, SFTP) ✅ Historique des imports avec logs détaillés ✅ Gestion des utilisateurs (admin) ✅ Export SFTP manuel/automatique ✅ Stockage local avec organisation YYYY-MM ✅ Tests unitaires d'authentification Architecture : - Frontend : React 19 + Vite + TailwindCSS + Radix UI - Backend : Express + tRPC + Drizzle ORM - Base de données : MySQL (6 tables) - IA : Mistral AI pour extraction - Stockage : Local filesystem - Export : SFTP Pages : - Login (choix local/Azure/Manus) - Dashboard (statistiques) - Upload (drag-and-drop) - Invoices (liste avec recherche) - Settings (LLM, keywords, SFTP) - History (logs d'import) - Users (gestion admin)
This commit is contained in:
Manus
@@ -35,6 +35,9 @@ async function startServer() {
|
||||
app.use(express.urlencoded({ limit: "50mb", extended: true }));
|
||||
// OAuth callback under /api/oauth/callback
|
||||
registerOAuthRoutes(app);
|
||||
// Serve local storage files
|
||||
app.use("/storage", express.static("storage"));
|
||||
|
||||
// tRPC API
|
||||
app.use(
|
||||
"/api/trpc",
|
||||
|
||||
@@ -31,8 +31,8 @@ export function registerOAuthRoutes(app: Express) {
|
||||
await db.upsertUser({
|
||||
openId: userInfo.openId,
|
||||
name: userInfo.name || null,
|
||||
email: userInfo.email ?? null,
|
||||
loginMethod: userInfo.loginMethod ?? userInfo.platform ?? null,
|
||||
email: userInfo.email ?? "unknown@example.com",
|
||||
loginMethod: (userInfo.loginMethod ?? userInfo.platform ?? "manus") as "manus" | "local" | "azure-ad",
|
||||
lastSignedIn: new Date(),
|
||||
});
|
||||
|
||||
|
||||
@@ -277,8 +277,8 @@ class SDKServer {
|
||||
await db.upsertUser({
|
||||
openId: userInfo.openId,
|
||||
name: userInfo.name || null,
|
||||
email: userInfo.email ?? null,
|
||||
loginMethod: userInfo.loginMethod ?? userInfo.platform ?? null,
|
||||
email: userInfo.email ?? "unknown@example.com",
|
||||
loginMethod: (userInfo.loginMethod ?? userInfo.platform ?? "manus") as "manus" | "local" | "azure-ad",
|
||||
lastSignedIn: signedInAt,
|
||||
});
|
||||
user = await db.getUserByOpenId(userInfo.openId);
|
||||
@@ -294,6 +294,8 @@ class SDKServer {
|
||||
|
||||
await db.upsertUser({
|
||||
openId: user.openId,
|
||||
email: user.email,
|
||||
loginMethod: user.loginMethod,
|
||||
lastSignedIn: signedInAt,
|
||||
});
|
||||
|
||||
|
||||
@@ -49,9 +49,16 @@ describe("auth.logout", () => {
|
||||
const result = await caller.auth.logout();
|
||||
|
||||
expect(result).toEqual({ success: true });
|
||||
expect(clearedCookies).toHaveLength(1);
|
||||
expect(clearedCookies[0]?.name).toBe(COOKIE_NAME);
|
||||
expect(clearedCookies[0]?.options).toMatchObject({
|
||||
expect(clearedCookies).toHaveLength(2); // COOKIE_NAME + auth_token
|
||||
|
||||
// Check that both cookies are cleared
|
||||
const cookieNames = clearedCookies.map(c => c.name);
|
||||
expect(cookieNames).toContain(COOKIE_NAME);
|
||||
expect(cookieNames).toContain("auth_token");
|
||||
|
||||
// Check options for the main cookie
|
||||
const mainCookie = clearedCookies.find(c => c.name === COOKIE_NAME);
|
||||
expect(mainCookie?.options).toMatchObject({
|
||||
maxAge: -1,
|
||||
secure: true,
|
||||
sameSite: "none",
|
||||
|
||||
68
server/auth.test.ts
Normal file
68
server/auth.test.ts
Normal file
@@ -0,0 +1,68 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { hashPassword, verifyPassword, generateToken, verifyToken } from "./auth";
|
||||
|
||||
describe("Authentication", () => {
|
||||
describe("Password hashing", () => {
|
||||
it("should hash a password", async () => {
|
||||
const password = "testpassword123";
|
||||
const hash = await hashPassword(password);
|
||||
|
||||
expect(hash).toBeDefined();
|
||||
expect(hash).not.toBe(password);
|
||||
expect(hash.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("should verify a correct password", async () => {
|
||||
const password = "testpassword123";
|
||||
const hash = await hashPassword(password);
|
||||
|
||||
const isValid = await verifyPassword(password, hash);
|
||||
expect(isValid).toBe(true);
|
||||
});
|
||||
|
||||
it("should reject an incorrect password", async () => {
|
||||
const password = "testpassword123";
|
||||
const hash = await hashPassword(password);
|
||||
|
||||
const isValid = await verifyPassword("wrongpassword", hash);
|
||||
expect(isValid).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("JWT tokens", () => {
|
||||
it("should generate a valid JWT token", () => {
|
||||
const user = {
|
||||
id: 1,
|
||||
email: "test@example.com",
|
||||
role: "user",
|
||||
};
|
||||
|
||||
const token = generateToken(user);
|
||||
|
||||
expect(token).toBeDefined();
|
||||
expect(typeof token).toBe("string");
|
||||
expect(token.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("should verify and decode a valid token", () => {
|
||||
const user = {
|
||||
id: 1,
|
||||
email: "test@example.com",
|
||||
role: "user",
|
||||
};
|
||||
|
||||
const token = generateToken(user);
|
||||
const decoded = verifyToken(token);
|
||||
|
||||
expect(decoded).toBeDefined();
|
||||
expect(decoded?.userId).toBe(user.id);
|
||||
expect(decoded?.email).toBe(user.email);
|
||||
expect(decoded?.role).toBe(user.role);
|
||||
});
|
||||
|
||||
it("should reject an invalid token", () => {
|
||||
const decoded = verifyToken("invalid-token");
|
||||
expect(decoded).toBeNull();
|
||||
});
|
||||
});
|
||||
});
|
||||
166
server/auth.ts
Normal file
166
server/auth.ts
Normal file
@@ -0,0 +1,166 @@
|
||||
import bcrypt from "bcrypt";
|
||||
import { ConfidentialClientApplication } from "@azure/msal-node";
|
||||
import { getUserByEmail, getUserByAzureAdId } from "./db";
|
||||
import jwt from "jsonwebtoken";
|
||||
|
||||
const SALT_ROUNDS = 10;
|
||||
|
||||
// ============= LOCAL AUTHENTICATION =============
|
||||
|
||||
/**
|
||||
* Hash a password using bcrypt
|
||||
*/
|
||||
export async function hashPassword(password: string): Promise<string> {
|
||||
return bcrypt.hash(password, SALT_ROUNDS);
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify a password against a hash
|
||||
*/
|
||||
export async function verifyPassword(password: string, hash: string): Promise<boolean> {
|
||||
return bcrypt.compare(password, hash);
|
||||
}
|
||||
|
||||
/**
|
||||
* Authenticate a user with email and password (local auth)
|
||||
* Returns user and JWT token if successful, null otherwise
|
||||
*/
|
||||
export async function loginLocal(email: string, password: string) {
|
||||
const user = await getUserByEmail(email);
|
||||
|
||||
if (!user) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Check if user is active
|
||||
if (user.isActive === 0) {
|
||||
throw new Error("Account is inactive");
|
||||
}
|
||||
|
||||
// Check if user has a password (local auth)
|
||||
if (!user.passwordHash) {
|
||||
throw new Error("This account does not support local authentication");
|
||||
}
|
||||
|
||||
// Verify password
|
||||
const isValid = await verifyPassword(password, user.passwordHash);
|
||||
if (!isValid) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Generate JWT token
|
||||
const token = generateToken(user);
|
||||
|
||||
return { user, token };
|
||||
}
|
||||
|
||||
// ============= JWT TOKEN GENERATION =============
|
||||
|
||||
/**
|
||||
* Generate a JWT token for a user
|
||||
*/
|
||||
export function generateToken(user: { id: number; email: string; role: string }): string {
|
||||
const secret = process.env.JWT_SECRET || "default-secret-change-in-production";
|
||||
|
||||
return jwt.sign(
|
||||
{
|
||||
userId: user.id,
|
||||
email: user.email,
|
||||
role: user.role,
|
||||
},
|
||||
secret,
|
||||
{ expiresIn: "7d" }
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify and decode a JWT token
|
||||
*/
|
||||
export function verifyToken(token: string): { userId: number; email: string; role: string } | null {
|
||||
try {
|
||||
const secret = process.env.JWT_SECRET || "default-secret-change-in-production";
|
||||
const decoded = jwt.verify(token, secret) as { userId: number; email: string; role: string };
|
||||
return decoded;
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ============= AZURE AD AUTHENTICATION =============
|
||||
|
||||
let msalClient: ConfidentialClientApplication | null = null;
|
||||
|
||||
/**
|
||||
* Check if Azure AD is configured
|
||||
*/
|
||||
export function isAzureAdConfigured(): boolean {
|
||||
return !!(
|
||||
process.env.AZURE_AD_TENANT_ID &&
|
||||
process.env.AZURE_AD_CLIENT_ID &&
|
||||
process.env.AZURE_AD_CLIENT_SECRET
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get MSAL client instance (lazy initialization)
|
||||
*/
|
||||
function getMsalClient(): ConfidentialClientApplication {
|
||||
if (!isAzureAdConfigured()) {
|
||||
throw new Error("Azure AD is not configured");
|
||||
}
|
||||
|
||||
if (!msalClient) {
|
||||
msalClient = new ConfidentialClientApplication({
|
||||
auth: {
|
||||
clientId: process.env.AZURE_AD_CLIENT_ID!,
|
||||
authority: `https://login.microsoftonline.com/${process.env.AZURE_AD_TENANT_ID}`,
|
||||
clientSecret: process.env.AZURE_AD_CLIENT_SECRET!,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return msalClient;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Azure AD authorization URL for user login
|
||||
*/
|
||||
export async function getAzureAuthUrl(): Promise<string> {
|
||||
const client = getMsalClient();
|
||||
|
||||
const redirectUri = process.env.AZURE_AD_REDIRECT_URI || "http://localhost:3000/api/auth/azure/callback";
|
||||
|
||||
const authCodeUrlParameters = {
|
||||
scopes: ["user.read"],
|
||||
redirectUri,
|
||||
};
|
||||
|
||||
return client.getAuthCodeUrl(authCodeUrlParameters);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle Azure AD callback and exchange code for tokens
|
||||
*/
|
||||
export async function handleAzureCallback(code: string) {
|
||||
const client = getMsalClient();
|
||||
|
||||
const redirectUri = process.env.AZURE_AD_REDIRECT_URI || "http://localhost:3000/api/auth/azure/callback";
|
||||
|
||||
const tokenRequest = {
|
||||
code,
|
||||
scopes: ["user.read"],
|
||||
redirectUri,
|
||||
};
|
||||
|
||||
const response = await client.acquireTokenByCode(tokenRequest);
|
||||
|
||||
if (!response || !response.account) {
|
||||
throw new Error("Failed to acquire token from Azure AD");
|
||||
}
|
||||
|
||||
return {
|
||||
azureAdId: response.account.homeAccountId,
|
||||
email: response.account.username,
|
||||
name: response.account.name || response.account.username,
|
||||
};
|
||||
}
|
||||
317
server/db.ts
317
server/db.ts
@@ -1,11 +1,28 @@
|
||||
import { eq } from "drizzle-orm";
|
||||
import { eq, and, desc, sql } from "drizzle-orm";
|
||||
import { drizzle } from "drizzle-orm/mysql2";
|
||||
import { InsertUser, users } from "../drizzle/schema";
|
||||
import {
|
||||
InsertUser,
|
||||
users,
|
||||
sourceFiles,
|
||||
InsertSourceFile,
|
||||
SourceFile,
|
||||
invoices,
|
||||
InsertInvoice,
|
||||
Invoice,
|
||||
userSettings,
|
||||
InsertUserSettings,
|
||||
UserSettings,
|
||||
importLogs,
|
||||
InsertImportLog,
|
||||
ImportLog,
|
||||
llmLogs,
|
||||
InsertLlmLog,
|
||||
LlmLog
|
||||
} from "../drizzle/schema";
|
||||
import { ENV } from './_core/env';
|
||||
|
||||
let _db: ReturnType<typeof drizzle> | null = null;
|
||||
|
||||
// Lazily create the drizzle instance so local tooling can run without a DB.
|
||||
export async function getDb() {
|
||||
if (!_db && process.env.DATABASE_URL) {
|
||||
try {
|
||||
@@ -18,11 +35,9 @@ export async function getDb() {
|
||||
return _db;
|
||||
}
|
||||
|
||||
export async function upsertUser(user: InsertUser): Promise<void> {
|
||||
if (!user.openId) {
|
||||
throw new Error("User openId is required for upsert");
|
||||
}
|
||||
// ============= USER OPERATIONS =============
|
||||
|
||||
export async function upsertUser(user: InsertUser): Promise<void> {
|
||||
const db = await getDb();
|
||||
if (!db) {
|
||||
console.warn("[Database] Cannot upsert user: database not available");
|
||||
@@ -31,23 +46,31 @@ export async function upsertUser(user: InsertUser): Promise<void> {
|
||||
|
||||
try {
|
||||
const values: InsertUser = {
|
||||
openId: user.openId,
|
||||
email: user.email,
|
||||
loginMethod: user.loginMethod,
|
||||
};
|
||||
const updateSet: Record<string, unknown> = {};
|
||||
|
||||
const textFields = ["name", "email", "loginMethod"] as const;
|
||||
type TextField = (typeof textFields)[number];
|
||||
|
||||
const assignNullable = (field: TextField) => {
|
||||
const value = user[field];
|
||||
if (value === undefined) return;
|
||||
const normalized = value ?? null;
|
||||
values[field] = normalized;
|
||||
updateSet[field] = normalized;
|
||||
};
|
||||
|
||||
textFields.forEach(assignNullable);
|
||||
|
||||
if (user.openId !== undefined) {
|
||||
values.openId = user.openId;
|
||||
updateSet.openId = user.openId;
|
||||
}
|
||||
if (user.azureAdId !== undefined) {
|
||||
values.azureAdId = user.azureAdId;
|
||||
updateSet.azureAdId = user.azureAdId;
|
||||
}
|
||||
if (user.name !== undefined) {
|
||||
values.name = user.name;
|
||||
updateSet.name = user.name;
|
||||
}
|
||||
if (user.passwordHash !== undefined) {
|
||||
values.passwordHash = user.passwordHash;
|
||||
updateSet.passwordHash = user.passwordHash;
|
||||
}
|
||||
if (user.isActive !== undefined) {
|
||||
values.isActive = user.isActive;
|
||||
updateSet.isActive = user.isActive;
|
||||
}
|
||||
if (user.lastSignedIn !== undefined) {
|
||||
values.lastSignedIn = user.lastSignedIn;
|
||||
updateSet.lastSignedIn = user.lastSignedIn;
|
||||
@@ -79,14 +102,252 @@ export async function upsertUser(user: InsertUser): Promise<void> {
|
||||
|
||||
export async function getUserByOpenId(openId: string) {
|
||||
const db = await getDb();
|
||||
if (!db) {
|
||||
console.warn("[Database] Cannot get user: database not available");
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (!db) return undefined;
|
||||
const result = await db.select().from(users).where(eq(users.openId, openId)).limit(1);
|
||||
|
||||
return result.length > 0 ? result[0] : undefined;
|
||||
}
|
||||
|
||||
// TODO: add feature queries here as your schema grows.
|
||||
export async function getUserByEmail(email: string) {
|
||||
const db = await getDb();
|
||||
if (!db) return undefined;
|
||||
const result = await db.select().from(users).where(eq(users.email, email)).limit(1);
|
||||
return result.length > 0 ? result[0] : undefined;
|
||||
}
|
||||
|
||||
export async function getUserByAzureAdId(azureAdId: string) {
|
||||
const db = await getDb();
|
||||
if (!db) return undefined;
|
||||
const result = await db.select().from(users).where(eq(users.azureAdId, azureAdId)).limit(1);
|
||||
return result.length > 0 ? result[0] : undefined;
|
||||
}
|
||||
|
||||
export async function createLocalUser(email: string, passwordHash: string, name: string, role: "user" | "admin" = "user") {
|
||||
const db = await getDb();
|
||||
if (!db) throw new Error("Database not available");
|
||||
|
||||
await db.insert(users).values({
|
||||
email,
|
||||
passwordHash,
|
||||
name,
|
||||
loginMethod: "local",
|
||||
role,
|
||||
isActive: 1,
|
||||
});
|
||||
|
||||
return getUserByEmail(email);
|
||||
}
|
||||
|
||||
export async function getAllUsers() {
|
||||
const db = await getDb();
|
||||
if (!db) return [];
|
||||
return db.select().from(users).orderBy(desc(users.createdAt));
|
||||
}
|
||||
|
||||
export async function updateUserPassword(userId: number, newPasswordHash: string) {
|
||||
const db = await getDb();
|
||||
if (!db) throw new Error("Database not available");
|
||||
await db.update(users).set({ passwordHash: newPasswordHash }).where(eq(users.id, userId));
|
||||
}
|
||||
|
||||
export async function toggleUserActive(userId: number, isActive: number) {
|
||||
const db = await getDb();
|
||||
if (!db) throw new Error("Database not available");
|
||||
await db.update(users).set({ isActive }).where(eq(users.id, userId));
|
||||
}
|
||||
|
||||
export async function deleteUser(userId: number) {
|
||||
const db = await getDb();
|
||||
if (!db) throw new Error("Database not available");
|
||||
await db.delete(users).where(eq(users.id, userId));
|
||||
}
|
||||
|
||||
// ============= SOURCE FILE OPERATIONS =============
|
||||
|
||||
export async function createSourceFile(data: InsertSourceFile): Promise<SourceFile> {
|
||||
const db = await getDb();
|
||||
if (!db) throw new Error("Database not available");
|
||||
|
||||
const result = await db.insert(sourceFiles).values(data);
|
||||
const insertedId = Number(result[0].insertId);
|
||||
|
||||
const inserted = await db.select().from(sourceFiles).where(eq(sourceFiles.id, insertedId)).limit(1);
|
||||
return inserted[0]!;
|
||||
}
|
||||
|
||||
export async function getSourceFileById(id: number): Promise<SourceFile | undefined> {
|
||||
const db = await getDb();
|
||||
if (!db) return undefined;
|
||||
const result = await db.select().from(sourceFiles).where(eq(sourceFiles.id, id)).limit(1);
|
||||
return result[0];
|
||||
}
|
||||
|
||||
export async function updateSourceFile(id: number, data: Partial<SourceFile>) {
|
||||
const db = await getDb();
|
||||
if (!db) throw new Error("Database not available");
|
||||
await db.update(sourceFiles).set(data).where(eq(sourceFiles.id, id));
|
||||
}
|
||||
|
||||
export async function getSourceFilesByUserId(userId: number): Promise<SourceFile[]> {
|
||||
const db = await getDb();
|
||||
if (!db) return [];
|
||||
return db.select().from(sourceFiles).where(eq(sourceFiles.userId, userId)).orderBy(desc(sourceFiles.createdAt));
|
||||
}
|
||||
|
||||
// ============= INVOICE OPERATIONS =============
|
||||
|
||||
export async function createInvoice(data: InsertInvoice): Promise<Invoice> {
|
||||
const db = await getDb();
|
||||
if (!db) throw new Error("Database not available");
|
||||
|
||||
const result = await db.insert(invoices).values(data);
|
||||
const insertedId = Number(result[0].insertId);
|
||||
|
||||
const inserted = await db.select().from(invoices).where(eq(invoices.id, insertedId)).limit(1);
|
||||
return inserted[0]!;
|
||||
}
|
||||
|
||||
export async function getInvoiceById(id: number): Promise<Invoice | undefined> {
|
||||
const db = await getDb();
|
||||
if (!db) return undefined;
|
||||
const result = await db.select().from(invoices).where(eq(invoices.id, id)).limit(1);
|
||||
return result[0];
|
||||
}
|
||||
|
||||
export async function getInvoicesByUserId(userId: number): Promise<Invoice[]> {
|
||||
const db = await getDb();
|
||||
if (!db) return [];
|
||||
return db.select().from(invoices).where(eq(invoices.userId, userId)).orderBy(desc(invoices.createdAt));
|
||||
}
|
||||
|
||||
export async function getInvoicesByUser(userId: number): Promise<Invoice[]> {
|
||||
return getInvoicesByUserId(userId);
|
||||
}
|
||||
|
||||
export async function updateInvoice(id: number, data: Partial<Invoice>) {
|
||||
const db = await getDb();
|
||||
if (!db) throw new Error("Database not available");
|
||||
await db.update(invoices).set(data).where(eq(invoices.id, id));
|
||||
}
|
||||
|
||||
export async function deleteInvoice(id: number) {
|
||||
const db = await getDb();
|
||||
if (!db) throw new Error("Database not available");
|
||||
await db.delete(invoices).where(eq(invoices.id, id));
|
||||
}
|
||||
|
||||
export async function searchInvoices(userId: number, query: string): Promise<Invoice[]> {
|
||||
const db = await getDb();
|
||||
if (!db) return [];
|
||||
|
||||
const searchPattern = `%${query}%`;
|
||||
return db.select().from(invoices)
|
||||
.where(
|
||||
and(
|
||||
eq(invoices.userId, userId),
|
||||
sql`(${invoices.supplierName} LIKE ${searchPattern} OR ${invoices.invoiceNumber} LIKE ${searchPattern})`
|
||||
)
|
||||
)
|
||||
.orderBy(desc(invoices.createdAt));
|
||||
}
|
||||
|
||||
export async function getInvoiceStats(userId: number) {
|
||||
const db = await getDb();
|
||||
if (!db) return { total: 0, completed: 0, processing: 0, error: 0 };
|
||||
|
||||
const allInvoices = await getInvoicesByUserId(userId);
|
||||
return {
|
||||
total: allInvoices.length,
|
||||
completed: allInvoices.filter(i => i.status === "completed").length,
|
||||
processing: allInvoices.filter(i => i.status === "processing").length,
|
||||
error: allInvoices.filter(i => i.status === "error").length,
|
||||
};
|
||||
}
|
||||
|
||||
export async function findDuplicateInvoice(
|
||||
supplierName: string | null,
|
||||
invoiceNumber: string | null,
|
||||
invoiceDate: Date | null
|
||||
): Promise<Invoice | undefined> {
|
||||
if (!supplierName || !invoiceNumber || !invoiceDate) return undefined;
|
||||
|
||||
const db = await getDb();
|
||||
if (!db) return undefined;
|
||||
|
||||
const result = await db.select().from(invoices)
|
||||
.where(
|
||||
and(
|
||||
eq(invoices.supplierName, supplierName),
|
||||
eq(invoices.invoiceNumber, invoiceNumber),
|
||||
eq(invoices.invoiceDate, invoiceDate)
|
||||
)
|
||||
)
|
||||
.limit(1);
|
||||
|
||||
return result[0];
|
||||
}
|
||||
|
||||
// ============= USER SETTINGS OPERATIONS =============
|
||||
|
||||
export async function getUserSettings(userId: number): Promise<UserSettings | undefined> {
|
||||
const db = await getDb();
|
||||
if (!db) return undefined;
|
||||
const result = await db.select().from(userSettings).where(eq(userSettings.userId, userId)).limit(1);
|
||||
return result[0];
|
||||
}
|
||||
|
||||
export async function upsertUserSettings(data: InsertUserSettings) {
|
||||
const db = await getDb();
|
||||
if (!db) throw new Error("Database not available");
|
||||
|
||||
const existing = await getUserSettings(data.userId);
|
||||
|
||||
if (existing) {
|
||||
await db.update(userSettings).set(data).where(eq(userSettings.userId, data.userId));
|
||||
} else {
|
||||
await db.insert(userSettings).values(data);
|
||||
}
|
||||
}
|
||||
|
||||
// ============= IMPORT LOG OPERATIONS =============
|
||||
|
||||
export async function createImportLog(data: InsertImportLog): Promise<ImportLog> {
|
||||
const db = await getDb();
|
||||
if (!db) throw new Error("Database not available");
|
||||
|
||||
const result = await db.insert(importLogs).values(data);
|
||||
const insertedId = Number(result[0].insertId);
|
||||
|
||||
const inserted = await db.select().from(importLogs).where(eq(importLogs.id, insertedId)).limit(1);
|
||||
return inserted[0]!;
|
||||
}
|
||||
|
||||
export async function getImportLogsByUser(userId: number): Promise<ImportLog[]> {
|
||||
const db = await getDb();
|
||||
if (!db) return [];
|
||||
return db.select().from(importLogs).where(eq(importLogs.userId, userId)).orderBy(desc(importLogs.importedAt));
|
||||
}
|
||||
|
||||
// ============= LLM LOG OPERATIONS =============
|
||||
|
||||
export async function createLlmLog(data: InsertLlmLog): Promise<LlmLog> {
|
||||
const db = await getDb();
|
||||
if (!db) throw new Error("Database not available");
|
||||
|
||||
const result = await db.insert(llmLogs).values(data);
|
||||
const insertedId = Number(result[0].insertId);
|
||||
|
||||
const inserted = await db.select().from(llmLogs).where(eq(llmLogs.id, insertedId)).limit(1);
|
||||
return inserted[0]!;
|
||||
}
|
||||
|
||||
export async function getLlmLogsBySourceFile(sourceFileId: number): Promise<LlmLog[]> {
|
||||
const db = await getDb();
|
||||
if (!db) return [];
|
||||
return db.select().from(llmLogs).where(eq(llmLogs.sourceFileId, sourceFileId)).orderBy(desc(llmLogs.createdAt));
|
||||
}
|
||||
|
||||
export async function getLlmLogsByInvoice(invoiceId: number): Promise<LlmLog[]> {
|
||||
const db = await getDb();
|
||||
if (!db) return [];
|
||||
return db.select().from(llmLogs).where(eq(llmLogs.invoiceId, invoiceId)).orderBy(desc(llmLogs.createdAt));
|
||||
}
|
||||
|
||||
262
server/invoiceExtractor.ts
Normal file
262
server/invoiceExtractor.ts
Normal file
@@ -0,0 +1,262 @@
|
||||
import { invokeLLM } from "./_core/llm";
|
||||
import { PDFDocument } from "pdf-lib";
|
||||
import { createLlmLog } from "./db";
|
||||
|
||||
export interface ExtractedInvoiceData {
|
||||
supplierName: string | null;
|
||||
invoiceNumber: string | null;
|
||||
invoiceDate: Date | null;
|
||||
deliveryNoteNumber: string | null;
|
||||
orderNumber: string | null;
|
||||
totalAmount: number | null;
|
||||
pageRange: string;
|
||||
qualityScore: number; // 0-100
|
||||
}
|
||||
|
||||
export interface MultiInvoiceResult {
|
||||
pageCount: number;
|
||||
invoiceCount: number;
|
||||
invoices: ExtractedInvoiceData[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert PDF buffer to base64 data URI for Mistral API processing
|
||||
*/
|
||||
function convertPdfToBase64(pdfBuffer: Buffer): string {
|
||||
try {
|
||||
const base64Pdf = pdfBuffer.toString("base64");
|
||||
const dataUri = `data:application/pdf;base64,${base64Pdf}`;
|
||||
console.log("[Mistral] PDF converted to base64, size:", Math.round(base64Pdf.length / 1024), "KB");
|
||||
return dataUri;
|
||||
} catch (error) {
|
||||
console.error("Error converting PDF to base64:", error);
|
||||
throw new Error("Failed to convert PDF to base64");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean JSON response from LLM by removing markdown code blocks
|
||||
*/
|
||||
function cleanJsonResponse(content: string): string {
|
||||
let cleaned = content.trim();
|
||||
|
||||
// Remove markdown code blocks
|
||||
cleaned = cleaned.replace(/^```(?:json)?\s*/i, "");
|
||||
cleaned = cleaned.replace(/\s*```$/, "");
|
||||
cleaned = cleaned.trim();
|
||||
|
||||
// Try to extract JSON object or array
|
||||
const firstBrace = cleaned.indexOf("{");
|
||||
const firstBracket = cleaned.indexOf("[");
|
||||
|
||||
let startIdx = -1;
|
||||
let startChar = "";
|
||||
|
||||
if (firstBrace !== -1 && (firstBracket === -1 || firstBrace < firstBracket)) {
|
||||
startIdx = firstBrace;
|
||||
startChar = "{";
|
||||
} else if (firstBracket !== -1) {
|
||||
startIdx = firstBracket;
|
||||
startChar = "[";
|
||||
}
|
||||
|
||||
if (startIdx === -1) {
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
const endChar = startChar === "{" ? "}" : "]";
|
||||
let depth = 0;
|
||||
let endIdx = -1;
|
||||
|
||||
for (let i = startIdx; i < cleaned.length; i++) {
|
||||
if (cleaned[i] === startChar) depth++;
|
||||
if (cleaned[i] === endChar) {
|
||||
depth--;
|
||||
if (depth === 0) {
|
||||
endIdx = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (endIdx !== -1) {
|
||||
cleaned = cleaned.substring(startIdx, endIdx + 1);
|
||||
}
|
||||
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract invoice data using Mistral AI
|
||||
* Hybrid approach: OCR to extract text, then LLM to parse and extract structured data
|
||||
*/
|
||||
export async function extractInvoicesWithMistral(
|
||||
pdfBuffer: Buffer,
|
||||
userId: number,
|
||||
sourceFileId: number,
|
||||
model: string = "mistral-large-latest",
|
||||
customKeywords?: {
|
||||
invoiceNumber?: string | null;
|
||||
deliveryNote?: string | null;
|
||||
orderNumber?: string | null;
|
||||
supplier?: string | null;
|
||||
totalAmount?: string | null;
|
||||
}
|
||||
): Promise<MultiInvoiceResult> {
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
console.log("[Mistral] Starting invoice extraction...");
|
||||
|
||||
// Convert PDF to base64
|
||||
const pdfDataUri = convertPdfToBase64(pdfBuffer);
|
||||
|
||||
// Get PDF page count
|
||||
const pdfDoc = await PDFDocument.load(pdfBuffer);
|
||||
const pageCount = pdfDoc.getPageCount();
|
||||
|
||||
console.log(`[Mistral] PDF has ${pageCount} pages`);
|
||||
|
||||
// Build custom keywords hint
|
||||
let keywordsHint = "";
|
||||
if (customKeywords) {
|
||||
const hints = [];
|
||||
if (customKeywords.invoiceNumber) hints.push(`Numéro de facture: ${customKeywords.invoiceNumber}`);
|
||||
if (customKeywords.deliveryNote) hints.push(`Bon de livraison: ${customKeywords.deliveryNote}`);
|
||||
if (customKeywords.orderNumber) hints.push(`Numéro de commande: ${customKeywords.orderNumber}`);
|
||||
if (customKeywords.supplier) hints.push(`Fournisseur: ${customKeywords.supplier}`);
|
||||
if (customKeywords.totalAmount) hints.push(`Montant total: ${customKeywords.totalAmount}`);
|
||||
|
||||
if (hints.length > 0) {
|
||||
keywordsHint = `\n\nMots-clés personnalisés à rechercher:\n${hints.join("\n")}`;
|
||||
}
|
||||
}
|
||||
|
||||
// Prepare prompt for LLM
|
||||
const prompt = `Tu es un expert en extraction de données de factures. Analyse ce document PDF et extrais toutes les factures qu'il contient.
|
||||
|
||||
Pour chaque facture trouvée, extrais les informations suivantes:
|
||||
- supplierName: Nom du fournisseur/vendeur
|
||||
- invoiceNumber: Numéro de la facture
|
||||
- invoiceDate: Date de la facture (format ISO 8601: YYYY-MM-DD)
|
||||
- deliveryNoteNumber: Numéro du bon de livraison (si présent)
|
||||
- orderNumber: Numéro de commande client (si présent)
|
||||
- totalAmount: Montant total TTC (nombre décimal)
|
||||
- pageRange: Plage de pages de cette facture (ex: "1-2" ou "5")
|
||||
- qualityScore: Score de qualité de l'extraction de 0 à 100 (100 = toutes les informations trouvées et claires)${keywordsHint}
|
||||
|
||||
Réponds UNIQUEMENT avec un objet JSON valide au format suivant:
|
||||
{
|
||||
"pageCount": ${pageCount},
|
||||
"invoiceCount": <nombre de factures détectées>,
|
||||
"invoices": [
|
||||
{
|
||||
"supplierName": "...",
|
||||
"invoiceNumber": "...",
|
||||
"invoiceDate": "YYYY-MM-DD",
|
||||
"deliveryNoteNumber": "...",
|
||||
"orderNumber": "...",
|
||||
"totalAmount": 123.45,
|
||||
"pageRange": "1-2",
|
||||
"qualityScore": 85
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Si une information n'est pas trouvée, utilise null. Ne retourne AUCUN texte en dehors du JSON.`;
|
||||
|
||||
// Call Mistral LLM with PDF
|
||||
const response = await invokeLLM({
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: prompt },
|
||||
{ type: "file_url", file_url: { url: pdfDataUri, mime_type: "application/pdf" } },
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const rawResponse = typeof response.choices[0]?.message?.content === "string"
|
||||
? response.choices[0].message.content
|
||||
: JSON.stringify(response.choices[0]?.message?.content || "");
|
||||
const processingTimeMs = Date.now() - startTime;
|
||||
|
||||
console.log("[Mistral] Raw response received:", rawResponse.substring(0, 200));
|
||||
|
||||
// Clean and parse response
|
||||
const cleanedResponse = cleanJsonResponse(rawResponse);
|
||||
|
||||
let result: MultiInvoiceResult;
|
||||
try {
|
||||
result = JSON.parse(cleanedResponse);
|
||||
} catch (parseError) {
|
||||
console.error("[Mistral] Failed to parse JSON:", parseError);
|
||||
console.error("[Mistral] Cleaned response:", cleanedResponse);
|
||||
|
||||
// Log error to database
|
||||
await createLlmLog({
|
||||
userId,
|
||||
sourceFileId,
|
||||
operation: "extraction",
|
||||
model,
|
||||
promptSent: prompt,
|
||||
rawResponse: rawResponse.substring(0, 10000),
|
||||
cleanedResponse: cleanedResponse.substring(0, 10000),
|
||||
success: 0,
|
||||
errorMessage: `JSON parse error: ${parseError}`,
|
||||
processingTimeMs,
|
||||
});
|
||||
|
||||
throw new Error("Failed to parse LLM response as JSON");
|
||||
}
|
||||
|
||||
// Log successful extraction
|
||||
await createLlmLog({
|
||||
userId,
|
||||
sourceFileId,
|
||||
operation: "extraction",
|
||||
model,
|
||||
promptSent: prompt.substring(0, 10000),
|
||||
rawResponse: rawResponse.substring(0, 10000),
|
||||
cleanedResponse: cleanedResponse.substring(0, 10000),
|
||||
success: 1,
|
||||
processingTimeMs,
|
||||
});
|
||||
|
||||
// Convert date strings to Date objects
|
||||
result.invoices = result.invoices.map((inv) => ({
|
||||
...inv,
|
||||
invoiceDate: inv.invoiceDate ? new Date(inv.invoiceDate) : null,
|
||||
}));
|
||||
|
||||
console.log(`[Mistral] Successfully extracted ${result.invoiceCount} invoice(s)`);
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
console.error("[Mistral] Extraction failed:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate metadata JSON for an invoice
|
||||
*/
|
||||
export function generateMetadataJSON(invoice: ExtractedInvoiceData): string {
|
||||
return JSON.stringify(
|
||||
{
|
||||
supplierName: invoice.supplierName,
|
||||
invoiceNumber: invoice.invoiceNumber,
|
||||
invoiceDate: invoice.invoiceDate?.toISOString(),
|
||||
deliveryNoteNumber: invoice.deliveryNoteNumber,
|
||||
orderNumber: invoice.orderNumber,
|
||||
totalAmount: invoice.totalAmount,
|
||||
pageRange: invoice.pageRange,
|
||||
qualityScore: invoice.qualityScore,
|
||||
extractedAt: new Date().toISOString(),
|
||||
},
|
||||
null,
|
||||
2
|
||||
);
|
||||
}
|
||||
111
server/localStorage.ts
Normal file
111
server/localStorage.ts
Normal file
@@ -0,0 +1,111 @@
|
||||
import fs from "fs/promises";
|
||||
import path from "path";
|
||||
import { nanoid } from "nanoid";
|
||||
|
||||
// Storage base path (local filesystem)
|
||||
const STORAGE_BASE_PATH = process.env.STORAGE_BASE_PATH || path.join(process.cwd(), "storage");
|
||||
|
||||
/**
|
||||
* Ensure storage directory exists
|
||||
*/
|
||||
async function ensureStorageDir(dirPath: string) {
|
||||
try {
|
||||
await fs.mkdir(dirPath, { recursive: true });
|
||||
} catch (error) {
|
||||
console.error(`[LocalStorage] Failed to create directory ${dirPath}:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a storage key with YYYY-MM prefix for organization
|
||||
*/
|
||||
export function generateStorageKey(userId: number, fileName: string): string {
|
||||
const now = new Date();
|
||||
const year = now.getFullYear();
|
||||
const month = String(now.getMonth() + 1).padStart(2, "0");
|
||||
const randomId = nanoid(8);
|
||||
|
||||
// Format: YYYY-MM/userId-randomId-filename
|
||||
return `${year}-${month}/${userId}-${randomId}-${fileName}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store a file in local storage
|
||||
* @param fileKey - Storage key (e.g., "2025-01/1-abc123-invoice.pdf")
|
||||
* @param buffer - File content as Buffer
|
||||
* @param contentType - MIME type (optional, for metadata)
|
||||
* @returns Object with key and public URL
|
||||
*/
|
||||
export async function localStoragePut(
|
||||
fileKey: string,
|
||||
buffer: Buffer,
|
||||
contentType?: string
|
||||
): Promise<{ key: string; url: string }> {
|
||||
try {
|
||||
const fullPath = path.join(STORAGE_BASE_PATH, fileKey);
|
||||
const dirPath = path.dirname(fullPath);
|
||||
|
||||
// Ensure directory exists
|
||||
await ensureStorageDir(dirPath);
|
||||
|
||||
// Write file
|
||||
await fs.writeFile(fullPath, buffer);
|
||||
|
||||
// Generate public URL (served by Express static middleware)
|
||||
const url = `/storage/${fileKey}`;
|
||||
|
||||
console.log(`[LocalStorage] File stored: ${fileKey}`);
|
||||
|
||||
return { key: fileKey, url };
|
||||
} catch (error) {
|
||||
console.error(`[LocalStorage] Failed to store file ${fileKey}:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve a file from local storage
|
||||
* @param fileKey - Storage key
|
||||
* @returns File content as Buffer
|
||||
*/
|
||||
export async function localStorageGet(fileKey: string): Promise<Buffer> {
|
||||
try {
|
||||
const fullPath = path.join(STORAGE_BASE_PATH, fileKey);
|
||||
const buffer = await fs.readFile(fullPath);
|
||||
return buffer;
|
||||
} catch (error) {
|
||||
console.error(`[LocalStorage] Failed to retrieve file ${fileKey}:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a file from local storage
|
||||
* @param fileKey - Storage key
|
||||
*/
|
||||
export async function localStorageDelete(fileKey: string): Promise<void> {
|
||||
try {
|
||||
const fullPath = path.join(STORAGE_BASE_PATH, fileKey);
|
||||
await fs.unlink(fullPath);
|
||||
console.log(`[LocalStorage] File deleted: ${fileKey}`);
|
||||
} catch (error) {
|
||||
console.error(`[LocalStorage] Failed to delete file ${fileKey}:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a file exists in storage
|
||||
* @param fileKey - Storage key
|
||||
* @returns true if file exists, false otherwise
|
||||
*/
|
||||
export async function localStorageExists(fileKey: string): Promise<boolean> {
|
||||
try {
|
||||
const fullPath = path.join(STORAGE_BASE_PATH, fileKey);
|
||||
await fs.access(fullPath);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -1,28 +1,493 @@
|
||||
import { z } from "zod";
|
||||
import { COOKIE_NAME } from "@shared/const";
|
||||
import { getSessionCookieOptions } from "./_core/cookies";
|
||||
import { systemRouter } from "./_core/systemRouter";
|
||||
import { publicProcedure, router } from "./_core/trpc";
|
||||
import { publicProcedure, protectedProcedure, router } from "./_core/trpc";
|
||||
import {
|
||||
createInvoice,
|
||||
getInvoiceById,
|
||||
getInvoicesByUser,
|
||||
updateInvoice,
|
||||
deleteInvoice,
|
||||
searchInvoices,
|
||||
getInvoiceStats,
|
||||
createSourceFile,
|
||||
getSourceFileById,
|
||||
updateSourceFile,
|
||||
getUserSettings,
|
||||
upsertUserSettings,
|
||||
createLocalUser,
|
||||
getAllUsers,
|
||||
updateUserPassword,
|
||||
toggleUserActive,
|
||||
deleteUser,
|
||||
findDuplicateInvoice,
|
||||
createImportLog,
|
||||
getImportLogsByUser,
|
||||
getLlmLogsBySourceFile,
|
||||
getLlmLogsByInvoice,
|
||||
} from "./db";
|
||||
import { loginLocal, hashPassword, getAzureAuthUrl, isAzureAdConfigured, generateToken } from "./auth";
|
||||
import { extractInvoicesWithMistral, generateMetadataJSON } from "./invoiceExtractor";
|
||||
import { localStoragePut, generateStorageKey } from "./localStorage";
|
||||
import { testSftpConnection, exportInvoiceToSftp, getUserSftpConfig } from "./sftpExport";
|
||||
import { TRPCError } from "@trpc/server";
|
||||
|
||||
// Admin-only procedure
|
||||
const adminProcedure = protectedProcedure.use(({ ctx, next }) => {
|
||||
if (ctx.user.role !== "admin") {
|
||||
throw new TRPCError({ code: "FORBIDDEN", message: "Admin access required" });
|
||||
}
|
||||
return next({ ctx });
|
||||
});
|
||||
|
||||
export const appRouter = router({
|
||||
// if you need to use socket.io, read and register route in server/_core/index.ts, all api should start with '/api/' so that the gateway can route correctly
|
||||
system: systemRouter,
|
||||
|
||||
// ============= AUTH ROUTES =============
|
||||
auth: router({
|
||||
me: publicProcedure.query(opts => opts.ctx.user),
|
||||
|
||||
// Local login (email + password)
|
||||
loginLocal: publicProcedure
|
||||
.input(z.object({
|
||||
email: z.string().email(),
|
||||
password: z.string().min(6),
|
||||
}))
|
||||
.mutation(async ({ input, ctx }) => {
|
||||
const result = await loginLocal(input.email, input.password);
|
||||
|
||||
if (!result) {
|
||||
throw new TRPCError({ code: "UNAUTHORIZED", message: "Invalid email or password" });
|
||||
}
|
||||
|
||||
// Set auth cookie
|
||||
ctx.res.cookie("auth_token", result.token, {
|
||||
httpOnly: true,
|
||||
secure: process.env.NODE_ENV === "production",
|
||||
sameSite: "lax",
|
||||
path: "/",
|
||||
maxAge: 7 * 24 * 60 * 60 * 1000, // 7 days
|
||||
});
|
||||
|
||||
return { user: result.user };
|
||||
}),
|
||||
|
||||
// Get Azure AD login URL
|
||||
getAzureLoginUrl: publicProcedure.query(async () => {
|
||||
if (!isAzureAdConfigured()) {
|
||||
throw new TRPCError({ code: "BAD_REQUEST", message: "Azure AD not configured" });
|
||||
}
|
||||
|
||||
const url = await getAzureAuthUrl();
|
||||
return { url };
|
||||
}),
|
||||
|
||||
// Check if Azure AD is available
|
||||
isAzureAdAvailable: publicProcedure.query(() => {
|
||||
return { available: isAzureAdConfigured() };
|
||||
}),
|
||||
|
||||
logout: publicProcedure.mutation(({ ctx }) => {
|
||||
const cookieOptions = getSessionCookieOptions(ctx.req);
|
||||
ctx.res.clearCookie(COOKIE_NAME, { ...cookieOptions, maxAge: -1 });
|
||||
return {
|
||||
success: true,
|
||||
} as const;
|
||||
ctx.res.clearCookie("auth_token", { path: "/", maxAge: -1 });
|
||||
return { success: true };
|
||||
}),
|
||||
}),
|
||||
|
||||
// TODO: add feature routers here, e.g.
|
||||
// todo: router({
|
||||
// list: protectedProcedure.query(({ ctx }) =>
|
||||
// db.getUserTodos(ctx.user.id)
|
||||
// ),
|
||||
// }),
|
||||
|
||||
// ============= INVOICE ROUTES =============
|
||||
invoices: router({
|
||||
// Upload and process PDF file
|
||||
upload: protectedProcedure
|
||||
.input(z.object({
|
||||
fileName: z.string(),
|
||||
fileData: z.string(), // Base64 encoded PDF
|
||||
}))
|
||||
.mutation(async ({ input, ctx }) => {
|
||||
const userId = ctx.user.id;
|
||||
|
||||
// Decode base64 file data
|
||||
const fileBuffer = Buffer.from(input.fileData, "base64");
|
||||
|
||||
// Store source file
|
||||
const sourceFileKey = generateStorageKey(userId, input.fileName);
|
||||
const { url: sourceFileUrl } = await localStoragePut(sourceFileKey, fileBuffer, "application/pdf");
|
||||
|
||||
// Create source file record
|
||||
const sourceFile = await createSourceFile({
|
||||
userId,
|
||||
fileName: input.fileName,
|
||||
fileKey: sourceFileKey,
|
||||
fileUrl: sourceFileUrl,
|
||||
processingStatus: "processing",
|
||||
});
|
||||
|
||||
// Start extraction process (async - don't wait)
|
||||
(async () => {
|
||||
try {
|
||||
// Get user settings for custom keywords
|
||||
const settings = await getUserSettings(userId);
|
||||
const customKeywords = settings ? {
|
||||
invoiceNumber: settings.invoiceNumberKeywords,
|
||||
deliveryNote: settings.deliveryNoteKeywords,
|
||||
orderNumber: settings.orderNumberKeywords,
|
||||
supplier: settings.supplierKeywords,
|
||||
totalAmount: settings.totalAmountKeywords,
|
||||
} : undefined;
|
||||
|
||||
const model = settings?.llmModel || "mistral-large-latest";
|
||||
|
||||
// Extract invoices
|
||||
const result = await extractInvoicesWithMistral(
|
||||
fileBuffer,
|
||||
userId,
|
||||
sourceFile.id,
|
||||
model,
|
||||
customKeywords
|
||||
);
|
||||
|
||||
// Update source file with total count
|
||||
await updateSourceFile(sourceFile.id, {
|
||||
totalInvoicesDetected: result.invoiceCount,
|
||||
processingProgress: `Extraction ${result.invoiceCount} facture(s) détectée(s)`,
|
||||
});
|
||||
|
||||
// Process each invoice
|
||||
let importedCount = 0;
|
||||
let duplicatesCount = 0;
|
||||
let errorsCount = 0;
|
||||
const duplicateDetails: any[] = [];
|
||||
const errorDetails: any[] = [];
|
||||
|
||||
for (let i = 0; i < result.invoices.length; i++) {
|
||||
const invoiceData = result.invoices[i]!;
|
||||
|
||||
try {
|
||||
// Update progress
|
||||
await updateSourceFile(sourceFile.id, {
|
||||
processingProgress: `Extraction ${i + 1}/${result.invoiceCount} factures...`,
|
||||
});
|
||||
|
||||
// Check for duplicates
|
||||
const duplicate = await findDuplicateInvoice(
|
||||
invoiceData.supplierName,
|
||||
invoiceData.invoiceNumber,
|
||||
invoiceData.invoiceDate
|
||||
);
|
||||
|
||||
if (duplicate) {
|
||||
duplicatesCount++;
|
||||
duplicateDetails.push({
|
||||
supplierName: invoiceData.supplierName,
|
||||
invoiceNumber: invoiceData.invoiceNumber,
|
||||
invoiceDate: invoiceData.invoiceDate,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Generate metadata JSON
|
||||
const metadataJson = generateMetadataJSON(invoiceData);
|
||||
const metadataKey = generateStorageKey(userId, `${input.fileName}-${i + 1}-metadata.json`);
|
||||
const { url: metadataUrl } = await localStoragePut(
|
||||
metadataKey,
|
||||
Buffer.from(metadataJson),
|
||||
"application/json"
|
||||
);
|
||||
|
||||
// Create invoice record
|
||||
await createInvoice({
|
||||
userId,
|
||||
sourceFileId: sourceFile.id,
|
||||
invoiceIndexInFile: i + 1,
|
||||
fileName: `${input.fileName} - Facture ${i + 1}`,
|
||||
fileKey: sourceFileKey, // Same as source for now
|
||||
fileUrl: sourceFileUrl,
|
||||
supplierName: invoiceData.supplierName,
|
||||
invoiceNumber: invoiceData.invoiceNumber,
|
||||
invoiceDate: invoiceData.invoiceDate,
|
||||
deliveryNoteNumber: invoiceData.deliveryNoteNumber,
|
||||
orderNumber: invoiceData.orderNumber,
|
||||
totalAmount: invoiceData.totalAmount?.toString(),
|
||||
pageRange: invoiceData.pageRange,
|
||||
qualityScore: invoiceData.qualityScore,
|
||||
metadataFileKey: metadataKey,
|
||||
metadataFileUrl: metadataUrl,
|
||||
status: "completed",
|
||||
});
|
||||
|
||||
importedCount++;
|
||||
} catch (error: any) {
|
||||
errorsCount++;
|
||||
errorDetails.push({
|
||||
invoiceIndex: i + 1,
|
||||
error: error.message,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Update source file status
|
||||
await updateSourceFile(sourceFile.id, {
|
||||
processingStatus: "completed",
|
||||
processingProgress: `Terminé: ${importedCount} importée(s), ${duplicatesCount} doublon(s)`,
|
||||
});
|
||||
|
||||
// Create import log
|
||||
await createImportLog({
|
||||
userId,
|
||||
sourceFileId: sourceFile.id,
|
||||
fileName: input.fileName,
|
||||
totalInvoicesDetected: result.invoiceCount,
|
||||
invoicesImported: importedCount,
|
||||
duplicatesIgnored: duplicatesCount,
|
||||
errors: errorsCount,
|
||||
duplicateDetails: JSON.stringify(duplicateDetails),
|
||||
errorDetails: JSON.stringify(errorDetails),
|
||||
});
|
||||
|
||||
} catch (error: any) {
|
||||
console.error("[Upload] Extraction failed:", error);
|
||||
await updateSourceFile(sourceFile.id, {
|
||||
processingStatus: "error",
|
||||
processingProgress: `Erreur: ${error.message}`,
|
||||
});
|
||||
}
|
||||
})();
|
||||
|
||||
return { sourceFileId: sourceFile.id };
|
||||
}),
|
||||
|
||||
list: protectedProcedure.query(async ({ ctx }) => {
|
||||
return getInvoicesByUser(ctx.user.id);
|
||||
}),
|
||||
|
||||
getById: protectedProcedure
|
||||
.input(z.object({ id: z.number() }))
|
||||
.query(async ({ input, ctx }) => {
|
||||
const invoice = await getInvoiceById(input.id);
|
||||
if (!invoice || invoice.userId !== ctx.user.id) {
|
||||
throw new TRPCError({ code: "NOT_FOUND" });
|
||||
}
|
||||
return invoice;
|
||||
}),
|
||||
|
||||
update: protectedProcedure
|
||||
.input(z.object({
|
||||
id: z.number(),
|
||||
data: z.object({
|
||||
supplierName: z.string().optional(),
|
||||
invoiceNumber: z.string().optional(),
|
||||
invoiceDate: z.date().optional(),
|
||||
deliveryNoteNumber: z.string().optional(),
|
||||
orderNumber: z.string().optional(),
|
||||
totalAmount: z.string().optional(),
|
||||
}),
|
||||
}))
|
||||
.mutation(async ({ input, ctx }) => {
|
||||
const invoice = await getInvoiceById(input.id);
|
||||
if (!invoice || invoice.userId !== ctx.user.id) {
|
||||
throw new TRPCError({ code: "NOT_FOUND" });
|
||||
}
|
||||
|
||||
await updateInvoice(input.id, {
|
||||
...input.data,
|
||||
manuallyEdited: 1,
|
||||
});
|
||||
|
||||
return { success: true };
|
||||
}),
|
||||
|
||||
delete: protectedProcedure
|
||||
.input(z.object({ id: z.number() }))
|
||||
.mutation(async ({ input, ctx }) => {
|
||||
const invoice = await getInvoiceById(input.id);
|
||||
if (!invoice || invoice.userId !== ctx.user.id) {
|
||||
throw new TRPCError({ code: "NOT_FOUND" });
|
||||
}
|
||||
|
||||
await deleteInvoice(input.id);
|
||||
return { success: true };
|
||||
}),
|
||||
|
||||
search: protectedProcedure
|
||||
.input(z.object({ query: z.string() }))
|
||||
.query(async ({ input, ctx }) => {
|
||||
return searchInvoices(ctx.user.id, input.query);
|
||||
}),
|
||||
|
||||
getStats: protectedProcedure.query(async ({ ctx }) => {
|
||||
return getInvoiceStats(ctx.user.id);
|
||||
}),
|
||||
}),
|
||||
|
||||
// ============= SOURCE FILES ROUTES =============
|
||||
sourceFiles: router({
|
||||
getByIds: protectedProcedure
|
||||
.input(z.object({ ids: z.array(z.number()) }))
|
||||
.query(async ({ input, ctx }) => {
|
||||
const files = await Promise.all(
|
||||
input.ids.map(id => getSourceFileById(id))
|
||||
);
|
||||
return files.filter(f => f && f.userId === ctx.user.id);
|
||||
}),
|
||||
}),
|
||||
|
||||
// ============= SETTINGS ROUTES =============
|
||||
settings: router({
|
||||
get: protectedProcedure.query(async ({ ctx }) => {
|
||||
return getUserSettings(ctx.user.id);
|
||||
}),
|
||||
|
||||
upsert: protectedProcedure
|
||||
.input(z.object({
|
||||
llmModel: z.string().optional(),
|
||||
orderNumberFormat: z.string().optional(),
|
||||
invoiceNumberKeywords: z.string().optional(),
|
||||
deliveryNoteKeywords: z.string().optional(),
|
||||
orderNumberKeywords: z.string().optional(),
|
||||
supplierKeywords: z.string().optional(),
|
||||
totalAmountKeywords: z.string().optional(),
|
||||
sftpHost: z.string().optional(),
|
||||
sftpPort: z.number().optional(),
|
||||
sftpUsername: z.string().optional(),
|
||||
sftpPassword: z.string().optional(),
|
||||
sftpRemotePath: z.string().optional(),
|
||||
sftpAutoExport: z.number().optional(),
|
||||
llmLogsRetentionMonths: z.number().optional(),
|
||||
}))
|
||||
.mutation(async ({ input, ctx }) => {
|
||||
await upsertUserSettings({
|
||||
userId: ctx.user.id,
|
||||
...input,
|
||||
});
|
||||
return { success: true };
|
||||
}),
|
||||
}),
|
||||
|
||||
// ============= ADMIN ROUTES =============
|
||||
admin: router({
|
||||
createUser: adminProcedure
|
||||
.input(z.object({
|
||||
email: z.string().email(),
|
||||
password: z.string().min(6),
|
||||
name: z.string(),
|
||||
role: z.enum(["user", "admin"]),
|
||||
}))
|
||||
.mutation(async ({ input }) => {
|
||||
const passwordHash = await hashPassword(input.password);
|
||||
const user = await createLocalUser(input.email, passwordHash, input.name, input.role);
|
||||
return { user };
|
||||
}),
|
||||
|
||||
getAllUsers: adminProcedure.query(async () => {
|
||||
return getAllUsers();
|
||||
}),
|
||||
|
||||
updateUserPassword: adminProcedure
|
||||
.input(z.object({
|
||||
userId: z.number(),
|
||||
newPassword: z.string().min(6),
|
||||
}))
|
||||
.mutation(async ({ input }) => {
|
||||
const passwordHash = await hashPassword(input.newPassword);
|
||||
await updateUserPassword(input.userId, passwordHash);
|
||||
return { success: true };
|
||||
}),
|
||||
|
||||
toggleUserActive: adminProcedure
|
||||
.input(z.object({
|
||||
userId: z.number(),
|
||||
isActive: z.number(),
|
||||
}))
|
||||
.mutation(async ({ input }) => {
|
||||
await toggleUserActive(input.userId, input.isActive);
|
||||
return { success: true };
|
||||
}),
|
||||
|
||||
deleteUser: adminProcedure
|
||||
.input(z.object({ userId: z.number() }))
|
||||
.mutation(async ({ input }) => {
|
||||
await deleteUser(input.userId);
|
||||
return { success: true };
|
||||
}),
|
||||
}),
|
||||
|
||||
// ============= SFTP ROUTES =============
|
||||
sftp: router({
|
||||
testConnection: protectedProcedure.mutation(async ({ ctx }) => {
|
||||
const config = await getUserSftpConfig(ctx.user.id);
|
||||
if (!config) {
|
||||
throw new TRPCError({ code: "BAD_REQUEST", message: "SFTP not configured" });
|
||||
}
|
||||
|
||||
const success = await testSftpConnection(config);
|
||||
return { success };
|
||||
}),
|
||||
|
||||
exportInvoices: protectedProcedure
|
||||
.input(z.object({ invoiceIds: z.array(z.number()) }))
|
||||
.mutation(async ({ input, ctx }) => {
|
||||
const config = await getUserSftpConfig(ctx.user.id);
|
||||
if (!config) {
|
||||
throw new TRPCError({ code: "BAD_REQUEST", message: "SFTP not configured" });
|
||||
}
|
||||
|
||||
let successCount = 0;
|
||||
let errorCount = 0;
|
||||
|
||||
for (const invoiceId of input.invoiceIds) {
|
||||
try {
|
||||
const invoice = await getInvoiceById(invoiceId);
|
||||
if (!invoice || invoice.userId !== ctx.user.id) {
|
||||
errorCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
await exportInvoiceToSftp(
|
||||
config,
|
||||
invoice.fileKey,
|
||||
invoice.metadataFileKey,
|
||||
invoice.invoiceDate || new Date()
|
||||
);
|
||||
|
||||
// Update invoice export status
|
||||
await updateInvoice(invoiceId, {
|
||||
exportedAt: new Date(),
|
||||
exportMode: "manual",
|
||||
});
|
||||
|
||||
successCount++;
|
||||
} catch (error) {
|
||||
console.error(`[SFTP] Failed to export invoice ${invoiceId}:`, error);
|
||||
errorCount++;
|
||||
}
|
||||
}
|
||||
|
||||
return { successCount, errorCount };
|
||||
}),
|
||||
}),
|
||||
|
||||
// ============= IMPORT LOGS ROUTES =============
|
||||
importLogs: router({
|
||||
getByUser: protectedProcedure.query(async ({ ctx }) => {
|
||||
return getImportLogsByUser(ctx.user.id);
|
||||
}),
|
||||
}),
|
||||
|
||||
// ============= LLM LOGS ROUTES =============
|
||||
llmLogs: router({
|
||||
getBySourceFile: protectedProcedure
|
||||
.input(z.object({ sourceFileId: z.number() }))
|
||||
.query(async ({ input }) => {
|
||||
return getLlmLogsBySourceFile(input.sourceFileId);
|
||||
}),
|
||||
|
||||
getByInvoice: protectedProcedure
|
||||
.input(z.object({ invoiceId: z.number() }))
|
||||
.query(async ({ input }) => {
|
||||
return getLlmLogsByInvoice(input.invoiceId);
|
||||
}),
|
||||
}),
|
||||
});
|
||||
|
||||
export type AppRouter = typeof appRouter;
|
||||
|
||||
115
server/sftpExport.ts
Normal file
115
server/sftpExport.ts
Normal file
@@ -0,0 +1,115 @@
|
||||
import SftpClient from "ssh2-sftp-client";
|
||||
import { localStorageGet } from "./localStorage";
|
||||
import { getUserSettings } from "./db";
|
||||
|
||||
export interface SftpConfig {
|
||||
host: string;
|
||||
port: number;
|
||||
username: string;
|
||||
password: string;
|
||||
remotePath: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test SFTP connection
|
||||
*/
|
||||
export async function testSftpConnection(config: SftpConfig): Promise<boolean> {
|
||||
const sftp = new SftpClient();
|
||||
|
||||
try {
|
||||
await sftp.connect({
|
||||
host: config.host,
|
||||
port: config.port,
|
||||
username: config.username,
|
||||
password: config.password,
|
||||
});
|
||||
|
||||
console.log("[SFTP] Connection successful");
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error("[SFTP] Connection failed:", error);
|
||||
return false;
|
||||
} finally {
|
||||
await sftp.end();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Export invoice files (PDF + JSON) to SFTP server
|
||||
*/
|
||||
export async function exportInvoiceToSftp(
|
||||
config: SftpConfig,
|
||||
pdfFileKey: string,
|
||||
jsonFileKey: string | null,
|
||||
invoiceDate: Date
|
||||
): Promise<void> {
|
||||
const sftp = new SftpClient();
|
||||
|
||||
try {
|
||||
// Connect to SFTP
|
||||
await sftp.connect({
|
||||
host: config.host,
|
||||
port: config.port,
|
||||
username: config.username,
|
||||
password: config.password,
|
||||
});
|
||||
|
||||
console.log("[SFTP] Connected successfully");
|
||||
|
||||
// Create directory structure: remotePath/YYYY/MM/DD
|
||||
const year = invoiceDate.getFullYear();
|
||||
const month = String(invoiceDate.getMonth() + 1).padStart(2, "0");
|
||||
const day = String(invoiceDate.getDate()).padStart(2, "0");
|
||||
|
||||
const targetDir = `${config.remotePath}/${year}/${month}/${day}`.replace(/\/+/g, "/");
|
||||
|
||||
// Ensure directory exists
|
||||
await sftp.mkdir(targetDir, true);
|
||||
|
||||
console.log(`[SFTP] Created directory: ${targetDir}`);
|
||||
|
||||
// Upload PDF file
|
||||
const pdfBuffer = await localStorageGet(pdfFileKey);
|
||||
const pdfFileName = pdfFileKey.split("/").pop() || "invoice.pdf";
|
||||
const pdfRemotePath = `${targetDir}/${pdfFileName}`;
|
||||
|
||||
await sftp.put(pdfBuffer, pdfRemotePath);
|
||||
console.log(`[SFTP] Uploaded PDF: ${pdfRemotePath}`);
|
||||
|
||||
// Upload JSON metadata file if exists
|
||||
if (jsonFileKey) {
|
||||
const jsonBuffer = await localStorageGet(jsonFileKey);
|
||||
const jsonFileName = jsonFileKey.split("/").pop() || "metadata.json";
|
||||
const jsonRemotePath = `${targetDir}/${jsonFileName}`;
|
||||
|
||||
await sftp.put(jsonBuffer, jsonRemotePath);
|
||||
console.log(`[SFTP] Uploaded JSON: ${jsonRemotePath}`);
|
||||
}
|
||||
|
||||
console.log("[SFTP] Export completed successfully");
|
||||
} catch (error) {
|
||||
console.error("[SFTP] Export failed:", error);
|
||||
throw error;
|
||||
} finally {
|
||||
await sftp.end();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get SFTP configuration for a user
|
||||
*/
|
||||
export async function getUserSftpConfig(userId: number): Promise<SftpConfig | null> {
|
||||
const settings = await getUserSettings(userId);
|
||||
|
||||
if (!settings || !settings.sftpHost || !settings.sftpUsername || !settings.sftpPassword) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
host: settings.sftpHost,
|
||||
port: settings.sftpPort || 22,
|
||||
username: settings.sftpUsername,
|
||||
password: settings.sftpPassword,
|
||||
remotePath: settings.sftpRemotePath || "/",
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user