manus-admin/demat-facturation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Checkpoint: Ajout de la configuration des champs obligatoires pour le score LLM

Browse Source 
Nouvelle fonctionnalité permettant de configurer quels champs sont obligatoires ou optionnels pour atteindre un score de reconnaissance de 100%.

Modifications:
- Nouvelle table llmFieldsConfig dans la base de données
- Routes tRPC pour gérer la configuration (getAll, updateField)
- Interface utilisateur dans la page Paramètres avec tableau et cases à cocher
- Modification du code d'extraction pour générer dynamiquement l'instruction de score
- Initialisation automatique des champs par défaut (supplierName, invoiceNumber, invoiceDate, totalAmount obligatoires)
- Tests unitaires pour valider la fonctionnalité

L'utilisateur peut maintenant personnaliser quels champs doivent être détectés pour qu'une facture atteigne 100% de score.
This commit is contained in:
Manus
2026-02-13 03:39:41 -05:00
parent 96f6367094
commit d32359fc10
14 changed files with 1903 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
server/_core/env.ts
View File

@@ -7,4 +7,5 @@ export const ENV = {
isProduction: process.env.NODE_ENV === "production",
forgeApiUrl: process.env.BUILT_IN_FORGE_API_URL ?? "",
forgeApiKey: process.env.BUILT_IN_FORGE_API_KEY ?? "",
mistralApiKey: process.env.MISTRAL_API_KEY ?? "",
};

40
server/_core/llm.ts
View File

@@ -209,17 +209,32 @@ const normalizeToolChoice = (
return toolChoice;
};
const resolveApiUrl = () =>
ENV.forgeApiUrl && ENV.forgeApiUrl.trim().length > 0
const resolveApiUrl = () => {
// If MISTRAL_API_KEY is set, use Mistral API directly
if (ENV.mistralApiKey && ENV.mistralApiKey.trim().length > 0) {
return "https://api.mistral.ai/v1/chat/completions";
}
// Otherwise use Manus Forge API
return ENV.forgeApiUrl && ENV.forgeApiUrl.trim().length > 0
? `${ENV.forgeApiUrl.replace(/\/$/, "")}/v1/chat/completions`
: "https://forge.manus.im/v1/chat/completions";
};
const assertApiKey = () => {
if (!ENV.forgeApiKey) {
throw new Error("OPENAI_API_KEY is not configured");
if (!ENV.mistralApiKey && !ENV.forgeApiKey) {
throw new Error("MISTRAL_API_KEY or OPENAI_API_KEY is not configured");
}
};
const getApiKey = () => {
// Prioritize MISTRAL_API_KEY if set
if (ENV.mistralApiKey && ENV.mistralApiKey.trim().length > 0) {
return ENV.mistralApiKey;
}
return ENV.forgeApiKey;
};
const normalizeResponseFormat = ({
responseFormat,
response_format,
@@ -279,8 +294,13 @@ export async function invokeLLM(params: InvokeParams): Promise<InvokeResult> {
response_format,
} = params;
// Use mistral-large-latest when MISTRAL_API_KEY is set, otherwise use gemini
const model = ENV.mistralApiKey && ENV.mistralApiKey.trim().length > 0
? "mistral-large-latest"
: "gemini-2.5-flash";
const payload: Record<string, unknown> = {
model: "gemini-2.5-flash",
model,
messages: messages.map(normalizeMessage),
};
@@ -297,8 +317,12 @@ export async function invokeLLM(params: InvokeParams): Promise<InvokeResult> {
}
payload.max_tokens = 32768
payload.thinking = {
"budget_tokens": 128
// Only add thinking parameter for Gemini models
if (!(ENV.mistralApiKey && ENV.mistralApiKey.trim().length > 0)) {
payload.thinking = {
"budget_tokens": 128
}
}
const normalizedResponseFormat = normalizeResponseFormat({
@@ -316,7 +340,7 @@ export async function invokeLLM(params: InvokeParams): Promise<InvokeResult> {
method: "POST",
headers: {
"content-type": "application/json",
authorization: `Bearer ${ENV.forgeApiKey}`,
authorization: `Bearer ${getApiKey()}`,
},
body: JSON.stringify(payload),
});

63
server/db.ts
View File

@@ -29,7 +29,10 @@ import {
AccountingAllocation,
automationRules,
InsertAutomationRule,
AutomationRule
AutomationRule,
llmFieldsConfig,
InsertLlmFieldConfig,
LlmFieldConfig
} from "../drizzle/schema";
import { ENV } from './_core/env';
@@ -606,3 +609,61 @@ export async function initializeDefaultLists(userId: number): Promise<void> {
}
}
}
// ============= LLM FIELDS CONFIG OPERATIONS =============
export async function getLlmFieldsConfigByUser(userId: number): Promise<LlmFieldConfig[]> {
const db = await getDb();
if (!db) return [];
return db.select().from(llmFieldsConfig).where(eq(llmFieldsConfig.userId, userId)).orderBy(llmFieldsConfig.displayOrder);
}
export async function upsertLlmFieldConfig(data: InsertLlmFieldConfig): Promise<LlmFieldConfig> {
const db = await getDb();
if (!db) throw new Error("Database not available");
const existing = await db.select().from(llmFieldsConfig)
.where(and(
eq(llmFieldsConfig.userId, data.userId),
eq(llmFieldsConfig.fieldName, data.fieldName)
))
.limit(1);
if (existing.length > 0) {
await db.update(llmFieldsConfig)
.set({ ...data, updatedAt: new Date() })
.where(eq(llmFieldsConfig.id, existing[0].id));
return (await db.select().from(llmFieldsConfig).where(eq(llmFieldsConfig.id, existing[0].id)))[0];
} else {
const result = await db.insert(llmFieldsConfig).values(data);
const insertedId = (result as any).insertId;
return (await db.select().from(llmFieldsConfig).where(eq(llmFieldsConfig.id, Number(insertedId))))[0];
}
}
export async function initializeDefaultLlmFields(userId: number): Promise<void> {
const db = await getDb();
if (!db) return;
const defaultFields = [
{ fieldName: "supplierName", displayName: "Nom du fournisseur", isRequired: 1, displayOrder: 1 },
{ fieldName: "invoiceNumber", displayName: "Numéro de facture", isRequired: 1, displayOrder: 2 },
{ fieldName: "invoiceDate", displayName: "Date de facture", isRequired: 1, displayOrder: 3 },
{ fieldName: "totalAmount", displayName: "Montant total TTC", isRequired: 1, displayOrder: 4 },
{ fieldName: "deliveryNoteNumber", displayName: "Numéro de bon de livraison", isRequired: 0, displayOrder: 5 },
{ fieldName: "orderNumber", displayName: "Numéro de commande", isRequired: 0, displayOrder: 6 },
];
const existingFields = await db.select().from(llmFieldsConfig).where(eq(llmFieldsConfig.userId, userId));
const existingFieldNames = new Set(existingFields.map(f => f.fieldName));
for (const field of defaultFields) {
if (!existingFieldNames.has(field.fieldName)) {
try {
await db.insert(llmFieldsConfig).values({ userId, ...field });
} catch (error) {
// Ignore errors
}
}
}
}

90
server/invoiceExtractor.ts
View File

@@ -1,6 +1,7 @@
import { invokeLLM } from "./_core/llm";
import { PDFDocument } from "pdf-lib";
import { createLlmLog } from "./db";
import PDFParser from "pdf2json";
export interface ExtractedInvoiceData {
supplierName: string | null;
@@ -22,18 +23,54 @@ export interface MultiInvoiceResult {
}
/**
* Convert PDF buffer to base64 data URI for Mistral API processing
* Extract text from PDF buffer using pdf2json
*/
function convertPdfToBase64(pdfBuffer: Buffer): string {
try {
const base64Pdf = pdfBuffer.toString("base64");
const dataUri = `data:application/pdf;base64,${base64Pdf}`;
console.log("[Mistral] PDF converted to base64, size:", Math.round(base64Pdf.length / 1024), "KB");
return dataUri;
} catch (error) {
console.error("Error converting PDF to base64:", error);
throw new Error("Failed to convert PDF to base64");
}
async function extractTextFromPdf(pdfBuffer: Buffer): Promise<string> {
return new Promise((resolve, reject) => {
const pdfParser = new (PDFParser as any)(null, 1);
pdfParser.on("pdfParser_dataError", (errData: any) => {
console.error("Error parsing PDF:", errData.parserError);
reject(new Error("Failed to parse PDF"));
});
pdfParser.on("pdfParser_dataReady", (pdfData: any) => {
try {
let text = "";
// Extract text from all pages
if (pdfData.Pages) {
for (const page of pdfData.Pages) {
if (page.Texts) {
for (const textItem of page.Texts) {
if (textItem.R) {
for (const run of textItem.R) {
if (run.T) {
try {
text += decodeURIComponent(run.T) + " ";
} catch (e) {
// If decodeURIComponent fails, use the raw text
text += run.T + " ";
}
}
}
}
}
text += "\n";
}
}
}
console.log("[PDF] Text extracted, length:", text.length, "characters");
resolve(text);
} catch (error) {
console.error("Error extracting text from PDF data:", error);
reject(new Error("Failed to extract text from PDF"));
}
});
pdfParser.parseBuffer(pdfBuffer);
});
}
/**
@@ -106,13 +143,30 @@ export async function extractInvoicesWithMistral(
subscription?: string | null;
}
): Promise<MultiInvoiceResult> {
// Load user's field configuration
const { getLlmFieldsConfigByUser } = await import("./db");
const fieldsConfig = await getLlmFieldsConfigByUser(userId);
// Build quality score instruction based on required fields
const requiredFields = fieldsConfig.filter(f => f.isRequired === 1);
const optionalFields = fieldsConfig.filter(f => f.isRequired === 0);
let qualityScoreInstruction = "- qualityScore: Score de qualité de l'extraction de 0 à 100";
if (requiredFields.length > 0) {
const requiredFieldNames = requiredFields.map(f => f.displayName).join(", ");
qualityScoreInstruction += ` (100 = tous les champs obligatoires trouvés: ${requiredFieldNames})`;
}
if (optionalFields.length > 0) {
const optionalFieldNames = optionalFields.map(f => f.displayName).join(", ");
qualityScoreInstruction += `. Champs optionnels (n'affectent pas le score): ${optionalFieldNames}`;
}
const startTime = Date.now();
try {
console.log("[Mistral] Starting invoice extraction...");
// Convert PDF to base64
const pdfDataUri = convertPdfToBase64(pdfBuffer);
// Extract text from PDF
const pdfText = await extractTextFromPdf(pdfBuffer);
// Get PDF page count
const pdfDoc = await PDFDocument.load(pdfBuffer);
@@ -153,7 +207,7 @@ Pour chaque facture trouvée, extrais les informations suivantes:
- orderNumber: Numéro de commande client (si présent)
- totalAmount: Montant total TTC (nombre décimal)
- pageRange: Plage de pages de cette facture (ex: "1-2" ou "5")
- qualityScore: Score de qualité de l'extraction de 0 à 100 (100 = toutes les informations trouvées et claires)
${qualityScoreInstruction}
- extractedText: Texte complet extrait de la facture (tout le texte visible sur les pages de cette facture)
${subscriptionInstruction}${keywordsHint}
@@ -179,15 +233,13 @@ Réponds UNIQUEMENT avec un objet JSON valide au format suivant:
Si une information n'est pas trouvée, utilise null. Ne retourne AUCUN texte en dehors du JSON.`;
// Call Mistral LLM with PDF
// Call Mistral LLM with extracted text
const fullPrompt = `${prompt}\n\nTexte extrait du PDF:\n${pdfText}`;
const response = await invokeLLM({
messages: [
{
role: "user",
content: [
{ type: "text", text: prompt },
{ type: "file_url", file_url: { url: pdfDataUri, mime_type: "application/pdf" } },
],
content: fullPrompt,
},
],
});

78
server/llmFieldsConfig.test.ts Normal file
View File

@@ -0,0 +1,78 @@
import { describe, it, expect, beforeAll } from "vitest";
import {
getLlmFieldsConfigByUser,
upsertLlmFieldConfig,
initializeDefaultLlmFields
} from "./db";
describe("LLM Fields Configuration", () => {
const testUserId = 99999; // Use a high ID to avoid conflicts
beforeAll(async () => {
// Initialize default fields for test user
await initializeDefaultLlmFields(testUserId);
});
it("should initialize default fields for a new user", async () => {
const fields = await getLlmFieldsConfigByUser(testUserId);
expect(fields).toBeDefined();
expect(fields.length).toBeGreaterThan(0);
// Check that default required fields exist
const supplierName = fields.find(f => f.fieldName === "supplierName");
const invoiceNumber = fields.find(f => f.fieldName === "invoiceNumber");
const invoiceDate = fields.find(f => f.fieldName === "invoiceDate");
const totalAmount = fields.find(f => f.fieldName === "totalAmount");
expect(supplierName).toBeDefined();
expect(supplierName?.isRequired).toBe(1);
expect(invoiceNumber).toBeDefined();
expect(invoiceNumber?.isRequired).toBe(1);
expect(invoiceDate).toBeDefined();
expect(invoiceDate?.isRequired).toBe(1);
expect(totalAmount).toBeDefined();
expect(totalAmount?.isRequired).toBe(1);
});
it("should update field configuration", async () => {
// Make deliveryNoteNumber required
await upsertLlmFieldConfig({
userId: testUserId,
fieldName: "deliveryNoteNumber",
displayName: "Numéro de bon de livraison",
isRequired: 1,
displayOrder: 5,
});
const fields = await getLlmFieldsConfigByUser(testUserId);
const deliveryNote = fields.find(f => f.fieldName === "deliveryNoteNumber");
expect(deliveryNote).toBeDefined();
expect(deliveryNote?.isRequired).toBe(1);
// Make it optional again
await upsertLlmFieldConfig({
userId: testUserId,
fieldName: "deliveryNoteNumber",
displayName: "Numéro de bon de livraison",
isRequired: 0,
displayOrder: 5,
});
const fieldsAfter = await getLlmFieldsConfigByUser(testUserId);
const deliveryNoteAfter = fieldsAfter.find(f => f.fieldName === "deliveryNoteNumber");
expect(deliveryNoteAfter).toBeDefined();
expect(deliveryNoteAfter?.isRequired).toBe(0);
});
it("should return fields in correct display order", async () => {
const fields = await getLlmFieldsConfigByUser(testUserId);
// Check that fields are sorted by displayOrder
for (let i = 1; i < fields.length; i++) {
expect(fields[i].displayOrder).toBeGreaterThanOrEqual(fields[i - 1].displayOrder);
}
});
});

65
server/routers.ts
View File

@@ -48,6 +48,9 @@ import {
createAccountingAllocation,
deleteAccountingAllocation,
initializeDefaultLists,
getLlmFieldsConfigByUser,
upsertLlmFieldConfig,
initializeDefaultLlmFields,
getAutomationRulesByUser,
getAutomationRuleById,
createAutomationRule,
@@ -93,7 +96,7 @@ export const appRouter = router({
// Set auth cookie
ctx.res.cookie("auth_token", result.token, {
httpOnly: true,
secure: process.env.NODE_ENV === "production",
secure: false, // Désactivé pour VPS sans HTTPS
sameSite: "lax",
path: "/",
maxAge: 7 * 24 * 60 * 60 * 1000, // 7 days
@@ -304,9 +307,11 @@ export const appRouter = router({
} catch (error: any) {
console.error("[Upload] Extraction failed:", error);
// Limit error message to 200 characters to avoid database field overflow
const errorMsg = error.message ? String(error.message).substring(0, 200) : "Erreur inconnue";
await updateSourceFile(sourceFile.id, {
processingStatus: "error",
processingProgress: `Erreur: ${error.message}`,
processingProgress: `Erreur: ${errorMsg}`,
});
}
})();
@@ -737,6 +742,15 @@ export const appRouter = router({
await initializeDefaultLists(ctx.user.id);
}
// Check if department already exists
const duplicate = existing.find(d => d.name.toLowerCase() === input.name.toLowerCase());
if (duplicate) {
throw new TRPCError({
code: "CONFLICT",
message: `Le service "${input.name}" existe déjà`
});
}
return await createDepartment({
userId: ctx.user.id,
name: input.name,
@@ -770,6 +784,15 @@ export const appRouter = router({
await initializeDefaultLists(ctx.user.id);
}
// Check if allocation already exists
const duplicate = existing.find(a => a.name.toLowerCase() === input.name.toLowerCase());
if (duplicate) {
throw new TRPCError({
code: "CONFLICT",
message: `La ventilation comptable "${input.name}" existe déjà`
});
}
return await createAccountingAllocation({
userId: ctx.user.id,
name: input.name,
@@ -1013,6 +1036,44 @@ export const appRouter = router({
};
}),
}),
// ============= LLM FIELDS CONFIG ROUTES =============
llmFieldsConfig: router({
getAll: protectedProcedure
.query(async ({ ctx }) => {
// Initialize default fields if none exist
const existing = await getLlmFieldsConfigByUser(ctx.user.id);
if (existing.length === 0) {
await initializeDefaultLlmFields(ctx.user.id);
return await getLlmFieldsConfigByUser(ctx.user.id);
}
return existing;
}),
updateField: protectedProcedure
.input(z.object({
fieldName: z.string(),
isRequired: z.number().min(0).max(1),
}))
.mutation(async ({ input, ctx }) => {
// Get existing field config
const existing = await getLlmFieldsConfigByUser(ctx.user.id);
const field = existing.find(f => f.fieldName === input.fieldName);
if (!field) {
throw new TRPCError({ code: "NOT_FOUND", message: "Field not found" });
}
// Update the field
return await upsertLlmFieldConfig({
userId: ctx.user.id,
fieldName: input.fieldName,
displayName: field.displayName,
isRequired: input.isRequired,
displayOrder: field.displayOrder,
});
}),
}),
});
export type AppRouter = typeof appRouter;