fix: recalculer les scores de qualité à l import

This commit is contained in:
Manus
2026-10-02 20:59:23 +00:00
parent 70f867d62b
commit ba4e42c7f1
6 changed files with 102 additions and 6 deletions

View File

@@ -19,6 +19,7 @@ import { handleAzureCallback, isAzureAdConfigured, generateToken, verifyToken }
import { createDatabaseBackup } from "../databaseBackup"; import { createDatabaseBackup } from "../databaseBackup";
import { generateStorageKey, localStorageDelete, localStoragePut } from "../localStorage"; import { generateStorageKey, localStorageDelete, localStoragePut } from "../localStorage";
import { calculateFileSha256 } from "../fileFingerprint"; import { calculateFileSha256 } from "../fileFingerprint";
import { withCalculatedInvoiceQuality } from "@shared/invoiceQuality";
const MAX_WEB_IMPORT_BYTES = 20 * 1024 * 1024; const MAX_WEB_IMPORT_BYTES = 20 * 1024 * 1024;
@@ -306,7 +307,7 @@ async function startServer() {
return; return;
} }
const { getWebImportSourceByToken, createInvoice, findDuplicateInvoice, isInvoiceBlacklisted, updateWebImportSourceStatus, createSourceFile, getSourceFileByContentHash } = await import('../db'); const { getWebImportSourceByToken, createInvoice, findDuplicateInvoice, isInvoiceBlacklisted, updateWebImportSourceStatus, createSourceFile, getSourceFileByContentHash, getLlmFieldsConfigByUser } = await import('../db');
const source = await getWebImportSourceByToken(apiToken); const source = await getWebImportSourceByToken(apiToken);
if (!source) { if (!source) {
res.status(401).json({ error: "Token invalide" }); res.status(401).json({ error: "Token invalide" });
@@ -348,6 +349,7 @@ async function startServer() {
throw error; throw error;
} }
const userSettings = await getUserSettings(source.userId); const userSettings = await getUserSettings(source.userId);
const qualityFieldsConfig = await getLlmFieldsConfigByUser(source.userId);
const aiSettings = { const aiSettings = {
aiProvider: userSettings?.aiProvider || "manus", aiProvider: userSettings?.aiProvider || "manus",
mistralApiKey: userSettings?.mistralApiKey || undefined, mistralApiKey: userSettings?.mistralApiKey || undefined,
@@ -364,7 +366,8 @@ async function startServer() {
if (blacklisted) { duplicates++; continue; } if (blacklisted) { duplicates++; continue; }
const dup = await findDuplicateInvoice(inv.invoiceNumber || null, String(inv.totalAmount ?? ''), source.userId); const dup = await findDuplicateInvoice(inv.invoiceNumber || null, String(inv.totalAmount ?? ''), source.userId);
if (dup) { duplicates++; continue; } if (dup) { duplicates++; continue; }
await createInvoice({ ...inv, userId: source.userId, sourceFileId: sourceFile.id } as any); const invoiceWithQuality = withCalculatedInvoiceQuality(inv, qualityFieldsConfig);
await createInvoice({ ...invoiceWithQuality, userId: source.userId, sourceFileId: sourceFile.id } as any);
imported++; imported++;
} }
await updateWebImportSourceStatus(source.id, 'success', imported, true); await updateWebImportSourceStatus(source.id, 'success', imported, true);

View File

@@ -5,6 +5,7 @@ import {
createSourceFile, createSourceFile,
updateSourceFile, updateSourceFile,
getUserSettings, getUserSettings,
getLlmFieldsConfigByUser,
getSourceFileByContentHash, getSourceFileByContentHash,
findDuplicateInvoice, findDuplicateInvoice,
isInvoiceBlacklisted, isInvoiceBlacklisted,
@@ -17,6 +18,7 @@ import { calculateFileSha256 } from "./fileFingerprint";
import { sendImportNotification } from "./notificationService"; import { sendImportNotification } from "./notificationService";
import { getOffice365ImapToken } from "./office365OAuth"; import { getOffice365ImapToken } from "./office365OAuth";
import { applyAutomationRules } from "./automationEngine"; import { applyAutomationRules } from "./automationEngine";
import { withCalculatedInvoiceQuality } from "@shared/invoiceQuality";
export interface EmailImportConfig { export interface EmailImportConfig {
userId: number; userId: number;
@@ -219,6 +221,7 @@ async function processEmailAttachment(
manusForgeApiUrl: settings.manusForgeApiUrl, manusForgeApiUrl: settings.manusForgeApiUrl,
geminiApiKey: (settings as any).geminiApiKey || null, geminiApiKey: (settings as any).geminiApiKey || null,
} : undefined; } : undefined;
const qualityFieldsConfig = await getLlmFieldsConfigByUser(userId);
// Extract invoices // Extract invoices
console.log(`[EmailImport] Starting invoice extraction (provider: ${aiSettings?.aiProvider || 'mistral'})...`); console.log(`[EmailImport] Starting invoice extraction (provider: ${aiSettings?.aiProvider || 'mistral'})...`);
@@ -248,6 +251,7 @@ async function processEmailAttachment(
for (let i = 0; i < result.invoices.length; i++) { for (let i = 0; i < result.invoices.length; i++) {
const invoiceData = result.invoices[i]!; const invoiceData = result.invoices[i]!;
const invoiceWithQuality = withCalculatedInvoiceQuality(invoiceData, qualityFieldsConfig);
try { try {
// Update progress // Update progress
@@ -310,7 +314,7 @@ async function processEmailAttachment(
totalAmount: invoiceData.totalAmount?.toString(), totalAmount: invoiceData.totalAmount?.toString(),
recipientName: invoiceData.recipientName, recipientName: invoiceData.recipientName,
pageRange: invoiceData.pageRange, pageRange: invoiceData.pageRange,
qualityScore: invoiceData.qualityScore, qualityScore: invoiceWithQuality.qualityScore,
extractedText: invoiceData.extractedText, extractedText: invoiceData.extractedText,
metadataFileKey: metadataKey, metadataFileKey: metadataKey,
metadataFileUrl: metadataUrl, metadataFileUrl: metadataUrl,

View File

@@ -7,6 +7,7 @@ import {
getSourceFileByContentHash, getSourceFileByContentHash,
updateSourceFile, updateSourceFile,
getUserSettings, getUserSettings,
getLlmFieldsConfigByUser,
findDuplicateInvoice, findDuplicateInvoice,
createInvoice, createInvoice,
createImportLog, createImportLog,
@@ -15,6 +16,7 @@ import { extractInvoicesWithMistral, generateMetadataJSON } from "./invoiceExtra
import { localStorageDelete, localStoragePut, generateStorageKey } from "./localStorage"; import { localStorageDelete, localStoragePut, generateStorageKey } from "./localStorage";
import { calculateFileSha256 } from "./fileFingerprint"; import { calculateFileSha256 } from "./fileFingerprint";
import { sendImportNotification } from "./notificationService"; import { sendImportNotification } from "./notificationService";
import { withCalculatedInvoiceQuality } from "@shared/invoiceQuality";
interface FolderImportConfig { interface FolderImportConfig {
userId: number; userId: number;
@@ -99,6 +101,7 @@ async function processFolderFile(
} : undefined; } : undefined;
const model = settings?.llmModel || "mistral-large-latest"; const model = settings?.llmModel || "mistral-large-latest";
const qualityFieldsConfig = await getLlmFieldsConfigByUser(userId);
// Extract invoices // Extract invoices
console.log(`[FolderImport] Starting invoice extraction...`); console.log(`[FolderImport] Starting invoice extraction...`);
@@ -127,6 +130,7 @@ async function processFolderFile(
for (let i = 0; i < result.invoices.length; i++) { for (let i = 0; i < result.invoices.length; i++) {
const invoiceData = result.invoices[i]!; const invoiceData = result.invoices[i]!;
const invoiceWithQuality = withCalculatedInvoiceQuality(invoiceData, qualityFieldsConfig);
try { try {
// Update progress // Update progress
@@ -176,7 +180,7 @@ async function processFolderFile(
totalAmount: invoiceData.totalAmount?.toString(), totalAmount: invoiceData.totalAmount?.toString(),
recipientName: invoiceData.recipientName, recipientName: invoiceData.recipientName,
pageRange: invoiceData.pageRange, pageRange: invoiceData.pageRange,
qualityScore: invoiceData.qualityScore, qualityScore: invoiceWithQuality.qualityScore,
extractedText: invoiceData.extractedText, extractedText: invoiceData.extractedText,
metadataFileKey: metadataKey, metadataFileKey: metadataKey,
metadataFileUrl: metadataUrl, metadataFileUrl: metadataUrl,

View File

@@ -0,0 +1,64 @@
import { readFileSync } from "node:fs";
import { resolve } from "node:path";
import { describe, expect, it } from "vitest";
import { calculateInvoiceQualityScore, withCalculatedInvoiceQuality } from "@shared/invoiceQuality";
const routerSource = readFileSync(resolve(process.cwd(), "server/routers.ts"), "utf8");
const emailImportSource = readFileSync(resolve(process.cwd(), "server/emailImportService.ts"), "utf8");
const folderImportSource = readFileSync(resolve(process.cwd(), "server/folderImportService.ts"), "utf8");
const webImportSource = readFileSync(resolve(process.cwd(), "server/_core/index.ts"), "utf8");
describe("score de qualité à l'import", () => {
it("calcule 100 % pour la facture lorsque tous les champs obligatoires configurés sont présents", () => {
const qualityScore = calculateInvoiceQualityScore({
supplierName: "Manus Butterfly",
invoiceNumber: "J6LNGDAL0021",
invoiceDate: new Date("2026-08-28"),
totalAmount: "120.00",
recipientName: "PAREIGE OLIVIER",
}, [
{ fieldName: "supplierName", isRequired: 1 },
{ fieldName: "invoiceNumber", isRequired: 1 },
{ fieldName: "invoiceDate", isRequired: 1 },
{ fieldName: "totalAmount", isRequired: 1 },
{ fieldName: "recipientName", isRequired: 1 },
{ fieldName: "orderNumber", isRequired: 0 },
]);
expect(qualityScore).toBe(100);
});
it("utilise le calcul serveur déterministe au lieu du score proposé par le modèle", () => {
expect(routerSource).toContain("withCalculatedInvoiceQuality(invoiceData, qualityFieldsConfig)");
expect(routerSource).toContain("qualityScore: invoiceWithQuality.qualityScore");
});
it("exécute le même recalcul pour chaque flux d’import", () => {
const requiredFields = [
{ fieldName: "supplierName", isRequired: 1 },
{ fieldName: "invoiceNumber", isRequired: 1 },
{ fieldName: "invoiceDate", isRequired: 1 },
{ fieldName: "totalAmount", isRequired: 1 },
{ fieldName: "recipientName", isRequired: 1 },
];
for (const rawModelScore of [0, 42, 90, 99]) {
const persistedInvoice = withCalculatedInvoiceQuality({
supplierName: "Manus Butterfly",
invoiceNumber: "J6LNGDAL0021",
invoiceDate: new Date("2026-08-28"),
totalAmount: "120.00",
recipientName: "PAREIGE OLIVIER",
qualityScore: rawModelScore,
}, requiredFields);
expect(persistedInvoice.qualityScore).toBe(100);
}
});
it("branche chaque flux sur le calcul partagé", () => {
for (const source of [routerSource, emailImportSource, folderImportSource, webImportSource]) {
expect(source).toContain("withCalculatedInvoiceQuality(");
expect(source).toContain("qualityFieldsConfig");
}
});
});

View File

@@ -1,7 +1,7 @@
import { z } from "zod"; import { z } from "zod";
import { COOKIE_NAME } from "@shared/const"; import { COOKIE_NAME } from "@shared/const";
import { BAP_MIN_QUALITY_SCORE, meetsBapQualityThreshold } from "@shared/bapEligibility"; import { BAP_MIN_QUALITY_SCORE, meetsBapQualityThreshold } from "@shared/bapEligibility";
import { calculateInvoiceQualityScore } from "@shared/invoiceQuality"; import { calculateInvoiceQualityScore, withCalculatedInvoiceQuality } from "@shared/invoiceQuality";
import { webPortalUrlSchema } from "./webPortalUrl"; import { webPortalUrlSchema } from "./webPortalUrl";
interface Condition { interface Condition {
@@ -287,6 +287,7 @@ export const appRouter = router({
manusForgeApiUrl: (settings as any).manusForgeApiUrl || null, manusForgeApiUrl: (settings as any).manusForgeApiUrl || null,
geminiApiKey: (settings as any).geminiApiKey || null, geminiApiKey: (settings as any).geminiApiKey || null,
} : undefined; } : undefined;
const qualityFieldsConfig = await getLlmFieldsConfigByUser(userId);
// Extract invoices // Extract invoices
const result = await extractInvoicesWithMistral( const result = await extractInvoicesWithMistral(
@@ -313,6 +314,9 @@ export const appRouter = router({
for (let i = 0; i < result.invoices.length; i++) { for (let i = 0; i < result.invoices.length; i++) {
const invoiceData = result.invoices[i]!; const invoiceData = result.invoices[i]!;
// Le score affiché est déterministe : les champs obligatoires
// configurés prévalent sur l'estimation fournie par le modèle.
const invoiceWithQuality = withCalculatedInvoiceQuality(invoiceData, qualityFieldsConfig);
try { try {
// Update progress // Update progress
@@ -375,7 +379,7 @@ export const appRouter = router({
totalAmount: invoiceData.totalAmount?.toString(), totalAmount: invoiceData.totalAmount?.toString(),
recipientName: invoiceData.recipientName, recipientName: invoiceData.recipientName,
pageRange: invoiceData.pageRange, pageRange: invoiceData.pageRange,
qualityScore: invoiceData.qualityScore, qualityScore: invoiceWithQuality.qualityScore,
extractedText: invoiceData.extractedText, extractedText: invoiceData.extractedText,
isSubscription: invoiceData.isSubscription ? 1 : 0, isSubscription: invoiceData.isSubscription ? 1 : 0,
metadataFileKey: metadataKey, metadataFileKey: metadataKey,

View File

@@ -40,3 +40,20 @@ export function calculateInvoiceQualityScore(
const completedFields = requiredFields.filter((fieldName) => hasValue(invoice[fieldName])).length; const completedFields = requiredFields.filter((fieldName) => hasValue(invoice[fieldName])).length;
return Math.round((completedFields / requiredFields.length) * 100); return Math.round((completedFields / requiredFields.length) * 100);
} }
/**
* Prépare une extraction IA pour la persistance. Le score brut du modèle est
* volontairement remplacé par le calcul déterministe commun à tous les flux.
*/
export function withCalculatedInvoiceQuality<T extends object>(
invoice: T,
fieldsConfiguration: readonly QualityFieldConfig[],
): T & { qualityScore: number } {
return {
...invoice,
qualityScore: calculateInvoiceQualityScore(
invoice as Record<string, unknown>,
fieldsConfiguration,
),
};
}