fix: recalculer les scores de qualité à l import
This commit is contained in:
@@ -19,6 +19,7 @@ import { handleAzureCallback, isAzureAdConfigured, generateToken, verifyToken }
|
|||||||
import { createDatabaseBackup } from "../databaseBackup";
|
import { createDatabaseBackup } from "../databaseBackup";
|
||||||
import { generateStorageKey, localStorageDelete, localStoragePut } from "../localStorage";
|
import { generateStorageKey, localStorageDelete, localStoragePut } from "../localStorage";
|
||||||
import { calculateFileSha256 } from "../fileFingerprint";
|
import { calculateFileSha256 } from "../fileFingerprint";
|
||||||
|
import { withCalculatedInvoiceQuality } from "@shared/invoiceQuality";
|
||||||
|
|
||||||
const MAX_WEB_IMPORT_BYTES = 20 * 1024 * 1024;
|
const MAX_WEB_IMPORT_BYTES = 20 * 1024 * 1024;
|
||||||
|
|
||||||
@@ -306,7 +307,7 @@ async function startServer() {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const { getWebImportSourceByToken, createInvoice, findDuplicateInvoice, isInvoiceBlacklisted, updateWebImportSourceStatus, createSourceFile, getSourceFileByContentHash } = await import('../db');
|
const { getWebImportSourceByToken, createInvoice, findDuplicateInvoice, isInvoiceBlacklisted, updateWebImportSourceStatus, createSourceFile, getSourceFileByContentHash, getLlmFieldsConfigByUser } = await import('../db');
|
||||||
const source = await getWebImportSourceByToken(apiToken);
|
const source = await getWebImportSourceByToken(apiToken);
|
||||||
if (!source) {
|
if (!source) {
|
||||||
res.status(401).json({ error: "Token invalide" });
|
res.status(401).json({ error: "Token invalide" });
|
||||||
@@ -348,6 +349,7 @@ async function startServer() {
|
|||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
const userSettings = await getUserSettings(source.userId);
|
const userSettings = await getUserSettings(source.userId);
|
||||||
|
const qualityFieldsConfig = await getLlmFieldsConfigByUser(source.userId);
|
||||||
const aiSettings = {
|
const aiSettings = {
|
||||||
aiProvider: userSettings?.aiProvider || "manus",
|
aiProvider: userSettings?.aiProvider || "manus",
|
||||||
mistralApiKey: userSettings?.mistralApiKey || undefined,
|
mistralApiKey: userSettings?.mistralApiKey || undefined,
|
||||||
@@ -364,7 +366,8 @@ async function startServer() {
|
|||||||
if (blacklisted) { duplicates++; continue; }
|
if (blacklisted) { duplicates++; continue; }
|
||||||
const dup = await findDuplicateInvoice(inv.invoiceNumber || null, String(inv.totalAmount ?? ''), source.userId);
|
const dup = await findDuplicateInvoice(inv.invoiceNumber || null, String(inv.totalAmount ?? ''), source.userId);
|
||||||
if (dup) { duplicates++; continue; }
|
if (dup) { duplicates++; continue; }
|
||||||
await createInvoice({ ...inv, userId: source.userId, sourceFileId: sourceFile.id } as any);
|
const invoiceWithQuality = withCalculatedInvoiceQuality(inv, qualityFieldsConfig);
|
||||||
|
await createInvoice({ ...invoiceWithQuality, userId: source.userId, sourceFileId: sourceFile.id } as any);
|
||||||
imported++;
|
imported++;
|
||||||
}
|
}
|
||||||
await updateWebImportSourceStatus(source.id, 'success', imported, true);
|
await updateWebImportSourceStatus(source.id, 'success', imported, true);
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import {
|
|||||||
createSourceFile,
|
createSourceFile,
|
||||||
updateSourceFile,
|
updateSourceFile,
|
||||||
getUserSettings,
|
getUserSettings,
|
||||||
|
getLlmFieldsConfigByUser,
|
||||||
getSourceFileByContentHash,
|
getSourceFileByContentHash,
|
||||||
findDuplicateInvoice,
|
findDuplicateInvoice,
|
||||||
isInvoiceBlacklisted,
|
isInvoiceBlacklisted,
|
||||||
@@ -17,6 +18,7 @@ import { calculateFileSha256 } from "./fileFingerprint";
|
|||||||
import { sendImportNotification } from "./notificationService";
|
import { sendImportNotification } from "./notificationService";
|
||||||
import { getOffice365ImapToken } from "./office365OAuth";
|
import { getOffice365ImapToken } from "./office365OAuth";
|
||||||
import { applyAutomationRules } from "./automationEngine";
|
import { applyAutomationRules } from "./automationEngine";
|
||||||
|
import { withCalculatedInvoiceQuality } from "@shared/invoiceQuality";
|
||||||
|
|
||||||
export interface EmailImportConfig {
|
export interface EmailImportConfig {
|
||||||
userId: number;
|
userId: number;
|
||||||
@@ -219,6 +221,7 @@ async function processEmailAttachment(
|
|||||||
manusForgeApiUrl: settings.manusForgeApiUrl,
|
manusForgeApiUrl: settings.manusForgeApiUrl,
|
||||||
geminiApiKey: (settings as any).geminiApiKey || null,
|
geminiApiKey: (settings as any).geminiApiKey || null,
|
||||||
} : undefined;
|
} : undefined;
|
||||||
|
const qualityFieldsConfig = await getLlmFieldsConfigByUser(userId);
|
||||||
|
|
||||||
// Extract invoices
|
// Extract invoices
|
||||||
console.log(`[EmailImport] Starting invoice extraction (provider: ${aiSettings?.aiProvider || 'mistral'})...`);
|
console.log(`[EmailImport] Starting invoice extraction (provider: ${aiSettings?.aiProvider || 'mistral'})...`);
|
||||||
@@ -248,6 +251,7 @@ async function processEmailAttachment(
|
|||||||
|
|
||||||
for (let i = 0; i < result.invoices.length; i++) {
|
for (let i = 0; i < result.invoices.length; i++) {
|
||||||
const invoiceData = result.invoices[i]!;
|
const invoiceData = result.invoices[i]!;
|
||||||
|
const invoiceWithQuality = withCalculatedInvoiceQuality(invoiceData, qualityFieldsConfig);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Update progress
|
// Update progress
|
||||||
@@ -310,7 +314,7 @@ async function processEmailAttachment(
|
|||||||
totalAmount: invoiceData.totalAmount?.toString(),
|
totalAmount: invoiceData.totalAmount?.toString(),
|
||||||
recipientName: invoiceData.recipientName,
|
recipientName: invoiceData.recipientName,
|
||||||
pageRange: invoiceData.pageRange,
|
pageRange: invoiceData.pageRange,
|
||||||
qualityScore: invoiceData.qualityScore,
|
qualityScore: invoiceWithQuality.qualityScore,
|
||||||
extractedText: invoiceData.extractedText,
|
extractedText: invoiceData.extractedText,
|
||||||
metadataFileKey: metadataKey,
|
metadataFileKey: metadataKey,
|
||||||
metadataFileUrl: metadataUrl,
|
metadataFileUrl: metadataUrl,
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import {
|
|||||||
getSourceFileByContentHash,
|
getSourceFileByContentHash,
|
||||||
updateSourceFile,
|
updateSourceFile,
|
||||||
getUserSettings,
|
getUserSettings,
|
||||||
|
getLlmFieldsConfigByUser,
|
||||||
findDuplicateInvoice,
|
findDuplicateInvoice,
|
||||||
createInvoice,
|
createInvoice,
|
||||||
createImportLog,
|
createImportLog,
|
||||||
@@ -15,6 +16,7 @@ import { extractInvoicesWithMistral, generateMetadataJSON } from "./invoiceExtra
|
|||||||
import { localStorageDelete, localStoragePut, generateStorageKey } from "./localStorage";
|
import { localStorageDelete, localStoragePut, generateStorageKey } from "./localStorage";
|
||||||
import { calculateFileSha256 } from "./fileFingerprint";
|
import { calculateFileSha256 } from "./fileFingerprint";
|
||||||
import { sendImportNotification } from "./notificationService";
|
import { sendImportNotification } from "./notificationService";
|
||||||
|
import { withCalculatedInvoiceQuality } from "@shared/invoiceQuality";
|
||||||
|
|
||||||
interface FolderImportConfig {
|
interface FolderImportConfig {
|
||||||
userId: number;
|
userId: number;
|
||||||
@@ -99,6 +101,7 @@ async function processFolderFile(
|
|||||||
} : undefined;
|
} : undefined;
|
||||||
|
|
||||||
const model = settings?.llmModel || "mistral-large-latest";
|
const model = settings?.llmModel || "mistral-large-latest";
|
||||||
|
const qualityFieldsConfig = await getLlmFieldsConfigByUser(userId);
|
||||||
|
|
||||||
// Extract invoices
|
// Extract invoices
|
||||||
console.log(`[FolderImport] Starting invoice extraction...`);
|
console.log(`[FolderImport] Starting invoice extraction...`);
|
||||||
@@ -127,6 +130,7 @@ async function processFolderFile(
|
|||||||
|
|
||||||
for (let i = 0; i < result.invoices.length; i++) {
|
for (let i = 0; i < result.invoices.length; i++) {
|
||||||
const invoiceData = result.invoices[i]!;
|
const invoiceData = result.invoices[i]!;
|
||||||
|
const invoiceWithQuality = withCalculatedInvoiceQuality(invoiceData, qualityFieldsConfig);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Update progress
|
// Update progress
|
||||||
@@ -176,7 +180,7 @@ async function processFolderFile(
|
|||||||
totalAmount: invoiceData.totalAmount?.toString(),
|
totalAmount: invoiceData.totalAmount?.toString(),
|
||||||
recipientName: invoiceData.recipientName,
|
recipientName: invoiceData.recipientName,
|
||||||
pageRange: invoiceData.pageRange,
|
pageRange: invoiceData.pageRange,
|
||||||
qualityScore: invoiceData.qualityScore,
|
qualityScore: invoiceWithQuality.qualityScore,
|
||||||
extractedText: invoiceData.extractedText,
|
extractedText: invoiceData.extractedText,
|
||||||
metadataFileKey: metadataKey,
|
metadataFileKey: metadataKey,
|
||||||
metadataFileUrl: metadataUrl,
|
metadataFileUrl: metadataUrl,
|
||||||
|
|||||||
64
server/invoiceImportQuality.test.ts
Normal file
64
server/invoiceImportQuality.test.ts
Normal file
@@ -0,0 +1,64 @@
|
|||||||
|
import { readFileSync } from "node:fs";
|
||||||
|
import { resolve } from "node:path";
|
||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { calculateInvoiceQualityScore, withCalculatedInvoiceQuality } from "@shared/invoiceQuality";
|
||||||
|
|
||||||
|
const routerSource = readFileSync(resolve(process.cwd(), "server/routers.ts"), "utf8");
|
||||||
|
const emailImportSource = readFileSync(resolve(process.cwd(), "server/emailImportService.ts"), "utf8");
|
||||||
|
const folderImportSource = readFileSync(resolve(process.cwd(), "server/folderImportService.ts"), "utf8");
|
||||||
|
const webImportSource = readFileSync(resolve(process.cwd(), "server/_core/index.ts"), "utf8");
|
||||||
|
|
||||||
|
describe("score de qualité à l'import", () => {
|
||||||
|
it("calcule 100 % pour la facture lorsque tous les champs obligatoires configurés sont présents", () => {
|
||||||
|
const qualityScore = calculateInvoiceQualityScore({
|
||||||
|
supplierName: "Manus Butterfly",
|
||||||
|
invoiceNumber: "J6LNGDAL0021",
|
||||||
|
invoiceDate: new Date("2026-08-28"),
|
||||||
|
totalAmount: "120.00",
|
||||||
|
recipientName: "PAREIGE OLIVIER",
|
||||||
|
}, [
|
||||||
|
{ fieldName: "supplierName", isRequired: 1 },
|
||||||
|
{ fieldName: "invoiceNumber", isRequired: 1 },
|
||||||
|
{ fieldName: "invoiceDate", isRequired: 1 },
|
||||||
|
{ fieldName: "totalAmount", isRequired: 1 },
|
||||||
|
{ fieldName: "recipientName", isRequired: 1 },
|
||||||
|
{ fieldName: "orderNumber", isRequired: 0 },
|
||||||
|
]);
|
||||||
|
|
||||||
|
expect(qualityScore).toBe(100);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("utilise le calcul serveur déterministe au lieu du score proposé par le modèle", () => {
|
||||||
|
expect(routerSource).toContain("withCalculatedInvoiceQuality(invoiceData, qualityFieldsConfig)");
|
||||||
|
expect(routerSource).toContain("qualityScore: invoiceWithQuality.qualityScore");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("exécute le même recalcul pour chaque flux d’import", () => {
|
||||||
|
const requiredFields = [
|
||||||
|
{ fieldName: "supplierName", isRequired: 1 },
|
||||||
|
{ fieldName: "invoiceNumber", isRequired: 1 },
|
||||||
|
{ fieldName: "invoiceDate", isRequired: 1 },
|
||||||
|
{ fieldName: "totalAmount", isRequired: 1 },
|
||||||
|
{ fieldName: "recipientName", isRequired: 1 },
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const rawModelScore of [0, 42, 90, 99]) {
|
||||||
|
const persistedInvoice = withCalculatedInvoiceQuality({
|
||||||
|
supplierName: "Manus Butterfly",
|
||||||
|
invoiceNumber: "J6LNGDAL0021",
|
||||||
|
invoiceDate: new Date("2026-08-28"),
|
||||||
|
totalAmount: "120.00",
|
||||||
|
recipientName: "PAREIGE OLIVIER",
|
||||||
|
qualityScore: rawModelScore,
|
||||||
|
}, requiredFields);
|
||||||
|
expect(persistedInvoice.qualityScore).toBe(100);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("branche chaque flux sur le calcul partagé", () => {
|
||||||
|
for (const source of [routerSource, emailImportSource, folderImportSource, webImportSource]) {
|
||||||
|
expect(source).toContain("withCalculatedInvoiceQuality(");
|
||||||
|
expect(source).toContain("qualityFieldsConfig");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
import { COOKIE_NAME } from "@shared/const";
|
import { COOKIE_NAME } from "@shared/const";
|
||||||
import { BAP_MIN_QUALITY_SCORE, meetsBapQualityThreshold } from "@shared/bapEligibility";
|
import { BAP_MIN_QUALITY_SCORE, meetsBapQualityThreshold } from "@shared/bapEligibility";
|
||||||
import { calculateInvoiceQualityScore } from "@shared/invoiceQuality";
|
import { calculateInvoiceQualityScore, withCalculatedInvoiceQuality } from "@shared/invoiceQuality";
|
||||||
import { webPortalUrlSchema } from "./webPortalUrl";
|
import { webPortalUrlSchema } from "./webPortalUrl";
|
||||||
|
|
||||||
interface Condition {
|
interface Condition {
|
||||||
@@ -287,6 +287,7 @@ export const appRouter = router({
|
|||||||
manusForgeApiUrl: (settings as any).manusForgeApiUrl || null,
|
manusForgeApiUrl: (settings as any).manusForgeApiUrl || null,
|
||||||
geminiApiKey: (settings as any).geminiApiKey || null,
|
geminiApiKey: (settings as any).geminiApiKey || null,
|
||||||
} : undefined;
|
} : undefined;
|
||||||
|
const qualityFieldsConfig = await getLlmFieldsConfigByUser(userId);
|
||||||
|
|
||||||
// Extract invoices
|
// Extract invoices
|
||||||
const result = await extractInvoicesWithMistral(
|
const result = await extractInvoicesWithMistral(
|
||||||
@@ -313,6 +314,9 @@ export const appRouter = router({
|
|||||||
|
|
||||||
for (let i = 0; i < result.invoices.length; i++) {
|
for (let i = 0; i < result.invoices.length; i++) {
|
||||||
const invoiceData = result.invoices[i]!;
|
const invoiceData = result.invoices[i]!;
|
||||||
|
// Le score affiché est déterministe : les champs obligatoires
|
||||||
|
// configurés prévalent sur l'estimation fournie par le modèle.
|
||||||
|
const invoiceWithQuality = withCalculatedInvoiceQuality(invoiceData, qualityFieldsConfig);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Update progress
|
// Update progress
|
||||||
@@ -375,7 +379,7 @@ export const appRouter = router({
|
|||||||
totalAmount: invoiceData.totalAmount?.toString(),
|
totalAmount: invoiceData.totalAmount?.toString(),
|
||||||
recipientName: invoiceData.recipientName,
|
recipientName: invoiceData.recipientName,
|
||||||
pageRange: invoiceData.pageRange,
|
pageRange: invoiceData.pageRange,
|
||||||
qualityScore: invoiceData.qualityScore,
|
qualityScore: invoiceWithQuality.qualityScore,
|
||||||
extractedText: invoiceData.extractedText,
|
extractedText: invoiceData.extractedText,
|
||||||
isSubscription: invoiceData.isSubscription ? 1 : 0,
|
isSubscription: invoiceData.isSubscription ? 1 : 0,
|
||||||
metadataFileKey: metadataKey,
|
metadataFileKey: metadataKey,
|
||||||
|
|||||||
@@ -40,3 +40,20 @@ export function calculateInvoiceQualityScore(
|
|||||||
const completedFields = requiredFields.filter((fieldName) => hasValue(invoice[fieldName])).length;
|
const completedFields = requiredFields.filter((fieldName) => hasValue(invoice[fieldName])).length;
|
||||||
return Math.round((completedFields / requiredFields.length) * 100);
|
return Math.round((completedFields / requiredFields.length) * 100);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Prépare une extraction IA pour la persistance. Le score brut du modèle est
|
||||||
|
* volontairement remplacé par le calcul déterministe commun à tous les flux.
|
||||||
|
*/
|
||||||
|
export function withCalculatedInvoiceQuality<T extends object>(
|
||||||
|
invoice: T,
|
||||||
|
fieldsConfiguration: readonly QualityFieldConfig[],
|
||||||
|
): T & { qualityScore: number } {
|
||||||
|
return {
|
||||||
|
...invoice,
|
||||||
|
qualityScore: calculateInvoiceQualityScore(
|
||||||
|
invoice as Record<string, unknown>,
|
||||||
|
fieldsConfiguration,
|
||||||
|
),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user