diff --git a/client/src/pages/Settings.tsx b/client/src/pages/Settings.tsx index 0017ae4..ced9f5e 100644 --- a/client/src/pages/Settings.tsx +++ b/client/src/pages/Settings.tsx @@ -9,6 +9,70 @@ import { Switch } from "@/components/ui/switch"; import { trpc } from "@/lib/trpc"; import { Loader2, Save, CheckCircle } from "lucide-react"; import { toast } from "sonner"; +import { Checkbox } from "@/components/ui/checkbox"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; + +function LlmFieldsConfigSection() { + const { data: fields, isLoading } = trpc.llmFieldsConfig.getAll.useQuery(); + const utils = trpc.useUtils(); + + const updateFieldMutation = trpc.llmFieldsConfig.updateField.useMutation({ + onSuccess: () => { + toast.success("Configuration mise à jour"); + utils.llmFieldsConfig.getAll.invalidate(); + }, + onError: (error) => { + toast.error(error.message || "Erreur lors de la mise à jour"); + }, + }); + + const handleToggle = (fieldName: string, currentValue: number) => { + updateFieldMutation.mutate({ + fieldName, + isRequired: currentValue === 1 ? 0 : 1, + }); + }; + + if (isLoading) { + return ( +
+ +
+ ); + } + + return ( +
+

+ Les champs marqués comme obligatoires doivent être détectés pour atteindre un score de 100%. + Les champs optionnels n'affectent pas le score. +

+ + + + + Champ + Obligatoire + + + + {fields?.map((field) => ( + + {field.displayName} + + handleToggle(field.fieldName, field.isRequired)} + disabled={updateFieldMutation.isPending} + /> + + + ))} + +
+
+ ); +} export default function Settings() { const { data: settings, isLoading } = trpc.settings.get.useQuery(); @@ -145,6 +209,19 @@ export default function Settings() { + {/* LLM Fields Configuration */} + + + Champs de détection + + Configurez quels champs sont obligatoires pour atteindre un score de reconnaissance de 100% + + + + + + + {/* Keywords Configuration */} diff --git a/drizzle/0011_woozy_sabretooth.sql b/drizzle/0011_woozy_sabretooth.sql new file mode 100644 index 0000000..1946ff7 --- /dev/null +++ b/drizzle/0011_woozy_sabretooth.sql @@ -0,0 +1,11 @@ +CREATE TABLE `llmFieldsConfig` ( + `id` int AUTO_INCREMENT NOT NULL, + `userId` int NOT NULL, + `fieldName` varchar(100) NOT NULL, + `displayName` varchar(255) NOT NULL, + `isRequired` int NOT NULL DEFAULT 1, + `displayOrder` int NOT NULL DEFAULT 0, + `createdAt` timestamp NOT NULL DEFAULT (now()), + `updatedAt` timestamp NOT NULL DEFAULT (now()) ON UPDATE CURRENT_TIMESTAMP, + CONSTRAINT `llmFieldsConfig_id` PRIMARY KEY(`id`) +); diff --git a/drizzle/meta/0011_snapshot.json b/drizzle/meta/0011_snapshot.json new file mode 100644 index 0000000..8febca0 --- /dev/null +++ b/drizzle/meta/0011_snapshot.json @@ -0,0 +1,1288 @@ +{ + "version": "5", + "dialect": "mysql", + "id": "f7e17fc5-7768-4e65-85a4-0c5d15c39ef6", + "prevId": "7c77c0f0-2c86-4da4-9148-0da6e99c1324", + "tables": { + "accountingAllocationList": { + "name": "accountingAllocationList", + "columns": { + "id": { + "name": "id", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": true + }, + "userId": { + "name": "userId", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "user_allocation_unique": { + "name": "user_allocation_unique", + "columns": [ + "userId", + "name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "accountingAllocationList_id": { + "name": "accountingAllocationList_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "automationRules": { + "name": "automationRules", + "columns": { + "id": { + "name": "id", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": true + }, + "userId": { + "name": "userId", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "isActive": { + "name": "isActive", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "priority": { + "name": "priority", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "conditions": { + "name": "conditions", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "conditionsLogic": { + "name": "conditionsLogic", + "type": "enum('AND','OR')", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'AND'" + }, + "actions": { + "name": "actions", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "automationRules_id": { + "name": "automationRules_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "departmentList": { + "name": "departmentList", + "columns": { + "id": { + "name": "id", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": true + }, + "userId": { + "name": "userId", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "user_department_unique": { + "name": "user_department_unique", + "columns": [ + "userId", + "name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "departmentList_id": { + "name": "departmentList_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "importLogs": { + "name": "importLogs", + "columns": { + "id": { + "name": "id", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": true + }, + "userId": { + "name": "userId", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sourceFileId": { + "name": "sourceFileId", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fileName": { + "name": "fileName", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "totalInvoicesDetected": { + "name": "totalInvoicesDetected", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "invoicesImported": { + "name": "invoicesImported", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "duplicatesIgnored": { + "name": "duplicatesIgnored", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "errors": { + "name": "errors", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "duplicateDetails": { + "name": "duplicateDetails", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "errorDetails": { + "name": "errorDetails", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "importedAt": { + "name": "importedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "importLogs_id": { + "name": "importLogs_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "importSettings": { + "name": "importSettings", + "columns": { + "id": { + "name": "id", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": true + }, + "userId": { + "name": "userId", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "manualImportEnabled": { + "name": "manualImportEnabled", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "autoImportEnabled": { + "name": "autoImportEnabled", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "autoImportSourcePath": { + "name": "autoImportSourcePath", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "autoImportFrequency": { + "name": "autoImportFrequency", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 60 + }, + "emailImportEnabled": { + "name": "emailImportEnabled", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "emailImportAddress": { + "name": "emailImportAddress", + "type": "varchar(320)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "emailImportPassword": { + "name": "emailImportPassword", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "emailImportHost": { + "name": "emailImportHost", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "emailImportPort": { + "name": "emailImportPort", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 993 + }, + "emailImportFrequency": { + "name": "emailImportFrequency", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 30 + }, + "exportFolder": { + "name": "exportFolder", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "importSettings_id": { + "name": "importSettings_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": { + "importSettings_userId_unique": { + "name": "importSettings_userId_unique", + "columns": [ + "userId" + ] + } + }, + "checkConstraint": {} + }, + "invoices": { + "name": "invoices", + "columns": { + "id": { + "name": "id", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": true + }, + "userId": { + "name": "userId", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sourceFileId": { + "name": "sourceFileId", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "invoiceIndexInFile": { + "name": "invoiceIndexInFile", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "fileName": { + "name": "fileName", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fileKey": { + "name": "fileKey", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fileUrl": { + "name": "fileUrl", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "supplierName": { + "name": "supplierName", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "invoiceNumber": { + "name": "invoiceNumber", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "invoiceDate": { + "name": "invoiceDate", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deliveryNoteNumber": { + "name": "deliveryNoteNumber", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "orderNumber": { + "name": "orderNumber", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "totalAmount": { + "name": "totalAmount", + "type": "decimal(10,2)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pageRange": { + "name": "pageRange", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "qualityScore": { + "name": "qualityScore", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadataFileKey": { + "name": "metadataFileKey", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadataFileUrl": { + "name": "metadataFileUrl", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "enum('processing','completed','error')", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'processing'" + }, + "errorMessage": { + "name": "errorMessage", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "exportStatus": { + "name": "exportStatus", + "type": "enum('not_exported','exported','export_error')", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'not_exported'" + }, + "manuallyEdited": { + "name": "manuallyEdited", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "serviceConcerne": { + "name": "serviceConcerne", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "typeAchat": { + "name": "typeAchat", + "type": "enum('CAPEX','OPEX')", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ventilationComptable": { + "name": "ventilationComptable", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "autoFilledFields": { + "name": "autoFilledFields", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "extractedText": { + "name": "extractedText", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "isSubscription": { + "name": "isSubscription", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "exportedAt": { + "name": "exportedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "exportMode": { + "name": "exportMode", + "type": "enum('manual','automatic')", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "supplier_invoice_date_unique": { + "name": "supplier_invoice_date_unique", + "columns": [ + "supplierName", + "invoiceNumber", + "invoiceDate" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "invoices_id": { + "name": "invoices_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "llmFieldsConfig": { + "name": "llmFieldsConfig", + "columns": { + "id": { + "name": "id", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": true + }, + "userId": { + "name": "userId", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fieldName": { + "name": "fieldName", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "displayName": { + "name": "displayName", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "isRequired": { + "name": "isRequired", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "displayOrder": { + "name": "displayOrder", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "llmFieldsConfig_id": { + "name": "llmFieldsConfig_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "llmLogs": { + "name": "llmLogs", + "columns": { + "id": { + "name": "id", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": true + }, + "userId": { + "name": "userId", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sourceFileId": { + "name": "sourceFileId", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "invoiceId": { + "name": "invoiceId", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "operation": { + "name": "operation", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "promptSent": { + "name": "promptSent", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rawResponse": { + "name": "rawResponse", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cleanedResponse": { + "name": "cleanedResponse", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "success": { + "name": "success", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "errorMessage": { + "name": "errorMessage", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "processingTimeMs": { + "name": "processingTimeMs", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pageRange": { + "name": "pageRange", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "llmLogs_id": { + "name": "llmLogs_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "sourceFiles": { + "name": "sourceFiles", + "columns": { + "id": { + "name": "id", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": true + }, + "userId": { + "name": "userId", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fileName": { + "name": "fileName", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fileKey": { + "name": "fileKey", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fileUrl": { + "name": "fileUrl", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "totalInvoicesDetected": { + "name": "totalInvoicesDetected", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "processingStatus": { + "name": "processingStatus", + "type": "enum('processing','completed','error')", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'processing'" + }, + "processingProgress": { + "name": "processingProgress", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "sourceFiles_id": { + "name": "sourceFiles_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "userSettings": { + "name": "userSettings", + "columns": { + "id": { + "name": "id", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": true + }, + "userId": { + "name": "userId", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "llmModel": { + "name": "llmModel", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'mistral-large-latest'" + }, + "orderNumberFormat": { + "name": "orderNumberFormat", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "invoiceNumberKeywords": { + "name": "invoiceNumberKeywords", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deliveryNoteKeywords": { + "name": "deliveryNoteKeywords", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "orderNumberKeywords": { + "name": "orderNumberKeywords", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "supplierKeywords": { + "name": "supplierKeywords", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "totalAmountKeywords": { + "name": "totalAmountKeywords", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "subscriptionKeywords": { + "name": "subscriptionKeywords", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sftpHost": { + "name": "sftpHost", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sftpPort": { + "name": "sftpPort", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 22 + }, + "sftpUsername": { + "name": "sftpUsername", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sftpPassword": { + "name": "sftpPassword", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sftpRemotePath": { + "name": "sftpRemotePath", + "type": "varchar(500)", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'/'" + }, + "sftpAutoExport": { + "name": "sftpAutoExport", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "llmLogsRetentionMonths": { + "name": "llmLogsRetentionMonths", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 3 + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "userSettings_id": { + "name": "userSettings_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": { + "userSettings_userId_unique": { + "name": "userSettings_userId_unique", + "columns": [ + "userId" + ] + } + }, + "checkConstraint": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": true + }, + "openId": { + "name": "openId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "azureAdId": { + "name": "azureAdId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "varchar(320)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "passwordHash": { + "name": "passwordHash", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "loginMethod": { + "name": "loginMethod", + "type": "enum('manus','local','azure-ad')", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "enum('user','admin')", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'user'" + }, + "isActive": { + "name": "isActive", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "lastSignedIn": { + "name": "lastSignedIn", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "users_id": { + "name": "users_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": { + "users_openId_unique": { + "name": "users_openId_unique", + "columns": [ + "openId" + ] + }, + "users_azureAdId_unique": { + "name": "users_azureAdId_unique", + "columns": [ + "azureAdId" + ] + }, + "users_email_unique": { + "name": "users_email_unique", + "columns": [ + "email" + ] + } + }, + "checkConstraint": {} + } + }, + "views": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "tables": {}, + "indexes": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index ecd9cf8..933b015 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -78,6 +78,13 @@ "when": 1770831408967, "tag": "0010_late_thor_girl", "breakpoints": true + }, + { + "idx": 11, + "version": "5", + "when": 1770971673626, + "tag": "0011_woozy_sabretooth", + "breakpoints": true } ] } \ No newline at end of file diff --git a/drizzle/schema.ts b/drizzle/schema.ts index 3adc077..d620437 100644 --- a/drizzle/schema.ts +++ b/drizzle/schema.ts @@ -283,3 +283,22 @@ export const automationRules = mysqlTable("automationRules", { export type AutomationRule = typeof automationRules.$inferSelect; export type InsertAutomationRule = typeof automationRules.$inferInsert; + +/** + * LLM Fields Configuration table + * Stores configuration for each field used in invoice extraction + * Allows users to define which fields are required for quality score calculation + */ +export const llmFieldsConfig = mysqlTable("llmFieldsConfig", { + id: int("id").autoincrement().primaryKey(), + userId: int("userId").notNull(), // Each user has their own configuration + fieldName: varchar("fieldName", { length: 100 }).notNull(), // Field identifier (supplierName, invoiceNumber, etc.) + displayName: varchar("displayName", { length: 255 }).notNull(), // Human-readable field name + isRequired: int("isRequired").default(1).notNull(), // 1 = required for 100% score, 0 = optional + displayOrder: int("displayOrder").default(0).notNull(), // Order in UI + createdAt: timestamp("createdAt").defaultNow().notNull(), + updatedAt: timestamp("updatedAt").defaultNow().onUpdateNow().notNull(), +}); + +export type LlmFieldConfig = typeof llmFieldsConfig.$inferSelect; +export type InsertLlmFieldConfig = typeof llmFieldsConfig.$inferInsert; diff --git a/package.json b/package.json index f3ec63c..cbfcaa6 100644 --- a/package.json +++ b/package.json @@ -76,6 +76,9 @@ "nanoid": "^5.1.5", "next-themes": "^0.4.6", "pdf-lib": "^1.17.1", + "pdf-parse": "^2.4.5", + "pdf2json": "^4.0.2", + "pdfjs-dist": "^5.4.624", "react": "^19.2.1", "react-day-picker": "^9.11.1", "react-dom": "^19.2.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f80ab1e..b00c4d1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -205,6 +205,15 @@ importers: pdf-lib: specifier: ^1.17.1 version: 1.17.1 + pdf-parse: + specifier: ^2.4.5 + version: 2.4.5 + pdf2json: + specifier: ^4.0.2 + version: 4.0.2 + pdfjs-dist: + specifier: ^5.4.624 + version: 5.4.624 react: specifier: ^19.2.1 version: 19.2.1 @@ -1111,54 +1120,108 @@ packages: '@mermaid-js/parser@0.6.3': resolution: {integrity: sha512-lnjOhe7zyHjc+If7yT4zoedx2vo4sHaTmtkl1+or8BRTnCtDmcTpAjpzDSfCZrshM5bCoz0GyidzadJAH1xobA==} + '@napi-rs/canvas-android-arm64@0.1.80': + resolution: {integrity: sha512-sk7xhN/MoXeuExlggf91pNziBxLPVUqF2CAVnB57KLG/pz7+U5TKG8eXdc3pm0d7Od0WreB6ZKLj37sX9muGOQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + '@napi-rs/canvas-android-arm64@0.1.88': resolution: {integrity: sha512-KEaClPnZuVxJ8smUWjV1wWFkByBO/D+vy4lN+Dm5DFH514oqwukxKGeck9xcKJhaWJGjfruGmYGiwRe//+/zQQ==} engines: {node: '>= 10'} cpu: [arm64] os: [android] + '@napi-rs/canvas-darwin-arm64@0.1.80': + resolution: {integrity: sha512-O64APRTXRUiAz0P8gErkfEr3lipLJgM6pjATwavZ22ebhjYl/SUbpgM0xcWPQBNMP1n29afAC/Us5PX1vg+JNQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + '@napi-rs/canvas-darwin-arm64@0.1.88': resolution: {integrity: sha512-Xgywz0dDxOKSgx3eZnK85WgGMmGrQEW7ZLA/E7raZdlEE+xXCozobgqz2ZvYigpB6DJFYkqnwHjqCOTSDGlFdg==} engines: {node: '>= 10'} cpu: [arm64] os: [darwin] + '@napi-rs/canvas-darwin-x64@0.1.80': + resolution: {integrity: sha512-FqqSU7qFce0Cp3pwnTjVkKjjOtxMqRe6lmINxpIZYaZNnVI0H5FtsaraZJ36SiTHNjZlUB69/HhxNDT1Aaa9vA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + '@napi-rs/canvas-darwin-x64@0.1.88': resolution: {integrity: sha512-Yz4wSCIQOUgNucgk+8NFtQxQxZV5NO8VKRl9ePKE6XoNyNVC8JDqtvhh3b3TPqKK8W5p2EQpAr1rjjm0mfBxdg==} engines: {node: '>= 10'} cpu: [x64] os: [darwin] + '@napi-rs/canvas-linux-arm-gnueabihf@0.1.80': + resolution: {integrity: sha512-eyWz0ddBDQc7/JbAtY4OtZ5SpK8tR4JsCYEZjCE3dI8pqoWUC8oMwYSBGCYfsx2w47cQgQCgMVRVTFiiO38hHQ==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + '@napi-rs/canvas-linux-arm-gnueabihf@0.1.88': resolution: {integrity: sha512-9gQM2SlTo76hYhxHi2XxWTAqpTOb+JtxMPEIr+H5nAhHhyEtNmTSDRtz93SP7mGd2G3Ojf2oF5tP9OdgtgXyKg==} engines: {node: '>= 10'} cpu: [arm] os: [linux] + '@napi-rs/canvas-linux-arm64-gnu@0.1.80': + resolution: {integrity: sha512-qwA63t8A86bnxhuA/GwOkK3jvb+XTQaTiVML0vAWoHyoZYTjNs7BzoOONDgTnNtr8/yHrq64XXzUoLqDzU+Uuw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + '@napi-rs/canvas-linux-arm64-gnu@0.1.88': resolution: {integrity: sha512-7qgaOBMXuVRk9Fzztzr3BchQKXDxGbY+nwsovD3I/Sx81e+sX0ReEDYHTItNb0Je4NHbAl7D0MKyd4SvUc04sg==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] + '@napi-rs/canvas-linux-arm64-musl@0.1.80': + resolution: {integrity: sha512-1XbCOz/ymhj24lFaIXtWnwv/6eFHXDrjP0jYkc6iHQ9q8oXKzUX1Lc6bu+wuGiLhGh2GS/2JlfORC5ZcXimRcg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + '@napi-rs/canvas-linux-arm64-musl@0.1.88': resolution: {integrity: sha512-kYyNrUsHLkoGHBc77u4Unh067GrfiCUMbGHC2+OTxbeWfZkPt2o32UOQkhnSswKd9Fko/wSqqGkY956bIUzruA==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] + '@napi-rs/canvas-linux-riscv64-gnu@0.1.80': + resolution: {integrity: sha512-XTzR125w5ZMs0lJcxRlS1K3P5RaZ9RmUsPtd1uGt+EfDyYMu4c6SEROYsxyatbbu/2+lPe7MPHOO/0a0x7L/gw==} + engines: {node: '>= 10'} + cpu: [riscv64] + os: [linux] + '@napi-rs/canvas-linux-riscv64-gnu@0.1.88': resolution: {integrity: sha512-HVuH7QgzB0yavYdNZDRyAsn/ejoXB0hn8twwFnOqUbCCdkV+REna7RXjSR7+PdfW0qMQ2YYWsLvVBT5iL/mGpw==} engines: {node: '>= 10'} cpu: [riscv64] os: [linux] + '@napi-rs/canvas-linux-x64-gnu@0.1.80': + resolution: {integrity: sha512-BeXAmhKg1kX3UCrJsYbdQd3hIMDH/K6HnP/pG2LuITaXhXBiNdh//TVVVVCBbJzVQaV5gK/4ZOCMrQW9mvuTqA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + '@napi-rs/canvas-linux-x64-gnu@0.1.88': resolution: {integrity: sha512-hvcvKIcPEQrvvJtJnwD35B3qk6umFJ8dFIr8bSymfrSMem0EQsfn1ztys8ETIFndTwdNWJKWluvxztA41ivsEw==} engines: {node: '>= 10'} cpu: [x64] os: [linux] + '@napi-rs/canvas-linux-x64-musl@0.1.80': + resolution: {integrity: sha512-x0XvZWdHbkgdgucJsRxprX/4o4sEed7qo9rCQA9ugiS9qE2QvP0RIiEugtZhfLH3cyI+jIRFJHV4Fuz+1BHHMg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + '@napi-rs/canvas-linux-x64-musl@0.1.88': resolution: {integrity: sha512-eSMpGYY2xnZSQ6UxYJ6plDboxq4KeJ4zT5HaVkUnbObNN6DlbJe0Mclh3wifAmquXfrlgTZt6zhHsUgz++AK6g==} engines: {node: '>= 10'} @@ -1171,12 +1234,22 @@ packages: cpu: [arm64] os: [win32] + '@napi-rs/canvas-win32-x64-msvc@0.1.80': + resolution: {integrity: sha512-Z8jPsM6df5V8B1HrCHB05+bDiCxjE9QA//3YrkKIdVDEwn5RKaqOxCJDRJkl48cJbylcrJbW4HxZbTte8juuPg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + '@napi-rs/canvas-win32-x64-msvc@0.1.88': resolution: {integrity: sha512-ROVqbfS4QyZxYkqmaIBBpbz/BQvAR+05FXM5PAtTYVc0uyY8Y4BHJSMdGAaMf6TdIVRsQsiq+FG/dH9XhvWCFQ==} engines: {node: '>= 10'} cpu: [x64] os: [win32] + '@napi-rs/canvas@0.1.80': + resolution: {integrity: sha512-DxuT1ClnIPts1kQx8FBmkk4BQDTfI5kIzywAaMjQSXfNnra5UFU9PwurXrl+Je3bJ6BGsp/zmshVVFbCmyI+ww==} + engines: {node: '>= 10'} + '@napi-rs/canvas@0.1.88': resolution: {integrity: sha512-/p08f93LEbsL5mDZFQ3DBxcPv/I4QG9EDYRRq1WNlCOXVfAHBTHMSVMwxlqG/AtnSfUr9+vgfN7MKiyDo0+Weg==} engines: {node: '>= 10'} @@ -3951,6 +4024,9 @@ packages: resolution: {integrity: sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==} hasBin: true + node-readable-to-web-readable-stream@0.4.2: + resolution: {integrity: sha512-/cMZNI34v//jUTrI+UIo4ieHAB5EZRY/+7OmXZgBxaWBMcW2tGdceIw06RFxWxrKZ5Jp3sI2i5TsRo+CBhtVLQ==} + node-releases@2.0.23: resolution: {integrity: sha512-cCmFDMSm26S6tQSDpBCg/NR8NENrVPhAJSf+XbxBG4rPFaaonlEoE9wHQmun+cls499TQGSb7ZyPBRlzgKfpeg==} @@ -4018,10 +4094,24 @@ packages: pdf-lib@1.17.1: resolution: {integrity: sha512-V/mpyJAoTsN4cnP31vc0wfNA1+p20evqqnap0KLoRUN0Yk/p3wN52DOEsL4oBFcLdb76hlpKPtzJIgo67j/XLw==} + pdf-parse@2.4.5: + resolution: {integrity: sha512-mHU89HGh7v+4u2ubfnevJ03lmPgQ5WU4CxAVmTSh/sxVTEDYd1er/dKS/A6vg77NX47KTEoihq8jZBLr8Cxuwg==} + engines: {node: '>=20.16.0 <21 || >=22.3.0'} + hasBin: true + + pdf2json@4.0.2: + resolution: {integrity: sha512-iiRSuRmLihoEJ4YGkoqSq3/r4MR0OmkMTYDda0Pq7DAWqJwMylTilXu46T16gfS3DUp3fhiVuz7NtRMbk3uBhw==} + engines: {node: '>=20.18.0'} + hasBin: true + pdfjs-dist@5.4.296: resolution: {integrity: sha512-DlOzet0HO7OEnmUmB6wWGJrrdvbyJKftI1bhMitK7O2N8W2gc757yyYBbINy9IDafXAV9wmKr9t7xsTaNKRG5Q==} engines: {node: '>=20.16.0 || >=22.3.0'} + pdfjs-dist@5.4.624: + resolution: {integrity: sha512-sm6TxKTtWv1Oh6n3C6J6a8odejb5uO4A4zo/2dgkHuC0iu8ZMAXOezEODkVaoVp8nX1Xzr+0WxFJJmUr45hQzg==} + engines: {node: '>=20.16.0 || >=22.3.0'} + peberminta@0.9.0: resolution: {integrity: sha512-XIxfHpEuSJbITd1H3EeQwpcZbTLHc+VVr8ANI9t5sit565tsI4/xK3KWTUFE2e6QiangUkh3B0jihzmGnNrRsQ==} @@ -5718,39 +5808,82 @@ snapshots: dependencies: langium: 3.3.1 + '@napi-rs/canvas-android-arm64@0.1.80': + optional: true + '@napi-rs/canvas-android-arm64@0.1.88': optional: true + '@napi-rs/canvas-darwin-arm64@0.1.80': + optional: true + '@napi-rs/canvas-darwin-arm64@0.1.88': optional: true + '@napi-rs/canvas-darwin-x64@0.1.80': + optional: true + '@napi-rs/canvas-darwin-x64@0.1.88': optional: true + '@napi-rs/canvas-linux-arm-gnueabihf@0.1.80': + optional: true + '@napi-rs/canvas-linux-arm-gnueabihf@0.1.88': optional: true + '@napi-rs/canvas-linux-arm64-gnu@0.1.80': + optional: true + '@napi-rs/canvas-linux-arm64-gnu@0.1.88': optional: true + '@napi-rs/canvas-linux-arm64-musl@0.1.80': + optional: true + '@napi-rs/canvas-linux-arm64-musl@0.1.88': optional: true + '@napi-rs/canvas-linux-riscv64-gnu@0.1.80': + optional: true + '@napi-rs/canvas-linux-riscv64-gnu@0.1.88': optional: true + '@napi-rs/canvas-linux-x64-gnu@0.1.80': + optional: true + '@napi-rs/canvas-linux-x64-gnu@0.1.88': optional: true + '@napi-rs/canvas-linux-x64-musl@0.1.80': + optional: true + '@napi-rs/canvas-linux-x64-musl@0.1.88': optional: true '@napi-rs/canvas-win32-arm64-msvc@0.1.88': optional: true + '@napi-rs/canvas-win32-x64-msvc@0.1.80': + optional: true + '@napi-rs/canvas-win32-x64-msvc@0.1.88': optional: true + '@napi-rs/canvas@0.1.80': + optionalDependencies: + '@napi-rs/canvas-android-arm64': 0.1.80 + '@napi-rs/canvas-darwin-arm64': 0.1.80 + '@napi-rs/canvas-darwin-x64': 0.1.80 + '@napi-rs/canvas-linux-arm-gnueabihf': 0.1.80 + '@napi-rs/canvas-linux-arm64-gnu': 0.1.80 + '@napi-rs/canvas-linux-arm64-musl': 0.1.80 + '@napi-rs/canvas-linux-riscv64-gnu': 0.1.80 + '@napi-rs/canvas-linux-x64-gnu': 0.1.80 + '@napi-rs/canvas-linux-x64-musl': 0.1.80 + '@napi-rs/canvas-win32-x64-msvc': 0.1.80 + '@napi-rs/canvas@0.1.88': optionalDependencies: '@napi-rs/canvas-android-arm64': 0.1.88 @@ -8976,6 +9109,9 @@ snapshots: node-gyp-build@4.8.4: {} + node-readable-to-web-readable-stream@0.4.2: + optional: true + node-releases@2.0.23: {} nodemailer@7.0.13: {} @@ -9040,10 +9176,22 @@ snapshots: pako: 1.0.11 tslib: 1.14.1 + pdf-parse@2.4.5: + dependencies: + '@napi-rs/canvas': 0.1.80 + pdfjs-dist: 5.4.296 + + pdf2json@4.0.2: {} + pdfjs-dist@5.4.296: optionalDependencies: '@napi-rs/canvas': 0.1.88 + pdfjs-dist@5.4.624: + optionalDependencies: + '@napi-rs/canvas': 0.1.88 + node-readable-to-web-readable-stream: 0.4.2 + peberminta@0.9.0: {} picocolors@1.1.1: {} diff --git a/server/_core/env.ts b/server/_core/env.ts index 2792b99..f67a574 100644 --- a/server/_core/env.ts +++ b/server/_core/env.ts @@ -7,4 +7,5 @@ export const ENV = { isProduction: process.env.NODE_ENV === "production", forgeApiUrl: process.env.BUILT_IN_FORGE_API_URL ?? "", forgeApiKey: process.env.BUILT_IN_FORGE_API_KEY ?? "", + mistralApiKey: process.env.MISTRAL_API_KEY ?? "", }; diff --git a/server/_core/llm.ts b/server/_core/llm.ts index 8ea4c4a..da5820b 100644 --- a/server/_core/llm.ts +++ b/server/_core/llm.ts @@ -209,17 +209,32 @@ const normalizeToolChoice = ( return toolChoice; }; -const resolveApiUrl = () => - ENV.forgeApiUrl && ENV.forgeApiUrl.trim().length > 0 +const resolveApiUrl = () => { + // If MISTRAL_API_KEY is set, use Mistral API directly + if (ENV.mistralApiKey && ENV.mistralApiKey.trim().length > 0) { + return "https://api.mistral.ai/v1/chat/completions"; + } + + // Otherwise use Manus Forge API + return ENV.forgeApiUrl && ENV.forgeApiUrl.trim().length > 0 ? `${ENV.forgeApiUrl.replace(/\/$/, "")}/v1/chat/completions` : "https://forge.manus.im/v1/chat/completions"; +}; const assertApiKey = () => { - if (!ENV.forgeApiKey) { - throw new Error("OPENAI_API_KEY is not configured"); + if (!ENV.mistralApiKey && !ENV.forgeApiKey) { + throw new Error("MISTRAL_API_KEY or OPENAI_API_KEY is not configured"); } }; +const getApiKey = () => { + // Prioritize MISTRAL_API_KEY if set + if (ENV.mistralApiKey && ENV.mistralApiKey.trim().length > 0) { + return ENV.mistralApiKey; + } + return ENV.forgeApiKey; +}; + const normalizeResponseFormat = ({ responseFormat, response_format, @@ -279,8 +294,13 @@ export async function invokeLLM(params: InvokeParams): Promise { response_format, } = params; + // Use mistral-large-latest when MISTRAL_API_KEY is set, otherwise use gemini + const model = ENV.mistralApiKey && ENV.mistralApiKey.trim().length > 0 + ? "mistral-large-latest" + : "gemini-2.5-flash"; + const payload: Record = { - model: "gemini-2.5-flash", + model, messages: messages.map(normalizeMessage), }; @@ -297,8 +317,12 @@ export async function invokeLLM(params: InvokeParams): Promise { } payload.max_tokens = 32768 - payload.thinking = { - "budget_tokens": 128 + + // Only add thinking parameter for Gemini models + if (!(ENV.mistralApiKey && ENV.mistralApiKey.trim().length > 0)) { + payload.thinking = { + "budget_tokens": 128 + } } const normalizedResponseFormat = normalizeResponseFormat({ @@ -316,7 +340,7 @@ export async function invokeLLM(params: InvokeParams): Promise { method: "POST", headers: { "content-type": "application/json", - authorization: `Bearer ${ENV.forgeApiKey}`, + authorization: `Bearer ${getApiKey()}`, }, body: JSON.stringify(payload), }); diff --git a/server/db.ts b/server/db.ts index efd07e2..d43e2a0 100644 --- a/server/db.ts +++ b/server/db.ts @@ -29,7 +29,10 @@ import { AccountingAllocation, automationRules, InsertAutomationRule, - AutomationRule + AutomationRule, + llmFieldsConfig, + InsertLlmFieldConfig, + LlmFieldConfig } from "../drizzle/schema"; import { ENV } from './_core/env'; @@ -606,3 +609,61 @@ export async function initializeDefaultLists(userId: number): Promise { } } } + +// ============= LLM FIELDS CONFIG OPERATIONS ============= + +export async function getLlmFieldsConfigByUser(userId: number): Promise { + const db = await getDb(); + if (!db) return []; + return db.select().from(llmFieldsConfig).where(eq(llmFieldsConfig.userId, userId)).orderBy(llmFieldsConfig.displayOrder); +} + +export async function upsertLlmFieldConfig(data: InsertLlmFieldConfig): Promise { + const db = await getDb(); + if (!db) throw new Error("Database not available"); + + const existing = await db.select().from(llmFieldsConfig) + .where(and( + eq(llmFieldsConfig.userId, data.userId), + eq(llmFieldsConfig.fieldName, data.fieldName) + )) + .limit(1); + + if (existing.length > 0) { + await db.update(llmFieldsConfig) + .set({ ...data, updatedAt: new Date() }) + .where(eq(llmFieldsConfig.id, existing[0].id)); + return (await db.select().from(llmFieldsConfig).where(eq(llmFieldsConfig.id, existing[0].id)))[0]; + } else { + const result = await db.insert(llmFieldsConfig).values(data); + const insertedId = (result as any).insertId; + return (await db.select().from(llmFieldsConfig).where(eq(llmFieldsConfig.id, Number(insertedId))))[0]; + } +} + +export async function initializeDefaultLlmFields(userId: number): Promise { + const db = await getDb(); + if (!db) return; + + const defaultFields = [ + { fieldName: "supplierName", displayName: "Nom du fournisseur", isRequired: 1, displayOrder: 1 }, + { fieldName: "invoiceNumber", displayName: "Numéro de facture", isRequired: 1, displayOrder: 2 }, + { fieldName: "invoiceDate", displayName: "Date de facture", isRequired: 1, displayOrder: 3 }, + { fieldName: "totalAmount", displayName: "Montant total TTC", isRequired: 1, displayOrder: 4 }, + { fieldName: "deliveryNoteNumber", displayName: "Numéro de bon de livraison", isRequired: 0, displayOrder: 5 }, + { fieldName: "orderNumber", displayName: "Numéro de commande", isRequired: 0, displayOrder: 6 }, + ]; + + const existingFields = await db.select().from(llmFieldsConfig).where(eq(llmFieldsConfig.userId, userId)); + const existingFieldNames = new Set(existingFields.map(f => f.fieldName)); + + for (const field of defaultFields) { + if (!existingFieldNames.has(field.fieldName)) { + try { + await db.insert(llmFieldsConfig).values({ userId, ...field }); + } catch (error) { + // Ignore errors + } + } + } +} diff --git a/server/invoiceExtractor.ts b/server/invoiceExtractor.ts index 3517dd6..157fdbf 100644 --- a/server/invoiceExtractor.ts +++ b/server/invoiceExtractor.ts @@ -1,6 +1,7 @@ import { invokeLLM } from "./_core/llm"; import { PDFDocument } from "pdf-lib"; import { createLlmLog } from "./db"; +import PDFParser from "pdf2json"; export interface ExtractedInvoiceData { supplierName: string | null; @@ -22,18 +23,54 @@ export interface MultiInvoiceResult { } /** - * Convert PDF buffer to base64 data URI for Mistral API processing + * Extract text from PDF buffer using pdf2json */ -function convertPdfToBase64(pdfBuffer: Buffer): string { - try { - const base64Pdf = pdfBuffer.toString("base64"); - const dataUri = `data:application/pdf;base64,${base64Pdf}`; - console.log("[Mistral] PDF converted to base64, size:", Math.round(base64Pdf.length / 1024), "KB"); - return dataUri; - } catch (error) { - console.error("Error converting PDF to base64:", error); - throw new Error("Failed to convert PDF to base64"); - } +async function extractTextFromPdf(pdfBuffer: Buffer): Promise { + return new Promise((resolve, reject) => { + const pdfParser = new (PDFParser as any)(null, 1); + + pdfParser.on("pdfParser_dataError", (errData: any) => { + console.error("Error parsing PDF:", errData.parserError); + reject(new Error("Failed to parse PDF")); + }); + + pdfParser.on("pdfParser_dataReady", (pdfData: any) => { + try { + let text = ""; + + // Extract text from all pages + if (pdfData.Pages) { + for (const page of pdfData.Pages) { + if (page.Texts) { + for (const textItem of page.Texts) { + if (textItem.R) { + for (const run of textItem.R) { + if (run.T) { + try { + text += decodeURIComponent(run.T) + " "; + } catch (e) { + // If decodeURIComponent fails, use the raw text + text += run.T + " "; + } + } + } + } + } + text += "\n"; + } + } + } + + console.log("[PDF] Text extracted, length:", text.length, "characters"); + resolve(text); + } catch (error) { + console.error("Error extracting text from PDF data:", error); + reject(new Error("Failed to extract text from PDF")); + } + }); + + pdfParser.parseBuffer(pdfBuffer); + }); } /** @@ -106,13 +143,30 @@ export async function extractInvoicesWithMistral( subscription?: string | null; } ): Promise { + // Load user's field configuration + const { getLlmFieldsConfigByUser } = await import("./db"); + const fieldsConfig = await getLlmFieldsConfigByUser(userId); + + // Build quality score instruction based on required fields + const requiredFields = fieldsConfig.filter(f => f.isRequired === 1); + const optionalFields = fieldsConfig.filter(f => f.isRequired === 0); + + let qualityScoreInstruction = "- qualityScore: Score de qualité de l'extraction de 0 à 100"; + if (requiredFields.length > 0) { + const requiredFieldNames = requiredFields.map(f => f.displayName).join(", "); + qualityScoreInstruction += ` (100 = tous les champs obligatoires trouvés: ${requiredFieldNames})`; + } + if (optionalFields.length > 0) { + const optionalFieldNames = optionalFields.map(f => f.displayName).join(", "); + qualityScoreInstruction += `. Champs optionnels (n'affectent pas le score): ${optionalFieldNames}`; + } const startTime = Date.now(); try { console.log("[Mistral] Starting invoice extraction..."); - // Convert PDF to base64 - const pdfDataUri = convertPdfToBase64(pdfBuffer); + // Extract text from PDF + const pdfText = await extractTextFromPdf(pdfBuffer); // Get PDF page count const pdfDoc = await PDFDocument.load(pdfBuffer); @@ -153,7 +207,7 @@ Pour chaque facture trouvée, extrais les informations suivantes: - orderNumber: Numéro de commande client (si présent) - totalAmount: Montant total TTC (nombre décimal) - pageRange: Plage de pages de cette facture (ex: "1-2" ou "5") -- qualityScore: Score de qualité de l'extraction de 0 à 100 (100 = toutes les informations trouvées et claires) +${qualityScoreInstruction} - extractedText: Texte complet extrait de la facture (tout le texte visible sur les pages de cette facture) ${subscriptionInstruction}${keywordsHint} @@ -179,15 +233,13 @@ Réponds UNIQUEMENT avec un objet JSON valide au format suivant: Si une information n'est pas trouvée, utilise null. Ne retourne AUCUN texte en dehors du JSON.`; - // Call Mistral LLM with PDF + // Call Mistral LLM with extracted text + const fullPrompt = `${prompt}\n\nTexte extrait du PDF:\n${pdfText}`; const response = await invokeLLM({ messages: [ { role: "user", - content: [ - { type: "text", text: prompt }, - { type: "file_url", file_url: { url: pdfDataUri, mime_type: "application/pdf" } }, - ], + content: fullPrompt, }, ], }); diff --git a/server/llmFieldsConfig.test.ts b/server/llmFieldsConfig.test.ts new file mode 100644 index 0000000..d5c9c58 --- /dev/null +++ b/server/llmFieldsConfig.test.ts @@ -0,0 +1,78 @@ +import { describe, it, expect, beforeAll } from "vitest"; +import { + getLlmFieldsConfigByUser, + upsertLlmFieldConfig, + initializeDefaultLlmFields +} from "./db"; + +describe("LLM Fields Configuration", () => { + const testUserId = 99999; // Use a high ID to avoid conflicts + + beforeAll(async () => { + // Initialize default fields for test user + await initializeDefaultLlmFields(testUserId); + }); + + it("should initialize default fields for a new user", async () => { + const fields = await getLlmFieldsConfigByUser(testUserId); + + expect(fields).toBeDefined(); + expect(fields.length).toBeGreaterThan(0); + + // Check that default required fields exist + const supplierName = fields.find(f => f.fieldName === "supplierName"); + const invoiceNumber = fields.find(f => f.fieldName === "invoiceNumber"); + const invoiceDate = fields.find(f => f.fieldName === "invoiceDate"); + const totalAmount = fields.find(f => f.fieldName === "totalAmount"); + + expect(supplierName).toBeDefined(); + expect(supplierName?.isRequired).toBe(1); + expect(invoiceNumber).toBeDefined(); + expect(invoiceNumber?.isRequired).toBe(1); + expect(invoiceDate).toBeDefined(); + expect(invoiceDate?.isRequired).toBe(1); + expect(totalAmount).toBeDefined(); + expect(totalAmount?.isRequired).toBe(1); + }); + + it("should update field configuration", async () => { + // Make deliveryNoteNumber required + await upsertLlmFieldConfig({ + userId: testUserId, + fieldName: "deliveryNoteNumber", + displayName: "Numéro de bon de livraison", + isRequired: 1, + displayOrder: 5, + }); + + const fields = await getLlmFieldsConfigByUser(testUserId); + const deliveryNote = fields.find(f => f.fieldName === "deliveryNoteNumber"); + + expect(deliveryNote).toBeDefined(); + expect(deliveryNote?.isRequired).toBe(1); + + // Make it optional again + await upsertLlmFieldConfig({ + userId: testUserId, + fieldName: "deliveryNoteNumber", + displayName: "Numéro de bon de livraison", + isRequired: 0, + displayOrder: 5, + }); + + const fieldsAfter = await getLlmFieldsConfigByUser(testUserId); + const deliveryNoteAfter = fieldsAfter.find(f => f.fieldName === "deliveryNoteNumber"); + + expect(deliveryNoteAfter).toBeDefined(); + expect(deliveryNoteAfter?.isRequired).toBe(0); + }); + + it("should return fields in correct display order", async () => { + const fields = await getLlmFieldsConfigByUser(testUserId); + + // Check that fields are sorted by displayOrder + for (let i = 1; i < fields.length; i++) { + expect(fields[i].displayOrder).toBeGreaterThanOrEqual(fields[i - 1].displayOrder); + } + }); +}); diff --git a/server/routers.ts b/server/routers.ts index 51d2b33..00a4c3e 100644 --- a/server/routers.ts +++ b/server/routers.ts @@ -48,6 +48,9 @@ import { createAccountingAllocation, deleteAccountingAllocation, initializeDefaultLists, + getLlmFieldsConfigByUser, + upsertLlmFieldConfig, + initializeDefaultLlmFields, getAutomationRulesByUser, getAutomationRuleById, createAutomationRule, @@ -93,7 +96,7 @@ export const appRouter = router({ // Set auth cookie ctx.res.cookie("auth_token", result.token, { httpOnly: true, - secure: process.env.NODE_ENV === "production", + secure: false, // Désactivé pour VPS sans HTTPS sameSite: "lax", path: "/", maxAge: 7 * 24 * 60 * 60 * 1000, // 7 days @@ -304,9 +307,11 @@ export const appRouter = router({ } catch (error: any) { console.error("[Upload] Extraction failed:", error); + // Limit error message to 200 characters to avoid database field overflow + const errorMsg = error.message ? String(error.message).substring(0, 200) : "Erreur inconnue"; await updateSourceFile(sourceFile.id, { processingStatus: "error", - processingProgress: `Erreur: ${error.message}`, + processingProgress: `Erreur: ${errorMsg}`, }); } })(); @@ -737,6 +742,15 @@ export const appRouter = router({ await initializeDefaultLists(ctx.user.id); } + // Check if department already exists + const duplicate = existing.find(d => d.name.toLowerCase() === input.name.toLowerCase()); + if (duplicate) { + throw new TRPCError({ + code: "CONFLICT", + message: `Le service "${input.name}" existe déjà` + }); + } + return await createDepartment({ userId: ctx.user.id, name: input.name, @@ -770,6 +784,15 @@ export const appRouter = router({ await initializeDefaultLists(ctx.user.id); } + // Check if allocation already exists + const duplicate = existing.find(a => a.name.toLowerCase() === input.name.toLowerCase()); + if (duplicate) { + throw new TRPCError({ + code: "CONFLICT", + message: `La ventilation comptable "${input.name}" existe déjà` + }); + } + return await createAccountingAllocation({ userId: ctx.user.id, name: input.name, @@ -1013,6 +1036,44 @@ export const appRouter = router({ }; }), }), + + // ============= LLM FIELDS CONFIG ROUTES ============= + llmFieldsConfig: router({ + getAll: protectedProcedure + .query(async ({ ctx }) => { + // Initialize default fields if none exist + const existing = await getLlmFieldsConfigByUser(ctx.user.id); + if (existing.length === 0) { + await initializeDefaultLlmFields(ctx.user.id); + return await getLlmFieldsConfigByUser(ctx.user.id); + } + return existing; + }), + + updateField: protectedProcedure + .input(z.object({ + fieldName: z.string(), + isRequired: z.number().min(0).max(1), + })) + .mutation(async ({ input, ctx }) => { + // Get existing field config + const existing = await getLlmFieldsConfigByUser(ctx.user.id); + const field = existing.find(f => f.fieldName === input.fieldName); + + if (!field) { + throw new TRPCError({ code: "NOT_FOUND", message: "Field not found" }); + } + + // Update the field + return await upsertLlmFieldConfig({ + userId: ctx.user.id, + fieldName: input.fieldName, + displayName: field.displayName, + isRequired: input.isRequired, + displayOrder: field.displayOrder, + }); + }), + }), }); export type AppRouter = typeof appRouter; diff --git a/todo.md b/todo.md index 0b2ed2b..ac073ea 100644 --- a/todo.md +++ b/todo.md @@ -341,3 +341,46 @@ - [x] Intégrer l'appel automatique après activation d'une règle - [x] Ajouter une notification avec le nombre de factures mises à jour - [x] Tester avec différents scénarios (création, modification, activation) + +## Création utilisateur admin local sur VPS +- [x] Créer l'utilisateur admin dans la base de données MySQL +- [x] Hacher le mot de passe avec bcrypt +- [x] Vérifier la création de l'utilisateur + +## Mise à jour email admin +- [x] Mettre à jour l'email de admin@local à o.pareige@itinova.org + +## Bug connexion locale VPS +- [x] Vérifier les logs PM2 pour identifier l'erreur +- [x] Diagnostiquer le problème d'authentification +- [x] Corriger le bug et tester la connexion + +## Bug ajout service +- [x] Identifier le problème (contrainte d'unicité) +- [x] Améliorer la gestion des erreurs pour afficher un message convivial +- [x] Tester l'ajout de service +- [x] Déployer la correction sur le VPS + +## Configuration clé API Mistral sur VPS +- [x] Ajouter MISTRAL_API_KEY dans le fichier .env du VPS +- [x] Modifier le code pour utiliser Mistral API directement +- [x] Redémarrer l'application PM2 +- [x] Tester l'extraction de factures + +## Bug extraction PDF qui ne se termine pas sur VPS +- [x] Vérifier les logs PM2 pour identifier l'erreur +- [x] Diagnostiquer le problème (Data too long for column 'processingProgress') +- [ ] Modifier le code pour stocker les PDF dans un dossier local +- [ ] Configurer Nginx pour servir les fichiers PDF +- [ ] Tester et redéployer sur le VPS + +## Configuration des champs obligatoires pour le score LLM +- [x] Créer table llmFieldsConfig pour stocker la configuration des champs (nom, obligatoire/optionnel) +- [x] Ajouter routes tRPC pour gérer la configuration des champs (get, update) +- [x] Créer interface utilisateur dans les paramètres LLM pour afficher tous les champs +- [x] Ajouter cases à cocher pour marquer chaque champ comme obligatoire ou optionnel +- [x] Modifier le code d'extraction (invoiceExtractor.ts) pour utiliser la configuration +- [x] Adapter le calcul du qualityScore selon les champs obligatoires configurés +- [x] Initialiser les valeurs par défaut (supplierName, invoiceNumber, invoiceDate, totalAmount obligatoires) +- [ ] Tester la configuration et vérifier le calcul du score +- [ ] Déployer sur le VPS