Checkpoint: Checkpoint saved: Implémentation complète de l'import automatique depuis dossier et du système de notifications.
Nouvelles fonctionnalités : **1. Import automatique depuis dossier (folderImportService.ts)** ✅ Surveillance périodique d'un dossier local configuré ✅ Détection automatique des nouveaux fichiers PDF ✅ Traitement identique à l'upload manuel (extraction Mistral AI, détection doublons, stockage) ✅ Déplacement automatique des fichiers traités vers un sous-dossier "processed" ✅ Scheduler configurable (fréquence en minutes) ✅ Routes tRPC pour démarrer/arrêter/vérifier le statut du service ✅ Boutons de contrôle dans la page Paramètres de réception **2. Système de notifications (notificationService.ts)** ✅ Notifications automatiques après chaque import (email ou dossier) ✅ Résumé détaillé : nombre de factures importées, doublons ignorés, erreurs ✅ Format adapté selon la source (email avec nom de fichier, dossier avec nombre de fichiers) ✅ Intégration avec le système de notifications Manus (notifyOwner) ✅ Notifications envoyées au propriétaire du projet **Fonctionnement de l'import dossier :** 1. L'utilisateur configure le chemin du dossier source dans "Paramètres de réception" 2. Il active l'import automatique et configure la fréquence de vérification 3. Il clique sur "Démarrer le service" pour lancer la surveillance 4. Le service scanne le dossier selon la fréquence configurée 5. Il détecte les nouveaux fichiers PDF (non encore traités) 6. Il traite chaque PDF avec la même logique que l'upload manuel 7. Les fichiers traités sont déplacés vers un sous-dossier "processed" 8. Une notification est envoyée avec le résumé des résultats **Fonctionnement des notifications :** - Après chaque import automatique (email ou dossier), une notification est envoyée - La notification contient : * Source de l'import (📧 email ou 📁 dossier) * Nom du fichier (email) ou nombre de fichiers (dossier) * ✅ Nombre de factures importées * 🔄 Nombre de doublons ignorés * ❌ Nombre d'erreurs * Total de factures détectées - Les notifications apparaissent dans l'interface Manus du propriétaire **Interface utilisateur :** - Chaque carte (Import dossier, Import email) dispose maintenant de : * Indicateur visuel "Service actif" avec point vert animé * Bouton "Démarrer le service" (désactivé si l'import n'est pas activé) * Bouton "Arrêter le service" (rouge) pour stopper la surveillance * Messages toast pour confirmer le démarrage/arrêt **Dépendances ajoutées :** - chokidar : Surveillance de fichiers et dossiers pour Node.js **Architecture :** - folderImportService.ts : Service d'import automatique depuis dossier - notificationService.ts : Service de notifications centralisé - Routes tRPC : folderImportService.{start, stop, status} - Intégration des notifications dans emailImportService et folderImportService Les deux services d'import automatique (email et dossier) sont maintenant opérationnels avec notifications intégrées.
This commit is contained in:
Manus
398
server/folderImportService.ts
Normal file
398
server/folderImportService.ts
Normal file
@@ -0,0 +1,398 @@
|
||||
import chokidar from "chokidar";
|
||||
import fs from "fs/promises";
|
||||
import path from "path";
|
||||
import {
|
||||
getImportSettingsByUser,
|
||||
createSourceFile,
|
||||
updateSourceFile,
|
||||
getUserSettings,
|
||||
findDuplicateInvoice,
|
||||
createInvoice,
|
||||
createImportLog,
|
||||
} from "./db";
|
||||
import { extractInvoicesWithMistral, generateMetadataJSON } from "./invoiceExtractor";
|
||||
import { localStoragePut, generateStorageKey } from "./localStorage";
|
||||
import { sendImportNotification } from "./notificationService";
|
||||
|
||||
interface FolderImportConfig {
|
||||
userId: number;
|
||||
folderPath: string;
|
||||
frequency: number; // in minutes
|
||||
}
|
||||
|
||||
// Store active watchers for each user
|
||||
const activeWatchers = new Map<number, { watcher: any; interval: NodeJS.Timeout }>();
|
||||
|
||||
// Track processed files to avoid reprocessing
|
||||
const processedFiles = new Map<number, Set<string>>();
|
||||
|
||||
/**
|
||||
* Process a single PDF file from the watched folder
|
||||
*/
|
||||
async function processFolderFile(
|
||||
userId: number,
|
||||
filePath: string,
|
||||
folderPath: string
|
||||
): Promise<{ success: boolean; imported: number; duplicates: number; errors: number }> {
|
||||
const fileName = path.basename(filePath);
|
||||
console.log(`[FolderImport] Processing file: ${fileName} for user ${userId}`);
|
||||
|
||||
try {
|
||||
// Read the file
|
||||
const fileBuffer = await fs.readFile(filePath);
|
||||
console.log(`[FolderImport] File size: ${fileBuffer.length} bytes`);
|
||||
|
||||
// Store source file
|
||||
const sourceFileKey = generateStorageKey(userId, fileName);
|
||||
console.log(`[FolderImport] Generated storage key: ${sourceFileKey}`);
|
||||
|
||||
let sourceFileUrl: string;
|
||||
try {
|
||||
const result = await localStoragePut(sourceFileKey, fileBuffer, "application/pdf");
|
||||
sourceFileUrl = result.url;
|
||||
console.log(`[FolderImport] File stored successfully at: ${sourceFileUrl}`);
|
||||
} catch (error) {
|
||||
console.error(`[FolderImport] FAILED to store file:`, error);
|
||||
throw new Error("Failed to store PDF file");
|
||||
}
|
||||
|
||||
// Create source file record
|
||||
const sourceFile = await createSourceFile({
|
||||
userId,
|
||||
fileName,
|
||||
fileKey: sourceFileKey,
|
||||
fileUrl: sourceFileUrl,
|
||||
processingStatus: "processing",
|
||||
});
|
||||
|
||||
console.log(`[FolderImport] Source file record created with ID: ${sourceFile.id}`);
|
||||
|
||||
// Get user settings for custom keywords
|
||||
const settings = await getUserSettings(userId);
|
||||
const customKeywords = settings ? {
|
||||
invoiceNumber: settings.invoiceNumberKeywords,
|
||||
deliveryNote: settings.deliveryNoteKeywords,
|
||||
orderNumber: settings.orderNumberKeywords,
|
||||
supplier: settings.supplierKeywords,
|
||||
totalAmount: settings.totalAmountKeywords,
|
||||
} : undefined;
|
||||
|
||||
const model = settings?.llmModel || "mistral-large-latest";
|
||||
|
||||
// Extract invoices
|
||||
console.log(`[FolderImport] Starting invoice extraction...`);
|
||||
const result = await extractInvoicesWithMistral(
|
||||
fileBuffer,
|
||||
userId,
|
||||
sourceFile.id,
|
||||
model,
|
||||
customKeywords
|
||||
);
|
||||
|
||||
console.log(`[FolderImport] Extraction complete: ${result.invoiceCount} invoice(s) detected`);
|
||||
|
||||
// Update source file with total count
|
||||
await updateSourceFile(sourceFile.id, {
|
||||
totalInvoicesDetected: result.invoiceCount,
|
||||
processingProgress: `Extraction ${result.invoiceCount} facture(s) détectée(s)`,
|
||||
});
|
||||
|
||||
// Process each invoice
|
||||
let importedCount = 0;
|
||||
let duplicatesCount = 0;
|
||||
let errorsCount = 0;
|
||||
const duplicateDetails: any[] = [];
|
||||
const errorDetails: any[] = [];
|
||||
|
||||
for (let i = 0; i < result.invoices.length; i++) {
|
||||
const invoiceData = result.invoices[i]!;
|
||||
|
||||
try {
|
||||
// Update progress
|
||||
await updateSourceFile(sourceFile.id, {
|
||||
processingProgress: `Extraction ${i + 1}/${result.invoiceCount} factures...`,
|
||||
});
|
||||
|
||||
// Check for duplicates
|
||||
const duplicate = await findDuplicateInvoice(
|
||||
invoiceData.supplierName,
|
||||
invoiceData.invoiceNumber,
|
||||
invoiceData.invoiceDate
|
||||
);
|
||||
|
||||
if (duplicate) {
|
||||
duplicatesCount++;
|
||||
duplicateDetails.push({
|
||||
supplierName: invoiceData.supplierName,
|
||||
invoiceNumber: invoiceData.invoiceNumber,
|
||||
invoiceDate: invoiceData.invoiceDate,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Generate metadata JSON
|
||||
const metadataJson = generateMetadataJSON(invoiceData);
|
||||
const metadataKey = generateStorageKey(userId, `${fileName}-${i + 1}-metadata.json`);
|
||||
const { url: metadataUrl } = await localStoragePut(
|
||||
metadataKey,
|
||||
Buffer.from(metadataJson),
|
||||
"application/json"
|
||||
);
|
||||
|
||||
// Create invoice record
|
||||
await createInvoice({
|
||||
userId,
|
||||
sourceFileId: sourceFile.id,
|
||||
invoiceIndexInFile: i + 1,
|
||||
fileName: `${fileName} - Facture ${i + 1}`,
|
||||
fileKey: sourceFileKey,
|
||||
fileUrl: sourceFileUrl,
|
||||
supplierName: invoiceData.supplierName,
|
||||
invoiceNumber: invoiceData.invoiceNumber,
|
||||
invoiceDate: invoiceData.invoiceDate,
|
||||
deliveryNoteNumber: invoiceData.deliveryNoteNumber,
|
||||
orderNumber: invoiceData.orderNumber,
|
||||
totalAmount: invoiceData.totalAmount?.toString(),
|
||||
pageRange: invoiceData.pageRange,
|
||||
qualityScore: invoiceData.qualityScore,
|
||||
metadataFileKey: metadataKey,
|
||||
metadataFileUrl: metadataUrl,
|
||||
status: "completed",
|
||||
});
|
||||
|
||||
importedCount++;
|
||||
} catch (error: any) {
|
||||
errorsCount++;
|
||||
errorDetails.push({
|
||||
invoiceIndex: i + 1,
|
||||
error: error.message,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Update source file status
|
||||
await updateSourceFile(sourceFile.id, {
|
||||
processingStatus: "completed",
|
||||
processingProgress: `Terminé: ${importedCount} importée(s), ${duplicatesCount} doublon(s)`,
|
||||
});
|
||||
|
||||
// Create import log
|
||||
await createImportLog({
|
||||
userId,
|
||||
sourceFileId: sourceFile.id,
|
||||
fileName,
|
||||
totalInvoicesDetected: result.invoiceCount,
|
||||
invoicesImported: importedCount,
|
||||
duplicatesIgnored: duplicatesCount,
|
||||
errors: errorsCount,
|
||||
duplicateDetails: duplicateDetails.length > 0 ? JSON.stringify(duplicateDetails) : null,
|
||||
errorDetails: errorDetails.length > 0 ? JSON.stringify(errorDetails) : null,
|
||||
});
|
||||
|
||||
// Move file to processed folder
|
||||
try {
|
||||
const processedDir = path.join(folderPath, "processed");
|
||||
await fs.mkdir(processedDir, { recursive: true });
|
||||
const newPath = path.join(processedDir, fileName);
|
||||
await fs.rename(filePath, newPath);
|
||||
console.log(`[FolderImport] File moved to processed folder: ${newPath}`);
|
||||
} catch (error) {
|
||||
console.error(`[FolderImport] Failed to move file to processed folder:`, error);
|
||||
}
|
||||
|
||||
console.log(`[FolderImport] Successfully processed file: ${fileName}`);
|
||||
console.log(`[FolderImport] Results: ${importedCount} imported, ${duplicatesCount} duplicates, ${errorsCount} errors`);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
imported: importedCount,
|
||||
duplicates: duplicatesCount,
|
||||
errors: errorsCount,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error(`[FolderImport] Error processing file ${fileName}:`, error);
|
||||
return {
|
||||
success: false,
|
||||
imported: 0,
|
||||
duplicates: 0,
|
||||
errors: 1,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Scan folder for PDF files and process them
|
||||
*/
|
||||
async function scanAndProcessFolder(config: FolderImportConfig): Promise<void> {
|
||||
console.log(`[FolderImport] Scanning folder: ${config.folderPath} for user ${config.userId}`);
|
||||
|
||||
try {
|
||||
// Check if folder exists
|
||||
try {
|
||||
await fs.access(config.folderPath);
|
||||
} catch (error) {
|
||||
console.error(`[FolderImport] Folder does not exist: ${config.folderPath}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Read all files in the folder
|
||||
const files = await fs.readdir(config.folderPath);
|
||||
const pdfFiles = files.filter(file => file.toLowerCase().endsWith('.pdf'));
|
||||
|
||||
if (pdfFiles.length === 0) {
|
||||
console.log(`[FolderImport] No PDF files found in folder for user ${config.userId}`);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[FolderImport] Found ${pdfFiles.length} PDF file(s) for user ${config.userId}`);
|
||||
|
||||
// Get or create processed files set for this user
|
||||
if (!processedFiles.has(config.userId)) {
|
||||
processedFiles.set(config.userId, new Set());
|
||||
}
|
||||
const userProcessedFiles = processedFiles.get(config.userId)!;
|
||||
|
||||
// Track totals for notification
|
||||
let totalImported = 0;
|
||||
let totalDuplicates = 0;
|
||||
let totalErrors = 0;
|
||||
let totalInvoices = 0;
|
||||
let processedCount = 0;
|
||||
|
||||
// Process each PDF file
|
||||
for (const file of pdfFiles) {
|
||||
const filePath = path.join(config.folderPath, file);
|
||||
|
||||
// Skip if already processed
|
||||
if (userProcessedFiles.has(filePath)) {
|
||||
console.log(`[FolderImport] Skipping already processed file: ${file}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Process the file
|
||||
const result = await processFolderFile(config.userId, filePath, config.folderPath);
|
||||
|
||||
// Accumulate results
|
||||
if (result.success) {
|
||||
totalImported += result.imported;
|
||||
totalDuplicates += result.duplicates;
|
||||
totalErrors += result.errors;
|
||||
totalInvoices += result.imported + result.duplicates;
|
||||
processedCount++;
|
||||
}
|
||||
|
||||
// Mark as processed
|
||||
userProcessedFiles.add(filePath);
|
||||
}
|
||||
|
||||
// Send notification if any files were processed
|
||||
if (processedCount > 0) {
|
||||
await sendImportNotification(config.userId, {
|
||||
source: "folder",
|
||||
totalFiles: processedCount,
|
||||
totalInvoices,
|
||||
imported: totalImported,
|
||||
duplicates: totalDuplicates,
|
||||
errors: totalErrors,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`[FolderImport] Error scanning folder for user ${config.userId}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start folder import service for a user
|
||||
*/
|
||||
export async function startFolderImportService(userId: number): Promise<boolean> {
|
||||
try {
|
||||
// Get user's import settings
|
||||
const settings = await getImportSettingsByUser(userId);
|
||||
|
||||
if (!settings || settings.autoImportEnabled !== 1) {
|
||||
console.log(`[FolderImport] Auto import not enabled for user ${userId}`);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!settings.autoImportSourcePath) {
|
||||
console.log(`[FolderImport] Folder path not configured for user ${userId}`);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Stop existing service if running
|
||||
stopFolderImportService(userId);
|
||||
|
||||
const config: FolderImportConfig = {
|
||||
userId,
|
||||
folderPath: settings.autoImportSourcePath,
|
||||
frequency: settings.autoImportFrequency || 30,
|
||||
};
|
||||
|
||||
const frequencyMs = config.frequency * 60 * 1000; // Convert minutes to milliseconds
|
||||
|
||||
console.log(
|
||||
`[FolderImport] Starting folder import service for user ${userId} with frequency ${config.frequency} minutes`
|
||||
);
|
||||
|
||||
// Run immediately on start
|
||||
scanAndProcessFolder(config).catch((error) => {
|
||||
console.error(`[FolderImport] Error scanning folder for user ${userId}:`, error);
|
||||
});
|
||||
|
||||
// Set up interval for periodic scans
|
||||
const interval = setInterval(() => {
|
||||
scanAndProcessFolder(config).catch((error) => {
|
||||
console.error(`[FolderImport] Error scanning folder for user ${userId}:`, error);
|
||||
});
|
||||
}, frequencyMs);
|
||||
|
||||
// Note: We're not using chokidar watcher for now, just periodic scans
|
||||
// This is simpler and more reliable for the initial implementation
|
||||
const dummyWatcher = chokidar.watch(config.folderPath, { ignored: /processed/ });
|
||||
|
||||
activeWatchers.set(userId, { watcher: dummyWatcher, interval });
|
||||
console.log(`[FolderImport] Folder import service started for user ${userId}`);
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error(`[FolderImport] Error starting folder import service for user ${userId}:`, error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop folder import service for a user
|
||||
*/
|
||||
export function stopFolderImportService(userId: number): void {
|
||||
const service = activeWatchers.get(userId);
|
||||
if (service) {
|
||||
clearInterval(service.interval);
|
||||
service.watcher.close();
|
||||
activeWatchers.delete(userId);
|
||||
|
||||
// Clear processed files tracking
|
||||
processedFiles.delete(userId);
|
||||
|
||||
console.log(`[FolderImport] Folder import service stopped for user ${userId}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if folder import service is running for a user
|
||||
*/
|
||||
export function isFolderImportServiceRunning(userId: number): boolean {
|
||||
return activeWatchers.has(userId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop all folder import services
|
||||
*/
|
||||
export function stopAllFolderImportServices(): void {
|
||||
activeWatchers.forEach((service, userId) => {
|
||||
clearInterval(service.interval);
|
||||
service.watcher.close();
|
||||
console.log(`[FolderImport] Stopped folder import service for user ${userId}`);
|
||||
});
|
||||
activeWatchers.clear();
|
||||
processedFiles.clear();
|
||||
}
|
||||
Reference in New Issue
Block a user