423 lines
14 KiB
TypeScript
423 lines
14 KiB
TypeScript
import chokidar from "chokidar";
|
|
import fs from "fs/promises";
|
|
import path from "path";
|
|
import {
|
|
getImportSettingsByUser,
|
|
createSourceFile,
|
|
getSourceFileByContentHash,
|
|
updateSourceFile,
|
|
getUserSettings,
|
|
findDuplicateInvoice,
|
|
createInvoice,
|
|
createImportLog,
|
|
} from "./db";
|
|
import { extractInvoicesWithMistral, generateMetadataJSON } from "./invoiceExtractor";
|
|
import { localStorageDelete, localStoragePut, generateStorageKey } from "./localStorage";
|
|
import { calculateFileSha256 } from "./fileFingerprint";
|
|
import { sendImportNotification } from "./notificationService";
|
|
|
|
interface FolderImportConfig {
|
|
userId: number;
|
|
folderPath: string;
|
|
frequency: number; // in minutes
|
|
}
|
|
|
|
// Store active watchers for each user
|
|
const activeWatchers = new Map<number, { watcher: any; interval: NodeJS.Timeout }>();
|
|
|
|
// Track processed files to avoid reprocessing
|
|
const processedFiles = new Map<number, Set<string>>();
|
|
|
|
/**
|
|
* Process a single PDF file from the watched folder
|
|
*/
|
|
async function processFolderFile(
|
|
userId: number,
|
|
filePath: string,
|
|
folderPath: string
|
|
): Promise<{ success: boolean; imported: number; duplicates: number; errors: number }> {
|
|
const fileName = path.basename(filePath);
|
|
console.log(`[FolderImport] Processing file: ${fileName} for user ${userId}`);
|
|
|
|
try {
|
|
// Read the file
|
|
const fileBuffer = await fs.readFile(filePath);
|
|
console.log(`[FolderImport] File size: ${fileBuffer.length} bytes`);
|
|
|
|
const contentHash = calculateFileSha256(fileBuffer);
|
|
const existingSource = await getSourceFileByContentHash(contentHash);
|
|
if (existingSource) {
|
|
console.log(`[FolderImport] PDF déjà importé, fichier ignoré: ${fileName}`);
|
|
return { success: true, imported: 0, duplicates: 1, errors: 0 };
|
|
}
|
|
|
|
// Store source file
|
|
const sourceFileKey = generateStorageKey(userId, fileName);
|
|
console.log(`[FolderImport] Generated storage key: ${sourceFileKey}`);
|
|
|
|
let sourceFileUrl: string;
|
|
try {
|
|
const result = await localStoragePut(sourceFileKey, fileBuffer, "application/pdf");
|
|
sourceFileUrl = result.url;
|
|
console.log(`[FolderImport] File stored successfully at: ${sourceFileUrl}`);
|
|
} catch (error) {
|
|
console.error(`[FolderImport] FAILED to store file:`, error);
|
|
throw new Error("Failed to store PDF file");
|
|
}
|
|
|
|
// Create source file record
|
|
let sourceFile;
|
|
try {
|
|
sourceFile = await createSourceFile({
|
|
userId,
|
|
fileName,
|
|
fileKey: sourceFileKey,
|
|
fileUrl: sourceFileUrl,
|
|
contentHash,
|
|
processingStatus: "processing",
|
|
});
|
|
} catch (error: any) {
|
|
if (error?.code === "ER_DUP_ENTRY" || error?.errno === 1062) {
|
|
await localStorageDelete(sourceFileKey).catch(() => undefined);
|
|
return { success: true, imported: 0, duplicates: 1, errors: 0 };
|
|
}
|
|
throw error;
|
|
}
|
|
|
|
console.log(`[FolderImport] Source file record created with ID: ${sourceFile.id}`);
|
|
|
|
// Get user settings for custom keywords
|
|
const settings = await getUserSettings(userId);
|
|
const customKeywords = settings ? {
|
|
invoiceNumber: settings.invoiceNumberKeywords,
|
|
deliveryNote: settings.deliveryNoteKeywords,
|
|
orderNumber: settings.orderNumberKeywords,
|
|
supplier: settings.supplierKeywords,
|
|
totalAmount: settings.totalAmountKeywords,
|
|
subscription: settings.subscriptionKeywords,
|
|
recipient: settings.recipientKeywords,
|
|
} : undefined;
|
|
|
|
const model = settings?.llmModel || "mistral-large-latest";
|
|
|
|
// Extract invoices
|
|
console.log(`[FolderImport] Starting invoice extraction...`);
|
|
const result = await extractInvoicesWithMistral(
|
|
fileBuffer,
|
|
userId,
|
|
sourceFile.id,
|
|
model,
|
|
customKeywords
|
|
);
|
|
|
|
console.log(`[FolderImport] Extraction complete: ${result.invoiceCount} invoice(s) detected`);
|
|
|
|
// Update source file with total count
|
|
await updateSourceFile(sourceFile.id, {
|
|
totalInvoicesDetected: result.invoiceCount,
|
|
processingProgress: `Extraction ${result.invoiceCount} facture(s) détectée(s)`,
|
|
});
|
|
|
|
// Process each invoice
|
|
let importedCount = 0;
|
|
let duplicatesCount = 0;
|
|
let errorsCount = 0;
|
|
const duplicateDetails: any[] = [];
|
|
const errorDetails: any[] = [];
|
|
|
|
for (let i = 0; i < result.invoices.length; i++) {
|
|
const invoiceData = result.invoices[i]!;
|
|
|
|
try {
|
|
// Update progress
|
|
await updateSourceFile(sourceFile.id, {
|
|
processingProgress: `Extraction ${i + 1}/${result.invoiceCount} factures...`,
|
|
});
|
|
|
|
// Check for duplicates (numéro de facture + montant)
|
|
const duplicate = await findDuplicateInvoice(
|
|
invoiceData.invoiceNumber,
|
|
invoiceData.totalAmount?.toString() ?? null,
|
|
userId
|
|
);
|
|
|
|
if (duplicate) {
|
|
duplicatesCount++;
|
|
duplicateDetails.push({
|
|
supplierName: invoiceData.supplierName,
|
|
invoiceNumber: invoiceData.invoiceNumber,
|
|
totalAmount: invoiceData.totalAmount,
|
|
});
|
|
continue;
|
|
}
|
|
|
|
// Generate metadata JSON
|
|
const metadataJson = generateMetadataJSON(invoiceData);
|
|
const metadataKey = generateStorageKey(userId, `${fileName}-${i + 1}-metadata.json`);
|
|
const { url: metadataUrl } = await localStoragePut(
|
|
metadataKey,
|
|
Buffer.from(metadataJson),
|
|
"application/json"
|
|
);
|
|
|
|
// Create invoice record
|
|
await createInvoice({
|
|
userId,
|
|
sourceFileId: sourceFile.id,
|
|
invoiceIndexInFile: i + 1,
|
|
fileName: `${fileName} - Facture ${i + 1}`,
|
|
fileKey: sourceFileKey,
|
|
fileUrl: sourceFileUrl,
|
|
supplierName: invoiceData.supplierName,
|
|
invoiceNumber: invoiceData.invoiceNumber,
|
|
invoiceDate: invoiceData.invoiceDate,
|
|
deliveryNoteNumber: invoiceData.deliveryNoteNumber,
|
|
orderNumber: invoiceData.orderNumber,
|
|
totalAmount: invoiceData.totalAmount?.toString(),
|
|
recipientName: invoiceData.recipientName,
|
|
pageRange: invoiceData.pageRange,
|
|
qualityScore: invoiceData.qualityScore,
|
|
extractedText: invoiceData.extractedText,
|
|
metadataFileKey: metadataKey,
|
|
metadataFileUrl: metadataUrl,
|
|
status: "completed",
|
|
});
|
|
|
|
importedCount++;
|
|
} catch (error: any) {
|
|
errorsCount++;
|
|
errorDetails.push({
|
|
invoiceIndex: i + 1,
|
|
error: error.message,
|
|
});
|
|
}
|
|
}
|
|
|
|
// Update source file status
|
|
await updateSourceFile(sourceFile.id, {
|
|
processingStatus: "completed",
|
|
processingProgress: `Terminé: ${importedCount} importée(s), ${duplicatesCount} doublon(s)`,
|
|
});
|
|
|
|
// Create import log
|
|
await createImportLog({
|
|
userId,
|
|
sourceFileId: sourceFile.id,
|
|
fileName,
|
|
totalInvoicesDetected: result.invoiceCount,
|
|
invoicesImported: importedCount,
|
|
duplicatesIgnored: duplicatesCount,
|
|
errors: errorsCount,
|
|
duplicateDetails: duplicateDetails.length > 0 ? JSON.stringify(duplicateDetails) : null,
|
|
errorDetails: errorDetails.length > 0 ? JSON.stringify(errorDetails) : null,
|
|
importSource: "folder",
|
|
});
|
|
|
|
// Move file to processed folder
|
|
try {
|
|
const processedDir = path.join(folderPath, "processed");
|
|
await fs.mkdir(processedDir, { recursive: true });
|
|
const newPath = path.join(processedDir, fileName);
|
|
await fs.rename(filePath, newPath);
|
|
console.log(`[FolderImport] File moved to processed folder: ${newPath}`);
|
|
} catch (error) {
|
|
console.error(`[FolderImport] Failed to move file to processed folder:`, error);
|
|
}
|
|
|
|
console.log(`[FolderImport] Successfully processed file: ${fileName}`);
|
|
console.log(`[FolderImport] Results: ${importedCount} imported, ${duplicatesCount} duplicates, ${errorsCount} errors`);
|
|
|
|
return {
|
|
success: true,
|
|
imported: importedCount,
|
|
duplicates: duplicatesCount,
|
|
errors: errorsCount,
|
|
};
|
|
} catch (error) {
|
|
console.error(`[FolderImport] Error processing file ${fileName}:`, error);
|
|
return {
|
|
success: false,
|
|
imported: 0,
|
|
duplicates: 0,
|
|
errors: 1,
|
|
};
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Scan folder for PDF files and process them
|
|
*/
|
|
async function scanAndProcessFolder(config: FolderImportConfig): Promise<void> {
|
|
console.log(`[FolderImport] Scanning folder: ${config.folderPath} for user ${config.userId}`);
|
|
|
|
try {
|
|
// Check if folder exists
|
|
try {
|
|
await fs.access(config.folderPath);
|
|
} catch (error) {
|
|
console.error(`[FolderImport] Folder does not exist: ${config.folderPath}`);
|
|
return;
|
|
}
|
|
|
|
// Read all files in the folder
|
|
const files = await fs.readdir(config.folderPath);
|
|
const pdfFiles = files.filter(file => file.toLowerCase().endsWith('.pdf'));
|
|
|
|
if (pdfFiles.length === 0) {
|
|
console.log(`[FolderImport] No PDF files found in folder for user ${config.userId}`);
|
|
return;
|
|
}
|
|
|
|
console.log(`[FolderImport] Found ${pdfFiles.length} PDF file(s) for user ${config.userId}`);
|
|
|
|
// Get or create processed files set for this user
|
|
if (!processedFiles.has(config.userId)) {
|
|
processedFiles.set(config.userId, new Set());
|
|
}
|
|
const userProcessedFiles = processedFiles.get(config.userId)!;
|
|
|
|
// Track totals for notification
|
|
let totalImported = 0;
|
|
let totalDuplicates = 0;
|
|
let totalErrors = 0;
|
|
let totalInvoices = 0;
|
|
let processedCount = 0;
|
|
|
|
// Process each PDF file
|
|
for (const file of pdfFiles) {
|
|
const filePath = path.join(config.folderPath, file);
|
|
|
|
// Skip if already processed
|
|
if (userProcessedFiles.has(filePath)) {
|
|
console.log(`[FolderImport] Skipping already processed file: ${file}`);
|
|
continue;
|
|
}
|
|
|
|
// Process the file
|
|
const result = await processFolderFile(config.userId, filePath, config.folderPath);
|
|
|
|
// Accumulate results
|
|
if (result.success) {
|
|
totalImported += result.imported;
|
|
totalDuplicates += result.duplicates;
|
|
totalErrors += result.errors;
|
|
totalInvoices += result.imported + result.duplicates;
|
|
processedCount++;
|
|
}
|
|
|
|
// Mark as processed
|
|
userProcessedFiles.add(filePath);
|
|
}
|
|
|
|
// Send notification if any files were processed
|
|
if (processedCount > 0) {
|
|
await sendImportNotification(config.userId, {
|
|
source: "folder",
|
|
totalFiles: processedCount,
|
|
totalInvoices,
|
|
imported: totalImported,
|
|
duplicates: totalDuplicates,
|
|
errors: totalErrors,
|
|
});
|
|
}
|
|
} catch (error) {
|
|
console.error(`[FolderImport] Error scanning folder for user ${config.userId}:`, error);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Start folder import service for a user
|
|
*/
|
|
export async function startFolderImportService(userId: number): Promise<boolean> {
|
|
try {
|
|
// Get user's import settings
|
|
const settings = await getImportSettingsByUser(userId);
|
|
|
|
if (!settings || settings.autoImportEnabled !== 1) {
|
|
console.log(`[FolderImport] Auto import not enabled for user ${userId}`);
|
|
return false;
|
|
}
|
|
|
|
if (!settings.autoImportSourcePath) {
|
|
console.log(`[FolderImport] Folder path not configured for user ${userId}`);
|
|
return false;
|
|
}
|
|
|
|
// Stop existing service if running
|
|
stopFolderImportService(userId);
|
|
|
|
const config: FolderImportConfig = {
|
|
userId,
|
|
folderPath: settings.autoImportSourcePath,
|
|
frequency: settings.autoImportFrequency || 30,
|
|
};
|
|
|
|
const frequencyMs = config.frequency * 60 * 1000; // Convert minutes to milliseconds
|
|
|
|
console.log(
|
|
`[FolderImport] Starting folder import service for user ${userId} with frequency ${config.frequency} minutes`
|
|
);
|
|
|
|
// Run immediately on start
|
|
scanAndProcessFolder(config).catch((error) => {
|
|
console.error(`[FolderImport] Error scanning folder for user ${userId}:`, error);
|
|
});
|
|
|
|
// Set up interval for periodic scans
|
|
const interval = setInterval(() => {
|
|
scanAndProcessFolder(config).catch((error) => {
|
|
console.error(`[FolderImport] Error scanning folder for user ${userId}:`, error);
|
|
});
|
|
}, frequencyMs);
|
|
|
|
// Note: We're not using chokidar watcher for now, just periodic scans
|
|
// This is simpler and more reliable for the initial implementation
|
|
const dummyWatcher = chokidar.watch(config.folderPath, { ignored: /processed/ });
|
|
|
|
activeWatchers.set(userId, { watcher: dummyWatcher, interval });
|
|
console.log(`[FolderImport] Folder import service started for user ${userId}`);
|
|
|
|
return true;
|
|
} catch (error) {
|
|
console.error(`[FolderImport] Error starting folder import service for user ${userId}:`, error);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Stop folder import service for a user
|
|
*/
|
|
export function stopFolderImportService(userId: number): void {
|
|
const service = activeWatchers.get(userId);
|
|
if (service) {
|
|
clearInterval(service.interval);
|
|
service.watcher.close();
|
|
activeWatchers.delete(userId);
|
|
|
|
// Clear processed files tracking
|
|
processedFiles.delete(userId);
|
|
|
|
console.log(`[FolderImport] Folder import service stopped for user ${userId}`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check if folder import service is running for a user
|
|
*/
|
|
export function isFolderImportServiceRunning(userId: number): boolean {
|
|
return activeWatchers.has(userId);
|
|
}
|
|
|
|
/**
|
|
* Stop all folder import services
|
|
*/
|
|
export function stopAllFolderImportServices(): void {
|
|
activeWatchers.forEach((service, userId) => {
|
|
clearInterval(service.interval);
|
|
service.watcher.close();
|
|
console.log(`[FolderImport] Stopped folder import service for user ${userId}`);
|
|
});
|
|
activeWatchers.clear();
|
|
processedFiles.clear();
|
|
}
|