demat-facturation/server/invoiceExtractor.ts

import { invokeLLM } from "./_core/llm";
import { PDFDocument } from "pdf-lib";
import { createLlmLog } from "./db";

export interface ExtractedInvoiceData {
  supplierName: string | null;
  invoiceNumber: string | null;
  invoiceDate: Date | null;
  deliveryNoteNumber: string | null;
  orderNumber: string | null;
  totalAmount: number | null;
  pageRange: string;
  qualityScore: number; // 0-100
  extractedText: string | null; // Full text extracted from the invoice PDF
}

export interface MultiInvoiceResult {
  pageCount: number;
  invoiceCount: number;
  invoices: ExtractedInvoiceData[];
}

/**
 * Convert PDF buffer to base64 data URI for Mistral API processing
 */
function convertPdfToBase64(pdfBuffer: Buffer): string {
  try {
    const base64Pdf = pdfBuffer.toString("base64");
    const dataUri = `data:application/pdf;base64,${base64Pdf}`;
    console.log("[Mistral] PDF converted to base64, size:", Math.round(base64Pdf.length / 1024), "KB");
    return dataUri;
  } catch (error) {
    console.error("Error converting PDF to base64:", error);
    throw new Error("Failed to convert PDF to base64");
  }
}

/**
 * Clean JSON response from LLM by removing markdown code blocks
 */
function cleanJsonResponse(content: string): string {
  let cleaned = content.trim();
  
  // Remove markdown code blocks
  cleaned = cleaned.replace(/^```(?:json)?\s*/i, "");
  cleaned = cleaned.replace(/\s*```$/, "");
  cleaned = cleaned.trim();
  
  // Try to extract JSON object or array
  const firstBrace = cleaned.indexOf("{");
  const firstBracket = cleaned.indexOf("[");
  
  let startIdx = -1;
  let startChar = "";
  
  if (firstBrace !== -1 && (firstBracket === -1 || firstBrace < firstBracket)) {
    startIdx = firstBrace;
    startChar = "{";
  } else if (firstBracket !== -1) {
    startIdx = firstBracket;
    startChar = "[";
  }
  
  if (startIdx === -1) {
    return cleaned;
  }
  
  const endChar = startChar === "{" ? "}" : "]";
  let depth = 0;
  let endIdx = -1;
  
  for (let i = startIdx; i < cleaned.length; i++) {
    if (cleaned[i] === startChar) depth++;
    if (cleaned[i] === endChar) {
      depth--;
      if (depth === 0) {
        endIdx = i;
        break;
      }
    }
  }
  
  if (endIdx !== -1) {
    cleaned = cleaned.substring(startIdx, endIdx + 1);
  }
  
  return cleaned;
}

/**
 * Extract invoice data using Mistral AI
 * Hybrid approach: OCR to extract text, then LLM to parse and extract structured data
 */
export async function extractInvoicesWithMistral(
  pdfBuffer: Buffer,
  userId: number,
  sourceFileId: number,
  model: string = "mistral-large-latest",
  customKeywords?: {
    invoiceNumber?: string | null;
    deliveryNote?: string | null;
    orderNumber?: string | null;
    supplier?: string | null;
    totalAmount?: string | null;
  }
): Promise<MultiInvoiceResult> {
  const startTime = Date.now();
  
  try {
    console.log("[Mistral] Starting invoice extraction...");
    
    // Convert PDF to base64
    const pdfDataUri = convertPdfToBase64(pdfBuffer);
    
    // Get PDF page count
    const pdfDoc = await PDFDocument.load(pdfBuffer);
    const pageCount = pdfDoc.getPageCount();
    
    console.log(`[Mistral] PDF has ${pageCount} pages`);
    
    // Build custom keywords hint
    let keywordsHint = "";
    if (customKeywords) {
      const hints = [];
      if (customKeywords.invoiceNumber) hints.push(`Numéro de facture: ${customKeywords.invoiceNumber}`);
      if (customKeywords.deliveryNote) hints.push(`Bon de livraison: ${customKeywords.deliveryNote}`);
      if (customKeywords.orderNumber) hints.push(`Numéro de commande: ${customKeywords.orderNumber}`);
      if (customKeywords.supplier) hints.push(`Fournisseur: ${customKeywords.supplier}`);
      if (customKeywords.totalAmount) hints.push(`Montant total: ${customKeywords.totalAmount}`);
      
      if (hints.length > 0) {
        keywordsHint = `\n\nMots-clés personnalisés à rechercher:\n${hints.join("\n")}`;
      }
    }
    
    // Prepare prompt for LLM
    const prompt = `Tu es un expert en extraction de données de factures. Analyse ce document PDF et extrais toutes les factures qu'il contient.

Pour chaque facture trouvée, extrais les informations suivantes:
- supplierName: Nom du fournisseur/vendeur
- invoiceNumber: Numéro de la facture
- invoiceDate: Date de la facture (format ISO 8601: YYYY-MM-DD)
- deliveryNoteNumber: Numéro du bon de livraison (si présent)
- orderNumber: Numéro de commande client (si présent)
- totalAmount: Montant total TTC (nombre décimal)
- pageRange: Plage de pages de cette facture (ex: "1-2" ou "5")
- qualityScore: Score de qualité de l'extraction de 0 à 100 (100 = toutes les informations trouvées et claires)
- extractedText: Texte complet extrait de la facture (tout le texte visible sur les pages de cette facture)${keywordsHint}

Réponds UNIQUEMENT avec un objet JSON valide au format suivant:
{
  "pageCount": ${pageCount},
  "invoiceCount": <nombre de factures détectées>,
  "invoices": [
    {
      "supplierName": "...",
      "invoiceNumber": "...",
      "invoiceDate": "YYYY-MM-DD",
      "deliveryNoteNumber": "...",
      "orderNumber": "...",
      "totalAmount": 123.45,
      "pageRange": "1-2",
      "qualityScore": 85,
      "extractedText": "Texte complet de la facture..."
    }
  ]
}

Si une information n'est pas trouvée, utilise null. Ne retourne AUCUN texte en dehors du JSON.`;
    
    // Call Mistral LLM with PDF
    const response = await invokeLLM({
      messages: [
        {
          role: "user",
          content: [
            { type: "text", text: prompt },
            { type: "file_url", file_url: { url: pdfDataUri, mime_type: "application/pdf" } },
          ],
        },
      ],
    });
    
    const rawResponse = typeof response.choices[0]?.message?.content === "string" 
      ? response.choices[0].message.content 
      : JSON.stringify(response.choices[0]?.message?.content || "");
    const processingTimeMs = Date.now() - startTime;
    
    console.log("[Mistral] Raw response received:", rawResponse.substring(0, 200));
    
    // Clean and parse response
    const cleanedResponse = cleanJsonResponse(rawResponse);
    
    let result: MultiInvoiceResult;
    try {
      result = JSON.parse(cleanedResponse);
    } catch (parseError) {
      console.error("[Mistral] Failed to parse JSON:", parseError);
      console.error("[Mistral] Cleaned response:", cleanedResponse);
      
      // Log error to database
      await createLlmLog({
        userId,
        sourceFileId,
        operation: "extraction",
        model,
        promptSent: prompt,
        rawResponse: rawResponse.substring(0, 10000),
        cleanedResponse: cleanedResponse.substring(0, 10000),
        success: 0,
        errorMessage: `JSON parse error: ${parseError}`,
        processingTimeMs,
      });
      
      throw new Error("Failed to parse LLM response as JSON");
    }
    
    // Log successful extraction
    await createLlmLog({
      userId,
      sourceFileId,
      operation: "extraction",
      model,
      promptSent: prompt.substring(0, 10000),
      rawResponse: rawResponse.substring(0, 10000),
      cleanedResponse: cleanedResponse.substring(0, 10000),
      success: 1,
      processingTimeMs,
    });
    
    // Convert date strings to Date objects
    result.invoices = result.invoices.map((inv) => ({
      ...inv,
      invoiceDate: inv.invoiceDate ? new Date(inv.invoiceDate) : null,
    }));
    
    console.log(`[Mistral] Successfully extracted ${result.invoiceCount} invoice(s)`);
    
    return result;
  } catch (error) {
    console.error("[Mistral] Extraction failed:", error);
    throw error;
  }
}

/**
 * Generate metadata JSON for an invoice
 */
export function generateMetadataJSON(invoice: ExtractedInvoiceData): string {
  return JSON.stringify(
    {
      supplierName: invoice.supplierName,
      invoiceNumber: invoice.invoiceNumber,
      invoiceDate: invoice.invoiceDate?.toISOString(),
      deliveryNoteNumber: invoice.deliveryNoteNumber,
      orderNumber: invoice.orderNumber,
      totalAmount: invoice.totalAmount,
      pageRange: invoice.pageRange,
      qualityScore: invoice.qualityScore,
      extractedAt: new Date().toISOString(),
    },
    null,
    2
  );
}