demat-facturation/server/invoiceExtractor.ts

import { invokeLLM } from "./_core/llm";
import { PDFDocument } from "pdf-lib";
import { createLlmLog } from "./db";
import PDFParser from "pdf2json";

export interface ExtractedInvoiceData {
  supplierName: string | null;
  invoiceNumber: string | null;
  invoiceDate: Date | null;
  deliveryNoteNumber: string | null;
  orderNumber: string | null;
  totalAmount: number | null;
  pageRange: string;
  qualityScore: number; // 0-100
  extractedText: string | null; // Full text extracted from the invoice PDF
  isSubscription: boolean; // True if subscription keywords detected
}

export interface MultiInvoiceResult {
  pageCount: number;
  invoiceCount: number;
  invoices: ExtractedInvoiceData[];
}

/**
 * Extract text from PDF buffer using pdf2json
 */
async function extractTextFromPdf(pdfBuffer: Buffer): Promise<string> {
  return new Promise((resolve, reject) => {
    const pdfParser = new (PDFParser as any)(null, 1);
    
    pdfParser.on("pdfParser_dataError", (errData: any) => {
      console.error("Error parsing PDF:", errData.parserError);
      reject(new Error("Failed to parse PDF"));
    });
    
    pdfParser.on("pdfParser_dataReady", (pdfData: any) => {
      try {
        let text = "";
        
        // Extract text from all pages
        if (pdfData.Pages) {
          for (const page of pdfData.Pages) {
            if (page.Texts) {
              for (const textItem of page.Texts) {
                if (textItem.R) {
                  for (const run of textItem.R) {
                    if (run.T) {
                      try {
                        text += decodeURIComponent(run.T) + " ";
                      } catch (e) {
                        // If decodeURIComponent fails, use the raw text
                        text += run.T + " ";
                      }
                    }
                  }
                }
              }
              text += "\n";
            }
          }
        }
        
        console.log("[PDF] Text extracted, length:", text.length, "characters");
        resolve(text);
      } catch (error) {
        console.error("Error extracting text from PDF data:", error);
        reject(new Error("Failed to extract text from PDF"));
      }
    });
    
    pdfParser.parseBuffer(pdfBuffer);
  });
}

/**
 * Clean JSON response from LLM by removing markdown code blocks
 */
function cleanJsonResponse(content: string): string {
  let cleaned = content.trim();
  
  // Remove markdown code blocks
  cleaned = cleaned.replace(/^```(?:json)?\s*/i, "");
  cleaned = cleaned.replace(/\s*```$/, "");
  cleaned = cleaned.trim();
  
  // Try to extract JSON object or array
  const firstBrace = cleaned.indexOf("{");
  const firstBracket = cleaned.indexOf("[");
  
  let startIdx = -1;
  let startChar = "";
  
  if (firstBrace !== -1 && (firstBracket === -1 || firstBrace < firstBracket)) {
    startIdx = firstBrace;
    startChar = "{";
  } else if (firstBracket !== -1) {
    startIdx = firstBracket;
    startChar = "[";
  }
  
  if (startIdx === -1) {
    return cleaned;
  }
  
  const endChar = startChar === "{" ? "}" : "]";
  let depth = 0;
  let endIdx = -1;
  
  for (let i = startIdx; i < cleaned.length; i++) {
    if (cleaned[i] === startChar) depth++;
    if (cleaned[i] === endChar) {
      depth--;
      if (depth === 0) {
        endIdx = i;
        break;
      }
    }
  }
  
  if (endIdx !== -1) {
    cleaned = cleaned.substring(startIdx, endIdx + 1);
  }
  
  return cleaned;
}

/**
 * Extract invoice data using Mistral AI
 * Hybrid approach: OCR to extract text, then LLM to parse and extract structured data
 */
export async function extractInvoicesWithMistral(
  pdfBuffer: Buffer,
  userId: number,
  sourceFileId: number,
  model: string = "mistral-large-latest",
  customKeywords?: {
    invoiceNumber?: string | null;
    deliveryNote?: string | null;
    orderNumber?: string | null;
    supplier?: string | null;
    totalAmount?: string | null;
    subscription?: string | null;
  }
): Promise<MultiInvoiceResult> {
  // Load user's field configuration
  const { getLlmFieldsConfigByUser } = await import("./db");
  const fieldsConfig = await getLlmFieldsConfigByUser(userId);
  
  // Build quality score instruction based on required fields
  const requiredFields = fieldsConfig.filter(f => f.isRequired === 1);
  const optionalFields = fieldsConfig.filter(f => f.isRequired === 0);
  
  let qualityScoreInstruction = "- qualityScore: Score de qualité de l'extraction de 0 à 100";
  if (requiredFields.length > 0) {
    const requiredFieldNames = requiredFields.map(f => f.displayName).join(", ");
    qualityScoreInstruction += ` (100 = tous les champs obligatoires trouvés: ${requiredFieldNames})`;
  }
  if (optionalFields.length > 0) {
    const optionalFieldNames = optionalFields.map(f => f.displayName).join(", ");
    qualityScoreInstruction += `. Champs optionnels (n'affectent pas le score): ${optionalFieldNames}`;
  }
  const startTime = Date.now();
  
  try {
    console.log("[Mistral] Starting invoice extraction...");
    
    // Extract text from PDF
    const pdfText = await extractTextFromPdf(pdfBuffer);
    
    // Get PDF page count
    const pdfDoc = await PDFDocument.load(pdfBuffer);
    const pageCount = pdfDoc.getPageCount();
    
    console.log(`[Mistral] PDF has ${pageCount} pages`);
    
    // Build custom keywords hint
    let keywordsHint = "";
    if (customKeywords) {
      const hints = [];
      if (customKeywords.invoiceNumber) hints.push(`Numéro de facture: ${customKeywords.invoiceNumber}`);
      if (customKeywords.deliveryNote) hints.push(`Bon de livraison: ${customKeywords.deliveryNote}`);
      if (customKeywords.orderNumber) hints.push(`Numéro de commande: ${customKeywords.orderNumber}`);
      if (customKeywords.supplier) hints.push(`Fournisseur: ${customKeywords.supplier}`);
      if (customKeywords.totalAmount) hints.push(`Montant total: ${customKeywords.totalAmount}`);
      if (customKeywords.subscription) hints.push(`Abonnement: ${customKeywords.subscription}`);
      
      if (hints.length > 0) {
        keywordsHint = `\n\nMots-clés personnalisés à rechercher:\n${hints.join("\n")}`;
      }
    }
    
    // Build subscription detection instruction
    let subscriptionInstruction = "- isSubscription: false (par défaut)";
    if (customKeywords?.subscription && customKeywords.subscription.trim()) {
      subscriptionInstruction = `- isSubscription: true UNIQUEMENT si la facture contient l'un de ces mots-clés exacts: ${customKeywords.subscription}, false sinon`;
    }
    
    // Prepare prompt for LLM
    const prompt = `Tu es un expert en extraction de données de factures. Analyse ce document PDF et extrais toutes les factures qu'il contient.

Pour chaque facture trouvée, extrais les informations suivantes:
- supplierName: Nom du fournisseur/vendeur
- invoiceNumber: Numéro de la facture
- invoiceDate: Date de la facture (format ISO 8601: YYYY-MM-DD)
- deliveryNoteNumber: Numéro du bon de livraison (si présent)
- orderNumber: Numéro de commande client (si présent)
- totalAmount: Montant total TTC (nombre décimal)
- pageRange: Plage de pages de cette facture (ex: "1-2" ou "5")
${qualityScoreInstruction}
- extractedText: Texte complet extrait de la facture (tout le texte visible sur les pages de cette facture)
${subscriptionInstruction}${keywordsHint}

Réponds UNIQUEMENT avec un objet JSON valide au format suivant:
{
  "pageCount": ${pageCount},
  "invoiceCount": <nombre de factures détectées>,
  "invoices": [
    {
      "supplierName": "...",
      "invoiceNumber": "...",
      "invoiceDate": "YYYY-MM-DD",
      "deliveryNoteNumber": "...",
      "orderNumber": "...",
      "totalAmount": 123.45,
      "pageRange": "1-2",
      "qualityScore": 85,
      "extractedText": "Texte complet de la facture...",
      "isSubscription": false
    }
  ]
}

Si une information n'est pas trouvée, utilise null. Ne retourne AUCUN texte en dehors du JSON.`;
    
    // Call Mistral LLM with extracted text
    const fullPrompt = `${prompt}\n\nTexte extrait du PDF:\n${pdfText}`;
    const response = await invokeLLM({
      messages: [
        {
          role: "user",
          content: fullPrompt,
        },
      ],
    });
    
    const rawResponse = typeof response.choices[0]?.message?.content === "string" 
      ? response.choices[0].message.content 
      : JSON.stringify(response.choices[0]?.message?.content || "");
    const processingTimeMs = Date.now() - startTime;
    
    console.log("[Mistral] Raw response received:", rawResponse.substring(0, 200));
    
    // Clean and parse response
    const cleanedResponse = cleanJsonResponse(rawResponse);
    
    let result: MultiInvoiceResult;
    try {
      result = JSON.parse(cleanedResponse);
    } catch (parseError) {
      console.error("[Mistral] Failed to parse JSON:", parseError);
      console.error("[Mistral] Cleaned response:", cleanedResponse);
      
      // Log error to database
      await createLlmLog({
        userId,
        sourceFileId,
        operation: "extraction",
        model,
        promptSent: prompt,
        rawResponse: rawResponse.substring(0, 10000),
        cleanedResponse: cleanedResponse.substring(0, 10000),
        success: 0,
        errorMessage: `JSON parse error: ${parseError}`,
        processingTimeMs,
      });
      
      throw new Error("Failed to parse LLM response as JSON");
    }
    
    // Log successful extraction
    await createLlmLog({
      userId,
      sourceFileId,
      operation: "extraction",
      model,
      promptSent: prompt.substring(0, 10000),
      rawResponse: rawResponse.substring(0, 10000),
      cleanedResponse: cleanedResponse.substring(0, 10000),
      success: 1,
      processingTimeMs,
    });
    
    // Convert date strings to Date objects
    result.invoices = result.invoices.map((inv) => ({
      ...inv,
      invoiceDate: inv.invoiceDate ? new Date(inv.invoiceDate) : null,
    }));
    
    console.log(`[Mistral] Successfully extracted ${result.invoiceCount} invoice(s)`);
    
    return result;
  } catch (error) {
    console.error("[Mistral] Extraction failed:", error);
    throw error;
  }
}

/**
 * Generate metadata JSON for an invoice
 */
export function generateMetadataJSON(invoice: ExtractedInvoiceData): string {
  return JSON.stringify(
    {
      supplierName: invoice.supplierName,
      invoiceNumber: invoice.invoiceNumber,
      invoiceDate: invoice.invoiceDate?.toISOString(),
      deliveryNoteNumber: invoice.deliveryNoteNumber,
      orderNumber: invoice.orderNumber,
      totalAmount: invoice.totalAmount,
      pageRange: invoice.pageRange,
      qualityScore: invoice.qualityScore,
      extractedAt: new Date().toISOString(),
    },
    null,
    2
  );
}