Checkpoint: Ajout du système de connecteurs web : table webImportSources, CRUD tRPC, page WebImportSources.tsx, endpoint /api/web-import/push-invoice, script cron SFR (scripts/web-import/sfr-connector.mjs)

This commit is contained in:
Manus
2026-07-30 13:53:07 +00:00
parent bcb307bde1
commit 711ce6b83a
13 changed files with 3314 additions and 2 deletions

View File

@@ -21,6 +21,7 @@ import BapHistory from "./pages/BapHistory";
import ImportReport from "./pages/ImportReport";
import LearningSettings from "./pages/LearningSettings";
import VentilationFreePro from "./pages/VentilationFreePro";
import WebImportSources from "./pages/WebImportSources";
function Router() {
return (
@@ -42,6 +43,7 @@ function Router() {
<Route path="/import-report" component={ImportReport} />
<Route path="/learning-settings" component={LearningSettings} />
<Route path="/ventilation-freepro" component={VentilationFreePro} />
<Route path="/web-import-sources" component={WebImportSources} />
<Route path="/404" component={NotFound} />
<Route component={NotFound} />
</Switch>

View File

@@ -25,7 +25,7 @@ import {
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible";
import { getLoginUrl } from "@/const";
import { useIsMobile } from "@/hooks/useMobile";
import { LayoutDashboard, LogOut, PanelLeft, Users, FileText, Upload, History, Settings, Download, List, Zap, ChevronDown, Receipt, Cog, ClipboardList, CheckSquare, Brain, BarChart2, BarChart3 } from "lucide-react";
import { LayoutDashboard, LogOut, PanelLeft, Users, FileText, Upload, History, Settings, Download, List, Zap, ChevronDown, Receipt, Cog, ClipboardList, CheckSquare, Brain, BarChart2, BarChart3, Globe } from "lucide-react";
import { CSSProperties, useEffect, useRef, useState } from "react";
import { useLocation } from "wouter";
import { DashboardLayoutSkeleton } from './DashboardLayoutSkeleton';
@@ -75,6 +75,7 @@ const menuStructure: MenuItem[] = [
{ icon: List, label: "Administration des listes", path: "/lists-admin" },
{ icon: Zap, label: "Automatismes", path: "/automation-rules" },
{ icon: Brain, label: "Apprentissages IA", path: "/learning-settings" },
{ icon: Globe, label: "Connecteurs web", path: "/web-import-sources" },
{ icon: Users, label: "Utilisateurs", path: "/users", adminOnly: true },
],
},

View File

@@ -0,0 +1,404 @@
import { useState } from "react";
import { trpc } from "@/lib/trpc";
import DashboardLayout from "@/components/DashboardLayout";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { Switch } from "@/components/ui/switch";
import { Badge } from "@/components/ui/badge";
import {
Dialog,
DialogContent,
DialogHeader,
DialogTitle,
DialogFooter,
} from "@/components/ui/dialog";
import {
AlertDialog,
AlertDialogAction,
AlertDialogCancel,
AlertDialogContent,
AlertDialogDescription,
AlertDialogFooter,
AlertDialogHeader,
AlertDialogTitle,
} from "@/components/ui/alert-dialog";
import { Globe, Plus, Pencil, Trash2, Key, CheckCircle, XCircle, Clock, RefreshCw } from "lucide-react";
import { toast } from "sonner";
const CONNECTOR_TYPES = [
{ value: "sfr", label: "SFR Pro", url: "https://www.sfr.fr/mon-espace-client/" },
{ value: "orange", label: "Orange Pro", url: "https://espaceclient.orange.fr/" },
{ value: "bouygues", label: "Bouygues Telecom", url: "https://www.bouyguestelecom.fr/mon-compte/" },
{ value: "free", label: "Free Pro", url: "https://pro.free.fr/" },
{ value: "starlink", label: "Starlink", url: "https://www.starlink.com/account/" },
{ value: "custom", label: "Autre (personnalisé)", url: "" },
];
const FREQUENCY_LABELS: Record<string, string> = {
manual: "Manuel",
daily: "Quotidien",
weekly: "Hebdomadaire",
monthly: "Mensuel",
};
type Source = {
id: number;
name: string;
connectorType: string;
portalUrl: string;
loginEmail: string;
loginPassword: string;
frequency: "manual" | "daily" | "weekly" | "monthly";
autoEnabled: number;
lastSuccessAt: Date | null;
lastStatus: string | null;
lastImportCount: number | null;
apiToken: string;
createdAt: Date;
updatedAt: Date;
};
type FormData = {
name: string;
connectorType: string;
portalUrl: string;
loginEmail: string;
loginPassword: string;
frequency: "manual" | "daily" | "weekly" | "monthly";
autoEnabled: number;
};
const emptyForm: FormData = {
name: "",
connectorType: "sfr",
portalUrl: "https://www.sfr.fr/mon-espace-client/",
loginEmail: "",
loginPassword: "",
frequency: "monthly",
autoEnabled: 0,
};
export default function WebImportSources() {
const [showForm, setShowForm] = useState(false);
const [editSource, setEditSource] = useState<Source | null>(null);
const [deleteId, setDeleteId] = useState<number | null>(null);
const [showToken, setShowToken] = useState<number | null>(null);
const [form, setForm] = useState<FormData>(emptyForm);
const { data: sources = [], refetch } = trpc.webImportSources.list.useQuery();
const { data: tokenData } = trpc.webImportSources.getToken.useQuery(
{ id: showToken! },
{ enabled: showToken !== null }
);
const createMutation = trpc.webImportSources.create.useMutation({
onSuccess: () => {
toast.success("Source créée", { description: "Le connecteur web a été ajouté." });
setShowForm(false);
setForm(emptyForm);
refetch();
},
onError: (e) => toast.error("Erreur", { description: e.message }),
});
const updateMutation = trpc.webImportSources.update.useMutation({
onSuccess: () => {
toast.success("Source mise à jour");
setEditSource(null);
setForm(emptyForm);
refetch();
},
onError: (e) => toast.error("Erreur", { description: e.message }),
});
const deleteMutation = trpc.webImportSources.delete.useMutation({
onSuccess: () => {
toast.success("Source supprimée");
setDeleteId(null);
refetch();
},
onError: (e) => toast.error("Erreur", { description: e.message }),
});
function openCreate() {
setForm(emptyForm);
setEditSource(null);
setShowForm(true);
}
function openEdit(source: Source) {
setForm({
name: source.name,
connectorType: source.connectorType,
portalUrl: source.portalUrl,
loginEmail: source.loginEmail,
loginPassword: "", // Ne pas pré-remplir le mot de passe
frequency: source.frequency,
autoEnabled: source.autoEnabled,
});
setEditSource(source);
setShowForm(true);
}
function handleConnectorTypeChange(value: string) {
const connector = CONNECTOR_TYPES.find(c => c.value === value);
setForm(f => ({
...f,
connectorType: value,
name: f.name || connector?.label || "",
portalUrl: connector?.url || f.portalUrl,
}));
}
function handleSubmit() {
if (!form.name || !form.loginEmail || (!editSource && !form.loginPassword)) {
toast.error("Champs requis", { description: "Nom, identifiant et mot de passe sont obligatoires." });
return;
}
if (editSource) {
const updateData: any = { id: editSource.id, ...form };
if (!form.loginPassword) delete updateData.loginPassword; // Ne pas écraser si vide
updateMutation.mutate(updateData);
} else {
createMutation.mutate(form);
}
}
return (
<DashboardLayout>
<div className="p-6 max-w-5xl mx-auto">
{/* En-tête */}
<div className="flex items-center justify-between mb-6">
<div className="flex items-center gap-3">
<div className="p-2 bg-indigo-100 rounded-lg">
<Globe className="w-6 h-6 text-indigo-600" />
</div>
<div>
<h1 className="text-2xl font-bold text-gray-900">Connecteurs web</h1>
<p className="text-sm text-gray-500">Import automatique de factures depuis des espaces clients</p>
</div>
</div>
<Button onClick={openCreate} className="bg-indigo-600 hover:bg-indigo-700 text-white gap-2">
<Plus className="w-4 h-4" />
Ajouter un connecteur
</Button>
</div>
{/* Info technique */}
<div className="bg-amber-50 border border-amber-200 rounded-lg p-4 mb-6 text-sm text-amber-800">
<strong>Fonctionnement :</strong> Un script cron tourne sur le serveur LWS et se connecte automatiquement aux espaces clients configurés pour télécharger les nouvelles factures. Le token API affiché ci-dessous est utilisé par ce script pour s'authentifier auprès de l'application.
</div>
{/* Liste des sources */}
{sources.length === 0 ? (
<div className="text-center py-16 text-gray-400">
<Globe className="w-12 h-12 mx-auto mb-3 opacity-30" />
<p className="text-lg font-medium">Aucun connecteur configuré</p>
<p className="text-sm mt-1">Ajoutez un connecteur pour importer automatiquement des factures depuis un espace client web.</p>
</div>
) : (
<div className="space-y-3">
{(sources as Source[]).map((source) => (
<div key={source.id} className="bg-white border border-gray-200 rounded-xl p-5 shadow-sm hover:shadow-md transition-shadow">
<div className="flex items-start justify-between gap-4">
<div className="flex items-start gap-4 flex-1 min-w-0">
<div className="p-2 bg-indigo-50 rounded-lg shrink-0">
<Globe className="w-5 h-5 text-indigo-500" />
</div>
<div className="flex-1 min-w-0">
<div className="flex items-center gap-2 flex-wrap">
<span className="font-semibold text-gray-900">{source.name}</span>
<Badge variant="outline" className="text-xs capitalize">{source.connectorType}</Badge>
<Badge variant={source.autoEnabled ? "default" : "secondary"} className="text-xs">
{source.autoEnabled ? "Auto activé" : "Manuel"}
</Badge>
<Badge variant="outline" className="text-xs">{FREQUENCY_LABELS[source.frequency]}</Badge>
</div>
<p className="text-sm text-gray-500 mt-1 truncate">{source.portalUrl}</p>
<p className="text-sm text-gray-600 mt-0.5">
<span className="font-medium">Identifiant :</span> {source.loginEmail}
</p>
<div className="flex items-center gap-4 mt-2 text-xs text-gray-400">
{source.lastSuccessAt ? (
<span className="flex items-center gap-1 text-green-600">
<CheckCircle className="w-3 h-3" />
Dernier import : {new Date(source.lastSuccessAt).toLocaleDateString("fr-FR")}
{source.lastImportCount !== null && ` (${source.lastImportCount} facture(s))`}
</span>
) : (
<span className="flex items-center gap-1 text-gray-400">
<Clock className="w-3 h-3" />
Jamais importé
</span>
)}
{source.lastStatus && !source.lastSuccessAt && (
<span className="flex items-center gap-1 text-red-500">
<XCircle className="w-3 h-3" />
{source.lastStatus}
</span>
)}
</div>
</div>
</div>
<div className="flex items-center gap-2 shrink-0">
<Button
variant="outline"
size="sm"
className="gap-1 text-xs"
onClick={() => setShowToken(showToken === source.id ? null : source.id)}
>
<Key className="w-3 h-3" />
Token
</Button>
<Button variant="outline" size="sm" onClick={() => openEdit(source)}>
<Pencil className="w-4 h-4" />
</Button>
<Button
variant="outline"
size="sm"
className="text-red-500 hover:text-red-700 hover:border-red-300"
onClick={() => setDeleteId(source.id)}
>
<Trash2 className="w-4 h-4" />
</Button>
</div>
</div>
{/* Token API affiché inline */}
{showToken === source.id && tokenData && (
<div className="mt-3 p-3 bg-gray-50 rounded-lg border border-gray-200">
<p className="text-xs text-gray-500 mb-1 font-medium">Token API (à configurer dans le script cron) :</p>
<code className="text-xs font-mono text-indigo-700 break-all select-all">{tokenData.apiToken}</code>
</div>
)}
</div>
))}
</div>
)}
{/* Dialog création / édition */}
<Dialog open={showForm} onOpenChange={(open) => { if (!open) { setShowForm(false); setEditSource(null); } }}>
<DialogContent className="max-w-lg">
<DialogHeader>
<DialogTitle>{editSource ? "Modifier le connecteur" : "Nouveau connecteur web"}</DialogTitle>
</DialogHeader>
<div className="space-y-4 py-2">
<div>
<Label>Type de connecteur</Label>
<Select value={form.connectorType} onValueChange={handleConnectorTypeChange}>
<SelectTrigger className="mt-1">
<SelectValue />
</SelectTrigger>
<SelectContent>
{CONNECTOR_TYPES.map(c => (
<SelectItem key={c.value} value={c.value}>{c.label}</SelectItem>
))}
</SelectContent>
</Select>
</div>
<div>
<Label>Nom affiché</Label>
<Input
className="mt-1"
value={form.name}
onChange={e => setForm(f => ({ ...f, name: e.target.value }))}
placeholder="Ex : SFR Pro - Itinova"
/>
</div>
<div>
<Label>URL de l'espace client</Label>
<Input
className="mt-1"
value={form.portalUrl}
onChange={e => setForm(f => ({ ...f, portalUrl: e.target.value }))}
placeholder="https://..."
/>
</div>
<div>
<Label>Identifiant (email ou login)</Label>
<Input
className="mt-1"
value={form.loginEmail}
onChange={e => setForm(f => ({ ...f, loginEmail: e.target.value }))}
placeholder="votre@email.com"
/>
</div>
<div>
<Label>{editSource ? "Nouveau mot de passe (laisser vide pour ne pas changer)" : "Mot de passe"}</Label>
<Input
className="mt-1"
type="password"
value={form.loginPassword}
onChange={e => setForm(f => ({ ...f, loginPassword: e.target.value }))}
placeholder={editSource ? "••••••••" : "Mot de passe"}
/>
</div>
<div className="grid grid-cols-2 gap-4">
<div>
<Label>Fréquence</Label>
<Select value={form.frequency} onValueChange={(v: any) => setForm(f => ({ ...f, frequency: v }))}>
<SelectTrigger className="mt-1">
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="manual">Manuel</SelectItem>
<SelectItem value="daily">Quotidien</SelectItem>
<SelectItem value="weekly">Hebdomadaire</SelectItem>
<SelectItem value="monthly">Mensuel</SelectItem>
</SelectContent>
</Select>
</div>
<div className="flex flex-col justify-end pb-1">
<div className="flex items-center gap-2">
<Switch
checked={form.autoEnabled === 1}
onCheckedChange={v => setForm(f => ({ ...f, autoEnabled: v ? 1 : 0 }))}
/>
<Label>Import auto activé</Label>
</div>
</div>
</div>
</div>
<DialogFooter>
<Button variant="outline" onClick={() => { setShowForm(false); setEditSource(null); }}>Annuler</Button>
<Button
onClick={handleSubmit}
disabled={createMutation.isPending || updateMutation.isPending}
className="bg-indigo-600 hover:bg-indigo-700 text-white"
>
{createMutation.isPending || updateMutation.isPending ? (
<RefreshCw className="w-4 h-4 animate-spin mr-2" />
) : null}
{editSource ? "Enregistrer" : "Créer"}
</Button>
</DialogFooter>
</DialogContent>
</Dialog>
{/* Confirmation suppression */}
<AlertDialog open={deleteId !== null} onOpenChange={(open) => { if (!open) setDeleteId(null); }}>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle>Supprimer ce connecteur ?</AlertDialogTitle>
<AlertDialogDescription>
Cette action est irréversible. Le connecteur et son token API seront supprimés.
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel>Annuler</AlertDialogCancel>
<AlertDialogAction
className="bg-red-600 hover:bg-red-700"
onClick={() => deleteId !== null && deleteMutation.mutate({ id: deleteId })}
>
Supprimer
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
</div>
</DashboardLayout>
);
}

View File

@@ -0,0 +1,18 @@
CREATE TABLE `webImportSources` (
`id` int AUTO_INCREMENT NOT NULL,
`userId` int NOT NULL,
`name` varchar(100) NOT NULL,
`connectorType` varchar(50) NOT NULL,
`portalUrl` varchar(500) NOT NULL,
`loginEmail` varchar(320) NOT NULL,
`loginPassword` text NOT NULL,
`frequency` enum('manual','daily','weekly','monthly') NOT NULL DEFAULT 'monthly',
`autoEnabled` int NOT NULL DEFAULT 0,
`lastSuccessAt` timestamp,
`lastStatus` text,
`lastImportCount` int DEFAULT 0,
`apiToken` varchar(128) NOT NULL,
`createdAt` timestamp NOT NULL DEFAULT (now()),
`updatedAt` timestamp NOT NULL DEFAULT (now()) ON UPDATE CURRENT_TIMESTAMP,
CONSTRAINT `webImportSources_id` PRIMARY KEY(`id`)
);

File diff suppressed because it is too large Load Diff

View File

@@ -253,6 +253,13 @@
"when": 1785404752594,
"tag": "0035_eminent_dreadnoughts",
"breakpoints": true
},
{
"idx": 36,
"version": "5",
"when": 1785419093588,
"tag": "0036_broken_rattler",
"breakpoints": true
}
]
}

View File

@@ -524,3 +524,39 @@ export const deletedInvoices = mysqlTable("deletedInvoices", {
});
export type DeletedInvoice = typeof deletedInvoices.$inferSelect;
export type InsertDeletedInvoice = typeof deletedInvoices.$inferInsert;
/**
* Web import sources — connecteurs web pour scraper des factures depuis des sites
* (ex: espace client SFR, Starlink, Orange...) avec login/mot de passe.
* Le scraping est exécuté par un script cron externe (Node.js + Playwright) sur LWS.
*/
export const webImportSources = mysqlTable("webImportSources", {
id: int("id").autoincrement().primaryKey(),
userId: int("userId").notNull(),
/** Nom affiché (ex: "SFR Pro", "Starlink") */
name: varchar("name", { length: 100 }).notNull(),
/** Type de connecteur — détermine le script Playwright à utiliser */
connectorType: varchar("connectorType", { length: 50 }).notNull(), // ex: "sfr", "starlink", "orange"
/** URL de l'espace client */
portalUrl: varchar("portalUrl", { length: 500 }).notNull(),
/** Identifiant de connexion (email ou login) */
loginEmail: varchar("loginEmail", { length: 320 }).notNull(),
/** Mot de passe chiffré (AES-256) */
loginPassword: text("loginPassword").notNull(),
/** Fréquence de vérification automatique */
frequency: mysqlEnum("frequency", ["manual", "daily", "weekly", "monthly"]).default("monthly").notNull(),
/** Activation de l'import automatique */
autoEnabled: int("autoEnabled").default(0).notNull(), // 0 = désactivé, 1 = activé
/** Date du dernier import réussi */
lastSuccessAt: timestamp("lastSuccessAt"),
/** Statut du dernier import */
lastStatus: text("lastStatus"),
/** Nombre de factures importées lors du dernier run */
lastImportCount: int("lastImportCount").default(0),
/** Token d'API pour que le script externe puisse s'authentifier */
apiToken: varchar("apiToken", { length: 128 }).notNull(),
createdAt: timestamp("createdAt").defaultNow().notNull(),
updatedAt: timestamp("updatedAt").defaultNow().onUpdateNow().notNull(),
});
export type WebImportSource = typeof webImportSources.$inferSelect;
export type InsertWebImportSource = typeof webImportSources.$inferInsert;

View File

@@ -0,0 +1,70 @@
# Connecteurs Web - Scripts d'import automatique
## Prérequis sur le serveur LWS
```bash
cd /opt/web-import
npm install playwright
npx playwright install chromium --with-deps
```
## Configuration
1. Depuis l'application : **Configuration > Connecteurs web** → créer une source SFR → copier le token API
2. Créer un fichier de configuration :
```bash
cp config.example.json config.json
# Éditer config.json avec vos valeurs
```
Contenu de `config.json` :
```json
{
"sfrLogin": "votre-login@sfr.fr",
"sfrPassword": "votre-mot-de-passe",
"appUrl": "https://demat-facturation.santinova-soft.org",
"apiToken": "votre-token-api-copié-depuis-lappli",
"downloadDir": "/tmp/sfr-invoices",
"processedFile": "/tmp/sfr-processed.json"
}
```
Ou utiliser des variables d'environnement :
```bash
export SFR_LOGIN=votre-login@sfr.fr
export SFR_PASSWORD=votre-mot-de-passe
export APP_URL=https://demat-facturation.santinova-soft.org
export API_TOKEN=votre-token-api
```
## Exécution manuelle
```bash
node sfr-connector.mjs
```
## Planification (cron)
Ajouter dans le crontab (`crontab -e`) :
```
# Import SFR le 5 de chaque mois à 8h00
0 8 5 * * /usr/bin/node /opt/web-import/sfr-connector.mjs >> /var/log/sfr-import.log 2>&1
```
## Ajouter un nouveau connecteur
Dupliquer `sfr-connector.mjs` et adapter :
1. L'URL du portail (`portalUrl`)
2. Les sélecteurs CSS pour le login et les liens de factures
3. Le nom du fichier de suivi (`processedFile`)
## Fonctionnement
1. Le script se connecte au site SFR avec les identifiants fournis
2. Il navigue vers la section factures
3. Il télécharge les PDFs non encore traités
4. Il les envoie à l'application via l'endpoint `/api/web-import/push-invoice`
5. L'application extrait les données avec l'IA et crée les factures
6. Le script marque les factures comme traitées pour éviter les doublons

View File

@@ -0,0 +1,216 @@
/**
* Connecteur SFR Pro - Script cron pour import automatique de factures
*
* Prérequis sur le serveur LWS :
* npm install playwright @playwright/test
* npx playwright install chromium
*
* Configuration :
* Copier .env.example en .env et remplir les variables
*
* Utilisation :
* node sfr-connector.mjs
*
* Cron (mensuel le 5 du mois à 8h) :
* 0 8 5 * * /usr/bin/node /opt/web-import/sfr-connector.mjs >> /var/log/sfr-import.log 2>&1
*/
import { chromium } from 'playwright';
import fs from 'fs';
import path from 'path';
import https from 'https';
import http from 'http';
import { fileURLToPath } from 'url';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
// ============ CONFIGURATION ============
// Ces variables peuvent être définies dans un fichier .env ou directement ici
const CONFIG = {
// URL de l'espace client SFR Pro
portalUrl: process.env.SFR_PORTAL_URL || 'https://www.sfr-business.fr/espace-client/',
// Identifiants SFR
login: process.env.SFR_LOGIN || '',
password: process.env.SFR_PASSWORD || '',
// URL de l'application de dématérialisation
appUrl: process.env.APP_URL || 'https://demat-facturation.santinova-soft.org',
// Token API de la source web (récupéré depuis l'interface Connecteurs web)
apiToken: process.env.API_TOKEN || '',
// Dossier temporaire pour les PDFs téléchargés
downloadDir: process.env.DOWNLOAD_DIR || '/tmp/sfr-invoices',
// Ne pas réimporter les factures déjà traitées (fichier de suivi)
processedFile: process.env.PROCESSED_FILE || '/tmp/sfr-processed.json',
};
// ============ HELPERS ============
function log(msg) {
console.log(`[${new Date().toISOString()}] [SFR] ${msg}`);
}
function loadProcessed() {
try {
if (fs.existsSync(CONFIG.processedFile)) {
return JSON.parse(fs.readFileSync(CONFIG.processedFile, 'utf8'));
}
} catch {}
return [];
}
function saveProcessed(list) {
fs.writeFileSync(CONFIG.processedFile, JSON.stringify(list, null, 2));
}
async function pushInvoiceToApp(filePath, fileName) {
const fileBuffer = fs.readFileSync(filePath);
const fileBase64 = fileBuffer.toString('base64');
const body = JSON.stringify({
apiToken: CONFIG.apiToken,
fileName,
fileBase64,
mimeType: 'application/pdf',
});
return new Promise((resolve, reject) => {
const url = new URL(`${CONFIG.appUrl}/api/web-import/push-invoice`);
const options = {
hostname: url.hostname,
port: url.port || (url.protocol === 'https:' ? 443 : 80),
path: url.pathname,
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Content-Length': Buffer.byteLength(body),
},
};
const lib = url.protocol === 'https:' ? https : http;
const req = lib.request(options, (res) => {
let data = '';
res.on('data', chunk => data += chunk);
res.on('end', () => {
try {
resolve({ status: res.statusCode, body: JSON.parse(data) });
} catch {
resolve({ status: res.statusCode, body: data });
}
});
});
req.on('error', reject);
req.write(body);
req.end();
});
}
// ============ CONNECTEUR SFR ============
async function runSfrConnector() {
log('Démarrage du connecteur SFR Pro');
if (!CONFIG.login || !CONFIG.password || !CONFIG.apiToken) {
log('ERREUR : SFR_LOGIN, SFR_PASSWORD et API_TOKEN sont requis');
process.exit(1);
}
// Créer le dossier de téléchargement
if (!fs.existsSync(CONFIG.downloadDir)) {
fs.mkdirSync(CONFIG.downloadDir, { recursive: true });
}
const processed = loadProcessed();
let newInvoices = 0;
const browser = await chromium.launch({ headless: true });
const context = await browser.newContext({
acceptDownloads: true,
});
const page = await context.newPage();
try {
// 1. Naviguer vers l'espace client SFR
log(`Navigation vers ${CONFIG.portalUrl}`);
await page.goto(CONFIG.portalUrl, { waitUntil: 'networkidle', timeout: 30000 });
// 2. Accepter les cookies si présent
try {
await page.click('[id*="accept"], [class*="accept-cookie"], #didomi-notice-agree-button', { timeout: 3000 });
log('Cookies acceptés');
} catch {}
// 3. Remplir le formulaire de connexion
log('Connexion en cours...');
await page.fill('input[type="email"], input[name="login"], input[id*="login"], input[id*="email"]', CONFIG.login);
await page.fill('input[type="password"], input[name="password"], input[id*="password"]', CONFIG.password);
await page.click('button[type="submit"], input[type="submit"], button:has-text("Connexion"), button:has-text("Se connecter")');
await page.waitForNavigation({ waitUntil: 'networkidle', timeout: 15000 }).catch(() => {});
log('Connecté');
// 4. Naviguer vers la section factures
// Adapter selon la structure réelle du site SFR Pro
await page.goto(`${CONFIG.portalUrl}factures`, { waitUntil: 'networkidle', timeout: 15000 }).catch(() => {});
// Chercher les liens de factures PDF
const invoiceLinks = await page.$$eval(
'a[href*=".pdf"], a[href*="facture"], a[href*="invoice"], a[download]',
links => links.map(a => ({
href: a.href,
text: a.textContent?.trim() || '',
download: a.getAttribute('download') || '',
}))
);
log(`${invoiceLinks.length} lien(s) de facture trouvé(s)`);
// 5. Télécharger et envoyer chaque facture
for (const link of invoiceLinks) {
const invoiceId = link.href || link.text;
if (processed.includes(invoiceId)) {
log(`Déjà traité : ${link.text}`);
continue;
}
try {
// Télécharger le PDF
const [download] = await Promise.all([
context.waitForEvent('download', { timeout: 15000 }),
page.click(`a[href="${link.href}"]`).catch(() => page.goto(link.href)),
]);
const fileName = download?.suggestedFilename() || `sfr-facture-${Date.now()}.pdf`;
const filePath = path.join(CONFIG.downloadDir, fileName);
await download?.saveAs(filePath);
log(`Téléchargé : ${fileName}`);
// Envoyer à l'application
const result = await pushInvoiceToApp(filePath, fileName);
log(`Envoyé : ${fileName}${JSON.stringify(result.body)}`);
// Marquer comme traité
processed.push(invoiceId);
saveProcessed(processed);
newInvoices++;
// Nettoyer le fichier temporaire
fs.unlinkSync(filePath);
} catch (err) {
log(`ERREUR sur ${link.text} : ${err.message}`);
}
}
} catch (err) {
log(`ERREUR FATALE : ${err.message}`);
await page.screenshot({ path: path.join(CONFIG.downloadDir, 'error-screenshot.png') }).catch(() => {});
throw err;
} finally {
await browser.close();
}
log(`Terminé : ${newInvoices} nouvelle(s) facture(s) importée(s)`);
return newInvoices;
}
// ============ POINT D'ENTRÉE ============
runSfrConnector().catch(err => {
console.error(`[FATAL] ${err.message}`);
process.exit(1);
});

View File

@@ -224,6 +224,56 @@ async function startServer() {
}
});
// ============= WEB IMPORT SOURCES - Endpoint pour script cron externe =============
app.post("/api/web-import/push-invoice", async (req, res) => {
try {
const { apiToken, fileName, fileBase64, mimeType } = req.body;
if (!apiToken || !fileName || !fileBase64) {
res.status(400).json({ error: "apiToken, fileName et fileBase64 sont requis" });
return;
}
const { getWebImportSourceByToken, getImportSettingsByUser, createInvoice, findDuplicateInvoice, isInvoiceBlacklisted, updateWebImportSourceStatus, createSourceFile } = await import('../db');
const source = await getWebImportSourceByToken(apiToken);
if (!source) {
res.status(401).json({ error: "Token invalide" });
return;
}
const pdfBuffer = Buffer.from(fileBase64, 'base64');
const fileMime = mimeType || 'application/pdf';
// Stocker le fichier source en DB
const sourceFile = await createSourceFile({
userId: source.userId,
fileName,
fileKey: `web-import/${source.userId}/${Date.now()}-${fileName}`,
fileUrl: '',
});
const importSettings = await getImportSettingsByUser(source.userId);
const aiSettings = {
aiProvider: importSettings?.aiProvider || 'manus',
mistralApiKey: importSettings?.mistralApiKey || undefined,
manusForgeApiUrl: importSettings?.manusForgeApiUrl || undefined,
manusForgeApiKey: importSettings?.manusForgeApiKey || undefined,
};
const { extractInvoicesWithMistral } = await import('../invoiceExtractor');
const extractResult = await extractInvoicesWithMistral(pdfBuffer, source.userId, sourceFile.id, 'mistral-large-latest', undefined, aiSettings);
let imported = 0;
let duplicates = 0;
for (const inv of extractResult.invoices || []) {
const blacklisted = await isInvoiceBlacklisted(inv.invoiceNumber || null, source.userId);
if (blacklisted) { duplicates++; continue; }
const dup = await findDuplicateInvoice(inv.invoiceNumber || null, String(inv.totalAmount ?? ''), source.userId);
if (dup) { duplicates++; continue; }
await createInvoice({ ...inv, userId: source.userId, sourceFileId: sourceFile.id } as any);
imported++;
}
await updateWebImportSourceStatus(source.id, 'success', imported, true);
res.json({ success: true, imported, duplicates, total: (extractResult.invoices || []).length });
} catch (err: any) {
console.error('[WebImport] Erreur push-invoice:', err.message);
res.status(500).json({ error: err.message });
}
});
// tRPC API
app.use(
"/api/trpc",

View File

@@ -47,7 +47,10 @@ import {
InvoiceLearning,
deletedInvoices,
InsertDeletedInvoice,
DeletedInvoice
DeletedInvoice,
webImportSources,
InsertWebImportSource,
WebImportSource
} from "../drizzle/schema";
import { ENV } from './_core/env';
@@ -1147,3 +1150,74 @@ export async function updateFreeproLastRun(
.set(update)
.where(eq(freeproSettings.userId, userId));
}
// ============= WEB IMPORT SOURCES =============
/** Génère un token API aléatoire de 64 caractères */
function generateApiToken(): string {
const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
let token = '';
for (let i = 0; i < 64; i++) {
token += chars.charAt(Math.floor(Math.random() * chars.length));
}
return token;
}
export async function getWebImportSourcesByUser(userId: number): Promise<WebImportSource[]> {
const db = await getDb();
if (!db) return [];
return db.select().from(webImportSources).where(eq(webImportSources.userId, userId)).orderBy(desc(webImportSources.createdAt));
}
export async function getWebImportSourceById(id: number): Promise<WebImportSource | undefined> {
const db = await getDb();
if (!db) return undefined;
const result = await db.select().from(webImportSources).where(eq(webImportSources.id, id)).limit(1);
return result[0];
}
export async function getWebImportSourceByToken(token: string): Promise<WebImportSource | undefined> {
const db = await getDb();
if (!db) return undefined;
const result = await db.select().from(webImportSources).where(eq(webImportSources.apiToken, token)).limit(1);
return result[0];
}
export async function createWebImportSource(data: Omit<InsertWebImportSource, 'apiToken'>): Promise<WebImportSource> {
const db = await getDb();
if (!db) throw new Error("Database not available");
const apiToken = generateApiToken();
const result = await db.insert(webImportSources).values({ ...data, apiToken });
const insertedId = Number(result[0].insertId);
const inserted = await db.select().from(webImportSources).where(eq(webImportSources.id, insertedId)).limit(1);
return inserted[0]!;
}
export async function updateWebImportSource(id: number, data: Partial<InsertWebImportSource>): Promise<void> {
const db = await getDb();
if (!db) throw new Error("Database not available");
await db.update(webImportSources).set({ ...data, updatedAt: new Date() }).where(eq(webImportSources.id, id));
}
export async function deleteWebImportSource(id: number): Promise<void> {
const db = await getDb();
if (!db) throw new Error("Database not available");
await db.delete(webImportSources).where(eq(webImportSources.id, id));
}
export async function updateWebImportSourceStatus(
id: number,
status: string,
importCount: number,
success: boolean
): Promise<void> {
const db = await getDb();
if (!db) return;
const update: Partial<InsertWebImportSource> = {
lastStatus: status,
lastImportCount: importCount,
updatedAt: new Date(),
};
if (success) update.lastSuccessAt = new Date();
await db.update(webImportSources).set(update).where(eq(webImportSources.id, id));
}

View File

@@ -77,6 +77,12 @@ import {
deleteLearning,
deleteAllLearnings,
getBapPdfUrlsByInvoiceIds,
getWebImportSourcesByUser,
getWebImportSourceById,
createWebImportSource,
updateWebImportSource,
deleteWebImportSource,
updateWebImportSourceStatus,
} from "./db";
import { loginLocal, hashPassword, getAzureAuthUrl, isAzureAdConfigured, generateToken } from "./auth";
import { extractInvoicesWithMistral, generateMetadataJSON } from "./invoiceExtractor";
@@ -2609,5 +2615,68 @@ export const appRouter = router({
return { success: true, webUrl: result.webUrl, fileName };
}),
}),
// ============= WEB IMPORT SOURCES =============
webImportSources: router({
list: protectedProcedure.query(async ({ ctx }) => {
return getWebImportSourcesByUser(ctx.user.id);
}),
create: protectedProcedure
.input(z.object({
name: z.string().min(1).max(100),
connectorType: z.string().min(1).max(50),
portalUrl: z.string().url(),
loginEmail: z.string().min(1),
loginPassword: z.string().min(1),
frequency: z.enum(['manual', 'daily', 'weekly', 'monthly']).default('monthly'),
autoEnabled: z.number().min(0).max(1).default(0),
}))
.mutation(async ({ input, ctx }) => {
return createWebImportSource({ ...input, userId: ctx.user.id });
}),
update: protectedProcedure
.input(z.object({
id: z.number(),
name: z.string().min(1).max(100).optional(),
connectorType: z.string().min(1).max(50).optional(),
portalUrl: z.string().url().optional(),
loginEmail: z.string().min(1).optional(),
loginPassword: z.string().optional(),
frequency: z.enum(['manual', 'daily', 'weekly', 'monthly']).optional(),
autoEnabled: z.number().min(0).max(1).optional(),
}))
.mutation(async ({ input, ctx }) => {
const source = await getWebImportSourceById(input.id);
if (!source || source.userId !== ctx.user.id) {
throw new TRPCError({ code: 'NOT_FOUND', message: 'Source introuvable' });
}
const { id, ...data } = input;
await updateWebImportSource(id, data);
return { success: true };
}),
delete: protectedProcedure
.input(z.object({ id: z.number() }))
.mutation(async ({ input, ctx }) => {
const source = await getWebImportSourceById(input.id);
if (!source || source.userId !== ctx.user.id) {
throw new TRPCError({ code: 'NOT_FOUND', message: 'Source introuvable' });
}
await deleteWebImportSource(input.id);
return { success: true };
}),
getToken: protectedProcedure
.input(z.object({ id: z.number() }))
.query(async ({ input, ctx }) => {
const source = await getWebImportSourceById(input.id);
if (!source || source.userId !== ctx.user.id) {
throw new TRPCError({ code: 'NOT_FOUND', message: 'Source introuvable' });
}
return { apiToken: source.apiToken };
}),
}),
});
export type AppRouter = typeof appRouter;

View File

@@ -684,3 +684,11 @@
- [ ] Modifier getServiceSignaturesByUser → retourner toutes les signatures de service (sans filtre userId)
- [ ] Migrer les données en production : dédoublonner les listes fusionnées
- [ ] Déployer en production
## Connecteurs web (scraping login/mdp)
- [ ] Table webImportSources dans le schéma DB
- [ ] Procédures tRPC CRUD pour webImportSources
- [ ] Interface de gestion des sources web dans les paramètres
- [ ] Endpoint API sécurisé pour déclencher l'import et recevoir les PDFs
- [ ] Script cron externe Node.js + Playwright pour SFR
- [ ] Framework connecteur générique extensible