From b6d89e10872ed2bd5f0e5f0dd3e3389a44809415 Mon Sep 17 00:00:00 2001 From: Manus Date: Thu, 2 Jul 2026 03:49:58 -0400 Subject: [PATCH] =?UTF-8?q?Checkpoint:=20Ajout=20de=20la=20table=20process?= =?UTF-8?q?ed=5Fdedup=5Fkeys=20:=20chaque=20article=20trait=C3=A9=20par=20?= =?UTF-8?q?le=20moteur=20RSS=20est=20enregistr=C3=A9=20dans=20cette=20tabl?= =?UTF-8?q?e.=20Lors=20des=20passages=20suivants=20du=20cron,=20si=20la=20?= =?UTF-8?q?dedupKey=20est=20pr=C3=A9sente=20dans=20cette=20table,=20l'arti?= =?UTF-8?q?cle=20est=20ignor=C3=A9=20m=C3=AAme=20s'il=20a=20=C3=A9t=C3=A9?= =?UTF-8?q?=20supprim=C3=A9=20de=20veille=5Fitems=20ou=20aap=5Fitems.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/public/__manus__/version.json | 4 +- drizzle/0010_graceful_nova.sql | 8 + drizzle/meta/0010_snapshot.json | 1038 ++++++++++++++++++++++++++ drizzle/meta/_journal.json | 7 + drizzle/schema.ts | 14 + server/rssEngine.ts | 14 + 6 files changed, 1083 insertions(+), 2 deletions(-) create mode 100644 drizzle/0010_graceful_nova.sql create mode 100644 drizzle/meta/0010_snapshot.json diff --git a/client/public/__manus__/version.json b/client/public/__manus__/version.json index d607bd2..5af8a5a 100644 --- a/client/public/__manus__/version.json +++ b/client/public/__manus__/version.json @@ -1,4 +1,4 @@ { - "version": "0f628f8c", - "timestamp": 1782891184609 + "version": "78f2d166", + "timestamp": 1782978598907 } \ No newline at end of file diff --git a/drizzle/0010_graceful_nova.sql b/drizzle/0010_graceful_nova.sql new file mode 100644 index 0000000..9c9945a --- /dev/null +++ b/drizzle/0010_graceful_nova.sql @@ -0,0 +1,8 @@ +CREATE TABLE `processed_dedup_keys` ( + `id` int AUTO_INCREMENT NOT NULL, + `dedupKey` varchar(64) NOT NULL, + `feedType` enum('veille','aap') NOT NULL, + `processedAt` timestamp NOT NULL DEFAULT (now()), + CONSTRAINT `processed_dedup_keys_id` PRIMARY KEY(`id`), + CONSTRAINT `processed_dedup_keys_dedupKey_unique` UNIQUE(`dedupKey`) +); diff --git a/drizzle/meta/0010_snapshot.json b/drizzle/meta/0010_snapshot.json new file mode 100644 index 0000000..664a27b --- /dev/null +++ b/drizzle/meta/0010_snapshot.json @@ -0,0 +1,1038 @@ +{ + "version": "5", + "dialect": "mysql", + "id": "b5b3cc6f-d9fc-4020-b148-0cbcd8b25335", + "prevId": "60818888-9a29-4da2-b430-ea765431d740", + "tables": { + "aap_items": { + "name": "aap_items", + "columns": { + "id": { + "name": "id", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": true + }, + "dedupKey": { + "name": "dedupKey", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "titre": { + "name": "titre", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "categorie": { + "name": "categorie", + "type": "enum('Handicap','PA','Enfance','Précarité','Sanitaire','Autre')", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "region": { + "name": "region", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "departement": { + "name": "departement", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "departements": { + "name": "departements", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dateCloture": { + "name": "dateCloture", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "datePublication": { + "name": "datePublication", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lien": { + "name": "lien", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "importedAt": { + "name": "importedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "iaRelevant": { + "name": "iaRelevant", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "iaCategorie": { + "name": "iaCategorie", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "iaClassifiedBy": { + "name": "iaClassifiedBy", + "type": "enum('ia','rules')", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "iaReason": { + "name": "iaReason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "iaResume": { + "name": "iaResume", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "aap_items_id": { + "name": "aap_items_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": { + "aap_items_dedupKey_unique": { + "name": "aap_items_dedupKey_unique", + "columns": [ + "dedupKey" + ] + } + }, + "checkConstraint": {} + }, + "app_settings": { + "name": "app_settings", + "columns": { + "id": { + "name": "id", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": true + }, + "key": { + "name": "key", + "type": "varchar(128)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "app_settings_id": { + "name": "app_settings_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": { + "app_settings_key_unique": { + "name": "app_settings_key_unique", + "columns": [ + "key" + ] + } + }, + "checkConstraint": {} + }, + "article_reads": { + "name": "article_reads", + "columns": { + "id": { + "name": "id", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": true + }, + "userId": { + "name": "userId", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "articleType": { + "name": "articleType", + "type": "enum('veille','aap')", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "articleId": { + "name": "articleId", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "readAt": { + "name": "readAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "article_reads_id": { + "name": "article_reads_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "ideas": { + "name": "ideas", + "columns": { + "id": { + "name": "id", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": true + }, + "userId": { + "name": "userId", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userName": { + "name": "userName", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "titre": { + "name": "titre", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "statut": { + "name": "statut", + "type": "enum('ouvert','en_cours','resolu','ferme')", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'ouvert'" + }, + "reponseAdmin": { + "name": "reponseAdmin", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reponduPar": { + "name": "reponduPar", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reponduAt": { + "name": "reponduAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "ideas_id": { + "name": "ideas_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "import_logs": { + "name": "import_logs", + "columns": { + "id": { + "name": "id", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": true + }, + "fileType": { + "name": "fileType", + "type": "enum('veille','aap')", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "enum('success','partial','error')", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "totalRows": { + "name": "totalRows", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "newRows": { + "name": "newRows", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "skippedRows": { + "name": "skippedRows", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "errorMessage": { + "name": "errorMessage", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "details": { + "name": "details", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "startedAt": { + "name": "startedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "completedAt": { + "name": "completedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "import_logs_id": { + "name": "import_logs_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "local_users": { + "name": "local_users", + "columns": { + "id": { + "name": "id", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "username": { + "name": "username", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "varchar(320)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "passwordHash": { + "name": "passwordHash", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "enum('admin','user','readonly')", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'user'" + }, + "isActive": { + "name": "isActive", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "lastSignedIn": { + "name": "lastSignedIn", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "local_users_id": { + "name": "local_users_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": { + "local_users_username_unique": { + "name": "local_users_username_unique", + "columns": [ + "username" + ] + } + }, + "checkConstraint": {} + }, + "processed_dedup_keys": { + "name": "processed_dedup_keys", + "columns": { + "id": { + "name": "id", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": true + }, + "dedupKey": { + "name": "dedupKey", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "feedType": { + "name": "feedType", + "type": "enum('veille','aap')", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processedAt": { + "name": "processedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "processed_dedup_keys_id": { + "name": "processed_dedup_keys_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": { + "processed_dedup_keys_dedupKey_unique": { + "name": "processed_dedup_keys_dedupKey_unique", + "columns": [ + "dedupKey" + ] + } + }, + "checkConstraint": {} + }, + "rss_feeds": { + "name": "rss_feeds", + "columns": { + "id": { + "name": "id", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "feedType": { + "name": "feedType", + "type": "enum('veille','aap')", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "defaultTypeVeille": { + "name": "defaultTypeVeille", + "type": "enum('reglementaire','concurrentielle','technologique','informationnelle')", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "defaultCategorieAap": { + "name": "defaultCategorieAap", + "type": "enum('Handicap','PA','Enfance','Précarité','Sanitaire','Autre')", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "autoRules": { + "name": "autoRules", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "isActive": { + "name": "isActive", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "lastFetchedAt": { + "name": "lastFetchedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lastFetchStatus": { + "name": "lastFetchStatus", + "type": "enum('ok','error','pending')", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'pending'" + }, + "lastFetchError": { + "name": "lastFetchError", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "rss_feeds_id": { + "name": "rss_feeds_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "rss_settings": { + "name": "rss_settings", + "columns": { + "id": { + "name": "id", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": true + }, + "fetchIntervalMinutes": { + "name": "fetchIntervalMinutes", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 360 + }, + "scheduledTime": { + "name": "scheduledTime", + "type": "varchar(5)", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'06:00'" + }, + "fetchMode": { + "name": "fetchMode", + "type": "enum('interval','scheduled')", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'scheduled'" + }, + "autoFetchEnabled": { + "name": "autoFetchEnabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "rss_settings_id": { + "name": "rss_settings_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": true + }, + "openId": { + "name": "openId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "varchar(320)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "loginMethod": { + "name": "loginMethod", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "enum('user','admin')", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'user'" + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "lastSignedIn": { + "name": "lastSignedIn", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "users_id": { + "name": "users_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": { + "users_openId_unique": { + "name": "users_openId_unique", + "columns": [ + "openId" + ] + } + }, + "checkConstraint": {} + }, + "veille_items": { + "name": "veille_items", + "columns": { + "id": { + "name": "id", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": true + }, + "dedupKey": { + "name": "dedupKey", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "titre": { + "name": "titre", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "categorie": { + "name": "categorie", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "niveau": { + "name": "niveau", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "territoire": { + "name": "territoire", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "territoires": { + "name": "territoires", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "resume": { + "name": "resume", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "passage": { + "name": "passage", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lien": { + "name": "lien", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "typeVeille": { + "name": "typeVeille", + "type": "enum('reglementaire','concurrentielle','technologique','informationnelle')", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "datePublication": { + "name": "datePublication", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "importedAt": { + "name": "importedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "iaRelevant": { + "name": "iaRelevant", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "iaCategorie": { + "name": "iaCategorie", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "iaClassifiedBy": { + "name": "iaClassifiedBy", + "type": "enum('ia','rules')", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "iaReason": { + "name": "iaReason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "iaResume": { + "name": "iaResume", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "veille_items_id": { + "name": "veille_items_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": { + "veille_items_dedupKey_unique": { + "name": "veille_items_dedupKey_unique", + "columns": [ + "dedupKey" + ] + } + }, + "checkConstraint": {} + } + }, + "views": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "tables": {}, + "indexes": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 94b48e0..2850d1c 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -71,6 +71,13 @@ "when": 1781683138994, "tag": "0009_fat_rocket_racer", "breakpoints": true + }, + { + "idx": 10, + "version": "5", + "when": 1782978530081, + "tag": "0010_graceful_nova", + "breakpoints": true } ] } \ No newline at end of file diff --git a/drizzle/schema.ts b/drizzle/schema.ts index 90eb35c..3346f95 100644 --- a/drizzle/schema.ts +++ b/drizzle/schema.ts @@ -213,3 +213,17 @@ export const articleReads = mysqlTable("article_reads", { export type ArticleRead = typeof articleReads.$inferSelect; export type InsertArticleRead = typeof articleReads.$inferInsert; + +// ─── Tombstones : articles déjà traités (empêche la réinsertion après purge) ── + +export const processedDedupKeys = mysqlTable("processed_dedup_keys", { + id: int("id").autoincrement().primaryKey(), + dedupKey: varchar("dedupKey", { length: 64 }).notNull().unique(), + // Type de contenu (veille ou aap) + feedType: mysqlEnum("feedType", ["veille", "aap"]).notNull(), + // Date à laquelle l'article a été traité pour la première fois + processedAt: timestamp("processedAt").defaultNow().notNull(), +}); + +export type ProcessedDedupKey = typeof processedDedupKeys.$inferSelect; +export type InsertProcessedDedupKey = typeof processedDedupKeys.$inferInsert; diff --git a/server/rssEngine.ts b/server/rssEngine.ts index c4f58d2..999458c 100644 --- a/server/rssEngine.ts +++ b/server/rssEngine.ts @@ -19,6 +19,7 @@ import { rssFeeds, veilleItems, aapItems, + processedDedupKeys, type RssFeed, } from "../drizzle/schema"; import { eq, sql } from "drizzle-orm"; @@ -525,6 +526,15 @@ async function processFeed(feed: RssFeed): Promise { const normalizedTitle = buildMergeKey(title); const dedupKey = dedupHash(normalizedTitle + "|" + (feed.feedType ?? "")); + // ─── Tombstone : ignorer les articles déjà traités (même supprimés) ───────── + const tombstone = await db.select().from(processedDedupKeys) + .where(eq(processedDedupKeys.dedupKey, dedupKey)) + .limit(1); + if (tombstone.length > 0) { + result.skippedItems++; + continue; + } + if (feed.feedType === "veille") { // ── Classification IA (avec fallback sur mots-clés) ────────────────────────────────────── const { niveau, territoire } = detectVeilleNiveauTerritoire(fullText); @@ -578,6 +588,8 @@ async function processFeed(feed: RssFeed): Promise { iaReason: aiResult.reason, iaResume: iaResume || null, }); + // Enregistrer le tombstone pour éviter la réinsertion après purge + await db.insert(processedDedupKeys).values({ dedupKey, feedType: "veille" }).onDuplicateKeyUpdate({ set: { dedupKey } }); result.newItems++; } catch (e: any) { if (e?.code === "ER_DUP_ENTRY" || e?.cause?.code === "ER_DUP_ENTRY" || e?.message?.includes("Duplicate entry") || e?.cause?.message?.includes("Duplicate entry")) { @@ -652,6 +664,8 @@ async function processFeed(feed: RssFeed): Promise { iaReason: aiResult.reason, iaResume: iaResume || null, }); + // Enregistrer le tombstone pour éviter la réinsertion après purge + await db.insert(processedDedupKeys).values({ dedupKey, feedType: "aap" }).onDuplicateKeyUpdate({ set: { dedupKey } }); result.newItems++; } catch (e: any) { if (e?.code === "ER_DUP_ENTRY" || e?.cause?.code === "ER_DUP_ENTRY" || e?.message?.includes("Duplicate entry") || e?.cause?.message?.includes("Duplicate entry")) {