Checkpoint: Ajout de la table processed_dedup_keys : chaque article traité par le moteur RSS est enregistré dans cette table. Lors des passages suivants du cron, si la dedupKey est présente dans cette table, l'article est ignoré même s'il a été supprimé de veille_items ou aap_items.

This commit is contained in:
Manus
2026-07-02 03:49:58 -04:00
parent c6b89c5b46
commit b6d89e1087
6 changed files with 1083 additions and 2 deletions

View File

@@ -1,4 +1,4 @@
{ {
"version": "0f628f8c", "version": "78f2d166",
"timestamp": 1782891184609 "timestamp": 1782978598907
} }

View File

@@ -0,0 +1,8 @@
CREATE TABLE `processed_dedup_keys` (
`id` int AUTO_INCREMENT NOT NULL,
`dedupKey` varchar(64) NOT NULL,
`feedType` enum('veille','aap') NOT NULL,
`processedAt` timestamp NOT NULL DEFAULT (now()),
CONSTRAINT `processed_dedup_keys_id` PRIMARY KEY(`id`),
CONSTRAINT `processed_dedup_keys_dedupKey_unique` UNIQUE(`dedupKey`)
);

File diff suppressed because it is too large Load Diff

View File

@@ -71,6 +71,13 @@
"when": 1781683138994, "when": 1781683138994,
"tag": "0009_fat_rocket_racer", "tag": "0009_fat_rocket_racer",
"breakpoints": true "breakpoints": true
},
{
"idx": 10,
"version": "5",
"when": 1782978530081,
"tag": "0010_graceful_nova",
"breakpoints": true
} }
] ]
} }

View File

@@ -213,3 +213,17 @@ export const articleReads = mysqlTable("article_reads", {
export type ArticleRead = typeof articleReads.$inferSelect; export type ArticleRead = typeof articleReads.$inferSelect;
export type InsertArticleRead = typeof articleReads.$inferInsert; export type InsertArticleRead = typeof articleReads.$inferInsert;
// ─── Tombstones : articles déjà traités (empêche la réinsertion après purge) ──
export const processedDedupKeys = mysqlTable("processed_dedup_keys", {
id: int("id").autoincrement().primaryKey(),
dedupKey: varchar("dedupKey", { length: 64 }).notNull().unique(),
// Type de contenu (veille ou aap)
feedType: mysqlEnum("feedType", ["veille", "aap"]).notNull(),
// Date à laquelle l'article a été traité pour la première fois
processedAt: timestamp("processedAt").defaultNow().notNull(),
});
export type ProcessedDedupKey = typeof processedDedupKeys.$inferSelect;
export type InsertProcessedDedupKey = typeof processedDedupKeys.$inferInsert;

View File

@@ -19,6 +19,7 @@ import {
rssFeeds, rssFeeds,
veilleItems, veilleItems,
aapItems, aapItems,
processedDedupKeys,
type RssFeed, type RssFeed,
} from "../drizzle/schema"; } from "../drizzle/schema";
import { eq, sql } from "drizzle-orm"; import { eq, sql } from "drizzle-orm";
@@ -525,6 +526,15 @@ async function processFeed(feed: RssFeed): Promise<FetchResult> {
const normalizedTitle = buildMergeKey(title); const normalizedTitle = buildMergeKey(title);
const dedupKey = dedupHash(normalizedTitle + "|" + (feed.feedType ?? "")); const dedupKey = dedupHash(normalizedTitle + "|" + (feed.feedType ?? ""));
// ─── Tombstone : ignorer les articles déjà traités (même supprimés) ─────────
const tombstone = await db.select().from(processedDedupKeys)
.where(eq(processedDedupKeys.dedupKey, dedupKey))
.limit(1);
if (tombstone.length > 0) {
result.skippedItems++;
continue;
}
if (feed.feedType === "veille") { if (feed.feedType === "veille") {
// ── Classification IA (avec fallback sur mots-clés) ────────────────────────────────────── // ── Classification IA (avec fallback sur mots-clés) ──────────────────────────────────────
const { niveau, territoire } = detectVeilleNiveauTerritoire(fullText); const { niveau, territoire } = detectVeilleNiveauTerritoire(fullText);
@@ -578,6 +588,8 @@ async function processFeed(feed: RssFeed): Promise<FetchResult> {
iaReason: aiResult.reason, iaReason: aiResult.reason,
iaResume: iaResume || null, iaResume: iaResume || null,
}); });
// Enregistrer le tombstone pour éviter la réinsertion après purge
await db.insert(processedDedupKeys).values({ dedupKey, feedType: "veille" }).onDuplicateKeyUpdate({ set: { dedupKey } });
result.newItems++; result.newItems++;
} catch (e: any) { } catch (e: any) {
if (e?.code === "ER_DUP_ENTRY" || e?.cause?.code === "ER_DUP_ENTRY" || e?.message?.includes("Duplicate entry") || e?.cause?.message?.includes("Duplicate entry")) { if (e?.code === "ER_DUP_ENTRY" || e?.cause?.code === "ER_DUP_ENTRY" || e?.message?.includes("Duplicate entry") || e?.cause?.message?.includes("Duplicate entry")) {
@@ -652,6 +664,8 @@ async function processFeed(feed: RssFeed): Promise<FetchResult> {
iaReason: aiResult.reason, iaReason: aiResult.reason,
iaResume: iaResume || null, iaResume: iaResume || null,
}); });
// Enregistrer le tombstone pour éviter la réinsertion après purge
await db.insert(processedDedupKeys).values({ dedupKey, feedType: "aap" }).onDuplicateKeyUpdate({ set: { dedupKey } });
result.newItems++; result.newItems++;
} catch (e: any) { } catch (e: any) {
if (e?.code === "ER_DUP_ENTRY" || e?.cause?.code === "ER_DUP_ENTRY" || e?.message?.includes("Duplicate entry") || e?.cause?.message?.includes("Duplicate entry")) { if (e?.code === "ER_DUP_ENTRY" || e?.cause?.code === "ER_DUP_ENTRY" || e?.message?.includes("Duplicate entry") || e?.cause?.message?.includes("Duplicate entry")) {