Checkpoint: Ajout de la table processed_dedup_keys : chaque article traité par le moteur RSS est enregistré dans cette table. Lors des passages suivants du cron, si la dedupKey est présente dans cette table, l'article est ignoré même s'il a été supprimé de veille_items ou aap_items.
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
{
|
||||
"version": "0f628f8c",
|
||||
"timestamp": 1782891184609
|
||||
"version": "78f2d166",
|
||||
"timestamp": 1782978598907
|
||||
}
|
||||
8
drizzle/0010_graceful_nova.sql
Normal file
8
drizzle/0010_graceful_nova.sql
Normal file
@@ -0,0 +1,8 @@
|
||||
CREATE TABLE `processed_dedup_keys` (
|
||||
`id` int AUTO_INCREMENT NOT NULL,
|
||||
`dedupKey` varchar(64) NOT NULL,
|
||||
`feedType` enum('veille','aap') NOT NULL,
|
||||
`processedAt` timestamp NOT NULL DEFAULT (now()),
|
||||
CONSTRAINT `processed_dedup_keys_id` PRIMARY KEY(`id`),
|
||||
CONSTRAINT `processed_dedup_keys_dedupKey_unique` UNIQUE(`dedupKey`)
|
||||
);
|
||||
1038
drizzle/meta/0010_snapshot.json
Normal file
1038
drizzle/meta/0010_snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
@@ -71,6 +71,13 @@
|
||||
"when": 1781683138994,
|
||||
"tag": "0009_fat_rocket_racer",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 10,
|
||||
"version": "5",
|
||||
"when": 1782978530081,
|
||||
"tag": "0010_graceful_nova",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -213,3 +213,17 @@ export const articleReads = mysqlTable("article_reads", {
|
||||
|
||||
export type ArticleRead = typeof articleReads.$inferSelect;
|
||||
export type InsertArticleRead = typeof articleReads.$inferInsert;
|
||||
|
||||
// ─── Tombstones : articles déjà traités (empêche la réinsertion après purge) ──
|
||||
|
||||
export const processedDedupKeys = mysqlTable("processed_dedup_keys", {
|
||||
id: int("id").autoincrement().primaryKey(),
|
||||
dedupKey: varchar("dedupKey", { length: 64 }).notNull().unique(),
|
||||
// Type de contenu (veille ou aap)
|
||||
feedType: mysqlEnum("feedType", ["veille", "aap"]).notNull(),
|
||||
// Date à laquelle l'article a été traité pour la première fois
|
||||
processedAt: timestamp("processedAt").defaultNow().notNull(),
|
||||
});
|
||||
|
||||
export type ProcessedDedupKey = typeof processedDedupKeys.$inferSelect;
|
||||
export type InsertProcessedDedupKey = typeof processedDedupKeys.$inferInsert;
|
||||
|
||||
@@ -19,6 +19,7 @@ import {
|
||||
rssFeeds,
|
||||
veilleItems,
|
||||
aapItems,
|
||||
processedDedupKeys,
|
||||
type RssFeed,
|
||||
} from "../drizzle/schema";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
@@ -525,6 +526,15 @@ async function processFeed(feed: RssFeed): Promise<FetchResult> {
|
||||
const normalizedTitle = buildMergeKey(title);
|
||||
const dedupKey = dedupHash(normalizedTitle + "|" + (feed.feedType ?? ""));
|
||||
|
||||
// ─── Tombstone : ignorer les articles déjà traités (même supprimés) ─────────
|
||||
const tombstone = await db.select().from(processedDedupKeys)
|
||||
.where(eq(processedDedupKeys.dedupKey, dedupKey))
|
||||
.limit(1);
|
||||
if (tombstone.length > 0) {
|
||||
result.skippedItems++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (feed.feedType === "veille") {
|
||||
// ── Classification IA (avec fallback sur mots-clés) ──────────────────────────────────────
|
||||
const { niveau, territoire } = detectVeilleNiveauTerritoire(fullText);
|
||||
@@ -578,6 +588,8 @@ async function processFeed(feed: RssFeed): Promise<FetchResult> {
|
||||
iaReason: aiResult.reason,
|
||||
iaResume: iaResume || null,
|
||||
});
|
||||
// Enregistrer le tombstone pour éviter la réinsertion après purge
|
||||
await db.insert(processedDedupKeys).values({ dedupKey, feedType: "veille" }).onDuplicateKeyUpdate({ set: { dedupKey } });
|
||||
result.newItems++;
|
||||
} catch (e: any) {
|
||||
if (e?.code === "ER_DUP_ENTRY" || e?.cause?.code === "ER_DUP_ENTRY" || e?.message?.includes("Duplicate entry") || e?.cause?.message?.includes("Duplicate entry")) {
|
||||
@@ -652,6 +664,8 @@ async function processFeed(feed: RssFeed): Promise<FetchResult> {
|
||||
iaReason: aiResult.reason,
|
||||
iaResume: iaResume || null,
|
||||
});
|
||||
// Enregistrer le tombstone pour éviter la réinsertion après purge
|
||||
await db.insert(processedDedupKeys).values({ dedupKey, feedType: "aap" }).onDuplicateKeyUpdate({ set: { dedupKey } });
|
||||
result.newItems++;
|
||||
} catch (e: any) {
|
||||
if (e?.code === "ER_DUP_ENTRY" || e?.cause?.code === "ER_DUP_ENTRY" || e?.message?.includes("Duplicate entry") || e?.cause?.message?.includes("Duplicate entry")) {
|
||||
|
||||
Reference in New Issue
Block a user