Add cross-source duplicate detection, extend retention to 14 days

Exact URL/text matching missed the same story from a different outlet
(different URL, different wording). Add fuzzy title matching via
difflib, checked both within a single poll batch (the common case —
two feeds returning the same story in the same 5-minute cycle) and
against a rolling window of recently-sent titles in the seen-DB (the
cross-cycle case, e.g. follow-up coverage a few hours later).

seen_articles.db/seen_victims.db retention bumped from 7 to 14 days
(bot_config.SEEN_RETENTION_DAYS) — the window the new title matching
actually needs, and no reason to keep dedup history longer than that.

validation/run_validation.py: switch to the shared bot_config mapping,
and prune results.jsonl to the same 14-day window each poll instead of
growing forever — it's a testing aid, not an archive.

Also fixes a bug where a future-dated (bad clock/feed) article stayed
"recent" indefinitely instead of aging out after 48 hours, and drops
the legacy MD5 url-hash field that SHA-256 fingerprints replaced.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

validation/run_validation.py

@@ -14,7 +14,7 @@ import aiohttp
 import json
 import os
 import sys
-from datetime import datetime, timezone
+from datetime import datetime, timezone, timedelta
 from pathlib import Path
 
 ROOT = Path(__file__).parent.parent
@@ -33,19 +33,11 @@ if env_file.exists():
 from rss_manager import RSSFeedManager
 from content_classifier import ContentClassifier
 from ransomware_fetcher import RansomwareFetcher
-
-CATEGORY_CONFIG = {
-    "news":         "feeds/news_feeds.json",
-    "malware":      "feeds/malware_feeds.json",
-    "threat_intel": "feeds/threat_intel_feeds.json",
-    "osint":        "feeds/osint_feeds.json",
-    "research":     "feeds/research_feeds.json",
-}
+from bot_config import CATEGORY_FEEDS as CATEGORY_CONFIG, POLL_INTERVAL_SECONDS as POLL_INTERVAL, SEEN_RETENTION_DAYS
 
 OUTPUT_FILE        = Path(__file__).parent / "results.jsonl"
 SEEN_FILE          = str(Path(__file__).parent / "seen_validation.db")
 SEEN_VICTIMS_FILE  = str(Path(__file__).parent / "seen_validation_victims.db")
-POLL_INTERVAL      = 300
 
 
 def save_record(f, record: dict):
@@ -123,7 +115,7 @@ async def poll_victims(fetcher: RansomwareFetcher) -> int:
                 "source":           "ransomware.live",
                 "url":              article.get("url"),
                 "published":        article.get("published_human"),
-                "category":         "malware",
+                "category":         article.get("category"),
                 "cves":             [],
                 "threat_actors":    [],
                 "malware_families": article.get("malware_families", []),
@@ -135,6 +127,33 @@ async def poll_victims(fetcher: RansomwareFetcher) -> int:
     return len(new_victims)
 
 
+def prune_results_file():
+    """Keep results.jsonl bounded to the same window dedup actually needs — this is a
+    testing aid, not an archive, so nothing here needs to outlive SEEN_RETENTION_DAYS."""
+    if not OUTPUT_FILE.exists():
+        return
+    cutoff = (datetime.now(timezone.utc) - timedelta(days=SEEN_RETENTION_DAYS)).isoformat()
+    kept = []
+    dropped = 0
+    with open(OUTPUT_FILE) as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                record = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if record.get("run_at", "") >= cutoff:
+                kept.append(line)
+            else:
+                dropped += 1
+    if dropped:
+        with open(OUTPUT_FILE, "w") as f:
+            f.write("\n".join(kept) + ("\n" if kept else ""))
+        print(f"  Pruned {dropped} record(s) older than {SEEN_RETENTION_DAYS} days")
+
+
 async def main():
     classifier = ContentClassifier()
 
@@ -166,9 +185,11 @@ async def main():
             victim_count = await poll_victims(fetcher) if fetcher else 0
             total = rss_count + victim_count
             if total:
-                print(f"  Saved {rss_count} articles + {victim_count} victims\n")
+                print(f"  Saved {rss_count} articles + {victim_count} victims")
             else:
-                print(f"  Nothing new\n")
+                print(f"  Nothing new")
+            prune_results_file()
+            print()
         except Exception as e:
             print(f"  Error: {e}\n")