Ransomware.live integration, 48h window, CISA feeds, validation updates, README
This commit is contained in:
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Review validation results — pretty-prints results.jsonl with optional filters.
|
||||
|
||||
Usage:
|
||||
python3 validation/review.py # all results
|
||||
python3 validation/review.py --category research # filter by category
|
||||
python3 validation/review.py --severity critical # filter by severity
|
||||
python3 validation/review.py --has cves # only articles with CVEs
|
||||
python3 validation/review.py --has malware # only articles with malware hits
|
||||
python3 validation/review.py --has actors # only articles with threat actors
|
||||
python3 validation/review.py --today # only today's run
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
|
||||
RESULTS_FILE = Path(__file__).parent / "results.jsonl"
|
||||
|
||||
SEVERITY_ORDER = {"critical": 0, "high": 1, "medium": 2, "low": 3, "unknown": 4}
|
||||
|
||||
|
||||
def parse_args():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--category", help="Filter by category (news, malware, threat_intel, osint, research)")
|
||||
p.add_argument("--severity", help="Filter by severity (critical, high, medium, low)")
|
||||
p.add_argument("--has", choices=["cves", "malware", "actors"], help="Only show articles with these detections")
|
||||
p.add_argument("--today", action="store_true", help="Only show results from today")
|
||||
p.add_argument("--limit", type=int, default=0, help="Max articles to show (0 = all)")
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
def load_results(args):
|
||||
if not RESULTS_FILE.exists():
|
||||
print("No results.jsonl found — run run_validation.py first.")
|
||||
sys.exit(0)
|
||||
|
||||
today = datetime.now(timezone.utc).date().isoformat()
|
||||
records = []
|
||||
|
||||
with open(RESULTS_FILE) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
r = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
if args.today and not r.get("run_at", "").startswith(today):
|
||||
continue
|
||||
if args.category and r.get("category") != args.category:
|
||||
continue
|
||||
if args.severity and r.get("severity") != args.severity:
|
||||
continue
|
||||
if args.has == "cves" and not r.get("cves"):
|
||||
continue
|
||||
if args.has == "malware" and not r.get("malware_families"):
|
||||
continue
|
||||
if args.has == "actors" and not r.get("threat_actors"):
|
||||
continue
|
||||
|
||||
records.append(r)
|
||||
|
||||
records.sort(key=lambda x: SEVERITY_ORDER.get(x.get("severity", "unknown"), 4))
|
||||
return records
|
||||
|
||||
|
||||
def print_record(r):
|
||||
sev = r.get("severity", "unknown").upper()
|
||||
sev_icons = {"CRITICAL": "🚨", "HIGH": "🔴", "MEDIUM": "🟠", "LOW": "🟡"}
|
||||
icon = sev_icons.get(sev, "⚪")
|
||||
|
||||
print(f"\n{'─' * 80}")
|
||||
print(f"{icon} [{r.get('category', '?').upper()}] {r.get('title', 'No title')}")
|
||||
print(f" 📡 {r.get('source')} · {r.get('published')} · Score: {r.get('quality_score', '?')}/100")
|
||||
|
||||
if r.get("cves"):
|
||||
print(f" 🆔 {', '.join(r['cves'])}")
|
||||
if r.get("threat_actors"):
|
||||
print(f" 👤 {', '.join(r['threat_actors'])}")
|
||||
if r.get("malware_families"):
|
||||
print(f" 🦠 {', '.join(r['malware_families'])}")
|
||||
if r.get("mitre_techniques"):
|
||||
print(f" 🎯 {', '.join(r['mitre_techniques'][:5])}")
|
||||
if r.get("description"):
|
||||
print(f" {r['description'][:200]}...")
|
||||
print(f" 🔗 {r.get('url', '')}")
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
records = load_results(args)
|
||||
|
||||
if not records:
|
||||
print("No records match the filters.")
|
||||
return
|
||||
|
||||
if args.limit:
|
||||
records = records[:args.limit]
|
||||
|
||||
total = len(records)
|
||||
sev_counts = {}
|
||||
for r in records:
|
||||
s = r.get("severity", "unknown")
|
||||
sev_counts[s] = sev_counts.get(s, 0) + 1
|
||||
|
||||
print(f"\n=== Validation Results ({total} articles) ===")
|
||||
print(" " + " ".join(f"{s.upper()}: {c}" for s, c in sorted(sev_counts.items(), key=lambda x: SEVERITY_ORDER.get(x[0], 4))))
|
||||
|
||||
for r in records:
|
||||
print_record(r)
|
||||
|
||||
print(f"\n{'─' * 80}")
|
||||
print(f"Total: {total} articles")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,180 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Validation script — continuously fetches RSS feeds + ransomware.live victims,
|
||||
classifies articles, appends everything to results.jsonl.
|
||||
Runs independently from the bot using its own seen DBs.
|
||||
|
||||
Usage:
|
||||
python3 validation/run_validation.py
|
||||
Ctrl+C to stop
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
# Load .env from project root
|
||||
env_file = ROOT / ".env"
|
||||
if env_file.exists():
|
||||
with open(env_file) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#") and "=" in line:
|
||||
k, v = line.split("=", 1)
|
||||
os.environ.setdefault(k.strip(), v.strip().strip("\"'"))
|
||||
|
||||
from rss_manager import RSSFeedManager
|
||||
from content_classifier import ContentClassifier
|
||||
from ransomware_fetcher import RansomwareFetcher
|
||||
|
||||
CATEGORY_CONFIG = {
|
||||
"news": "feeds/news_feeds.json",
|
||||
"malware": "feeds/malware_feeds.json",
|
||||
"threat_intel": "feeds/threat_intel_feeds.json",
|
||||
"osint": "feeds/osint_feeds.json",
|
||||
"research": "feeds/research_feeds.json",
|
||||
}
|
||||
|
||||
OUTPUT_FILE = Path(__file__).parent / "results.jsonl"
|
||||
SEEN_FILE = str(Path(__file__).parent / "seen_validation.db")
|
||||
SEEN_VICTIMS_FILE = str(Path(__file__).parent / "seen_validation_victims.db")
|
||||
POLL_INTERVAL = 300
|
||||
|
||||
|
||||
def save_record(f, record: dict):
|
||||
f.write(json.dumps(record) + "\n")
|
||||
flags = []
|
||||
if record.get("cves"): flags.append(f"CVEs: {', '.join(record['cves'][:3])}")
|
||||
if record.get("threat_actors"): flags.append(f"Actors: {', '.join(record['threat_actors'][:2])}")
|
||||
if record.get("malware_families"): flags.append(f"Malware: {', '.join(record['malware_families'][:2])}")
|
||||
flag_str = f" → {' | '.join(flags)}" if flags else ""
|
||||
category = record.get("category", "?").upper()
|
||||
print(f" [{category:12s}] {record.get('title','')[:70]}{flag_str}")
|
||||
|
||||
|
||||
async def poll_rss(classifier: ContentClassifier) -> int:
|
||||
all_articles = []
|
||||
for category, feeds_file in CATEGORY_CONFIG.items():
|
||||
feeds_path = str(ROOT / feeds_file)
|
||||
async with RSSFeedManager(feeds_file=feeds_path, seen_file=SEEN_FILE, feed_type=category) as manager:
|
||||
articles = await manager.fetch_all_feeds(initial_run=False)
|
||||
all_articles.extend(articles)
|
||||
|
||||
if not all_articles:
|
||||
return 0
|
||||
|
||||
unique = []
|
||||
seen_urls, seen_content = set(), set()
|
||||
for article in all_articles:
|
||||
fp = RSSFeedManager.get_article_fingerprints(article)
|
||||
url_key, content_key = fp["url_key"], fp["content_key"]
|
||||
if (url_key and url_key in seen_urls) or content_key in seen_content:
|
||||
continue
|
||||
if url_key:
|
||||
seen_urls.add(url_key)
|
||||
seen_content.add(content_key)
|
||||
unique.append(article)
|
||||
|
||||
with open(OUTPUT_FILE, "a") as f:
|
||||
for article in unique:
|
||||
classified = classifier.classify_article(article)
|
||||
RSSFeedManager.mark_article_as_sent(classified, seen_file=SEEN_FILE)
|
||||
record = {
|
||||
"run_at": datetime.now(timezone.utc).isoformat(),
|
||||
"type": "article",
|
||||
"title": classified.get("title"),
|
||||
"source": classified.get("source"),
|
||||
"url": classified.get("url"),
|
||||
"published": classified.get("published_human"),
|
||||
"category": classified.get("category"),
|
||||
"cves": classified.get("cves", []),
|
||||
"threat_actors": classified.get("threat_actors", []),
|
||||
"malware_families": classified.get("malware_families", []),
|
||||
"mitre_techniques": classified.get("mitre_techniques", []),
|
||||
"description": classified.get("description", ""),
|
||||
}
|
||||
save_record(f, record)
|
||||
|
||||
return len(unique)
|
||||
|
||||
|
||||
async def poll_victims(fetcher: RansomwareFetcher) -> int:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
new_victims = await fetcher.fetch_new_victims(session)
|
||||
|
||||
if not new_victims:
|
||||
return 0
|
||||
|
||||
with open(OUTPUT_FILE, "a") as f:
|
||||
for victim in new_victims:
|
||||
article = RansomwareFetcher.to_article(victim)
|
||||
fetcher.mark_seen(victim.get("id", ""))
|
||||
record = {
|
||||
"run_at": datetime.now(timezone.utc).isoformat(),
|
||||
"type": "victim",
|
||||
"title": article.get("title"),
|
||||
"source": "ransomware.live",
|
||||
"url": article.get("url"),
|
||||
"published": article.get("published_human"),
|
||||
"category": "malware",
|
||||
"cves": [],
|
||||
"threat_actors": [],
|
||||
"malware_families": article.get("malware_families", []),
|
||||
"mitre_techniques": [],
|
||||
"description": article.get("description", ""),
|
||||
}
|
||||
save_record(f, record)
|
||||
|
||||
return len(new_victims)
|
||||
|
||||
|
||||
async def main():
|
||||
classifier = ContentClassifier()
|
||||
|
||||
ransomware_key = os.getenv("RANSOMWARE_LIVE_API_KEY", "")
|
||||
fetcher = RansomwareFetcher(ransomware_key, seen_file=SEEN_VICTIMS_FILE) if ransomware_key else None
|
||||
|
||||
print(f"Validation monitor started — polling every {POLL_INTERVAL}s. Ctrl+C to stop.")
|
||||
print(f"RSS feeds: {'enabled' if CATEGORY_CONFIG else 'disabled'}")
|
||||
print(f"ransomware.live: {'enabled' if fetcher else 'disabled (no API key)'}")
|
||||
print(f"Output: {OUTPUT_FILE}\n")
|
||||
|
||||
# Initial scan — mark everything currently visible as seen
|
||||
print(f"[{datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}] Initial scan...")
|
||||
for category, feeds_file in CATEGORY_CONFIG.items():
|
||||
feeds_path = str(ROOT / feeds_file)
|
||||
async with RSSFeedManager(feeds_file=feeds_path, seen_file=SEEN_FILE, feed_type=category) as manager:
|
||||
await manager.fetch_all_feeds(initial_run=True)
|
||||
if fetcher:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
await fetcher.fetch_new_victims(session, initial_run=True)
|
||||
print(" Done. Watching for new items from now on.\n")
|
||||
|
||||
while True:
|
||||
await asyncio.sleep(POLL_INTERVAL)
|
||||
ts = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
|
||||
print(f"[{ts}] Polling...")
|
||||
try:
|
||||
rss_count = await poll_rss(classifier)
|
||||
victim_count = await poll_victims(fetcher) if fetcher else 0
|
||||
total = rss_count + victim_count
|
||||
if total:
|
||||
print(f" Saved {rss_count} articles + {victim_count} victims\n")
|
||||
else:
|
||||
print(f" Nothing new\n")
|
||||
except Exception as e:
|
||||
print(f" Error: {e}\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
asyncio.run(main())
|
||||
except KeyboardInterrupt:
|
||||
print("\nStopped.")
|
||||
Reference in New Issue
Block a user