Drop unused IOC extraction

extract_iocs() (IPs, domains, hashes) ran on every classified article
but the result was never shown in Telegram alerts, never saved to
validation/results.jsonl, and never printed by review.py — pure wasted
work every poll cycle. MITRE technique extraction stays; that one is
actually used by the validation tooling.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
bot
2026-08-25 12:20:00 +03:00
parent ce3b813e8b
commit 1b2eb74cf1
+1 -23
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env python3
"""
Content Classifier for Threat Intelligence RSS Bot
Extracts CVEs, threat actors, malware families, MITRE techniques, and IOCs.
Extracts CVEs, threat actors, malware families, and MITRE technique IDs.
"""
import re
@@ -47,11 +47,6 @@ class ContentClassifier:
MITRE_PATTERN = re.compile(r'T\d{4}(?:\.\d{3})?', re.IGNORECASE)
CVE_PATTERN = re.compile(r'CVE-\d{4}-\d{4,7}', re.IGNORECASE)
IP_PATTERN = re.compile(r'\b(?:\d{1,3}\.){3}\d{1,3}\b')
DOMAIN_PATTERN = re.compile(r'\b[a-z0-9]+(?:[\-\.]{1}[a-z0-9]+)*\.[a-z]{2,6}\b', re.IGNORECASE)
HASH_MD5_PATTERN = re.compile(r'\b[a-f0-9]{32}\b', re.IGNORECASE)
HASH_SHA1_PATTERN = re.compile(r'\b[a-f0-9]{40}\b', re.IGNORECASE)
HASH_SHA256_PATTERN = re.compile(r'\b[a-f0-9]{64}\b', re.IGNORECASE)
def extract_cves(self, text: str) -> List[str]:
return list(set(cve.upper() for cve in self.CVE_PATTERN.findall(text)))
@@ -75,21 +70,6 @@ class ContentClassifier:
if re.search(r'\b' + re.escape(family.lower()) + r'\b', text_lower)
))
def extract_iocs(self, text: str) -> Dict[str, List[str]]:
iocs: Dict[str, List[str]] = {
'ips': [], 'domains': [], 'md5': [], 'sha1': [], 'sha256': []
}
for ip in self.IP_PATTERN.findall(text):
if all(0 <= int(o) <= 255 for o in ip.split('.')):
iocs['ips'].append(ip)
iocs['md5'] = self.HASH_MD5_PATTERN.findall(text)
iocs['sha1'] = self.HASH_SHA1_PATTERN.findall(text)
iocs['sha256'] = self.HASH_SHA256_PATTERN.findall(text)
iocs['domains'] = [m.group(0).lower() for m in self.DOMAIN_PATTERN.finditer(text)]
for key in iocs:
iocs[key] = list(set(iocs[key]))[:5]
return iocs
def classify_article(self, article: Dict) -> Dict:
combined_text = f"{article.get('title', '')} {article.get('description', '')}"
@@ -97,13 +77,11 @@ class ContentClassifier:
mitre_techniques = self.extract_mitre_techniques(combined_text)
threat_actors = self.extract_threat_actors(combined_text)
malware_families = self.extract_malware_families(combined_text)
iocs = self.extract_iocs(combined_text)
article['cves'] = cves
article['mitre_techniques'] = mitre_techniques
article['threat_actors'] = threat_actors
article['malware_families'] = malware_families
article['iocs'] = iocs
logger.info(
f"Classified: {article.get('title', '')[:50]}... | "