Drop unused IOC extraction
extract_iocs() (IPs, domains, hashes) ran on every classified article but the result was never shown in Telegram alerts, never saved to validation/results.jsonl, and never printed by review.py — pure wasted work every poll cycle. MITRE technique extraction stays; that one is actually used by the validation tooling. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
+1
-23
@@ -1,7 +1,7 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Content Classifier for Threat Intelligence RSS Bot
|
||||
Extracts CVEs, threat actors, malware families, MITRE techniques, and IOCs.
|
||||
Extracts CVEs, threat actors, malware families, and MITRE technique IDs.
|
||||
"""
|
||||
|
||||
import re
|
||||
@@ -47,11 +47,6 @@ class ContentClassifier:
|
||||
|
||||
MITRE_PATTERN = re.compile(r'T\d{4}(?:\.\d{3})?', re.IGNORECASE)
|
||||
CVE_PATTERN = re.compile(r'CVE-\d{4}-\d{4,7}', re.IGNORECASE)
|
||||
IP_PATTERN = re.compile(r'\b(?:\d{1,3}\.){3}\d{1,3}\b')
|
||||
DOMAIN_PATTERN = re.compile(r'\b[a-z0-9]+(?:[\-\.]{1}[a-z0-9]+)*\.[a-z]{2,6}\b', re.IGNORECASE)
|
||||
HASH_MD5_PATTERN = re.compile(r'\b[a-f0-9]{32}\b', re.IGNORECASE)
|
||||
HASH_SHA1_PATTERN = re.compile(r'\b[a-f0-9]{40}\b', re.IGNORECASE)
|
||||
HASH_SHA256_PATTERN = re.compile(r'\b[a-f0-9]{64}\b', re.IGNORECASE)
|
||||
|
||||
def extract_cves(self, text: str) -> List[str]:
|
||||
return list(set(cve.upper() for cve in self.CVE_PATTERN.findall(text)))
|
||||
@@ -75,21 +70,6 @@ class ContentClassifier:
|
||||
if re.search(r'\b' + re.escape(family.lower()) + r'\b', text_lower)
|
||||
))
|
||||
|
||||
def extract_iocs(self, text: str) -> Dict[str, List[str]]:
|
||||
iocs: Dict[str, List[str]] = {
|
||||
'ips': [], 'domains': [], 'md5': [], 'sha1': [], 'sha256': []
|
||||
}
|
||||
for ip in self.IP_PATTERN.findall(text):
|
||||
if all(0 <= int(o) <= 255 for o in ip.split('.')):
|
||||
iocs['ips'].append(ip)
|
||||
iocs['md5'] = self.HASH_MD5_PATTERN.findall(text)
|
||||
iocs['sha1'] = self.HASH_SHA1_PATTERN.findall(text)
|
||||
iocs['sha256'] = self.HASH_SHA256_PATTERN.findall(text)
|
||||
iocs['domains'] = [m.group(0).lower() for m in self.DOMAIN_PATTERN.finditer(text)]
|
||||
for key in iocs:
|
||||
iocs[key] = list(set(iocs[key]))[:5]
|
||||
return iocs
|
||||
|
||||
def classify_article(self, article: Dict) -> Dict:
|
||||
combined_text = f"{article.get('title', '')} {article.get('description', '')}"
|
||||
|
||||
@@ -97,13 +77,11 @@ class ContentClassifier:
|
||||
mitre_techniques = self.extract_mitre_techniques(combined_text)
|
||||
threat_actors = self.extract_threat_actors(combined_text)
|
||||
malware_families = self.extract_malware_families(combined_text)
|
||||
iocs = self.extract_iocs(combined_text)
|
||||
|
||||
article['cves'] = cves
|
||||
article['mitre_techniques'] = mitre_techniques
|
||||
article['threat_actors'] = threat_actors
|
||||
article['malware_families'] = malware_families
|
||||
article['iocs'] = iocs
|
||||
|
||||
logger.info(
|
||||
f"Classified: {article.get('title', '')[:50]}... | "
|
||||
|
||||
Reference in New Issue
Block a user