Drop unused IOC extraction
extract_iocs() (IPs, domains, hashes) ran on every classified article but the result was never shown in Telegram alerts, never saved to validation/results.jsonl, and never printed by review.py — pure wasted work every poll cycle. MITRE technique extraction stays; that one is actually used by the validation tooling. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
+1
-23
@@ -1,7 +1,7 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""
|
"""
|
||||||
Content Classifier for Threat Intelligence RSS Bot
|
Content Classifier for Threat Intelligence RSS Bot
|
||||||
Extracts CVEs, threat actors, malware families, MITRE techniques, and IOCs.
|
Extracts CVEs, threat actors, malware families, and MITRE technique IDs.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import re
|
import re
|
||||||
@@ -47,11 +47,6 @@ class ContentClassifier:
|
|||||||
|
|
||||||
MITRE_PATTERN = re.compile(r'T\d{4}(?:\.\d{3})?', re.IGNORECASE)
|
MITRE_PATTERN = re.compile(r'T\d{4}(?:\.\d{3})?', re.IGNORECASE)
|
||||||
CVE_PATTERN = re.compile(r'CVE-\d{4}-\d{4,7}', re.IGNORECASE)
|
CVE_PATTERN = re.compile(r'CVE-\d{4}-\d{4,7}', re.IGNORECASE)
|
||||||
IP_PATTERN = re.compile(r'\b(?:\d{1,3}\.){3}\d{1,3}\b')
|
|
||||||
DOMAIN_PATTERN = re.compile(r'\b[a-z0-9]+(?:[\-\.]{1}[a-z0-9]+)*\.[a-z]{2,6}\b', re.IGNORECASE)
|
|
||||||
HASH_MD5_PATTERN = re.compile(r'\b[a-f0-9]{32}\b', re.IGNORECASE)
|
|
||||||
HASH_SHA1_PATTERN = re.compile(r'\b[a-f0-9]{40}\b', re.IGNORECASE)
|
|
||||||
HASH_SHA256_PATTERN = re.compile(r'\b[a-f0-9]{64}\b', re.IGNORECASE)
|
|
||||||
|
|
||||||
def extract_cves(self, text: str) -> List[str]:
|
def extract_cves(self, text: str) -> List[str]:
|
||||||
return list(set(cve.upper() for cve in self.CVE_PATTERN.findall(text)))
|
return list(set(cve.upper() for cve in self.CVE_PATTERN.findall(text)))
|
||||||
@@ -75,21 +70,6 @@ class ContentClassifier:
|
|||||||
if re.search(r'\b' + re.escape(family.lower()) + r'\b', text_lower)
|
if re.search(r'\b' + re.escape(family.lower()) + r'\b', text_lower)
|
||||||
))
|
))
|
||||||
|
|
||||||
def extract_iocs(self, text: str) -> Dict[str, List[str]]:
|
|
||||||
iocs: Dict[str, List[str]] = {
|
|
||||||
'ips': [], 'domains': [], 'md5': [], 'sha1': [], 'sha256': []
|
|
||||||
}
|
|
||||||
for ip in self.IP_PATTERN.findall(text):
|
|
||||||
if all(0 <= int(o) <= 255 for o in ip.split('.')):
|
|
||||||
iocs['ips'].append(ip)
|
|
||||||
iocs['md5'] = self.HASH_MD5_PATTERN.findall(text)
|
|
||||||
iocs['sha1'] = self.HASH_SHA1_PATTERN.findall(text)
|
|
||||||
iocs['sha256'] = self.HASH_SHA256_PATTERN.findall(text)
|
|
||||||
iocs['domains'] = [m.group(0).lower() for m in self.DOMAIN_PATTERN.finditer(text)]
|
|
||||||
for key in iocs:
|
|
||||||
iocs[key] = list(set(iocs[key]))[:5]
|
|
||||||
return iocs
|
|
||||||
|
|
||||||
def classify_article(self, article: Dict) -> Dict:
|
def classify_article(self, article: Dict) -> Dict:
|
||||||
combined_text = f"{article.get('title', '')} {article.get('description', '')}"
|
combined_text = f"{article.get('title', '')} {article.get('description', '')}"
|
||||||
|
|
||||||
@@ -97,13 +77,11 @@ class ContentClassifier:
|
|||||||
mitre_techniques = self.extract_mitre_techniques(combined_text)
|
mitre_techniques = self.extract_mitre_techniques(combined_text)
|
||||||
threat_actors = self.extract_threat_actors(combined_text)
|
threat_actors = self.extract_threat_actors(combined_text)
|
||||||
malware_families = self.extract_malware_families(combined_text)
|
malware_families = self.extract_malware_families(combined_text)
|
||||||
iocs = self.extract_iocs(combined_text)
|
|
||||||
|
|
||||||
article['cves'] = cves
|
article['cves'] = cves
|
||||||
article['mitre_techniques'] = mitre_techniques
|
article['mitre_techniques'] = mitre_techniques
|
||||||
article['threat_actors'] = threat_actors
|
article['threat_actors'] = threat_actors
|
||||||
article['malware_families'] = malware_families
|
article['malware_families'] = malware_families
|
||||||
article['iocs'] = iocs
|
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
f"Classified: {article.get('title', '')[:50]}... | "
|
f"Classified: {article.get('title', '')[:50]}... | "
|
||||||
|
|||||||
Reference in New Issue
Block a user