bot/esp32-cluster
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revive project at new apartment — fixes, SSID update, gitignore

Browse Source 
WiFi/firmware:
- SSID updated to botnet in both node and ble_node config.h
- Stale coordinator comment removed from node/config.h

Coordinator fixes:
- BLE-only nodes now visible in dashboard sidebar and node detail
  (build_nodes was querying beacon_events only; BLE nodes have no beacons)
- ble_events and heartbeat_events added to pruning cycle
- ble_events table added to ensure_schema() in dashboard
- confidence column dropped from beacon_events (always 'high', never queried)
- Ingestor commits batched per packet instead of per store call
- wal_autocheckpoint=500 added to ingestor DB connection (writer was missing it)
- python3 -u added to both service ExecStart lines (stdout was buffered,
  logs not appearing in journalctl)

Repo hygiene:
- .gitignore added (events.db, __pycache__, build artifacts)
- events.db removed from git tracking
- README updated: new apartment, node table, active investigations,
  BLE observations labelled as previous-location data, status checklist

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
bot
2026-05-21 15:30:15 +03:00
parent 1ca2dc21f5
commit 4924cf53e5
9 changed files with 210 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
coordinator/dashboard.py
+89 -11
View File
@@ -196,7 +196,9 @@ def build_nodes() -> list[dict]:
now = datetime.datetime.utcnow().timestamp()
if _nodes_cache and (now - _nodes_cache[0]) < _NODES_CACHE_TTL:
return _nodes_cache[1]
rows = query("""
# WiFi nodes: beacon-derived stats
beacon_rows = query("""
SELECT
node_id,
MAX(received_at) AS last_beacon,
@@ -204,9 +206,9 @@ def build_nodes() -> list[dict]:
MIN(received_at) AS first_seen
FROM beacon_events
GROUP BY node_id
ORDER BY node_id ASC
""")
# Latest heartbeat per node
# Latest heartbeat per node — covers ALL node types (WiFi and BLE)
hb_rows = query("""
SELECT node_id, received_at AS last_heartbeat,
uptime_ms, free_heap, wifi_rssi
@@ -214,6 +216,21 @@ def build_nodes() -> list[dict]:
WHERE id IN (SELECT MAX(id) FROM heartbeat_events GROUP BY node_id)
""")
hb_map = {r["node_id"]: r for r in hb_rows}
node_map: dict = {r["node_id"]: r for r in beacon_rows}
# BLE-only nodes have no beacon events — add them from heartbeat data
for node_id, hb in hb_map.items():
if node_id not in node_map:
node_map[node_id] = {
"node_id": node_id,
"last_beacon": None,
"total_events": 0,
"first_seen": hb.get("last_heartbeat"),
}
rows = sorted(node_map.values(), key=lambda r: r["node_id"])
for r in rows:
hb = hb_map.get(r["node_id"], {})
r["last_heartbeat"] = hb.get("last_heartbeat")
@@ -221,11 +238,11 @@ def build_nodes() -> list[dict]:
r["free_heap"] = hb.get("free_heap")
r["wifi_rssi"] = hb.get("wifi_rssi")
r["last_seen"] = r["last_heartbeat"] or r["last_beacon"]
# Online if heartbeat within 30s; fall back to beacon within 60s if no heartbeat yet
if r["last_heartbeat"]:
r["status"] = _node_status(r["last_heartbeat"], threshold=30)
else:
r["status"] = _node_status(r["last_beacon"], threshold=60)
_nodes_cache = (now, rows)
return rows
@@ -973,6 +990,7 @@ def build_search(q: str) -> dict:
def build_node_detail(node_id: str) -> dict | None:
# WiFi node path — beacon-derived stats
rows = query("""
SELECT
node_id,
@@ -988,9 +1006,51 @@ def build_node_detail(node_id: str) -> dict | None:
WHERE node_id = ?
GROUP BY node_id
""", (node_id,))
if not rows:
return None
detail = rows[0]
if rows:
detail = rows[0]
detail["node_type"] = "wifi"
else:
# BLE node path — stats from ble_events
ble_rows = query("""
SELECT
node_id,
MIN(received_at) AS first_seen,
MAX(received_at) AS last_seen,
COUNT(*) AS total_events,
COUNT(DISTINCT mac) AS unique_macs,
AVG(rssi) AS avg_rssi,
MIN(rssi) AS min_rssi,
MAX(rssi) AS max_rssi
FROM ble_events
WHERE node_id = ?
GROUP BY node_id
""", (node_id,))
if ble_rows:
detail = ble_rows[0]
else:
# Node exists (heartbeats only) but no scan data yet
hb_check = query(
"SELECT node_id FROM heartbeat_events WHERE node_id = ? LIMIT 1", (node_id,)
)
if not hb_check:
return None
detail = {
"node_id": node_id,
"first_seen": None,
"last_seen": None,
"total_events": 0,
"unique_macs": 0,
"avg_rssi": None,
"min_rssi": None,
"max_rssi": None,
}
detail["node_type"] = "ble"
detail["unique_ssids"] = 0
detail["unique_bssids"] = 0
hb_rows = query("""
SELECT received_at AS last_heartbeat, uptime_ms, free_heap, wifi_rssi
FROM heartbeat_events
@@ -1005,8 +1065,8 @@ def build_node_detail(node_id: str) -> dict | None:
if detail["last_heartbeat"]:
detail["status"] = _node_status(detail["last_heartbeat"], threshold=30)
else:
detail["status"] = _node_status(detail["last_seen"], threshold=60)
detail["avg_rssi"] = round(detail["avg_rssi"], 1) if detail["avg_rssi"] else None
detail["status"] = _node_status(detail.get("last_seen"), threshold=60)
detail["avg_rssi"] = round(detail["avg_rssi"], 1) if detail.get("avg_rssi") else None
detail["events"] = build_events(limit=EVENT_CAP, node_id=node_id)
return detail
@@ -1019,7 +1079,7 @@ def ensure_schema():
CREATE TABLE IF NOT EXISTS beacon_events (
id INTEGER PRIMARY KEY AUTOINCREMENT, received_at TEXT NOT NULL,
node_id TEXT, node_ts INTEGER, ssid TEXT, bssid TEXT,
rssi INTEGER, channel INTEGER, encryption TEXT, importance TEXT, confidence TEXT
rssi INTEGER, channel INTEGER, encryption TEXT, importance TEXT
)
""")
conn.execute("""
@@ -1063,6 +1123,20 @@ def ensure_schema():
rssi INTEGER
)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS ble_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
received_at TEXT NOT NULL,
node_id TEXT,
node_ts INTEGER,
mac TEXT,
addr_type TEXT,
name TEXT,
rssi INTEGER,
mfr_id INTEGER,
mfr_data TEXT
)
""")
conn.executescript("""
CREATE INDEX IF NOT EXISTS idx_beacon_received_at ON beacon_events (received_at);
CREATE INDEX IF NOT EXISTS idx_beacon_node_id ON beacon_events (node_id);
@@ -1083,6 +1157,10 @@ def ensure_schema():
CREATE INDEX IF NOT EXISTS idx_assoc_node_id ON assoc_events (node_id);
CREATE INDEX IF NOT EXISTS idx_assoc_src ON assoc_events (src);
CREATE INDEX IF NOT EXISTS idx_assoc_bssid ON assoc_events (bssid);
CREATE INDEX IF NOT EXISTS idx_ble_received_at ON ble_events (received_at);
CREATE INDEX IF NOT EXISTS idx_ble_node_id ON ble_events (node_id);
CREATE INDEX IF NOT EXISTS idx_ble_mac ON ble_events (mac);
CREATE INDEX IF NOT EXISTS idx_ble_mfr_id ON ble_events (mfr_id);
""")
conn.commit()
@@ -1094,7 +1172,7 @@ CHECKPOINT_INTERVAL_SECS = 30 * 60 # force WAL checkpoint every 30 minutes
def prune_old_events() -> None:
"""Delete events older than HOT_DAYS from all event tables, then checkpoint."""
cutoff = _hot_cutoff()
tables = ["beacon_events", "probe_events", "deauth_events", "assoc_events"]
tables = ["beacon_events", "probe_events", "deauth_events", "assoc_events", "ble_events", "heartbeat_events"]
with get_conn() as conn:
for table in tables:
conn.execute(f"DELETE FROM {table} WHERE received_at < ?", (cutoff,))
coordinator/dashboard.service
+1 -1
View File
@@ -5,7 +5,7 @@ After=network.target
[Service]
Type=simple
WorkingDirectory=/opt/esp32_cluster/coordinator
ExecStart=/usr/bin/python3 dashboard.py
ExecStart=/usr/bin/python3 -u dashboard.py
Restart=always
RestartSec=5
StandardOutput=journal
coordinator/events.db
BIN
View File
Binary file not shown.
coordinator/udp_ingest.py
+6 -10
View File
@@ -21,6 +21,7 @@ DB_PATH = os.path.join(os.path.dirname(__file__), "events.db")
def init_db(path: str) -> sqlite3.Connection:
conn = sqlite3.connect(path, check_same_thread=False)
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA wal_autocheckpoint=500")
conn.execute("""
CREATE TABLE IF NOT EXISTS beacon_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -32,8 +33,7 @@ def init_db(path: str) -> sqlite3.Connection:
rssi INTEGER,
channel INTEGER,
encryption TEXT,
importance TEXT,
confidence TEXT
importance TEXT
)
""")
conn.execute("""
@@ -139,8 +139,8 @@ def init_db(path: str) -> sqlite3.Connection:
def store_beacon(conn: sqlite3.Connection, ev: dict, received_at: str):
conn.execute("""
INSERT INTO beacon_events
(received_at, node_id, node_ts, ssid, bssid, rssi, channel, encryption, importance, confidence)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
(received_at, node_id, node_ts, ssid, bssid, rssi, channel, encryption, importance)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""", (
received_at,
ev.get("node_id"),
@@ -151,7 +151,6 @@ def store_beacon(conn: sqlite3.Connection, ev: dict, received_at: str):
ev.get("ch"),
ev.get("enc"),
ev.get("imp"),
ev.get("conf"),
))
conn.commit()
@@ -174,7 +173,6 @@ def store_heartbeat(conn: sqlite3.Connection, ev: dict, received_at: str):
ev.get("assoc_drops"),
ev.get("ble_drops"),
))
conn.commit()
def store_probe(conn: sqlite3.Connection, ev: dict, received_at: str):
@@ -191,7 +189,6 @@ def store_probe(conn: sqlite3.Connection, ev: dict, received_at: str):
ev.get("rssi"),
ev.get("imp"),
))
conn.commit()
# ─── Validation ──────────────────────────────────────────────────────────────
@@ -244,7 +241,6 @@ def store_deauth(conn: sqlite3.Connection, ev: dict, received_at: str):
ev.get("reason"),
ev.get("rssi"),
))
conn.commit()
def validate_deauth(ev: dict) -> str | None:
@@ -279,7 +275,6 @@ def store_assoc(conn: sqlite3.Connection, ev: dict, received_at: str):
ev.get("ssid", ""),
ev.get("rssi"),
))
conn.commit()
def validate_assoc(ev: dict) -> str | None:
@@ -312,7 +307,6 @@ def store_ble(conn: sqlite3.Connection, ev: dict, received_at: str):
mfr_id if mfr_id is not None and mfr_id >= 0 else None,
ev.get("mfr_data", ""),
))
conn.commit()
def validate_ble(ev: dict) -> str | None:
@@ -438,6 +432,8 @@ def handle_packet(data: bytes, addr: tuple, conn: sqlite3.Connection):
color = IMP_COLOR.get(imp, "")
print(f"{color}[{received_at}] {node} {ssid:<32} {bssid} ch{ch:<3} {rssi:>4}dBm {enc}{RESET}")
conn.commit()
# ─── Main ────────────────────────────────────────────────────────────────────
def main():
coordinator/udp_ingest.service
+1 -1
View File
@@ -5,7 +5,7 @@ After=network.target
[Service]
Type=simple
WorkingDirectory=/opt/esp32_cluster/coordinator
ExecStart=/usr/bin/python3 udp_ingest.py
ExecStart=/usr/bin/python3 -u udp_ingest.py
Restart=always
RestartSec=5
StandardOutput=journal