mirror of
https://github.com/CyberMind-FR/secubox-deb.git
synced 2026-06-30 12:01:24 +00:00
Compare commits
2 Commits
0fc5871169
...
0a05bed028
| Author | SHA1 | Date | |
|---|---|---|---|
| 0a05bed028 | |||
| fdfc404818 |
0
packages/secubox-threatmesh/api/__init__.py
Normal file
0
packages/secubox-threatmesh/api/__init__.py
Normal file
267
packages/secubox-threatmesh/api/main.py
Normal file
267
packages/secubox-threatmesh/api/main.py
Normal file
|
|
@ -0,0 +1,267 @@
|
||||||
|
# SPDX-License-Identifier: LicenseRef-CMSD-1.0
|
||||||
|
# Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
|
||||||
|
# Source-Disclosed License — All rights reserved except as expressly granted.
|
||||||
|
# See LICENCE-CMSD-1.0.md for terms.
|
||||||
|
"""
|
||||||
|
SecuBox-Deb :: secubox-threatmesh (#728)
|
||||||
|
CyberMind — https://cybermind.fr
|
||||||
|
|
||||||
|
Sovereign threat-intel control plane that replaces CrowdSec CAPI:
|
||||||
|
|
||||||
|
Phase 1 free public feeds (secubox-threatfeed timer) -> threat_intel
|
||||||
|
Phase 2 MESH distribution of LOCAL decisions over the SecuBox P2P mesh
|
||||||
|
(gossip to peers from secubox-p2p; ingest peer decisions) -> threat_intel
|
||||||
|
Phase 3 status / aggregate / CrowdSec-bouncer-compatible decisions API + UI
|
||||||
|
|
||||||
|
All IOCs land in the shared `threat_intel` table (toolbox.db); secubox-blacklist-sync
|
||||||
|
drains it to the nft blacklist. No central account, no CAPI, no paywall.
|
||||||
|
"""
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import sqlite3
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
import urllib.request
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from fastapi import FastAPI, APIRouter, Depends, Request
|
||||||
|
from fastapi.responses import JSONResponse
|
||||||
|
|
||||||
|
from secubox_core.auth import require_jwt
|
||||||
|
from secubox_core.logger import get_logger
|
||||||
|
|
||||||
|
log = get_logger("threatmesh")
|
||||||
|
|
||||||
|
TI_DB = Path("/var/lib/secubox/toolbox/toolbox.db")
|
||||||
|
P2P_PEERS = Path("/var/lib/secubox/p2p/peers.json")
|
||||||
|
P2P_NODE = Path("/var/lib/secubox/p2p/node_id")
|
||||||
|
MESH_INTERVAL = 600 # gossip every 10 min
|
||||||
|
LOCAL_ORIGINS = {"crowdsec", "cscli", "manual", "secubox-waf", "secubox"}
|
||||||
|
IPV4 = re.compile(r"^\d{1,3}(\.\d{1,3}){3}(/\d{1,2})?$")
|
||||||
|
|
||||||
|
app = FastAPI(title="secubox-threatmesh", version="1.0.0", root_path="/api/v1/threatmesh")
|
||||||
|
router = APIRouter()
|
||||||
|
_node_id = None
|
||||||
|
|
||||||
|
|
||||||
|
# ── store ───────────────────────────────────────────────────────────
|
||||||
|
def _conn():
|
||||||
|
c = sqlite3.connect(TI_DB, timeout=20)
|
||||||
|
c.row_factory = sqlite3.Row
|
||||||
|
return c
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure(c):
|
||||||
|
c.execute("""CREATE TABLE IF NOT EXISTS threat_intel (
|
||||||
|
ioc TEXT NOT NULL, ioc_type TEXT NOT NULL, source TEXT NOT NULL,
|
||||||
|
weight INTEGER NOT NULL DEFAULT 50, label TEXT,
|
||||||
|
first_seen INTEGER, last_seen INTEGER,
|
||||||
|
PRIMARY KEY (ioc, ioc_type, source))""")
|
||||||
|
|
||||||
|
|
||||||
|
def node_id() -> str:
|
||||||
|
global _node_id
|
||||||
|
if _node_id:
|
||||||
|
return _node_id
|
||||||
|
try:
|
||||||
|
_node_id = P2P_NODE.read_text().strip()
|
||||||
|
except Exception:
|
||||||
|
try:
|
||||||
|
_node_id = json.loads(P2P_PEERS.read_text())["peers"][0]["id"]
|
||||||
|
except Exception:
|
||||||
|
_node_id = "sb-local"
|
||||||
|
return _node_id
|
||||||
|
|
||||||
|
|
||||||
|
def upsert_iocs(rows, source, weight=60, label=None):
|
||||||
|
"""rows: iterable of (ioc, ioc_type). Returns count."""
|
||||||
|
now = int(time.time())
|
||||||
|
n = 0
|
||||||
|
with _conn() as c:
|
||||||
|
_ensure(c)
|
||||||
|
for ioc, t in rows:
|
||||||
|
c.execute(
|
||||||
|
"INSERT INTO threat_intel(ioc,ioc_type,source,weight,label,first_seen,last_seen) "
|
||||||
|
"VALUES(?,?,?,?,?,?,?) ON CONFLICT(ioc,ioc_type,source) DO UPDATE SET last_seen=excluded.last_seen",
|
||||||
|
(ioc, t, source, weight, label, now, now))
|
||||||
|
n += 1
|
||||||
|
c.commit()
|
||||||
|
return n
|
||||||
|
|
||||||
|
|
||||||
|
def counts_by_source():
|
||||||
|
with _conn() as c:
|
||||||
|
_ensure(c)
|
||||||
|
return {r["source"]: r["n"] for r in c.execute(
|
||||||
|
"SELECT source, COUNT(*) n FROM threat_intel GROUP BY source ORDER BY n DESC")}
|
||||||
|
|
||||||
|
|
||||||
|
def aggregate_ips(limit=50000):
|
||||||
|
"""Distinct bad IPs across ALL sources, with a consensus count."""
|
||||||
|
with _conn() as c:
|
||||||
|
_ensure(c)
|
||||||
|
return [dict(r) for r in c.execute(
|
||||||
|
"SELECT ioc, COUNT(DISTINCT source) consensus, MAX(weight) weight, "
|
||||||
|
"GROUP_CONCAT(DISTINCT source) sources FROM threat_intel "
|
||||||
|
"WHERE ioc_type='ip' GROUP BY ioc ORDER BY consensus DESC LIMIT ?", (limit,))]
|
||||||
|
|
||||||
|
|
||||||
|
# ── mesh (Phase 2) ──────────────────────────────────────────────────
|
||||||
|
def mesh_peers():
|
||||||
|
"""Other SecuBox nodes from secubox-p2p (skip self)."""
|
||||||
|
me = node_id()
|
||||||
|
out = []
|
||||||
|
try:
|
||||||
|
d = json.loads(P2P_PEERS.read_text())
|
||||||
|
for p in d.get("peers", []):
|
||||||
|
if p.get("id") == me:
|
||||||
|
continue
|
||||||
|
addr = None
|
||||||
|
for a in p.get("addresses", []):
|
||||||
|
if a.get("type") in ("wg", "wireguard", "mesh"):
|
||||||
|
addr = a.get("address"); break
|
||||||
|
addr = addr or (p.get("wg_addresses") or [None])[0] or p.get("address")
|
||||||
|
if addr:
|
||||||
|
out.append({"id": p.get("id"), "name": p.get("name"), "address": addr})
|
||||||
|
except Exception as e:
|
||||||
|
log.debug(f"peers read: {e}")
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def local_decisions():
|
||||||
|
"""Our OWN locally-detected bad IPs (CrowdSec LAPI bans of local origin)."""
|
||||||
|
try:
|
||||||
|
out = subprocess.run(["cscli", "decisions", "list", "-o", "json", "-a"],
|
||||||
|
capture_output=True, text=True, timeout=20)
|
||||||
|
data = json.loads(out.stdout or "[]") or []
|
||||||
|
except Exception as e:
|
||||||
|
log.debug(f"cscli decisions: {e}")
|
||||||
|
return []
|
||||||
|
ips = []
|
||||||
|
for alert in data:
|
||||||
|
for dec in (alert.get("decisions") or []):
|
||||||
|
if dec.get("type") == "ban" and dec.get("scope", "").lower() == "ip":
|
||||||
|
origin = (dec.get("origin") or "").lower()
|
||||||
|
val = dec.get("value", "")
|
||||||
|
if IPV4.match(val) and not origin.startswith(("feed:", "mesh:", "lists:")):
|
||||||
|
ips.append(val)
|
||||||
|
return sorted(set(ips))
|
||||||
|
|
||||||
|
|
||||||
|
async def _post_json(url, payload, timeout=15):
|
||||||
|
def _do():
|
||||||
|
req = urllib.request.Request(
|
||||||
|
url, data=json.dumps(payload).encode(),
|
||||||
|
headers={"Content-Type": "application/json", "User-Agent": "SecuBox-ThreatMesh/1.0"},
|
||||||
|
method="POST")
|
||||||
|
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||||
|
return r.status
|
||||||
|
return await asyncio.to_thread(_do)
|
||||||
|
|
||||||
|
|
||||||
|
async def mesh_gossip_once():
|
||||||
|
peers = mesh_peers()
|
||||||
|
decisions = local_decisions()
|
||||||
|
if not peers or not decisions:
|
||||||
|
return {"peers": len(peers), "shared": len(decisions), "pushed": 0}
|
||||||
|
payload = {"node": node_id(), "ts": int(time.time()), "ips": decisions}
|
||||||
|
pushed = 0
|
||||||
|
for p in peers:
|
||||||
|
url = f"http://{p['address']}:8780/api/v1/threatmesh/mesh/ingest"
|
||||||
|
try:
|
||||||
|
await _post_json(url, payload)
|
||||||
|
pushed += 1
|
||||||
|
except Exception as e:
|
||||||
|
log.debug(f"gossip to {p['id']} failed: {e}")
|
||||||
|
return {"peers": len(peers), "shared": len(decisions), "pushed": pushed}
|
||||||
|
|
||||||
|
|
||||||
|
async def _mesh_loop():
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
r = await mesh_gossip_once()
|
||||||
|
log.info(f"mesh gossip: {r}")
|
||||||
|
except Exception as e:
|
||||||
|
log.error(f"mesh loop: {e}")
|
||||||
|
await asyncio.sleep(MESH_INTERVAL)
|
||||||
|
|
||||||
|
|
||||||
|
@app.on_event("startup")
|
||||||
|
async def _startup():
|
||||||
|
asyncio.create_task(_mesh_loop())
|
||||||
|
|
||||||
|
|
||||||
|
# ── endpoints ───────────────────────────────────────────────────────
|
||||||
|
@app.get("/health")
|
||||||
|
async def health():
|
||||||
|
return {"status": "ok", "module": "deb"}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/status")
|
||||||
|
async def status():
|
||||||
|
src = counts_by_source()
|
||||||
|
feeds = {k: v for k, v in src.items() if k.startswith("feed:")}
|
||||||
|
mesh = {k: v for k, v in src.items() if k.startswith("mesh:")}
|
||||||
|
return {
|
||||||
|
"mode": "sovereign", # CAPI dropped (#728)
|
||||||
|
"node": node_id(),
|
||||||
|
"capi": False,
|
||||||
|
"feeds": feeds, "feed_total": sum(feeds.values()),
|
||||||
|
"mesh": mesh, "mesh_total": sum(mesh.values()),
|
||||||
|
"peers": len(mesh_peers()),
|
||||||
|
"sources": src,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/peers")
|
||||||
|
async def peers():
|
||||||
|
return {"node": node_id(), "peers": mesh_peers()}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/decisions")
|
||||||
|
async def decisions(min_consensus: int = 1, limit: int = 50000):
|
||||||
|
"""Aggregated sovereign blocklist (feeds + mesh + local). CrowdSec-bouncer
|
||||||
|
friendly shape so external consumers can poll OUR server, not crowdsec.net."""
|
||||||
|
rows = [r for r in aggregate_ips(limit) if r["consensus"] >= min_consensus]
|
||||||
|
return [{
|
||||||
|
"id": i, "origin": "secubox-threatmesh", "type": "ban", "scope": "Ip",
|
||||||
|
"value": r["ioc"], "duration": "24h",
|
||||||
|
"consensus": r["consensus"], "sources": r["sources"],
|
||||||
|
} for i, r in enumerate(rows)]
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/mesh/ingest")
|
||||||
|
async def mesh_ingest(request: Request):
|
||||||
|
"""Receive a peer's locally-detected decisions over the mesh -> threat_intel
|
||||||
|
tagged mesh:<node>. Trust boundary = the WireGuard mesh transport."""
|
||||||
|
try:
|
||||||
|
body = await request.json()
|
||||||
|
except Exception:
|
||||||
|
return JSONResponse({"ok": False, "error": "bad json"}, status_code=400)
|
||||||
|
node = str(body.get("node", "")).strip()[:64] or "unknown"
|
||||||
|
if not re.match(r"^[A-Za-z0-9._:-]+$", node):
|
||||||
|
return JSONResponse({"ok": False, "error": "bad node"}, status_code=400)
|
||||||
|
ips = [ip for ip in (body.get("ips") or []) if isinstance(ip, str) and IPV4.match(ip)]
|
||||||
|
n = upsert_iocs(((ip, "ip") for ip in ips[:20000]), f"mesh:{node}", weight=70, label="mesh")
|
||||||
|
log.info(f"mesh ingest from {node}: {n} ips")
|
||||||
|
return {"ok": True, "ingested": n}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/feeds/refresh", dependencies=[Depends(require_jwt)])
|
||||||
|
async def feeds_refresh():
|
||||||
|
"""Trigger the feed fetcher now (normally on a timer)."""
|
||||||
|
try:
|
||||||
|
subprocess.Popen(["/usr/sbin/secubox-threatfeed"])
|
||||||
|
return {"ok": True, "started": True}
|
||||||
|
except Exception as e:
|
||||||
|
return JSONResponse({"ok": False, "error": str(e)}, status_code=500)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/mesh/gossip", dependencies=[Depends(require_jwt)])
|
||||||
|
async def mesh_gossip_now():
|
||||||
|
return await mesh_gossip_once()
|
||||||
|
|
||||||
|
|
||||||
|
app.include_router(router)
|
||||||
13
packages/secubox-threatmesh/debian/changelog
Normal file
13
packages/secubox-threatmesh/debian/changelog
Normal file
|
|
@ -0,0 +1,13 @@
|
||||||
|
secubox-threatmesh (1.0.0-1~bookworm1) bookworm; urgency=medium
|
||||||
|
|
||||||
|
* Initial release (#728) — sovereign threat-intel, CrowdSec CAPI replacement.
|
||||||
|
- Phase 1: secubox-threatfeed timer pulls free public blocklists
|
||||||
|
(feodo/sslbl/firehol/spamhaus-drop/blocklist.de/cins/et/dshield) into the
|
||||||
|
shared threat_intel table -> secubox-blacklist-sync -> nft.
|
||||||
|
- Phase 2: MESH distribution — gossip locally-detected CrowdSec decisions to
|
||||||
|
SecuBox P2P peers over WireGuard; ingest peer decisions (tagged mesh:<node>),
|
||||||
|
consensus-counted. :8780 locked to wg*/loopback by an nft drop-in.
|
||||||
|
- Phase 3: /status, /peers, /decisions (bouncer-compatible aggregate),
|
||||||
|
/mesh/ingest, dashboard. CrowdSec LAPI kept; CAPI dropped.
|
||||||
|
|
||||||
|
-- Gerald KERMA <devel@cybermind.fr> Wed, 24 Jun 2026 16:00:00 +0000
|
||||||
18
packages/secubox-threatmesh/debian/control
Normal file
18
packages/secubox-threatmesh/debian/control
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
Source: secubox-threatmesh
|
||||||
|
Section: net
|
||||||
|
Priority: optional
|
||||||
|
Maintainer: Gerald KERMA <devel@cybermind.fr>
|
||||||
|
Build-Depends: debhelper-compat (= 13)
|
||||||
|
Standards-Version: 4.6.2
|
||||||
|
|
||||||
|
Package: secubox-threatmesh
|
||||||
|
Architecture: all
|
||||||
|
Depends: ${misc:Depends}, secubox-core (>= 1.0), python3, python3-uvicorn | python3-pip, secubox-vortex-firewall | secubox-toolbox
|
||||||
|
Recommends: secubox-p2p, crowdsec
|
||||||
|
Description: Sovereign threat-intel mesh for SecuBox (CrowdSec CAPI replacement)
|
||||||
|
Replaces the CrowdSec Central API with self-sourced free public blocklists
|
||||||
|
(Phase 1) and peer-to-peer MESH distribution of locally-detected decisions over
|
||||||
|
the SecuBox WireGuard mesh (Phase 2), plus a status/aggregate/bouncer-compatible
|
||||||
|
API and dashboard (Phase 3). All IOCs flow into the shared threat_intel table
|
||||||
|
that secubox-blacklist-sync drains into the nft blacklist. No central account,
|
||||||
|
no enrollment, no paywall, no IP-blocklisting by a third party.
|
||||||
23
packages/secubox-threatmesh/debian/postinst
Normal file
23
packages/secubox-threatmesh/debian/postinst
Normal file
|
|
@ -0,0 +1,23 @@
|
||||||
|
#!/bin/bash
|
||||||
|
set -e
|
||||||
|
case "$1" in
|
||||||
|
configure)
|
||||||
|
install -d -m 0755 /var/lib/secubox/threatmesh
|
||||||
|
systemctl daemon-reload 2>/dev/null || true
|
||||||
|
# mesh-port firewall (lock :8780 to wg*/lo under DEFAULT DROP)
|
||||||
|
if [ -f /usr/share/secubox/threatmesh/nftables.d/secubox-threatmesh.nft ]; then
|
||||||
|
nft -f /usr/share/secubox/threatmesh/nftables.d/secubox-threatmesh.nft 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
if [ "$(systemctl is-enabled secubox-threatmesh.service 2>/dev/null)" != "masked" ]; then
|
||||||
|
systemctl enable --now secubox-threatmesh.service 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
systemctl enable --now secubox-threatfeed.timer 2>/dev/null || true
|
||||||
|
# first feed population (background; logs to /tmp to avoid /var/log perms)
|
||||||
|
/usr/sbin/secubox-threatfeed >/tmp/secubox-threatfeed-firstrun.log 2>&1 &
|
||||||
|
if command -v nginx >/dev/null 2>&1 && nginx -t 2>/dev/null; then
|
||||||
|
systemctl reload nginx 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
#DEBHELPER#
|
||||||
|
exit 0
|
||||||
11
packages/secubox-threatmesh/debian/prerm
Normal file
11
packages/secubox-threatmesh/debian/prerm
Normal file
|
|
@ -0,0 +1,11 @@
|
||||||
|
#!/bin/bash
|
||||||
|
set -e
|
||||||
|
case "$1" in
|
||||||
|
remove|deconfigure)
|
||||||
|
systemctl stop secubox-threatmesh.service 2>/dev/null || true
|
||||||
|
systemctl disable secubox-threatmesh.service secubox-threatfeed.timer 2>/dev/null || true
|
||||||
|
nft delete table inet secubox_threatmesh 2>/dev/null || true
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
#DEBHELPER#
|
||||||
|
exit 0
|
||||||
23
packages/secubox-threatmesh/debian/rules
Executable file
23
packages/secubox-threatmesh/debian/rules
Executable file
|
|
@ -0,0 +1,23 @@
|
||||||
|
#!/usr/bin/make -f
|
||||||
|
%:
|
||||||
|
dh $@
|
||||||
|
|
||||||
|
override_dh_auto_install:
|
||||||
|
install -d debian/secubox-threatmesh/usr/lib/secubox/threatmesh/api
|
||||||
|
cp -r api/. debian/secubox-threatmesh/usr/lib/secubox/threatmesh/api/
|
||||||
|
install -d debian/secubox-threatmesh/usr/sbin
|
||||||
|
install -m 755 sbin/secubox-threatfeed debian/secubox-threatmesh/usr/sbin/
|
||||||
|
install -d debian/secubox-threatmesh/usr/lib/systemd/system
|
||||||
|
cp systemd/*.service systemd/*.timer debian/secubox-threatmesh/usr/lib/systemd/system/
|
||||||
|
install -d debian/secubox-threatmesh/usr/share/secubox/www
|
||||||
|
cp -r www/. debian/secubox-threatmesh/usr/share/secubox/www/
|
||||||
|
install -d debian/secubox-threatmesh/usr/share/secubox/menu.d
|
||||||
|
cp menu.d/. -r debian/secubox-threatmesh/usr/share/secubox/menu.d/
|
||||||
|
install -d debian/secubox-threatmesh/etc/nginx/secubox-routes.d debian/secubox-threatmesh/etc/nginx/secubox.d
|
||||||
|
cp nginx/threatmesh.conf debian/secubox-threatmesh/etc/nginx/secubox-routes.d/
|
||||||
|
cp nginx/threatmesh.conf debian/secubox-threatmesh/etc/nginx/secubox.d/
|
||||||
|
install -d debian/secubox-threatmesh/usr/share/secubox/threatmesh/nftables.d
|
||||||
|
cp nftables.d/secubox-threatmesh.nft debian/secubox-threatmesh/usr/share/secubox/threatmesh/nftables.d/
|
||||||
|
|
||||||
|
override_dh_installsystemd:
|
||||||
|
true
|
||||||
1
packages/secubox-threatmesh/debian/source/format
Normal file
1
packages/secubox-threatmesh/debian/source/format
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
3.0 (native)
|
||||||
9
packages/secubox-threatmesh/menu.d/37-threatmesh.json
Normal file
9
packages/secubox-threatmesh/menu.d/37-threatmesh.json
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
{
|
||||||
|
"id": "threatmesh",
|
||||||
|
"name": "ThreatMesh",
|
||||||
|
"icon": "🛰️",
|
||||||
|
"path": "/threatmesh/",
|
||||||
|
"category": "security",
|
||||||
|
"order": 37,
|
||||||
|
"description": "Sovereign threat-intel (feeds + mesh, no CAPI)"
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,10 @@
|
||||||
|
# ThreatMesh peer port (#728): only reachable over the WireGuard mesh + loopback.
|
||||||
|
# The base inet filter INPUT chain is DEFAULT DROP, so this only OPENS 8780 to wg.
|
||||||
|
table inet secubox_threatmesh {
|
||||||
|
chain input {
|
||||||
|
type filter hook input priority 0; policy accept;
|
||||||
|
iifname "lo" tcp dport 8780 accept
|
||||||
|
iifname "wg*" tcp dport 8780 accept
|
||||||
|
tcp dport 8780 drop
|
||||||
|
}
|
||||||
|
}
|
||||||
10
packages/secubox-threatmesh/nginx/threatmesh.conf
Normal file
10
packages/secubox-threatmesh/nginx/threatmesh.conf
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
# /etc/nginx/secubox-routes.d/threatmesh.conf — secubox-threatmesh (#728)
|
||||||
|
location /threatmesh/ {
|
||||||
|
alias /usr/share/secubox/www/threatmesh/;
|
||||||
|
index index.html;
|
||||||
|
try_files $uri $uri/ /threatmesh/index.html;
|
||||||
|
}
|
||||||
|
location /api/v1/threatmesh/ {
|
||||||
|
proxy_pass http://127.0.0.1:8780/api/v1/threatmesh/;
|
||||||
|
include /etc/nginx/snippets/secubox-proxy.conf;
|
||||||
|
}
|
||||||
121
packages/secubox-threatmesh/sbin/secubox-threatfeed
Normal file
121
packages/secubox-threatmesh/sbin/secubox-threatfeed
Normal file
|
|
@ -0,0 +1,121 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# SPDX-License-Identifier: LicenseRef-CMSD-1.0
|
||||||
|
# Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
|
||||||
|
# Source-Disclosed License — All rights reserved except as expressly granted.
|
||||||
|
# See LICENCE-CMSD-1.0.md for terms.
|
||||||
|
"""
|
||||||
|
SecuBox-Deb :: secubox-threatfeed (Phase 1 — sovereign threat-intel #728)
|
||||||
|
CyberMind — https://cybermind.fr
|
||||||
|
|
||||||
|
Pulls FREE, no-enrollment public IP/CIDR/domain blocklists and writes them into
|
||||||
|
the shared `threat_intel` table (toolbox.db) that secubox-blacklist-sync already
|
||||||
|
drains into the nft blacklist. This REPLACES the CrowdSec CAPI community feed
|
||||||
|
with self-sourced lists — no central account, no paywall, no IP blocklisting.
|
||||||
|
|
||||||
|
Pure stdlib (urllib) — no extra Debian deps. Idempotent, safe on a timer.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import urllib.request
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
DB = Path(os.environ.get("SECUBOX_TI_DB", "/var/lib/secubox/toolbox/toolbox.db"))
|
||||||
|
TIMEOUT = int(os.environ.get("SECUBOX_TF_TIMEOUT", "30"))
|
||||||
|
UA = "SecuBox-ThreatFeed/1.0 (+https://secubox.in)"
|
||||||
|
TTL_DAYS = int(os.environ.get("SECUBOX_TF_TTL_DAYS", "10"))
|
||||||
|
|
||||||
|
# (name, url, ioc_type, weight). All free + no enrollment. CIDRs allowed for ip.
|
||||||
|
FEEDS = [
|
||||||
|
("feodo", "https://feodotracker.abuse.ch/downloads/ipblocklist.txt", "ip", 90),
|
||||||
|
("sslbl", "https://sslbl.abuse.ch/blacklist/sslipblacklist.txt", "ip", 85),
|
||||||
|
("firehol1", "https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset", "ip", 80),
|
||||||
|
("spamhaus-drop", "https://www.spamhaus.org/drop/drop.txt", "ip", 85),
|
||||||
|
("blocklist-de", "https://lists.blocklist.de/lists/all.txt", "ip", 60),
|
||||||
|
("cins-army", "https://cinsscore.com/list/ci-badguys.txt", "ip", 70),
|
||||||
|
("et-compromised", "https://rules.emergingthreats.net/blockrules/compromised-ips.txt", "ip", 75),
|
||||||
|
("dshield", "https://feeds.dshield.org/block.txt", "ip", 65),
|
||||||
|
]
|
||||||
|
# Optional (set SECUBOX_TF_TOR=1 to also pull Tor exit nodes)
|
||||||
|
if os.environ.get("SECUBOX_TF_TOR") == "1":
|
||||||
|
FEEDS.append(("tor-exit", "https://check.torproject.org/torbulkexitlist", "ip", 40))
|
||||||
|
|
||||||
|
_IPV4 = re.compile(r"^\s*(\d{1,3}(?:\.\d{1,3}){3}(?:/\d{1,2})?)")
|
||||||
|
_IPV6 = re.compile(r"^\s*([0-9A-Fa-f:]{2,}(?:/\d{1,3})?)\s*$")
|
||||||
|
|
||||||
|
|
||||||
|
def log(msg):
|
||||||
|
print(f"[threatfeed] {msg}", file=sys.stderr, flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
def fetch(url):
|
||||||
|
req = urllib.request.Request(url, headers={"User-Agent": UA})
|
||||||
|
with urllib.request.urlopen(req, timeout=TIMEOUT) as r:
|
||||||
|
return r.read().decode("utf-8", "replace")
|
||||||
|
|
||||||
|
|
||||||
|
def parse_ips(text):
|
||||||
|
out = []
|
||||||
|
for line in text.splitlines():
|
||||||
|
s = line.strip()
|
||||||
|
if not s or s.startswith("#") or s.startswith(";"):
|
||||||
|
continue
|
||||||
|
# spamhaus DROP: "1.2.3.0/24 ; SBL..." -> take the first field
|
||||||
|
token = s.split(";")[0].split()[0].strip() if (";" in s or " " in s) else s
|
||||||
|
m = _IPV4.match(token)
|
||||||
|
if m:
|
||||||
|
out.append((m.group(1), "ip"))
|
||||||
|
continue
|
||||||
|
m = _IPV6.match(token)
|
||||||
|
if m and ":" in m.group(1):
|
||||||
|
out.append((m.group(1), "ip"))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_schema(c):
|
||||||
|
c.execute("""CREATE TABLE IF NOT EXISTS threat_intel (
|
||||||
|
ioc TEXT NOT NULL, ioc_type TEXT NOT NULL, source TEXT NOT NULL,
|
||||||
|
weight INTEGER NOT NULL DEFAULT 50, label TEXT,
|
||||||
|
first_seen INTEGER, last_seen INTEGER,
|
||||||
|
PRIMARY KEY (ioc, ioc_type, source))""")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
if not DB.exists():
|
||||||
|
log(f"db missing: {DB}")
|
||||||
|
return 1
|
||||||
|
now = int(time.time())
|
||||||
|
total = 0
|
||||||
|
with sqlite3.connect(DB, timeout=20) as c:
|
||||||
|
ensure_schema(c)
|
||||||
|
for name, url, ioc_type, weight in FEEDS:
|
||||||
|
src = f"feed:{name}"
|
||||||
|
try:
|
||||||
|
iocs = parse_ips(fetch(url))
|
||||||
|
except Exception as e:
|
||||||
|
log(f"{name}: fetch failed: {e}")
|
||||||
|
continue
|
||||||
|
n = 0
|
||||||
|
for ioc, t in iocs:
|
||||||
|
c.execute(
|
||||||
|
"INSERT INTO threat_intel(ioc,ioc_type,source,weight,label,first_seen,last_seen) "
|
||||||
|
"VALUES(?,?,?,?,?,?,?) "
|
||||||
|
"ON CONFLICT(ioc,ioc_type,source) DO UPDATE SET last_seen=excluded.last_seen, weight=excluded.weight",
|
||||||
|
(ioc, t, src, weight, name, now, now))
|
||||||
|
n += 1
|
||||||
|
log(f"{name}: {n} iocs")
|
||||||
|
total += n
|
||||||
|
# prune stale feed entries (older than TTL) — keeps the set fresh.
|
||||||
|
cutoff = now - TTL_DAYS * 86400
|
||||||
|
pruned = c.execute(
|
||||||
|
"DELETE FROM threat_intel WHERE source LIKE 'feed:%' AND last_seen < ?",
|
||||||
|
(cutoff,)).rowcount
|
||||||
|
c.commit()
|
||||||
|
log(f"done: {total} iocs across {len(FEEDS)} feeds, pruned {pruned} stale")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
[Unit]
|
||||||
|
Description=SecuBox ThreatFeed — pull free public blocklists into threat_intel (#728)
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
User=root
|
||||||
|
ExecStart=/usr/sbin/secubox-threatfeed
|
||||||
|
Nice=10
|
||||||
|
IOSchedulingClass=idle
|
||||||
|
TimeoutStartSec=600
|
||||||
11
packages/secubox-threatmesh/systemd/secubox-threatfeed.timer
Normal file
11
packages/secubox-threatmesh/systemd/secubox-threatfeed.timer
Normal file
|
|
@ -0,0 +1,11 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Refresh sovereign threat-intel feeds (#728)
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnBootSec=5min
|
||||||
|
OnUnitActiveSec=6h
|
||||||
|
RandomizedDelaySec=15min
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
|
|
@ -0,0 +1,22 @@
|
||||||
|
[Unit]
|
||||||
|
Description=SecuBox ThreatMesh — sovereign threat-intel mesh API (#728)
|
||||||
|
After=network.target secubox-runtime.service secubox-p2p.service
|
||||||
|
Wants=secubox-p2p.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=root
|
||||||
|
WorkingDirectory=/usr/lib/secubox/threatmesh
|
||||||
|
# Listens on :8780 — nginx proxies the dashboard; peers POST mesh decisions
|
||||||
|
# over the WireGuard mesh. nft drop-in restricts :8780 to wg* + loopback.
|
||||||
|
ExecStartPre=-/usr/sbin/nft -f /usr/share/secubox/threatmesh/nftables.d/secubox-threatmesh.nft
|
||||||
|
ExecStart=/usr/bin/python3 -m uvicorn api.main:app --host 0.0.0.0 --port 8780 --workers 1
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=5
|
||||||
|
NoNewPrivileges=no
|
||||||
|
ProtectSystem=full
|
||||||
|
PrivateTmp=true
|
||||||
|
ReadWritePaths=/run/secubox /var/lib/secubox /etc/secubox
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
87
packages/secubox-threatmesh/www/threatmesh/index.html
Normal file
87
packages/secubox-threatmesh/www/threatmesh/index.html
Normal file
|
|
@ -0,0 +1,87 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<!-- SPDX-License-Identifier: LicenseRef-CMSD-1.0 -->
|
||||||
|
<!-- SecuBox-Deb :: ThreatMesh — sovereign threat-intel (#728) — CyberMind -->
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>SecuBox · ThreatMesh</title>
|
||||||
|
<link rel="stylesheet" href="/shared/crt-light.css">
|
||||||
|
<link rel="stylesheet" href="/shared/sidebar-light.css">
|
||||||
|
<style>
|
||||||
|
:root{--cosmos-black:#0a0a0f;--gold:#c9a84c;--cinnabar:#e63946;--matrix:#00ff41;
|
||||||
|
--void:#6e40c9;--cyan:#00d4ff;--text:#e8e6d9;--muted:#6b6b7a;--panel:#13131c;--line:#23232f}
|
||||||
|
body{margin:0;background:var(--cosmos-black);color:var(--text);font-family:'JetBrains Mono',ui-monospace,monospace;display:flex}
|
||||||
|
.main{flex:1;margin-left:220px;min-height:100vh;padding:0 0 40px}
|
||||||
|
@media(max-width:900px){.sidebar{display:none}.main{margin-left:0}}
|
||||||
|
.topbar{display:flex;align-items:center;gap:14px;padding:14px 22px;border-bottom:1px solid var(--line);
|
||||||
|
background:linear-gradient(90deg,#0a0a0f,#13131c)}
|
||||||
|
.topbar h1{font-size:18px;margin:0;letter-spacing:1px;color:var(--gold)} .topbar .em{font-size:22px}
|
||||||
|
.pill{margin-left:auto;font-size:12px;padding:5px 12px;border-radius:20px;border:1px solid var(--matrix);color:var(--matrix)}
|
||||||
|
.wrap{padding:20px 22px;display:grid;gap:16px;grid-template-columns:repeat(auto-fit,minmax(260px,1fr))}
|
||||||
|
.card{background:var(--panel);border:1px solid var(--line);border-radius:12px;padding:16px}
|
||||||
|
.card h2{font-size:13px;color:var(--gold);margin:0 0 10px;letter-spacing:.5px}
|
||||||
|
.big{font-size:34px;color:var(--cyan);font-weight:700} .sub{color:var(--muted);font-size:12px}
|
||||||
|
.src{display:flex;justify-content:space-between;padding:4px 0;border-bottom:1px dashed var(--line);font-size:12px}
|
||||||
|
.src b{color:var(--cyan)} .src.mesh b{color:var(--void)}
|
||||||
|
.bar{display:flex;gap:8px;padding:0 22px 16px}
|
||||||
|
button{font-family:inherit;font-size:13px;background:#1a1a26;color:var(--text);border:1px solid var(--line);
|
||||||
|
border-radius:8px;padding:9px 14px;cursor:pointer} button:hover{border-color:var(--cyan);color:var(--cyan)}
|
||||||
|
button.go{background:var(--void);border-color:var(--void);color:#fff}
|
||||||
|
table{width:100%;border-collapse:collapse;font-size:12px} td,th{text-align:left;padding:5px 8px;border-bottom:1px solid var(--line)}
|
||||||
|
th{color:var(--muted)} .toast{position:fixed;bottom:18px;right:18px;background:#13131c;border:1px solid var(--cyan);
|
||||||
|
color:var(--cyan);padding:10px 14px;border-radius:8px;opacity:0;transition:.2s;font-size:12px}.toast.on{opacity:1}
|
||||||
|
.full{grid-column:1/-1}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<nav class="sidebar" id="sidebar"></nav>
|
||||||
|
<main class="main">
|
||||||
|
<div class="topbar"><span class="em">🛰️</span><h1>THREATMESH</h1>
|
||||||
|
<span class="pill" id="mode">sovereign · CAPI off</span></div>
|
||||||
|
<div class="bar">
|
||||||
|
<button class="go" onclick="refreshFeeds()">↻ Refresh feeds</button>
|
||||||
|
<button onclick="gossip()">🛰️ Gossip now</button>
|
||||||
|
<button onclick="load()">⟳</button>
|
||||||
|
</div>
|
||||||
|
<div class="wrap">
|
||||||
|
<div class="card"><h2>BLOCKED IPs (aggregate)</h2><div class="big" id="total">…</div><div class="sub">feeds + mesh + local → nft</div></div>
|
||||||
|
<div class="card"><h2>FREE FEEDS</h2><div class="big" id="feedtotal">…</div><div class="sub" id="feedn"></div></div>
|
||||||
|
<div class="card"><h2>MESH</h2><div class="big" id="meshtotal">…</div><div class="sub" id="peern"></div></div>
|
||||||
|
<div class="card"><h2>SOURCES</h2><div id="srclist"></div></div>
|
||||||
|
<div class="card full"><h2>MESH PEERS</h2><div id="peers" class="sub">…</div></div>
|
||||||
|
<div class="card full"><h2>TOP CONSENSUS IPs</h2><table id="agg"><thead><tr><th>IP</th><th>consensus</th><th>sources</th></tr></thead><tbody></tbody></table></div>
|
||||||
|
</div>
|
||||||
|
</main>
|
||||||
|
<div class="toast" id="toast"></div>
|
||||||
|
<script src="/shared/sidebar.js"></script>
|
||||||
|
<script>
|
||||||
|
const API="/api/v1/threatmesh";
|
||||||
|
function tok(){return localStorage.getItem("sbx_token")||""}
|
||||||
|
async function api(p,opt={}){opt.headers=Object.assign({},opt.headers||{});const t=tok();if(t)opt.headers.Authorization="Bearer "+t;
|
||||||
|
const r=await fetch(API+p,opt);if(r.status===401){toast("login via Hub");throw 0}if(!r.ok)throw 0;
|
||||||
|
const ct=r.headers.get("content-type")||"";return ct.includes("json")?r.json():r.text()}
|
||||||
|
function toast(m){const t=document.getElementById("toast");t.textContent=m;t.classList.add("on");clearTimeout(t._);t._=setTimeout(()=>t.classList.remove("on"),2600)}
|
||||||
|
function esc(s){return (s||"").replace(/[&<>"]/g,c=>({"&":"&","<":"<",">":">",'"':"""}[c]))}
|
||||||
|
async function load(){
|
||||||
|
let s;try{s=await fetch(API+"/status").then(r=>r.json())}catch(e){return}
|
||||||
|
document.getElementById("mode").textContent=(s.mode||"sovereign")+" · CAPI "+(s.capi?"on":"off")+" · node "+(s.node||"?");
|
||||||
|
document.getElementById("feedtotal").textContent=s.feed_total??0;
|
||||||
|
document.getElementById("feedn").textContent=Object.keys(s.feeds||{}).length+" feeds";
|
||||||
|
document.getElementById("meshtotal").textContent=s.mesh_total??0;
|
||||||
|
document.getElementById("peern").textContent=(s.peers||0)+" peers";
|
||||||
|
const all=s.sources||{};document.getElementById("total").textContent=Object.values(all).reduce((a,b)=>a+b,0);
|
||||||
|
document.getElementById("srclist").innerHTML=Object.entries(all).sort((a,b)=>b[1]-a[1]).slice(0,12)
|
||||||
|
.map(([k,v])=>`<div class="src ${k.startsWith('mesh:')?'mesh':''}"><span>${esc(k)}</span><b>${v}</b></div>`).join("")||'<div class="sub">no sources yet</div>';
|
||||||
|
try{const p=await fetch(API+"/peers").then(r=>r.json());
|
||||||
|
document.getElementById("peers").innerHTML=(p.peers||[]).length?p.peers.map(x=>`${esc(x.name||x.id)} <span style="color:var(--muted)">(${esc(x.address)})</span>`).join(" · "):"no mesh peers yet (single-node) — decisions will gossip when peers join";}catch(e){}
|
||||||
|
try{const d=await fetch(API+"/decisions?min_consensus=1&limit=40").then(r=>r.json());
|
||||||
|
d.sort((a,b)=>b.consensus-a.consensus);
|
||||||
|
document.querySelector("#agg tbody").innerHTML=d.slice(0,25).map(r=>`<tr><td>${esc(r.value)}</td><td>${r.consensus}</td><td style="color:var(--muted)">${esc((r.sources||'').slice(0,60))}</td></tr>`).join("");}catch(e){}
|
||||||
|
}
|
||||||
|
async function refreshFeeds(){try{await api("/feeds/refresh",{method:"POST"});toast("feed refresh started (~1 min)")}catch(e){toast("need Hub login")}}
|
||||||
|
async function gossip(){try{const r=await api("/mesh/gossip",{method:"POST"});toast(`gossip: ${r.pushed}/${r.peers} peers, ${r.shared} shared`)}catch(e){toast("need Hub login")}}
|
||||||
|
load();setInterval(load,8000);
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
Loading…
Reference in New Issue
Block a user