mirror of
https://github.com/CyberMind-FR/secubox-deb.git
synced 2026-06-30 08:00:54 +00:00
Compare commits
2 Commits
0fc5871169
...
0a05bed028
| Author | SHA1 | Date | |
|---|---|---|---|
| 0a05bed028 | |||
| fdfc404818 |
0
packages/secubox-threatmesh/api/__init__.py
Normal file
0
packages/secubox-threatmesh/api/__init__.py
Normal file
267
packages/secubox-threatmesh/api/main.py
Normal file
267
packages/secubox-threatmesh/api/main.py
Normal file
|
|
@ -0,0 +1,267 @@
|
|||
# SPDX-License-Identifier: LicenseRef-CMSD-1.0
|
||||
# Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
|
||||
# Source-Disclosed License — All rights reserved except as expressly granted.
|
||||
# See LICENCE-CMSD-1.0.md for terms.
|
||||
"""
|
||||
SecuBox-Deb :: secubox-threatmesh (#728)
|
||||
CyberMind — https://cybermind.fr
|
||||
|
||||
Sovereign threat-intel control plane that replaces CrowdSec CAPI:
|
||||
|
||||
Phase 1 free public feeds (secubox-threatfeed timer) -> threat_intel
|
||||
Phase 2 MESH distribution of LOCAL decisions over the SecuBox P2P mesh
|
||||
(gossip to peers from secubox-p2p; ingest peer decisions) -> threat_intel
|
||||
Phase 3 status / aggregate / CrowdSec-bouncer-compatible decisions API + UI
|
||||
|
||||
All IOCs land in the shared `threat_intel` table (toolbox.db); secubox-blacklist-sync
|
||||
drains it to the nft blacklist. No central account, no CAPI, no paywall.
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import FastAPI, APIRouter, Depends, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from secubox_core.auth import require_jwt
|
||||
from secubox_core.logger import get_logger
|
||||
|
||||
log = get_logger("threatmesh")
|
||||
|
||||
TI_DB = Path("/var/lib/secubox/toolbox/toolbox.db")
|
||||
P2P_PEERS = Path("/var/lib/secubox/p2p/peers.json")
|
||||
P2P_NODE = Path("/var/lib/secubox/p2p/node_id")
|
||||
MESH_INTERVAL = 600 # gossip every 10 min
|
||||
LOCAL_ORIGINS = {"crowdsec", "cscli", "manual", "secubox-waf", "secubox"}
|
||||
IPV4 = re.compile(r"^\d{1,3}(\.\d{1,3}){3}(/\d{1,2})?$")
|
||||
|
||||
app = FastAPI(title="secubox-threatmesh", version="1.0.0", root_path="/api/v1/threatmesh")
|
||||
router = APIRouter()
|
||||
_node_id = None
|
||||
|
||||
|
||||
# ── store ───────────────────────────────────────────────────────────
|
||||
def _conn():
|
||||
c = sqlite3.connect(TI_DB, timeout=20)
|
||||
c.row_factory = sqlite3.Row
|
||||
return c
|
||||
|
||||
|
||||
def _ensure(c):
|
||||
c.execute("""CREATE TABLE IF NOT EXISTS threat_intel (
|
||||
ioc TEXT NOT NULL, ioc_type TEXT NOT NULL, source TEXT NOT NULL,
|
||||
weight INTEGER NOT NULL DEFAULT 50, label TEXT,
|
||||
first_seen INTEGER, last_seen INTEGER,
|
||||
PRIMARY KEY (ioc, ioc_type, source))""")
|
||||
|
||||
|
||||
def node_id() -> str:
|
||||
global _node_id
|
||||
if _node_id:
|
||||
return _node_id
|
||||
try:
|
||||
_node_id = P2P_NODE.read_text().strip()
|
||||
except Exception:
|
||||
try:
|
||||
_node_id = json.loads(P2P_PEERS.read_text())["peers"][0]["id"]
|
||||
except Exception:
|
||||
_node_id = "sb-local"
|
||||
return _node_id
|
||||
|
||||
|
||||
def upsert_iocs(rows, source, weight=60, label=None):
|
||||
"""rows: iterable of (ioc, ioc_type). Returns count."""
|
||||
now = int(time.time())
|
||||
n = 0
|
||||
with _conn() as c:
|
||||
_ensure(c)
|
||||
for ioc, t in rows:
|
||||
c.execute(
|
||||
"INSERT INTO threat_intel(ioc,ioc_type,source,weight,label,first_seen,last_seen) "
|
||||
"VALUES(?,?,?,?,?,?,?) ON CONFLICT(ioc,ioc_type,source) DO UPDATE SET last_seen=excluded.last_seen",
|
||||
(ioc, t, source, weight, label, now, now))
|
||||
n += 1
|
||||
c.commit()
|
||||
return n
|
||||
|
||||
|
||||
def counts_by_source():
|
||||
with _conn() as c:
|
||||
_ensure(c)
|
||||
return {r["source"]: r["n"] for r in c.execute(
|
||||
"SELECT source, COUNT(*) n FROM threat_intel GROUP BY source ORDER BY n DESC")}
|
||||
|
||||
|
||||
def aggregate_ips(limit=50000):
|
||||
"""Distinct bad IPs across ALL sources, with a consensus count."""
|
||||
with _conn() as c:
|
||||
_ensure(c)
|
||||
return [dict(r) for r in c.execute(
|
||||
"SELECT ioc, COUNT(DISTINCT source) consensus, MAX(weight) weight, "
|
||||
"GROUP_CONCAT(DISTINCT source) sources FROM threat_intel "
|
||||
"WHERE ioc_type='ip' GROUP BY ioc ORDER BY consensus DESC LIMIT ?", (limit,))]
|
||||
|
||||
|
||||
# ── mesh (Phase 2) ──────────────────────────────────────────────────
|
||||
def mesh_peers():
|
||||
"""Other SecuBox nodes from secubox-p2p (skip self)."""
|
||||
me = node_id()
|
||||
out = []
|
||||
try:
|
||||
d = json.loads(P2P_PEERS.read_text())
|
||||
for p in d.get("peers", []):
|
||||
if p.get("id") == me:
|
||||
continue
|
||||
addr = None
|
||||
for a in p.get("addresses", []):
|
||||
if a.get("type") in ("wg", "wireguard", "mesh"):
|
||||
addr = a.get("address"); break
|
||||
addr = addr or (p.get("wg_addresses") or [None])[0] or p.get("address")
|
||||
if addr:
|
||||
out.append({"id": p.get("id"), "name": p.get("name"), "address": addr})
|
||||
except Exception as e:
|
||||
log.debug(f"peers read: {e}")
|
||||
return out
|
||||
|
||||
|
||||
def local_decisions():
|
||||
"""Our OWN locally-detected bad IPs (CrowdSec LAPI bans of local origin)."""
|
||||
try:
|
||||
out = subprocess.run(["cscli", "decisions", "list", "-o", "json", "-a"],
|
||||
capture_output=True, text=True, timeout=20)
|
||||
data = json.loads(out.stdout or "[]") or []
|
||||
except Exception as e:
|
||||
log.debug(f"cscli decisions: {e}")
|
||||
return []
|
||||
ips = []
|
||||
for alert in data:
|
||||
for dec in (alert.get("decisions") or []):
|
||||
if dec.get("type") == "ban" and dec.get("scope", "").lower() == "ip":
|
||||
origin = (dec.get("origin") or "").lower()
|
||||
val = dec.get("value", "")
|
||||
if IPV4.match(val) and not origin.startswith(("feed:", "mesh:", "lists:")):
|
||||
ips.append(val)
|
||||
return sorted(set(ips))
|
||||
|
||||
|
||||
async def _post_json(url, payload, timeout=15):
|
||||
def _do():
|
||||
req = urllib.request.Request(
|
||||
url, data=json.dumps(payload).encode(),
|
||||
headers={"Content-Type": "application/json", "User-Agent": "SecuBox-ThreatMesh/1.0"},
|
||||
method="POST")
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return r.status
|
||||
return await asyncio.to_thread(_do)
|
||||
|
||||
|
||||
async def mesh_gossip_once():
|
||||
peers = mesh_peers()
|
||||
decisions = local_decisions()
|
||||
if not peers or not decisions:
|
||||
return {"peers": len(peers), "shared": len(decisions), "pushed": 0}
|
||||
payload = {"node": node_id(), "ts": int(time.time()), "ips": decisions}
|
||||
pushed = 0
|
||||
for p in peers:
|
||||
url = f"http://{p['address']}:8780/api/v1/threatmesh/mesh/ingest"
|
||||
try:
|
||||
await _post_json(url, payload)
|
||||
pushed += 1
|
||||
except Exception as e:
|
||||
log.debug(f"gossip to {p['id']} failed: {e}")
|
||||
return {"peers": len(peers), "shared": len(decisions), "pushed": pushed}
|
||||
|
||||
|
||||
async def _mesh_loop():
|
||||
while True:
|
||||
try:
|
||||
r = await mesh_gossip_once()
|
||||
log.info(f"mesh gossip: {r}")
|
||||
except Exception as e:
|
||||
log.error(f"mesh loop: {e}")
|
||||
await asyncio.sleep(MESH_INTERVAL)
|
||||
|
||||
|
||||
@app.on_event("startup")
|
||||
async def _startup():
|
||||
asyncio.create_task(_mesh_loop())
|
||||
|
||||
|
||||
# ── endpoints ───────────────────────────────────────────────────────
|
||||
@app.get("/health")
|
||||
async def health():
|
||||
return {"status": "ok", "module": "deb"}
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
async def status():
|
||||
src = counts_by_source()
|
||||
feeds = {k: v for k, v in src.items() if k.startswith("feed:")}
|
||||
mesh = {k: v for k, v in src.items() if k.startswith("mesh:")}
|
||||
return {
|
||||
"mode": "sovereign", # CAPI dropped (#728)
|
||||
"node": node_id(),
|
||||
"capi": False,
|
||||
"feeds": feeds, "feed_total": sum(feeds.values()),
|
||||
"mesh": mesh, "mesh_total": sum(mesh.values()),
|
||||
"peers": len(mesh_peers()),
|
||||
"sources": src,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/peers")
|
||||
async def peers():
|
||||
return {"node": node_id(), "peers": mesh_peers()}
|
||||
|
||||
|
||||
@router.get("/decisions")
|
||||
async def decisions(min_consensus: int = 1, limit: int = 50000):
|
||||
"""Aggregated sovereign blocklist (feeds + mesh + local). CrowdSec-bouncer
|
||||
friendly shape so external consumers can poll OUR server, not crowdsec.net."""
|
||||
rows = [r for r in aggregate_ips(limit) if r["consensus"] >= min_consensus]
|
||||
return [{
|
||||
"id": i, "origin": "secubox-threatmesh", "type": "ban", "scope": "Ip",
|
||||
"value": r["ioc"], "duration": "24h",
|
||||
"consensus": r["consensus"], "sources": r["sources"],
|
||||
} for i, r in enumerate(rows)]
|
||||
|
||||
|
||||
@router.post("/mesh/ingest")
|
||||
async def mesh_ingest(request: Request):
|
||||
"""Receive a peer's locally-detected decisions over the mesh -> threat_intel
|
||||
tagged mesh:<node>. Trust boundary = the WireGuard mesh transport."""
|
||||
try:
|
||||
body = await request.json()
|
||||
except Exception:
|
||||
return JSONResponse({"ok": False, "error": "bad json"}, status_code=400)
|
||||
node = str(body.get("node", "")).strip()[:64] or "unknown"
|
||||
if not re.match(r"^[A-Za-z0-9._:-]+$", node):
|
||||
return JSONResponse({"ok": False, "error": "bad node"}, status_code=400)
|
||||
ips = [ip for ip in (body.get("ips") or []) if isinstance(ip, str) and IPV4.match(ip)]
|
||||
n = upsert_iocs(((ip, "ip") for ip in ips[:20000]), f"mesh:{node}", weight=70, label="mesh")
|
||||
log.info(f"mesh ingest from {node}: {n} ips")
|
||||
return {"ok": True, "ingested": n}
|
||||
|
||||
|
||||
@router.post("/feeds/refresh", dependencies=[Depends(require_jwt)])
|
||||
async def feeds_refresh():
|
||||
"""Trigger the feed fetcher now (normally on a timer)."""
|
||||
try:
|
||||
subprocess.Popen(["/usr/sbin/secubox-threatfeed"])
|
||||
return {"ok": True, "started": True}
|
||||
except Exception as e:
|
||||
return JSONResponse({"ok": False, "error": str(e)}, status_code=500)
|
||||
|
||||
|
||||
@router.post("/mesh/gossip", dependencies=[Depends(require_jwt)])
|
||||
async def mesh_gossip_now():
|
||||
return await mesh_gossip_once()
|
||||
|
||||
|
||||
app.include_router(router)
|
||||
13
packages/secubox-threatmesh/debian/changelog
Normal file
13
packages/secubox-threatmesh/debian/changelog
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
secubox-threatmesh (1.0.0-1~bookworm1) bookworm; urgency=medium
|
||||
|
||||
* Initial release (#728) — sovereign threat-intel, CrowdSec CAPI replacement.
|
||||
- Phase 1: secubox-threatfeed timer pulls free public blocklists
|
||||
(feodo/sslbl/firehol/spamhaus-drop/blocklist.de/cins/et/dshield) into the
|
||||
shared threat_intel table -> secubox-blacklist-sync -> nft.
|
||||
- Phase 2: MESH distribution — gossip locally-detected CrowdSec decisions to
|
||||
SecuBox P2P peers over WireGuard; ingest peer decisions (tagged mesh:<node>),
|
||||
consensus-counted. :8780 locked to wg*/loopback by an nft drop-in.
|
||||
- Phase 3: /status, /peers, /decisions (bouncer-compatible aggregate),
|
||||
/mesh/ingest, dashboard. CrowdSec LAPI kept; CAPI dropped.
|
||||
|
||||
-- Gerald KERMA <devel@cybermind.fr> Wed, 24 Jun 2026 16:00:00 +0000
|
||||
18
packages/secubox-threatmesh/debian/control
Normal file
18
packages/secubox-threatmesh/debian/control
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
Source: secubox-threatmesh
|
||||
Section: net
|
||||
Priority: optional
|
||||
Maintainer: Gerald KERMA <devel@cybermind.fr>
|
||||
Build-Depends: debhelper-compat (= 13)
|
||||
Standards-Version: 4.6.2
|
||||
|
||||
Package: secubox-threatmesh
|
||||
Architecture: all
|
||||
Depends: ${misc:Depends}, secubox-core (>= 1.0), python3, python3-uvicorn | python3-pip, secubox-vortex-firewall | secubox-toolbox
|
||||
Recommends: secubox-p2p, crowdsec
|
||||
Description: Sovereign threat-intel mesh for SecuBox (CrowdSec CAPI replacement)
|
||||
Replaces the CrowdSec Central API with self-sourced free public blocklists
|
||||
(Phase 1) and peer-to-peer MESH distribution of locally-detected decisions over
|
||||
the SecuBox WireGuard mesh (Phase 2), plus a status/aggregate/bouncer-compatible
|
||||
API and dashboard (Phase 3). All IOCs flow into the shared threat_intel table
|
||||
that secubox-blacklist-sync drains into the nft blacklist. No central account,
|
||||
no enrollment, no paywall, no IP-blocklisting by a third party.
|
||||
23
packages/secubox-threatmesh/debian/postinst
Normal file
23
packages/secubox-threatmesh/debian/postinst
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
case "$1" in
|
||||
configure)
|
||||
install -d -m 0755 /var/lib/secubox/threatmesh
|
||||
systemctl daemon-reload 2>/dev/null || true
|
||||
# mesh-port firewall (lock :8780 to wg*/lo under DEFAULT DROP)
|
||||
if [ -f /usr/share/secubox/threatmesh/nftables.d/secubox-threatmesh.nft ]; then
|
||||
nft -f /usr/share/secubox/threatmesh/nftables.d/secubox-threatmesh.nft 2>/dev/null || true
|
||||
fi
|
||||
if [ "$(systemctl is-enabled secubox-threatmesh.service 2>/dev/null)" != "masked" ]; then
|
||||
systemctl enable --now secubox-threatmesh.service 2>/dev/null || true
|
||||
fi
|
||||
systemctl enable --now secubox-threatfeed.timer 2>/dev/null || true
|
||||
# first feed population (background; logs to /tmp to avoid /var/log perms)
|
||||
/usr/sbin/secubox-threatfeed >/tmp/secubox-threatfeed-firstrun.log 2>&1 &
|
||||
if command -v nginx >/dev/null 2>&1 && nginx -t 2>/dev/null; then
|
||||
systemctl reload nginx 2>/dev/null || true
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
#DEBHELPER#
|
||||
exit 0
|
||||
11
packages/secubox-threatmesh/debian/prerm
Normal file
11
packages/secubox-threatmesh/debian/prerm
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
case "$1" in
|
||||
remove|deconfigure)
|
||||
systemctl stop secubox-threatmesh.service 2>/dev/null || true
|
||||
systemctl disable secubox-threatmesh.service secubox-threatfeed.timer 2>/dev/null || true
|
||||
nft delete table inet secubox_threatmesh 2>/dev/null || true
|
||||
;;
|
||||
esac
|
||||
#DEBHELPER#
|
||||
exit 0
|
||||
23
packages/secubox-threatmesh/debian/rules
Executable file
23
packages/secubox-threatmesh/debian/rules
Executable file
|
|
@ -0,0 +1,23 @@
|
|||
#!/usr/bin/make -f
|
||||
%:
|
||||
dh $@
|
||||
|
||||
override_dh_auto_install:
|
||||
install -d debian/secubox-threatmesh/usr/lib/secubox/threatmesh/api
|
||||
cp -r api/. debian/secubox-threatmesh/usr/lib/secubox/threatmesh/api/
|
||||
install -d debian/secubox-threatmesh/usr/sbin
|
||||
install -m 755 sbin/secubox-threatfeed debian/secubox-threatmesh/usr/sbin/
|
||||
install -d debian/secubox-threatmesh/usr/lib/systemd/system
|
||||
cp systemd/*.service systemd/*.timer debian/secubox-threatmesh/usr/lib/systemd/system/
|
||||
install -d debian/secubox-threatmesh/usr/share/secubox/www
|
||||
cp -r www/. debian/secubox-threatmesh/usr/share/secubox/www/
|
||||
install -d debian/secubox-threatmesh/usr/share/secubox/menu.d
|
||||
cp menu.d/. -r debian/secubox-threatmesh/usr/share/secubox/menu.d/
|
||||
install -d debian/secubox-threatmesh/etc/nginx/secubox-routes.d debian/secubox-threatmesh/etc/nginx/secubox.d
|
||||
cp nginx/threatmesh.conf debian/secubox-threatmesh/etc/nginx/secubox-routes.d/
|
||||
cp nginx/threatmesh.conf debian/secubox-threatmesh/etc/nginx/secubox.d/
|
||||
install -d debian/secubox-threatmesh/usr/share/secubox/threatmesh/nftables.d
|
||||
cp nftables.d/secubox-threatmesh.nft debian/secubox-threatmesh/usr/share/secubox/threatmesh/nftables.d/
|
||||
|
||||
override_dh_installsystemd:
|
||||
true
|
||||
1
packages/secubox-threatmesh/debian/source/format
Normal file
1
packages/secubox-threatmesh/debian/source/format
Normal file
|
|
@ -0,0 +1 @@
|
|||
3.0 (native)
|
||||
9
packages/secubox-threatmesh/menu.d/37-threatmesh.json
Normal file
9
packages/secubox-threatmesh/menu.d/37-threatmesh.json
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"id": "threatmesh",
|
||||
"name": "ThreatMesh",
|
||||
"icon": "🛰️",
|
||||
"path": "/threatmesh/",
|
||||
"category": "security",
|
||||
"order": 37,
|
||||
"description": "Sovereign threat-intel (feeds + mesh, no CAPI)"
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
# ThreatMesh peer port (#728): only reachable over the WireGuard mesh + loopback.
|
||||
# The base inet filter INPUT chain is DEFAULT DROP, so this only OPENS 8780 to wg.
|
||||
table inet secubox_threatmesh {
|
||||
chain input {
|
||||
type filter hook input priority 0; policy accept;
|
||||
iifname "lo" tcp dport 8780 accept
|
||||
iifname "wg*" tcp dport 8780 accept
|
||||
tcp dport 8780 drop
|
||||
}
|
||||
}
|
||||
10
packages/secubox-threatmesh/nginx/threatmesh.conf
Normal file
10
packages/secubox-threatmesh/nginx/threatmesh.conf
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
# /etc/nginx/secubox-routes.d/threatmesh.conf — secubox-threatmesh (#728)
|
||||
location /threatmesh/ {
|
||||
alias /usr/share/secubox/www/threatmesh/;
|
||||
index index.html;
|
||||
try_files $uri $uri/ /threatmesh/index.html;
|
||||
}
|
||||
location /api/v1/threatmesh/ {
|
||||
proxy_pass http://127.0.0.1:8780/api/v1/threatmesh/;
|
||||
include /etc/nginx/snippets/secubox-proxy.conf;
|
||||
}
|
||||
121
packages/secubox-threatmesh/sbin/secubox-threatfeed
Normal file
121
packages/secubox-threatmesh/sbin/secubox-threatfeed
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: LicenseRef-CMSD-1.0
|
||||
# Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
|
||||
# Source-Disclosed License — All rights reserved except as expressly granted.
|
||||
# See LICENCE-CMSD-1.0.md for terms.
|
||||
"""
|
||||
SecuBox-Deb :: secubox-threatfeed (Phase 1 — sovereign threat-intel #728)
|
||||
CyberMind — https://cybermind.fr
|
||||
|
||||
Pulls FREE, no-enrollment public IP/CIDR/domain blocklists and writes them into
|
||||
the shared `threat_intel` table (toolbox.db) that secubox-blacklist-sync already
|
||||
drains into the nft blacklist. This REPLACES the CrowdSec CAPI community feed
|
||||
with self-sourced lists — no central account, no paywall, no IP blocklisting.
|
||||
|
||||
Pure stdlib (urllib) — no extra Debian deps. Idempotent, safe on a timer.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
DB = Path(os.environ.get("SECUBOX_TI_DB", "/var/lib/secubox/toolbox/toolbox.db"))
|
||||
TIMEOUT = int(os.environ.get("SECUBOX_TF_TIMEOUT", "30"))
|
||||
UA = "SecuBox-ThreatFeed/1.0 (+https://secubox.in)"
|
||||
TTL_DAYS = int(os.environ.get("SECUBOX_TF_TTL_DAYS", "10"))
|
||||
|
||||
# (name, url, ioc_type, weight). All free + no enrollment. CIDRs allowed for ip.
|
||||
FEEDS = [
|
||||
("feodo", "https://feodotracker.abuse.ch/downloads/ipblocklist.txt", "ip", 90),
|
||||
("sslbl", "https://sslbl.abuse.ch/blacklist/sslipblacklist.txt", "ip", 85),
|
||||
("firehol1", "https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset", "ip", 80),
|
||||
("spamhaus-drop", "https://www.spamhaus.org/drop/drop.txt", "ip", 85),
|
||||
("blocklist-de", "https://lists.blocklist.de/lists/all.txt", "ip", 60),
|
||||
("cins-army", "https://cinsscore.com/list/ci-badguys.txt", "ip", 70),
|
||||
("et-compromised", "https://rules.emergingthreats.net/blockrules/compromised-ips.txt", "ip", 75),
|
||||
("dshield", "https://feeds.dshield.org/block.txt", "ip", 65),
|
||||
]
|
||||
# Optional (set SECUBOX_TF_TOR=1 to also pull Tor exit nodes)
|
||||
if os.environ.get("SECUBOX_TF_TOR") == "1":
|
||||
FEEDS.append(("tor-exit", "https://check.torproject.org/torbulkexitlist", "ip", 40))
|
||||
|
||||
_IPV4 = re.compile(r"^\s*(\d{1,3}(?:\.\d{1,3}){3}(?:/\d{1,2})?)")
|
||||
_IPV6 = re.compile(r"^\s*([0-9A-Fa-f:]{2,}(?:/\d{1,3})?)\s*$")
|
||||
|
||||
|
||||
def log(msg):
|
||||
print(f"[threatfeed] {msg}", file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
def fetch(url):
|
||||
req = urllib.request.Request(url, headers={"User-Agent": UA})
|
||||
with urllib.request.urlopen(req, timeout=TIMEOUT) as r:
|
||||
return r.read().decode("utf-8", "replace")
|
||||
|
||||
|
||||
def parse_ips(text):
|
||||
out = []
|
||||
for line in text.splitlines():
|
||||
s = line.strip()
|
||||
if not s or s.startswith("#") or s.startswith(";"):
|
||||
continue
|
||||
# spamhaus DROP: "1.2.3.0/24 ; SBL..." -> take the first field
|
||||
token = s.split(";")[0].split()[0].strip() if (";" in s or " " in s) else s
|
||||
m = _IPV4.match(token)
|
||||
if m:
|
||||
out.append((m.group(1), "ip"))
|
||||
continue
|
||||
m = _IPV6.match(token)
|
||||
if m and ":" in m.group(1):
|
||||
out.append((m.group(1), "ip"))
|
||||
return out
|
||||
|
||||
|
||||
def ensure_schema(c):
|
||||
c.execute("""CREATE TABLE IF NOT EXISTS threat_intel (
|
||||
ioc TEXT NOT NULL, ioc_type TEXT NOT NULL, source TEXT NOT NULL,
|
||||
weight INTEGER NOT NULL DEFAULT 50, label TEXT,
|
||||
first_seen INTEGER, last_seen INTEGER,
|
||||
PRIMARY KEY (ioc, ioc_type, source))""")
|
||||
|
||||
|
||||
def main():
|
||||
if not DB.exists():
|
||||
log(f"db missing: {DB}")
|
||||
return 1
|
||||
now = int(time.time())
|
||||
total = 0
|
||||
with sqlite3.connect(DB, timeout=20) as c:
|
||||
ensure_schema(c)
|
||||
for name, url, ioc_type, weight in FEEDS:
|
||||
src = f"feed:{name}"
|
||||
try:
|
||||
iocs = parse_ips(fetch(url))
|
||||
except Exception as e:
|
||||
log(f"{name}: fetch failed: {e}")
|
||||
continue
|
||||
n = 0
|
||||
for ioc, t in iocs:
|
||||
c.execute(
|
||||
"INSERT INTO threat_intel(ioc,ioc_type,source,weight,label,first_seen,last_seen) "
|
||||
"VALUES(?,?,?,?,?,?,?) "
|
||||
"ON CONFLICT(ioc,ioc_type,source) DO UPDATE SET last_seen=excluded.last_seen, weight=excluded.weight",
|
||||
(ioc, t, src, weight, name, now, now))
|
||||
n += 1
|
||||
log(f"{name}: {n} iocs")
|
||||
total += n
|
||||
# prune stale feed entries (older than TTL) — keeps the set fresh.
|
||||
cutoff = now - TTL_DAYS * 86400
|
||||
pruned = c.execute(
|
||||
"DELETE FROM threat_intel WHERE source LIKE 'feed:%' AND last_seen < ?",
|
||||
(cutoff,)).rowcount
|
||||
c.commit()
|
||||
log(f"done: {total} iocs across {len(FEEDS)} feeds, pruned {pruned} stale")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
[Unit]
|
||||
Description=SecuBox ThreatFeed — pull free public blocklists into threat_intel (#728)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=root
|
||||
ExecStart=/usr/sbin/secubox-threatfeed
|
||||
Nice=10
|
||||
IOSchedulingClass=idle
|
||||
TimeoutStartSec=600
|
||||
11
packages/secubox-threatmesh/systemd/secubox-threatfeed.timer
Normal file
11
packages/secubox-threatmesh/systemd/secubox-threatfeed.timer
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
[Unit]
|
||||
Description=Refresh sovereign threat-intel feeds (#728)
|
||||
|
||||
[Timer]
|
||||
OnBootSec=5min
|
||||
OnUnitActiveSec=6h
|
||||
RandomizedDelaySec=15min
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
[Unit]
|
||||
Description=SecuBox ThreatMesh — sovereign threat-intel mesh API (#728)
|
||||
After=network.target secubox-runtime.service secubox-p2p.service
|
||||
Wants=secubox-p2p.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=root
|
||||
WorkingDirectory=/usr/lib/secubox/threatmesh
|
||||
# Listens on :8780 — nginx proxies the dashboard; peers POST mesh decisions
|
||||
# over the WireGuard mesh. nft drop-in restricts :8780 to wg* + loopback.
|
||||
ExecStartPre=-/usr/sbin/nft -f /usr/share/secubox/threatmesh/nftables.d/secubox-threatmesh.nft
|
||||
ExecStart=/usr/bin/python3 -m uvicorn api.main:app --host 0.0.0.0 --port 8780 --workers 1
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
NoNewPrivileges=no
|
||||
ProtectSystem=full
|
||||
PrivateTmp=true
|
||||
ReadWritePaths=/run/secubox /var/lib/secubox /etc/secubox
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
87
packages/secubox-threatmesh/www/threatmesh/index.html
Normal file
87
packages/secubox-threatmesh/www/threatmesh/index.html
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
<!DOCTYPE html>
|
||||
<!-- SPDX-License-Identifier: LicenseRef-CMSD-1.0 -->
|
||||
<!-- SecuBox-Deb :: ThreatMesh — sovereign threat-intel (#728) — CyberMind -->
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>SecuBox · ThreatMesh</title>
|
||||
<link rel="stylesheet" href="/shared/crt-light.css">
|
||||
<link rel="stylesheet" href="/shared/sidebar-light.css">
|
||||
<style>
|
||||
:root{--cosmos-black:#0a0a0f;--gold:#c9a84c;--cinnabar:#e63946;--matrix:#00ff41;
|
||||
--void:#6e40c9;--cyan:#00d4ff;--text:#e8e6d9;--muted:#6b6b7a;--panel:#13131c;--line:#23232f}
|
||||
body{margin:0;background:var(--cosmos-black);color:var(--text);font-family:'JetBrains Mono',ui-monospace,monospace;display:flex}
|
||||
.main{flex:1;margin-left:220px;min-height:100vh;padding:0 0 40px}
|
||||
@media(max-width:900px){.sidebar{display:none}.main{margin-left:0}}
|
||||
.topbar{display:flex;align-items:center;gap:14px;padding:14px 22px;border-bottom:1px solid var(--line);
|
||||
background:linear-gradient(90deg,#0a0a0f,#13131c)}
|
||||
.topbar h1{font-size:18px;margin:0;letter-spacing:1px;color:var(--gold)} .topbar .em{font-size:22px}
|
||||
.pill{margin-left:auto;font-size:12px;padding:5px 12px;border-radius:20px;border:1px solid var(--matrix);color:var(--matrix)}
|
||||
.wrap{padding:20px 22px;display:grid;gap:16px;grid-template-columns:repeat(auto-fit,minmax(260px,1fr))}
|
||||
.card{background:var(--panel);border:1px solid var(--line);border-radius:12px;padding:16px}
|
||||
.card h2{font-size:13px;color:var(--gold);margin:0 0 10px;letter-spacing:.5px}
|
||||
.big{font-size:34px;color:var(--cyan);font-weight:700} .sub{color:var(--muted);font-size:12px}
|
||||
.src{display:flex;justify-content:space-between;padding:4px 0;border-bottom:1px dashed var(--line);font-size:12px}
|
||||
.src b{color:var(--cyan)} .src.mesh b{color:var(--void)}
|
||||
.bar{display:flex;gap:8px;padding:0 22px 16px}
|
||||
button{font-family:inherit;font-size:13px;background:#1a1a26;color:var(--text);border:1px solid var(--line);
|
||||
border-radius:8px;padding:9px 14px;cursor:pointer} button:hover{border-color:var(--cyan);color:var(--cyan)}
|
||||
button.go{background:var(--void);border-color:var(--void);color:#fff}
|
||||
table{width:100%;border-collapse:collapse;font-size:12px} td,th{text-align:left;padding:5px 8px;border-bottom:1px solid var(--line)}
|
||||
th{color:var(--muted)} .toast{position:fixed;bottom:18px;right:18px;background:#13131c;border:1px solid var(--cyan);
|
||||
color:var(--cyan);padding:10px 14px;border-radius:8px;opacity:0;transition:.2s;font-size:12px}.toast.on{opacity:1}
|
||||
.full{grid-column:1/-1}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<nav class="sidebar" id="sidebar"></nav>
|
||||
<main class="main">
|
||||
<div class="topbar"><span class="em">🛰️</span><h1>THREATMESH</h1>
|
||||
<span class="pill" id="mode">sovereign · CAPI off</span></div>
|
||||
<div class="bar">
|
||||
<button class="go" onclick="refreshFeeds()">↻ Refresh feeds</button>
|
||||
<button onclick="gossip()">🛰️ Gossip now</button>
|
||||
<button onclick="load()">⟳</button>
|
||||
</div>
|
||||
<div class="wrap">
|
||||
<div class="card"><h2>BLOCKED IPs (aggregate)</h2><div class="big" id="total">…</div><div class="sub">feeds + mesh + local → nft</div></div>
|
||||
<div class="card"><h2>FREE FEEDS</h2><div class="big" id="feedtotal">…</div><div class="sub" id="feedn"></div></div>
|
||||
<div class="card"><h2>MESH</h2><div class="big" id="meshtotal">…</div><div class="sub" id="peern"></div></div>
|
||||
<div class="card"><h2>SOURCES</h2><div id="srclist"></div></div>
|
||||
<div class="card full"><h2>MESH PEERS</h2><div id="peers" class="sub">…</div></div>
|
||||
<div class="card full"><h2>TOP CONSENSUS IPs</h2><table id="agg"><thead><tr><th>IP</th><th>consensus</th><th>sources</th></tr></thead><tbody></tbody></table></div>
|
||||
</div>
|
||||
</main>
|
||||
<div class="toast" id="toast"></div>
|
||||
<script src="/shared/sidebar.js"></script>
|
||||
<script>
|
||||
const API="/api/v1/threatmesh";
|
||||
function tok(){return localStorage.getItem("sbx_token")||""}
|
||||
async function api(p,opt={}){opt.headers=Object.assign({},opt.headers||{});const t=tok();if(t)opt.headers.Authorization="Bearer "+t;
|
||||
const r=await fetch(API+p,opt);if(r.status===401){toast("login via Hub");throw 0}if(!r.ok)throw 0;
|
||||
const ct=r.headers.get("content-type")||"";return ct.includes("json")?r.json():r.text()}
|
||||
function toast(m){const t=document.getElementById("toast");t.textContent=m;t.classList.add("on");clearTimeout(t._);t._=setTimeout(()=>t.classList.remove("on"),2600)}
|
||||
function esc(s){return (s||"").replace(/[&<>"]/g,c=>({"&":"&","<":"<",">":">",'"':"""}[c]))}
|
||||
async function load(){
|
||||
let s;try{s=await fetch(API+"/status").then(r=>r.json())}catch(e){return}
|
||||
document.getElementById("mode").textContent=(s.mode||"sovereign")+" · CAPI "+(s.capi?"on":"off")+" · node "+(s.node||"?");
|
||||
document.getElementById("feedtotal").textContent=s.feed_total??0;
|
||||
document.getElementById("feedn").textContent=Object.keys(s.feeds||{}).length+" feeds";
|
||||
document.getElementById("meshtotal").textContent=s.mesh_total??0;
|
||||
document.getElementById("peern").textContent=(s.peers||0)+" peers";
|
||||
const all=s.sources||{};document.getElementById("total").textContent=Object.values(all).reduce((a,b)=>a+b,0);
|
||||
document.getElementById("srclist").innerHTML=Object.entries(all).sort((a,b)=>b[1]-a[1]).slice(0,12)
|
||||
.map(([k,v])=>`<div class="src ${k.startsWith('mesh:')?'mesh':''}"><span>${esc(k)}</span><b>${v}</b></div>`).join("")||'<div class="sub">no sources yet</div>';
|
||||
try{const p=await fetch(API+"/peers").then(r=>r.json());
|
||||
document.getElementById("peers").innerHTML=(p.peers||[]).length?p.peers.map(x=>`${esc(x.name||x.id)} <span style="color:var(--muted)">(${esc(x.address)})</span>`).join(" · "):"no mesh peers yet (single-node) — decisions will gossip when peers join";}catch(e){}
|
||||
try{const d=await fetch(API+"/decisions?min_consensus=1&limit=40").then(r=>r.json());
|
||||
d.sort((a,b)=>b.consensus-a.consensus);
|
||||
document.querySelector("#agg tbody").innerHTML=d.slice(0,25).map(r=>`<tr><td>${esc(r.value)}</td><td>${r.consensus}</td><td style="color:var(--muted)">${esc((r.sources||'').slice(0,60))}</td></tr>`).join("");}catch(e){}
|
||||
}
|
||||
async function refreshFeeds(){try{await api("/feeds/refresh",{method:"POST"});toast("feed refresh started (~1 min)")}catch(e){toast("need Hub login")}}
|
||||
async function gossip(){try{const r=await api("/mesh/gossip",{method:"POST"});toast(`gossip: ${r.pushed}/${r.peers} peers, ${r.shared} shared`)}catch(e){toast("need Hub login")}}
|
||||
load();setInterval(load,8000);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
Loading…
Reference in New Issue
Block a user