mirror of
https://github.com/CyberMind-FR/secubox-deb.git
synced 2026-06-30 09:00:53 +00:00
Compare commits
No commits in common. "0a05bed0280b72544b18ef637f1c94eecb00770d" and "0fc5871169d94695037258b16d70a9cb012292db" have entirely different histories.
0a05bed028
...
0fc5871169
|
|
@ -1,267 +0,0 @@
|
||||||
# SPDX-License-Identifier: LicenseRef-CMSD-1.0
|
|
||||||
# Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
|
|
||||||
# Source-Disclosed License — All rights reserved except as expressly granted.
|
|
||||||
# See LICENCE-CMSD-1.0.md for terms.
|
|
||||||
"""
|
|
||||||
SecuBox-Deb :: secubox-threatmesh (#728)
|
|
||||||
CyberMind — https://cybermind.fr
|
|
||||||
|
|
||||||
Sovereign threat-intel control plane that replaces CrowdSec CAPI:
|
|
||||||
|
|
||||||
Phase 1 free public feeds (secubox-threatfeed timer) -> threat_intel
|
|
||||||
Phase 2 MESH distribution of LOCAL decisions over the SecuBox P2P mesh
|
|
||||||
(gossip to peers from secubox-p2p; ingest peer decisions) -> threat_intel
|
|
||||||
Phase 3 status / aggregate / CrowdSec-bouncer-compatible decisions API + UI
|
|
||||||
|
|
||||||
All IOCs land in the shared `threat_intel` table (toolbox.db); secubox-blacklist-sync
|
|
||||||
drains it to the nft blacklist. No central account, no CAPI, no paywall.
|
|
||||||
"""
|
|
||||||
import asyncio
|
|
||||||
import json
|
|
||||||
import re
|
|
||||||
import sqlite3
|
|
||||||
import subprocess
|
|
||||||
import time
|
|
||||||
import urllib.request
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Optional
|
|
||||||
|
|
||||||
from fastapi import FastAPI, APIRouter, Depends, Request
|
|
||||||
from fastapi.responses import JSONResponse
|
|
||||||
|
|
||||||
from secubox_core.auth import require_jwt
|
|
||||||
from secubox_core.logger import get_logger
|
|
||||||
|
|
||||||
log = get_logger("threatmesh")
|
|
||||||
|
|
||||||
TI_DB = Path("/var/lib/secubox/toolbox/toolbox.db")
|
|
||||||
P2P_PEERS = Path("/var/lib/secubox/p2p/peers.json")
|
|
||||||
P2P_NODE = Path("/var/lib/secubox/p2p/node_id")
|
|
||||||
MESH_INTERVAL = 600 # gossip every 10 min
|
|
||||||
LOCAL_ORIGINS = {"crowdsec", "cscli", "manual", "secubox-waf", "secubox"}
|
|
||||||
IPV4 = re.compile(r"^\d{1,3}(\.\d{1,3}){3}(/\d{1,2})?$")
|
|
||||||
|
|
||||||
app = FastAPI(title="secubox-threatmesh", version="1.0.0", root_path="/api/v1/threatmesh")
|
|
||||||
router = APIRouter()
|
|
||||||
_node_id = None
|
|
||||||
|
|
||||||
|
|
||||||
# ── store ───────────────────────────────────────────────────────────
|
|
||||||
def _conn():
|
|
||||||
c = sqlite3.connect(TI_DB, timeout=20)
|
|
||||||
c.row_factory = sqlite3.Row
|
|
||||||
return c
|
|
||||||
|
|
||||||
|
|
||||||
def _ensure(c):
|
|
||||||
c.execute("""CREATE TABLE IF NOT EXISTS threat_intel (
|
|
||||||
ioc TEXT NOT NULL, ioc_type TEXT NOT NULL, source TEXT NOT NULL,
|
|
||||||
weight INTEGER NOT NULL DEFAULT 50, label TEXT,
|
|
||||||
first_seen INTEGER, last_seen INTEGER,
|
|
||||||
PRIMARY KEY (ioc, ioc_type, source))""")
|
|
||||||
|
|
||||||
|
|
||||||
def node_id() -> str:
|
|
||||||
global _node_id
|
|
||||||
if _node_id:
|
|
||||||
return _node_id
|
|
||||||
try:
|
|
||||||
_node_id = P2P_NODE.read_text().strip()
|
|
||||||
except Exception:
|
|
||||||
try:
|
|
||||||
_node_id = json.loads(P2P_PEERS.read_text())["peers"][0]["id"]
|
|
||||||
except Exception:
|
|
||||||
_node_id = "sb-local"
|
|
||||||
return _node_id
|
|
||||||
|
|
||||||
|
|
||||||
def upsert_iocs(rows, source, weight=60, label=None):
|
|
||||||
"""rows: iterable of (ioc, ioc_type). Returns count."""
|
|
||||||
now = int(time.time())
|
|
||||||
n = 0
|
|
||||||
with _conn() as c:
|
|
||||||
_ensure(c)
|
|
||||||
for ioc, t in rows:
|
|
||||||
c.execute(
|
|
||||||
"INSERT INTO threat_intel(ioc,ioc_type,source,weight,label,first_seen,last_seen) "
|
|
||||||
"VALUES(?,?,?,?,?,?,?) ON CONFLICT(ioc,ioc_type,source) DO UPDATE SET last_seen=excluded.last_seen",
|
|
||||||
(ioc, t, source, weight, label, now, now))
|
|
||||||
n += 1
|
|
||||||
c.commit()
|
|
||||||
return n
|
|
||||||
|
|
||||||
|
|
||||||
def counts_by_source():
|
|
||||||
with _conn() as c:
|
|
||||||
_ensure(c)
|
|
||||||
return {r["source"]: r["n"] for r in c.execute(
|
|
||||||
"SELECT source, COUNT(*) n FROM threat_intel GROUP BY source ORDER BY n DESC")}
|
|
||||||
|
|
||||||
|
|
||||||
def aggregate_ips(limit=50000):
|
|
||||||
"""Distinct bad IPs across ALL sources, with a consensus count."""
|
|
||||||
with _conn() as c:
|
|
||||||
_ensure(c)
|
|
||||||
return [dict(r) for r in c.execute(
|
|
||||||
"SELECT ioc, COUNT(DISTINCT source) consensus, MAX(weight) weight, "
|
|
||||||
"GROUP_CONCAT(DISTINCT source) sources FROM threat_intel "
|
|
||||||
"WHERE ioc_type='ip' GROUP BY ioc ORDER BY consensus DESC LIMIT ?", (limit,))]
|
|
||||||
|
|
||||||
|
|
||||||
# ── mesh (Phase 2) ──────────────────────────────────────────────────
|
|
||||||
def mesh_peers():
|
|
||||||
"""Other SecuBox nodes from secubox-p2p (skip self)."""
|
|
||||||
me = node_id()
|
|
||||||
out = []
|
|
||||||
try:
|
|
||||||
d = json.loads(P2P_PEERS.read_text())
|
|
||||||
for p in d.get("peers", []):
|
|
||||||
if p.get("id") == me:
|
|
||||||
continue
|
|
||||||
addr = None
|
|
||||||
for a in p.get("addresses", []):
|
|
||||||
if a.get("type") in ("wg", "wireguard", "mesh"):
|
|
||||||
addr = a.get("address"); break
|
|
||||||
addr = addr or (p.get("wg_addresses") or [None])[0] or p.get("address")
|
|
||||||
if addr:
|
|
||||||
out.append({"id": p.get("id"), "name": p.get("name"), "address": addr})
|
|
||||||
except Exception as e:
|
|
||||||
log.debug(f"peers read: {e}")
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
def local_decisions():
|
|
||||||
"""Our OWN locally-detected bad IPs (CrowdSec LAPI bans of local origin)."""
|
|
||||||
try:
|
|
||||||
out = subprocess.run(["cscli", "decisions", "list", "-o", "json", "-a"],
|
|
||||||
capture_output=True, text=True, timeout=20)
|
|
||||||
data = json.loads(out.stdout or "[]") or []
|
|
||||||
except Exception as e:
|
|
||||||
log.debug(f"cscli decisions: {e}")
|
|
||||||
return []
|
|
||||||
ips = []
|
|
||||||
for alert in data:
|
|
||||||
for dec in (alert.get("decisions") or []):
|
|
||||||
if dec.get("type") == "ban" and dec.get("scope", "").lower() == "ip":
|
|
||||||
origin = (dec.get("origin") or "").lower()
|
|
||||||
val = dec.get("value", "")
|
|
||||||
if IPV4.match(val) and not origin.startswith(("feed:", "mesh:", "lists:")):
|
|
||||||
ips.append(val)
|
|
||||||
return sorted(set(ips))
|
|
||||||
|
|
||||||
|
|
||||||
async def _post_json(url, payload, timeout=15):
|
|
||||||
def _do():
|
|
||||||
req = urllib.request.Request(
|
|
||||||
url, data=json.dumps(payload).encode(),
|
|
||||||
headers={"Content-Type": "application/json", "User-Agent": "SecuBox-ThreatMesh/1.0"},
|
|
||||||
method="POST")
|
|
||||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
|
||||||
return r.status
|
|
||||||
return await asyncio.to_thread(_do)
|
|
||||||
|
|
||||||
|
|
||||||
async def mesh_gossip_once():
|
|
||||||
peers = mesh_peers()
|
|
||||||
decisions = local_decisions()
|
|
||||||
if not peers or not decisions:
|
|
||||||
return {"peers": len(peers), "shared": len(decisions), "pushed": 0}
|
|
||||||
payload = {"node": node_id(), "ts": int(time.time()), "ips": decisions}
|
|
||||||
pushed = 0
|
|
||||||
for p in peers:
|
|
||||||
url = f"http://{p['address']}:8780/api/v1/threatmesh/mesh/ingest"
|
|
||||||
try:
|
|
||||||
await _post_json(url, payload)
|
|
||||||
pushed += 1
|
|
||||||
except Exception as e:
|
|
||||||
log.debug(f"gossip to {p['id']} failed: {e}")
|
|
||||||
return {"peers": len(peers), "shared": len(decisions), "pushed": pushed}
|
|
||||||
|
|
||||||
|
|
||||||
async def _mesh_loop():
|
|
||||||
while True:
|
|
||||||
try:
|
|
||||||
r = await mesh_gossip_once()
|
|
||||||
log.info(f"mesh gossip: {r}")
|
|
||||||
except Exception as e:
|
|
||||||
log.error(f"mesh loop: {e}")
|
|
||||||
await asyncio.sleep(MESH_INTERVAL)
|
|
||||||
|
|
||||||
|
|
||||||
@app.on_event("startup")
|
|
||||||
async def _startup():
|
|
||||||
asyncio.create_task(_mesh_loop())
|
|
||||||
|
|
||||||
|
|
||||||
# ── endpoints ───────────────────────────────────────────────────────
|
|
||||||
@app.get("/health")
|
|
||||||
async def health():
|
|
||||||
return {"status": "ok", "module": "deb"}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/status")
|
|
||||||
async def status():
|
|
||||||
src = counts_by_source()
|
|
||||||
feeds = {k: v for k, v in src.items() if k.startswith("feed:")}
|
|
||||||
mesh = {k: v for k, v in src.items() if k.startswith("mesh:")}
|
|
||||||
return {
|
|
||||||
"mode": "sovereign", # CAPI dropped (#728)
|
|
||||||
"node": node_id(),
|
|
||||||
"capi": False,
|
|
||||||
"feeds": feeds, "feed_total": sum(feeds.values()),
|
|
||||||
"mesh": mesh, "mesh_total": sum(mesh.values()),
|
|
||||||
"peers": len(mesh_peers()),
|
|
||||||
"sources": src,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/peers")
|
|
||||||
async def peers():
|
|
||||||
return {"node": node_id(), "peers": mesh_peers()}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/decisions")
|
|
||||||
async def decisions(min_consensus: int = 1, limit: int = 50000):
|
|
||||||
"""Aggregated sovereign blocklist (feeds + mesh + local). CrowdSec-bouncer
|
|
||||||
friendly shape so external consumers can poll OUR server, not crowdsec.net."""
|
|
||||||
rows = [r for r in aggregate_ips(limit) if r["consensus"] >= min_consensus]
|
|
||||||
return [{
|
|
||||||
"id": i, "origin": "secubox-threatmesh", "type": "ban", "scope": "Ip",
|
|
||||||
"value": r["ioc"], "duration": "24h",
|
|
||||||
"consensus": r["consensus"], "sources": r["sources"],
|
|
||||||
} for i, r in enumerate(rows)]
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/mesh/ingest")
|
|
||||||
async def mesh_ingest(request: Request):
|
|
||||||
"""Receive a peer's locally-detected decisions over the mesh -> threat_intel
|
|
||||||
tagged mesh:<node>. Trust boundary = the WireGuard mesh transport."""
|
|
||||||
try:
|
|
||||||
body = await request.json()
|
|
||||||
except Exception:
|
|
||||||
return JSONResponse({"ok": False, "error": "bad json"}, status_code=400)
|
|
||||||
node = str(body.get("node", "")).strip()[:64] or "unknown"
|
|
||||||
if not re.match(r"^[A-Za-z0-9._:-]+$", node):
|
|
||||||
return JSONResponse({"ok": False, "error": "bad node"}, status_code=400)
|
|
||||||
ips = [ip for ip in (body.get("ips") or []) if isinstance(ip, str) and IPV4.match(ip)]
|
|
||||||
n = upsert_iocs(((ip, "ip") for ip in ips[:20000]), f"mesh:{node}", weight=70, label="mesh")
|
|
||||||
log.info(f"mesh ingest from {node}: {n} ips")
|
|
||||||
return {"ok": True, "ingested": n}
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/feeds/refresh", dependencies=[Depends(require_jwt)])
|
|
||||||
async def feeds_refresh():
|
|
||||||
"""Trigger the feed fetcher now (normally on a timer)."""
|
|
||||||
try:
|
|
||||||
subprocess.Popen(["/usr/sbin/secubox-threatfeed"])
|
|
||||||
return {"ok": True, "started": True}
|
|
||||||
except Exception as e:
|
|
||||||
return JSONResponse({"ok": False, "error": str(e)}, status_code=500)
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/mesh/gossip", dependencies=[Depends(require_jwt)])
|
|
||||||
async def mesh_gossip_now():
|
|
||||||
return await mesh_gossip_once()
|
|
||||||
|
|
||||||
|
|
||||||
app.include_router(router)
|
|
||||||
|
|
@ -1,13 +0,0 @@
|
||||||
secubox-threatmesh (1.0.0-1~bookworm1) bookworm; urgency=medium
|
|
||||||
|
|
||||||
* Initial release (#728) — sovereign threat-intel, CrowdSec CAPI replacement.
|
|
||||||
- Phase 1: secubox-threatfeed timer pulls free public blocklists
|
|
||||||
(feodo/sslbl/firehol/spamhaus-drop/blocklist.de/cins/et/dshield) into the
|
|
||||||
shared threat_intel table -> secubox-blacklist-sync -> nft.
|
|
||||||
- Phase 2: MESH distribution — gossip locally-detected CrowdSec decisions to
|
|
||||||
SecuBox P2P peers over WireGuard; ingest peer decisions (tagged mesh:<node>),
|
|
||||||
consensus-counted. :8780 locked to wg*/loopback by an nft drop-in.
|
|
||||||
- Phase 3: /status, /peers, /decisions (bouncer-compatible aggregate),
|
|
||||||
/mesh/ingest, dashboard. CrowdSec LAPI kept; CAPI dropped.
|
|
||||||
|
|
||||||
-- Gerald KERMA <devel@cybermind.fr> Wed, 24 Jun 2026 16:00:00 +0000
|
|
||||||
|
|
@ -1,18 +0,0 @@
|
||||||
Source: secubox-threatmesh
|
|
||||||
Section: net
|
|
||||||
Priority: optional
|
|
||||||
Maintainer: Gerald KERMA <devel@cybermind.fr>
|
|
||||||
Build-Depends: debhelper-compat (= 13)
|
|
||||||
Standards-Version: 4.6.2
|
|
||||||
|
|
||||||
Package: secubox-threatmesh
|
|
||||||
Architecture: all
|
|
||||||
Depends: ${misc:Depends}, secubox-core (>= 1.0), python3, python3-uvicorn | python3-pip, secubox-vortex-firewall | secubox-toolbox
|
|
||||||
Recommends: secubox-p2p, crowdsec
|
|
||||||
Description: Sovereign threat-intel mesh for SecuBox (CrowdSec CAPI replacement)
|
|
||||||
Replaces the CrowdSec Central API with self-sourced free public blocklists
|
|
||||||
(Phase 1) and peer-to-peer MESH distribution of locally-detected decisions over
|
|
||||||
the SecuBox WireGuard mesh (Phase 2), plus a status/aggregate/bouncer-compatible
|
|
||||||
API and dashboard (Phase 3). All IOCs flow into the shared threat_intel table
|
|
||||||
that secubox-blacklist-sync drains into the nft blacklist. No central account,
|
|
||||||
no enrollment, no paywall, no IP-blocklisting by a third party.
|
|
||||||
|
|
@ -1,23 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
set -e
|
|
||||||
case "$1" in
|
|
||||||
configure)
|
|
||||||
install -d -m 0755 /var/lib/secubox/threatmesh
|
|
||||||
systemctl daemon-reload 2>/dev/null || true
|
|
||||||
# mesh-port firewall (lock :8780 to wg*/lo under DEFAULT DROP)
|
|
||||||
if [ -f /usr/share/secubox/threatmesh/nftables.d/secubox-threatmesh.nft ]; then
|
|
||||||
nft -f /usr/share/secubox/threatmesh/nftables.d/secubox-threatmesh.nft 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
if [ "$(systemctl is-enabled secubox-threatmesh.service 2>/dev/null)" != "masked" ]; then
|
|
||||||
systemctl enable --now secubox-threatmesh.service 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
systemctl enable --now secubox-threatfeed.timer 2>/dev/null || true
|
|
||||||
# first feed population (background; logs to /tmp to avoid /var/log perms)
|
|
||||||
/usr/sbin/secubox-threatfeed >/tmp/secubox-threatfeed-firstrun.log 2>&1 &
|
|
||||||
if command -v nginx >/dev/null 2>&1 && nginx -t 2>/dev/null; then
|
|
||||||
systemctl reload nginx 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
#DEBHELPER#
|
|
||||||
exit 0
|
|
||||||
|
|
@ -1,11 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
set -e
|
|
||||||
case "$1" in
|
|
||||||
remove|deconfigure)
|
|
||||||
systemctl stop secubox-threatmesh.service 2>/dev/null || true
|
|
||||||
systemctl disable secubox-threatmesh.service secubox-threatfeed.timer 2>/dev/null || true
|
|
||||||
nft delete table inet secubox_threatmesh 2>/dev/null || true
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
#DEBHELPER#
|
|
||||||
exit 0
|
|
||||||
|
|
@ -1,23 +0,0 @@
|
||||||
#!/usr/bin/make -f
|
|
||||||
%:
|
|
||||||
dh $@
|
|
||||||
|
|
||||||
override_dh_auto_install:
|
|
||||||
install -d debian/secubox-threatmesh/usr/lib/secubox/threatmesh/api
|
|
||||||
cp -r api/. debian/secubox-threatmesh/usr/lib/secubox/threatmesh/api/
|
|
||||||
install -d debian/secubox-threatmesh/usr/sbin
|
|
||||||
install -m 755 sbin/secubox-threatfeed debian/secubox-threatmesh/usr/sbin/
|
|
||||||
install -d debian/secubox-threatmesh/usr/lib/systemd/system
|
|
||||||
cp systemd/*.service systemd/*.timer debian/secubox-threatmesh/usr/lib/systemd/system/
|
|
||||||
install -d debian/secubox-threatmesh/usr/share/secubox/www
|
|
||||||
cp -r www/. debian/secubox-threatmesh/usr/share/secubox/www/
|
|
||||||
install -d debian/secubox-threatmesh/usr/share/secubox/menu.d
|
|
||||||
cp menu.d/. -r debian/secubox-threatmesh/usr/share/secubox/menu.d/
|
|
||||||
install -d debian/secubox-threatmesh/etc/nginx/secubox-routes.d debian/secubox-threatmesh/etc/nginx/secubox.d
|
|
||||||
cp nginx/threatmesh.conf debian/secubox-threatmesh/etc/nginx/secubox-routes.d/
|
|
||||||
cp nginx/threatmesh.conf debian/secubox-threatmesh/etc/nginx/secubox.d/
|
|
||||||
install -d debian/secubox-threatmesh/usr/share/secubox/threatmesh/nftables.d
|
|
||||||
cp nftables.d/secubox-threatmesh.nft debian/secubox-threatmesh/usr/share/secubox/threatmesh/nftables.d/
|
|
||||||
|
|
||||||
override_dh_installsystemd:
|
|
||||||
true
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
3.0 (native)
|
|
||||||
|
|
@ -1,9 +0,0 @@
|
||||||
{
|
|
||||||
"id": "threatmesh",
|
|
||||||
"name": "ThreatMesh",
|
|
||||||
"icon": "🛰️",
|
|
||||||
"path": "/threatmesh/",
|
|
||||||
"category": "security",
|
|
||||||
"order": 37,
|
|
||||||
"description": "Sovereign threat-intel (feeds + mesh, no CAPI)"
|
|
||||||
}
|
|
||||||
|
|
@ -1,10 +0,0 @@
|
||||||
# ThreatMesh peer port (#728): only reachable over the WireGuard mesh + loopback.
|
|
||||||
# The base inet filter INPUT chain is DEFAULT DROP, so this only OPENS 8780 to wg.
|
|
||||||
table inet secubox_threatmesh {
|
|
||||||
chain input {
|
|
||||||
type filter hook input priority 0; policy accept;
|
|
||||||
iifname "lo" tcp dport 8780 accept
|
|
||||||
iifname "wg*" tcp dport 8780 accept
|
|
||||||
tcp dport 8780 drop
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,10 +0,0 @@
|
||||||
# /etc/nginx/secubox-routes.d/threatmesh.conf — secubox-threatmesh (#728)
|
|
||||||
location /threatmesh/ {
|
|
||||||
alias /usr/share/secubox/www/threatmesh/;
|
|
||||||
index index.html;
|
|
||||||
try_files $uri $uri/ /threatmesh/index.html;
|
|
||||||
}
|
|
||||||
location /api/v1/threatmesh/ {
|
|
||||||
proxy_pass http://127.0.0.1:8780/api/v1/threatmesh/;
|
|
||||||
include /etc/nginx/snippets/secubox-proxy.conf;
|
|
||||||
}
|
|
||||||
|
|
@ -1,121 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
# SPDX-License-Identifier: LicenseRef-CMSD-1.0
|
|
||||||
# Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
|
|
||||||
# Source-Disclosed License — All rights reserved except as expressly granted.
|
|
||||||
# See LICENCE-CMSD-1.0.md for terms.
|
|
||||||
"""
|
|
||||||
SecuBox-Deb :: secubox-threatfeed (Phase 1 — sovereign threat-intel #728)
|
|
||||||
CyberMind — https://cybermind.fr
|
|
||||||
|
|
||||||
Pulls FREE, no-enrollment public IP/CIDR/domain blocklists and writes them into
|
|
||||||
the shared `threat_intel` table (toolbox.db) that secubox-blacklist-sync already
|
|
||||||
drains into the nft blacklist. This REPLACES the CrowdSec CAPI community feed
|
|
||||||
with self-sourced lists — no central account, no paywall, no IP blocklisting.
|
|
||||||
|
|
||||||
Pure stdlib (urllib) — no extra Debian deps. Idempotent, safe on a timer.
|
|
||||||
"""
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
import sqlite3
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
import urllib.request
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
DB = Path(os.environ.get("SECUBOX_TI_DB", "/var/lib/secubox/toolbox/toolbox.db"))
|
|
||||||
TIMEOUT = int(os.environ.get("SECUBOX_TF_TIMEOUT", "30"))
|
|
||||||
UA = "SecuBox-ThreatFeed/1.0 (+https://secubox.in)"
|
|
||||||
TTL_DAYS = int(os.environ.get("SECUBOX_TF_TTL_DAYS", "10"))
|
|
||||||
|
|
||||||
# (name, url, ioc_type, weight). All free + no enrollment. CIDRs allowed for ip.
|
|
||||||
FEEDS = [
|
|
||||||
("feodo", "https://feodotracker.abuse.ch/downloads/ipblocklist.txt", "ip", 90),
|
|
||||||
("sslbl", "https://sslbl.abuse.ch/blacklist/sslipblacklist.txt", "ip", 85),
|
|
||||||
("firehol1", "https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset", "ip", 80),
|
|
||||||
("spamhaus-drop", "https://www.spamhaus.org/drop/drop.txt", "ip", 85),
|
|
||||||
("blocklist-de", "https://lists.blocklist.de/lists/all.txt", "ip", 60),
|
|
||||||
("cins-army", "https://cinsscore.com/list/ci-badguys.txt", "ip", 70),
|
|
||||||
("et-compromised", "https://rules.emergingthreats.net/blockrules/compromised-ips.txt", "ip", 75),
|
|
||||||
("dshield", "https://feeds.dshield.org/block.txt", "ip", 65),
|
|
||||||
]
|
|
||||||
# Optional (set SECUBOX_TF_TOR=1 to also pull Tor exit nodes)
|
|
||||||
if os.environ.get("SECUBOX_TF_TOR") == "1":
|
|
||||||
FEEDS.append(("tor-exit", "https://check.torproject.org/torbulkexitlist", "ip", 40))
|
|
||||||
|
|
||||||
_IPV4 = re.compile(r"^\s*(\d{1,3}(?:\.\d{1,3}){3}(?:/\d{1,2})?)")
|
|
||||||
_IPV6 = re.compile(r"^\s*([0-9A-Fa-f:]{2,}(?:/\d{1,3})?)\s*$")
|
|
||||||
|
|
||||||
|
|
||||||
def log(msg):
|
|
||||||
print(f"[threatfeed] {msg}", file=sys.stderr, flush=True)
|
|
||||||
|
|
||||||
|
|
||||||
def fetch(url):
|
|
||||||
req = urllib.request.Request(url, headers={"User-Agent": UA})
|
|
||||||
with urllib.request.urlopen(req, timeout=TIMEOUT) as r:
|
|
||||||
return r.read().decode("utf-8", "replace")
|
|
||||||
|
|
||||||
|
|
||||||
def parse_ips(text):
|
|
||||||
out = []
|
|
||||||
for line in text.splitlines():
|
|
||||||
s = line.strip()
|
|
||||||
if not s or s.startswith("#") or s.startswith(";"):
|
|
||||||
continue
|
|
||||||
# spamhaus DROP: "1.2.3.0/24 ; SBL..." -> take the first field
|
|
||||||
token = s.split(";")[0].split()[0].strip() if (";" in s or " " in s) else s
|
|
||||||
m = _IPV4.match(token)
|
|
||||||
if m:
|
|
||||||
out.append((m.group(1), "ip"))
|
|
||||||
continue
|
|
||||||
m = _IPV6.match(token)
|
|
||||||
if m and ":" in m.group(1):
|
|
||||||
out.append((m.group(1), "ip"))
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
def ensure_schema(c):
|
|
||||||
c.execute("""CREATE TABLE IF NOT EXISTS threat_intel (
|
|
||||||
ioc TEXT NOT NULL, ioc_type TEXT NOT NULL, source TEXT NOT NULL,
|
|
||||||
weight INTEGER NOT NULL DEFAULT 50, label TEXT,
|
|
||||||
first_seen INTEGER, last_seen INTEGER,
|
|
||||||
PRIMARY KEY (ioc, ioc_type, source))""")
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
if not DB.exists():
|
|
||||||
log(f"db missing: {DB}")
|
|
||||||
return 1
|
|
||||||
now = int(time.time())
|
|
||||||
total = 0
|
|
||||||
with sqlite3.connect(DB, timeout=20) as c:
|
|
||||||
ensure_schema(c)
|
|
||||||
for name, url, ioc_type, weight in FEEDS:
|
|
||||||
src = f"feed:{name}"
|
|
||||||
try:
|
|
||||||
iocs = parse_ips(fetch(url))
|
|
||||||
except Exception as e:
|
|
||||||
log(f"{name}: fetch failed: {e}")
|
|
||||||
continue
|
|
||||||
n = 0
|
|
||||||
for ioc, t in iocs:
|
|
||||||
c.execute(
|
|
||||||
"INSERT INTO threat_intel(ioc,ioc_type,source,weight,label,first_seen,last_seen) "
|
|
||||||
"VALUES(?,?,?,?,?,?,?) "
|
|
||||||
"ON CONFLICT(ioc,ioc_type,source) DO UPDATE SET last_seen=excluded.last_seen, weight=excluded.weight",
|
|
||||||
(ioc, t, src, weight, name, now, now))
|
|
||||||
n += 1
|
|
||||||
log(f"{name}: {n} iocs")
|
|
||||||
total += n
|
|
||||||
# prune stale feed entries (older than TTL) — keeps the set fresh.
|
|
||||||
cutoff = now - TTL_DAYS * 86400
|
|
||||||
pruned = c.execute(
|
|
||||||
"DELETE FROM threat_intel WHERE source LIKE 'feed:%' AND last_seen < ?",
|
|
||||||
(cutoff,)).rowcount
|
|
||||||
c.commit()
|
|
||||||
log(f"done: {total} iocs across {len(FEEDS)} feeds, pruned {pruned} stale")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
|
|
@ -1,12 +0,0 @@
|
||||||
[Unit]
|
|
||||||
Description=SecuBox ThreatFeed — pull free public blocklists into threat_intel (#728)
|
|
||||||
After=network-online.target
|
|
||||||
Wants=network-online.target
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Type=oneshot
|
|
||||||
User=root
|
|
||||||
ExecStart=/usr/sbin/secubox-threatfeed
|
|
||||||
Nice=10
|
|
||||||
IOSchedulingClass=idle
|
|
||||||
TimeoutStartSec=600
|
|
||||||
|
|
@ -1,11 +0,0 @@
|
||||||
[Unit]
|
|
||||||
Description=Refresh sovereign threat-intel feeds (#728)
|
|
||||||
|
|
||||||
[Timer]
|
|
||||||
OnBootSec=5min
|
|
||||||
OnUnitActiveSec=6h
|
|
||||||
RandomizedDelaySec=15min
|
|
||||||
Persistent=true
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=timers.target
|
|
||||||
|
|
@ -1,22 +0,0 @@
|
||||||
[Unit]
|
|
||||||
Description=SecuBox ThreatMesh — sovereign threat-intel mesh API (#728)
|
|
||||||
After=network.target secubox-runtime.service secubox-p2p.service
|
|
||||||
Wants=secubox-p2p.service
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Type=simple
|
|
||||||
User=root
|
|
||||||
WorkingDirectory=/usr/lib/secubox/threatmesh
|
|
||||||
# Listens on :8780 — nginx proxies the dashboard; peers POST mesh decisions
|
|
||||||
# over the WireGuard mesh. nft drop-in restricts :8780 to wg* + loopback.
|
|
||||||
ExecStartPre=-/usr/sbin/nft -f /usr/share/secubox/threatmesh/nftables.d/secubox-threatmesh.nft
|
|
||||||
ExecStart=/usr/bin/python3 -m uvicorn api.main:app --host 0.0.0.0 --port 8780 --workers 1
|
|
||||||
Restart=on-failure
|
|
||||||
RestartSec=5
|
|
||||||
NoNewPrivileges=no
|
|
||||||
ProtectSystem=full
|
|
||||||
PrivateTmp=true
|
|
||||||
ReadWritePaths=/run/secubox /var/lib/secubox /etc/secubox
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=multi-user.target
|
|
||||||
|
|
@ -1,87 +0,0 @@
|
||||||
<!DOCTYPE html>
|
|
||||||
<!-- SPDX-License-Identifier: LicenseRef-CMSD-1.0 -->
|
|
||||||
<!-- SecuBox-Deb :: ThreatMesh — sovereign threat-intel (#728) — CyberMind -->
|
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
<meta charset="utf-8">
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
||||||
<title>SecuBox · ThreatMesh</title>
|
|
||||||
<link rel="stylesheet" href="/shared/crt-light.css">
|
|
||||||
<link rel="stylesheet" href="/shared/sidebar-light.css">
|
|
||||||
<style>
|
|
||||||
:root{--cosmos-black:#0a0a0f;--gold:#c9a84c;--cinnabar:#e63946;--matrix:#00ff41;
|
|
||||||
--void:#6e40c9;--cyan:#00d4ff;--text:#e8e6d9;--muted:#6b6b7a;--panel:#13131c;--line:#23232f}
|
|
||||||
body{margin:0;background:var(--cosmos-black);color:var(--text);font-family:'JetBrains Mono',ui-monospace,monospace;display:flex}
|
|
||||||
.main{flex:1;margin-left:220px;min-height:100vh;padding:0 0 40px}
|
|
||||||
@media(max-width:900px){.sidebar{display:none}.main{margin-left:0}}
|
|
||||||
.topbar{display:flex;align-items:center;gap:14px;padding:14px 22px;border-bottom:1px solid var(--line);
|
|
||||||
background:linear-gradient(90deg,#0a0a0f,#13131c)}
|
|
||||||
.topbar h1{font-size:18px;margin:0;letter-spacing:1px;color:var(--gold)} .topbar .em{font-size:22px}
|
|
||||||
.pill{margin-left:auto;font-size:12px;padding:5px 12px;border-radius:20px;border:1px solid var(--matrix);color:var(--matrix)}
|
|
||||||
.wrap{padding:20px 22px;display:grid;gap:16px;grid-template-columns:repeat(auto-fit,minmax(260px,1fr))}
|
|
||||||
.card{background:var(--panel);border:1px solid var(--line);border-radius:12px;padding:16px}
|
|
||||||
.card h2{font-size:13px;color:var(--gold);margin:0 0 10px;letter-spacing:.5px}
|
|
||||||
.big{font-size:34px;color:var(--cyan);font-weight:700} .sub{color:var(--muted);font-size:12px}
|
|
||||||
.src{display:flex;justify-content:space-between;padding:4px 0;border-bottom:1px dashed var(--line);font-size:12px}
|
|
||||||
.src b{color:var(--cyan)} .src.mesh b{color:var(--void)}
|
|
||||||
.bar{display:flex;gap:8px;padding:0 22px 16px}
|
|
||||||
button{font-family:inherit;font-size:13px;background:#1a1a26;color:var(--text);border:1px solid var(--line);
|
|
||||||
border-radius:8px;padding:9px 14px;cursor:pointer} button:hover{border-color:var(--cyan);color:var(--cyan)}
|
|
||||||
button.go{background:var(--void);border-color:var(--void);color:#fff}
|
|
||||||
table{width:100%;border-collapse:collapse;font-size:12px} td,th{text-align:left;padding:5px 8px;border-bottom:1px solid var(--line)}
|
|
||||||
th{color:var(--muted)} .toast{position:fixed;bottom:18px;right:18px;background:#13131c;border:1px solid var(--cyan);
|
|
||||||
color:var(--cyan);padding:10px 14px;border-radius:8px;opacity:0;transition:.2s;font-size:12px}.toast.on{opacity:1}
|
|
||||||
.full{grid-column:1/-1}
|
|
||||||
</style>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<nav class="sidebar" id="sidebar"></nav>
|
|
||||||
<main class="main">
|
|
||||||
<div class="topbar"><span class="em">🛰️</span><h1>THREATMESH</h1>
|
|
||||||
<span class="pill" id="mode">sovereign · CAPI off</span></div>
|
|
||||||
<div class="bar">
|
|
||||||
<button class="go" onclick="refreshFeeds()">↻ Refresh feeds</button>
|
|
||||||
<button onclick="gossip()">🛰️ Gossip now</button>
|
|
||||||
<button onclick="load()">⟳</button>
|
|
||||||
</div>
|
|
||||||
<div class="wrap">
|
|
||||||
<div class="card"><h2>BLOCKED IPs (aggregate)</h2><div class="big" id="total">…</div><div class="sub">feeds + mesh + local → nft</div></div>
|
|
||||||
<div class="card"><h2>FREE FEEDS</h2><div class="big" id="feedtotal">…</div><div class="sub" id="feedn"></div></div>
|
|
||||||
<div class="card"><h2>MESH</h2><div class="big" id="meshtotal">…</div><div class="sub" id="peern"></div></div>
|
|
||||||
<div class="card"><h2>SOURCES</h2><div id="srclist"></div></div>
|
|
||||||
<div class="card full"><h2>MESH PEERS</h2><div id="peers" class="sub">…</div></div>
|
|
||||||
<div class="card full"><h2>TOP CONSENSUS IPs</h2><table id="agg"><thead><tr><th>IP</th><th>consensus</th><th>sources</th></tr></thead><tbody></tbody></table></div>
|
|
||||||
</div>
|
|
||||||
</main>
|
|
||||||
<div class="toast" id="toast"></div>
|
|
||||||
<script src="/shared/sidebar.js"></script>
|
|
||||||
<script>
|
|
||||||
const API="/api/v1/threatmesh";
|
|
||||||
function tok(){return localStorage.getItem("sbx_token")||""}
|
|
||||||
async function api(p,opt={}){opt.headers=Object.assign({},opt.headers||{});const t=tok();if(t)opt.headers.Authorization="Bearer "+t;
|
|
||||||
const r=await fetch(API+p,opt);if(r.status===401){toast("login via Hub");throw 0}if(!r.ok)throw 0;
|
|
||||||
const ct=r.headers.get("content-type")||"";return ct.includes("json")?r.json():r.text()}
|
|
||||||
function toast(m){const t=document.getElementById("toast");t.textContent=m;t.classList.add("on");clearTimeout(t._);t._=setTimeout(()=>t.classList.remove("on"),2600)}
|
|
||||||
function esc(s){return (s||"").replace(/[&<>"]/g,c=>({"&":"&","<":"<",">":">",'"':"""}[c]))}
|
|
||||||
async function load(){
|
|
||||||
let s;try{s=await fetch(API+"/status").then(r=>r.json())}catch(e){return}
|
|
||||||
document.getElementById("mode").textContent=(s.mode||"sovereign")+" · CAPI "+(s.capi?"on":"off")+" · node "+(s.node||"?");
|
|
||||||
document.getElementById("feedtotal").textContent=s.feed_total??0;
|
|
||||||
document.getElementById("feedn").textContent=Object.keys(s.feeds||{}).length+" feeds";
|
|
||||||
document.getElementById("meshtotal").textContent=s.mesh_total??0;
|
|
||||||
document.getElementById("peern").textContent=(s.peers||0)+" peers";
|
|
||||||
const all=s.sources||{};document.getElementById("total").textContent=Object.values(all).reduce((a,b)=>a+b,0);
|
|
||||||
document.getElementById("srclist").innerHTML=Object.entries(all).sort((a,b)=>b[1]-a[1]).slice(0,12)
|
|
||||||
.map(([k,v])=>`<div class="src ${k.startsWith('mesh:')?'mesh':''}"><span>${esc(k)}</span><b>${v}</b></div>`).join("")||'<div class="sub">no sources yet</div>';
|
|
||||||
try{const p=await fetch(API+"/peers").then(r=>r.json());
|
|
||||||
document.getElementById("peers").innerHTML=(p.peers||[]).length?p.peers.map(x=>`${esc(x.name||x.id)} <span style="color:var(--muted)">(${esc(x.address)})</span>`).join(" · "):"no mesh peers yet (single-node) — decisions will gossip when peers join";}catch(e){}
|
|
||||||
try{const d=await fetch(API+"/decisions?min_consensus=1&limit=40").then(r=>r.json());
|
|
||||||
d.sort((a,b)=>b.consensus-a.consensus);
|
|
||||||
document.querySelector("#agg tbody").innerHTML=d.slice(0,25).map(r=>`<tr><td>${esc(r.value)}</td><td>${r.consensus}</td><td style="color:var(--muted)">${esc((r.sources||'').slice(0,60))}</td></tr>`).join("");}catch(e){}
|
|
||||||
}
|
|
||||||
async function refreshFeeds(){try{await api("/feeds/refresh",{method:"POST"});toast("feed refresh started (~1 min)")}catch(e){toast("need Hub login")}}
|
|
||||||
async function gossip(){try{const r=await api("/mesh/gossip",{method:"POST"});toast(`gossip: ${r.pushed}/${r.peers} peers, ${r.shared} shared`)}catch(e){toast("need Hub login")}}
|
|
||||||
load();setInterval(load,8000);
|
|
||||||
</script>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
Loading…
Reference in New Issue
Block a user