mirror of
https://github.com/CyberMind-FR/secubox-deb.git
synced 2026-06-29 10:08:36 +00:00
Compare commits
16 Commits
39d7002b7a
...
4f8eb711f3
| Author | SHA1 | Date | |
|---|---|---|---|
| 4f8eb711f3 | |||
| 4cf7c85191 | |||
| 6286b83bda | |||
| 6907b95d11 | |||
| d6eaf52ce1 | |||
| 0a05bed028 | |||
| fdfc404818 | |||
| 0fc5871169 | |||
| 36cfb72e41 | |||
| 6e62c0166d | |||
| ff6fd7632f | |||
| 0566672615 | |||
| 9f5bec6a87 | |||
| 560b8d8213 | |||
| 7da61e8fd5 | |||
| f839c9260e |
|
|
@ -1,3 +1,14 @@
|
||||||
|
secubox-aggregator (0.2.3-1~bookworm1) bookworm; urgency=medium
|
||||||
|
|
||||||
|
* #727 auto-heal watchdog: ship secubox-aggregator-watchdog.{sh,service,timer}.
|
||||||
|
The in-process aggregator is the hub/auth/menu SPOF; under a host load spike
|
||||||
|
its event loop can wedge and the socket stops answering (board-wide 502/000:
|
||||||
|
sparse navbar, login errors). The timer probes aggregator.sock every 2 min
|
||||||
|
and restarts the service after 2 consecutive failures. Enabled in postinst
|
||||||
|
(respects operator masking). Packages the live fix from the 2026-06-24 incident.
|
||||||
|
|
||||||
|
-- Gerald KERMA <devel@cybermind.fr> Wed, 24 Jun 2026 15:10:00 +0000
|
||||||
|
|
||||||
secubox-aggregator (0.2.1-1~bookworm1) bookworm; urgency=medium
|
secubox-aggregator (0.2.1-1~bookworm1) bookworm; urgency=medium
|
||||||
|
|
||||||
* Phase 7 follow-up (#498) — relax hardening for module sudoers :
|
* Phase 7 follow-up (#498) — relax hardening for module sudoers :
|
||||||
|
|
|
||||||
|
|
@ -17,6 +17,12 @@ case "$1" in
|
||||||
systemctl enable secubox-aggregator.service
|
systemctl enable secubox-aggregator.service
|
||||||
systemctl start secubox-aggregator.service || true
|
systemctl start secubox-aggregator.service || true
|
||||||
|
|
||||||
|
# Auto-heal watchdog (#727): restart the aggregator if its socket wedges
|
||||||
|
# under load (the hub/auth/menu SPOF). Respect operator masking.
|
||||||
|
if [ "$(systemctl is-enabled secubox-aggregator-watchdog.timer 2>/dev/null)" != "masked" ]; then
|
||||||
|
systemctl enable --now secubox-aggregator-watchdog.timer 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
|
||||||
echo "secubox-aggregator: to migrate all installed SecuBox modules into"
|
echo "secubox-aggregator: to migrate all installed SecuBox modules into"
|
||||||
echo " the aggregator (replaces per-module uvicorn processes) run :"
|
echo " the aggregator (replaces per-module uvicorn processes) run :"
|
||||||
echo " sudo /usr/sbin/secubox-aggregator-migrate"
|
echo " sudo /usr/sbin/secubox-aggregator-migrate"
|
||||||
|
|
|
||||||
6
packages/secubox-aggregator/debian/rules
Normal file → Executable file
6
packages/secubox-aggregator/debian/rules
Normal file → Executable file
|
|
@ -14,6 +14,12 @@ override_dh_auto_install:
|
||||||
install -d $(CURDIR)/debian/secubox-aggregator/lib/systemd/system
|
install -d $(CURDIR)/debian/secubox-aggregator/lib/systemd/system
|
||||||
install -m 644 systemd/secubox-aggregator.service \
|
install -m 644 systemd/secubox-aggregator.service \
|
||||||
$(CURDIR)/debian/secubox-aggregator/lib/systemd/system/
|
$(CURDIR)/debian/secubox-aggregator/lib/systemd/system/
|
||||||
|
install -m 644 systemd/secubox-aggregator-watchdog.service \
|
||||||
|
$(CURDIR)/debian/secubox-aggregator/lib/systemd/system/
|
||||||
|
install -m 644 systemd/secubox-aggregator-watchdog.timer \
|
||||||
|
$(CURDIR)/debian/secubox-aggregator/lib/systemd/system/
|
||||||
install -d $(CURDIR)/debian/secubox-aggregator/usr/sbin
|
install -d $(CURDIR)/debian/secubox-aggregator/usr/sbin
|
||||||
install -m 755 sbin/secubox-aggregator-migrate \
|
install -m 755 sbin/secubox-aggregator-migrate \
|
||||||
$(CURDIR)/debian/secubox-aggregator/usr/sbin/
|
$(CURDIR)/debian/secubox-aggregator/usr/sbin/
|
||||||
|
install -m 755 sbin/secubox-aggregator-watchdog.sh \
|
||||||
|
$(CURDIR)/debian/secubox-aggregator/usr/sbin/
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,10 @@
|
||||||
|
|
||||||
|
# Automatically added by dh_python3
|
||||||
|
if command -v py3compile >/dev/null 2>&1; then
|
||||||
|
py3compile -p secubox-aggregator
|
||||||
|
fi
|
||||||
|
if command -v pypy3compile >/dev/null 2>&1; then
|
||||||
|
pypy3compile -p secubox-aggregator || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
# End automatically added section
|
||||||
|
|
@ -0,0 +1,10 @@
|
||||||
|
|
||||||
|
# Automatically added by dh_python3
|
||||||
|
if command -v py3clean >/dev/null 2>&1; then
|
||||||
|
py3clean -p secubox-aggregator
|
||||||
|
else
|
||||||
|
dpkg -L secubox-aggregator | sed -En -e '/^(.*)\/(.+)\.py$/s,,rm "\1/__pycache__/\2".*,e'
|
||||||
|
find /usr/lib/python3/dist-packages/ -type d -name __pycache__ -empty -print0 | xargs --null --no-run-if-empty rmdir
|
||||||
|
fi
|
||||||
|
|
||||||
|
# End automatically added section
|
||||||
48
packages/secubox-aggregator/sbin/secubox-aggregator-watchdog.sh
Executable file
48
packages/secubox-aggregator/sbin/secubox-aggregator-watchdog.sh
Executable file
|
|
@ -0,0 +1,48 @@
|
||||||
|
#!/bin/bash
|
||||||
|
# SPDX-License-Identifier: LicenseRef-CMSD-1.0
|
||||||
|
# Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
|
||||||
|
# Source-Disclosed License — All rights reserved except as expressly granted.
|
||||||
|
# See LICENCE-CMSD-1.0.md for terms.
|
||||||
|
#
|
||||||
|
# SecuBox-Deb :: secubox-aggregator-watchdog
|
||||||
|
#
|
||||||
|
# Auto-heal the in-process aggregator — the hub/auth/menu single point of
|
||||||
|
# failure. Under a host load spike its shared event loop can wedge and its
|
||||||
|
# socket stops answering, taking down the navbar, login and service status
|
||||||
|
# board-wide (incident 2026-06-24). Probe the socket; if /api/v1/hub/public/menu
|
||||||
|
# stops answering for N consecutive checks, restart the service. Idempotent,
|
||||||
|
# safe to run on a timer.
|
||||||
|
set -uo pipefail
|
||||||
|
readonly MODULE="secubox-aggregator-watchdog"
|
||||||
|
readonly VERSION="1.0"
|
||||||
|
|
||||||
|
SOCK="/run/secubox/aggregator.sock"
|
||||||
|
# State lives in /run (root-owned), NOT the shared sticky /run/secubox: that dir
|
||||||
|
# is 1777 and a stale secubox-owned file there can't be overwritten by this
|
||||||
|
# (CSPN-hardened, CAP_DAC_OVERRIDE-less) root — which would silently freeze the
|
||||||
|
# streak counter and stop the watchdog ever triggering.
|
||||||
|
STATE="/run/secubox-aggregator-watchdog.fails"
|
||||||
|
FAIL_THRESHOLD="${SECUBOX_AGG_WD_THRESHOLD:-2}"
|
||||||
|
TIMEOUT="${SECUBOX_AGG_WD_TIMEOUT:-12}"
|
||||||
|
|
||||||
|
# No socket yet (service still starting / not migrated) → nothing to heal.
|
||||||
|
[ -S "$SOCK" ] || exit 0
|
||||||
|
|
||||||
|
code=$(curl -s -o /dev/null -w "%{http_code}" --max-time "$TIMEOUT" \
|
||||||
|
--unix-socket "$SOCK" http://localhost/api/v1/hub/public/menu 2>/dev/null || echo 000)
|
||||||
|
|
||||||
|
if [ "$code" = "200" ]; then
|
||||||
|
echo 0 > "$STATE" 2>/dev/null || true
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
n=$(( $(cat "$STATE" 2>/dev/null || echo 0) + 1 ))
|
||||||
|
echo "$n" > "$STATE" 2>/dev/null || true
|
||||||
|
logger -t "$MODULE" "aggregator probe failed (code=$code, streak=$n/$FAIL_THRESHOLD)"
|
||||||
|
|
||||||
|
if [ "$n" -ge "$FAIL_THRESHOLD" ]; then
|
||||||
|
logger -t "$MODULE" "restarting secubox-aggregator (auto-heal)"
|
||||||
|
systemctl restart secubox-aggregator.service
|
||||||
|
echo 0 > "$STATE" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
exit 0
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
[Unit]
|
||||||
|
Description=SecuBox aggregator auto-heal watchdog
|
||||||
|
Documentation=https://github.com/CyberMind-FR/secubox-deb/issues/727
|
||||||
|
After=secubox-aggregator.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
ExecStart=/usr/sbin/secubox-aggregator-watchdog.sh
|
||||||
|
Nice=10
|
||||||
|
|
@ -0,0 +1,10 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Probe + auto-heal secubox-aggregator every 2 min
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnBootSec=2min
|
||||||
|
OnUnitActiveSec=2min
|
||||||
|
AccuracySec=20s
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
|
|
@ -268,27 +268,20 @@ async def _monitor_streams():
|
||||||
try:
|
try:
|
||||||
settings = _load_settings()
|
settings = _load_settings()
|
||||||
if settings.get("detection_enabled", True):
|
if settings.get("detection_enabled", True):
|
||||||
flows = await _dpi("/flows")
|
ex = await _dpi("/exfil")
|
||||||
media_stats: Dict[str, Dict[str, Any]] = {}
|
media_stats: Dict[str, Dict[str, Any]] = {}
|
||||||
|
|
||||||
for f in flows.get("flows", []):
|
for f in _exfil_media_flows(ex):
|
||||||
app_name = f.get("app_name", "Unknown")
|
name = f.get("service") or f.get("dst") or "Unknown"
|
||||||
if app_name in MEDIA_APPS:
|
if name not in media_stats:
|
||||||
if app_name not in media_stats:
|
media_stats[name] = {"name": name, "flows": 0, "bytes": 0}
|
||||||
media_stats[app_name] = {"name": app_name, "flows": 0, "bytes": 0}
|
media_stats[name]["flows"] += int(f.get("flows", 1) or 1)
|
||||||
media_stats[app_name]["flows"] += 1
|
media_stats[name]["bytes"] += int(f.get("up_bytes", 0) or 0) + int(f.get("down_bytes", 0) or 0)
|
||||||
media_stats[app_name]["bytes"] += f.get("bytes", 0)
|
|
||||||
|
|
||||||
# Check for new services
|
# Check for new services
|
||||||
if settings.get("alert_on_new_service") and app_name not in seen_services:
|
if settings.get("alert_on_new_service") and name not in seen_services:
|
||||||
seen_services.add(app_name)
|
seen_services.add(name)
|
||||||
await _notify_webhooks("new_service", {
|
await _notify_webhooks("new_service", {"service": name, "category": "media"})
|
||||||
"service": app_name,
|
|
||||||
"category": next(
|
|
||||||
(cat for cat, apps in STREAMING_CATEGORIES.items() if app_name in apps),
|
|
||||||
"other"
|
|
||||||
)
|
|
||||||
})
|
|
||||||
|
|
||||||
# Check alerts
|
# Check alerts
|
||||||
await _check_alerts(media_stats)
|
await _check_alerts(media_stats)
|
||||||
|
|
@ -323,16 +316,40 @@ async def health():
|
||||||
return {"status": "ok", "module": "mediaflow", "version": "2.0.0"}
|
return {"status": "ok", "module": "mediaflow", "version": "2.0.0"}
|
||||||
|
|
||||||
|
|
||||||
|
# The DPI engine now exposes a public, category-tagged exfil view (the netifyd
|
||||||
|
# /flows path is dead). Media = the exfil classifier's "media" category.
|
||||||
|
MEDIA_CATEGORIES = {"media"}
|
||||||
|
|
||||||
|
|
||||||
|
def _exfil_media_flows(exfil: Dict[str, Any]):
|
||||||
|
"""Flatten exfil devices->services + active_flows to category=='media' rows."""
|
||||||
|
rows = []
|
||||||
|
for dev in exfil.get("devices", []) or []:
|
||||||
|
for s in dev.get("services", []) or []:
|
||||||
|
if s.get("category") in MEDIA_CATEGORIES:
|
||||||
|
rows.append(s)
|
||||||
|
for f in exfil.get("active_flows", []) or []:
|
||||||
|
if f.get("category") in MEDIA_CATEGORIES:
|
||||||
|
rows.append(f)
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
@router.get("/status")
|
@router.get("/status")
|
||||||
async def status(user=Depends(require_jwt)):
|
async def status(user=Depends(require_jwt)):
|
||||||
try:
|
try:
|
||||||
s = await _dpi("/status")
|
ex = await _dpi("/exfil")
|
||||||
settings = _load_settings()
|
settings = _load_settings()
|
||||||
|
media = _exfil_media_flows(ex)
|
||||||
return {
|
return {
|
||||||
**s,
|
"running": bool(ex) and "error" not in ex,
|
||||||
|
"source": "dpi-exfil",
|
||||||
|
"devices": len(ex.get("devices", []) or []),
|
||||||
|
"active_flows": len(ex.get("active_flows", []) or []),
|
||||||
|
"media_flows": len(media),
|
||||||
"media_detection": settings.get("detection_enabled", True),
|
"media_detection": settings.get("detection_enabled", True),
|
||||||
"monitored_apps": len(MEDIA_APPS),
|
"monitored_categories": sorted(MEDIA_CATEGORIES),
|
||||||
"timestamp": datetime.now().isoformat()
|
"generated_at": ex.get("generated_at"),
|
||||||
|
"timestamp": datetime.now().isoformat(),
|
||||||
}
|
}
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
return {"running": False, "error": str(e)}
|
return {"running": False, "error": str(e)}
|
||||||
|
|
@ -340,42 +357,28 @@ async def status(user=Depends(require_jwt)):
|
||||||
|
|
||||||
@router.get("/services")
|
@router.get("/services")
|
||||||
async def services(user=Depends(require_jwt)):
|
async def services(user=Depends(require_jwt)):
|
||||||
"""Get active media services with statistics."""
|
"""Active media services (from the DPI exfil view), grouped by service/host."""
|
||||||
cached = stats_cache.get("services")
|
cached = stats_cache.get("services")
|
||||||
if cached:
|
if cached:
|
||||||
return cached
|
return cached
|
||||||
|
|
||||||
try:
|
try:
|
||||||
flows = await _dpi("/flows")
|
ex = await _dpi("/exfil")
|
||||||
media: Dict[str, Dict[str, Any]] = {}
|
media: Dict[str, Dict[str, Any]] = {}
|
||||||
|
for f in _exfil_media_flows(ex):
|
||||||
for f in flows.get("flows", []):
|
name = f.get("service") or f.get("dst") or "Unknown"
|
||||||
app_name = f.get("app_name", "Unknown")
|
if name not in media:
|
||||||
if app_name in MEDIA_APPS:
|
media[name] = {"name": name, "category": "media",
|
||||||
if app_name not in media:
|
"host": f.get("dst"), "cloud": f.get("cloud"),
|
||||||
category = next(
|
"flows": 0, "bytes": 0, "clients": set()}
|
||||||
(cat for cat, apps in STREAMING_CATEGORIES.items() if app_name in apps),
|
media[name]["flows"] += int(f.get("flows", 1) or 1)
|
||||||
"other"
|
media[name]["bytes"] += int(f.get("up_bytes", 0) or 0) + int(f.get("down_bytes", 0) or 0)
|
||||||
)
|
if f.get("device"):
|
||||||
media[app_name] = {
|
media[name]["clients"].add(f.get("device"))
|
||||||
"name": app_name,
|
|
||||||
"category": category,
|
|
||||||
"flows": 0,
|
|
||||||
"bytes": 0,
|
|
||||||
"clients": set()
|
|
||||||
}
|
|
||||||
media[app_name]["flows"] += 1
|
|
||||||
media[app_name]["bytes"] += f.get("bytes", 0)
|
|
||||||
if f.get("src_ip"):
|
|
||||||
media[app_name]["clients"].add(f.get("src_ip"))
|
|
||||||
|
|
||||||
# Convert sets to counts
|
|
||||||
result = []
|
result = []
|
||||||
for name, data in media.items():
|
for data in media.values():
|
||||||
data["clients"] = len(data["clients"])
|
data["clients"] = len(data["clients"])
|
||||||
data["bytes_human"] = _format_bytes(data["bytes"])
|
data["bytes_human"] = _format_bytes(data["bytes"])
|
||||||
result.append(data)
|
result.append(data)
|
||||||
|
|
||||||
result.sort(key=lambda x: x["bytes"], reverse=True)
|
result.sort(key=lambda x: x["bytes"], reverse=True)
|
||||||
stats_cache.set("services", result)
|
stats_cache.set("services", result)
|
||||||
return result
|
return result
|
||||||
|
|
@ -413,24 +416,38 @@ async def services_by_category(user=Depends(require_jwt)):
|
||||||
|
|
||||||
@router.get("/clients")
|
@router.get("/clients")
|
||||||
async def clients(user=Depends(require_jwt)):
|
async def clients(user=Depends(require_jwt)):
|
||||||
"""Get clients using media services."""
|
"""Get clients (devices) seen by the DPI exfil view, with totals."""
|
||||||
try:
|
try:
|
||||||
devices = await _dpi("/devices")
|
ex = await _dpi("/exfil")
|
||||||
return devices if isinstance(devices, list) else []
|
out = []
|
||||||
|
for d in ex.get("devices", []) or []:
|
||||||
|
tot = int(d.get("up_bytes", 0) or 0) + int(d.get("down_bytes", 0) or 0)
|
||||||
|
out.append({
|
||||||
|
"device": d.get("device"),
|
||||||
|
"flows": d.get("flows", 0),
|
||||||
|
"up_bytes": d.get("up_bytes", 0),
|
||||||
|
"down_bytes": d.get("down_bytes", 0),
|
||||||
|
"bytes": tot,
|
||||||
|
"bytes_human": _format_bytes(tot),
|
||||||
|
"media_flows": sum(1 for s in (d.get("services") or [])
|
||||||
|
if s.get("category") in MEDIA_CATEGORIES),
|
||||||
|
})
|
||||||
|
out.sort(key=lambda x: x["bytes"], reverse=True)
|
||||||
|
return out
|
||||||
except Exception:
|
except Exception:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
@router.get("/get_active_streams")
|
@router.get("/get_active_streams")
|
||||||
async def get_active_streams(user=Depends(require_jwt)):
|
async def get_active_streams(user=Depends(require_jwt)):
|
||||||
"""Get active media streams."""
|
"""Active media streams (category=='media') from the DPI exfil view."""
|
||||||
try:
|
try:
|
||||||
flows = await _dpi("/flows")
|
ex = await _dpi("/exfil")
|
||||||
streams = []
|
streams = []
|
||||||
for f in flows.get("flows", []):
|
for f in _exfil_media_flows(ex):
|
||||||
if f.get("app_name") in MEDIA_APPS:
|
b = int(f.get("up_bytes", 0) or 0) + int(f.get("down_bytes", 0) or 0)
|
||||||
f["bytes_human"] = _format_bytes(f.get("bytes", 0))
|
streams.append({**f, "bytes": b, "bytes_human": _format_bytes(b)})
|
||||||
streams.append(f)
|
streams.sort(key=lambda x: x["bytes"], reverse=True)
|
||||||
return streams
|
return streams
|
||||||
except Exception:
|
except Exception:
|
||||||
return []
|
return []
|
||||||
|
|
@ -440,23 +457,20 @@ async def get_active_streams(user=Depends(require_jwt)):
|
||||||
async def get_service_details(service: str, user=Depends(require_jwt)):
|
async def get_service_details(service: str, user=Depends(require_jwt)):
|
||||||
"""Get detailed info for a specific media service."""
|
"""Get detailed info for a specific media service."""
|
||||||
try:
|
try:
|
||||||
flows = await _dpi("/flows")
|
ex = await _dpi("/exfil")
|
||||||
service_flows = [f for f in flows.get("flows", []) if f.get("app_name") == service]
|
service_flows = [f for f in _exfil_media_flows(ex)
|
||||||
|
if (f.get("service") or f.get("dst")) == service]
|
||||||
total_bytes = sum(f.get("bytes", 0) for f in service_flows)
|
total_bytes = sum(int(f.get("up_bytes", 0) or 0) + int(f.get("down_bytes", 0) or 0)
|
||||||
clients = set(f.get("src_ip") for f in service_flows if f.get("src_ip"))
|
for f in service_flows)
|
||||||
|
clients = set(f.get("device") for f in service_flows if f.get("device"))
|
||||||
return {
|
return {
|
||||||
"service": service,
|
"service": service,
|
||||||
"category": next(
|
"category": "media",
|
||||||
(cat for cat, apps in STREAMING_CATEGORIES.items() if service in apps),
|
|
||||||
"other"
|
|
||||||
),
|
|
||||||
"active_flows": len(service_flows),
|
"active_flows": len(service_flows),
|
||||||
"total_bytes": total_bytes,
|
"total_bytes": total_bytes,
|
||||||
"total_bytes_human": _format_bytes(total_bytes),
|
"total_bytes_human": _format_bytes(total_bytes),
|
||||||
"unique_clients": len(clients),
|
"unique_clients": len(clients),
|
||||||
"flows": service_flows[:50] # Limit to 50 flows
|
"flows": service_flows[:50],
|
||||||
}
|
}
|
||||||
except Exception:
|
except Exception:
|
||||||
return {"service": service, "error": "Failed to fetch details"}
|
return {"service": service, "error": "Failed to fetch details"}
|
||||||
|
|
@ -592,8 +606,8 @@ async def stop_ndpid(user=Depends(require_jwt)):
|
||||||
async def summary(user=Depends(require_jwt)):
|
async def summary(user=Depends(require_jwt)):
|
||||||
"""Get mediaflow summary."""
|
"""Get mediaflow summary."""
|
||||||
try:
|
try:
|
||||||
dpi_status = await _dpi("/status")
|
ex = await _dpi("/exfil")
|
||||||
dpi_running = dpi_status.get("running", False)
|
dpi_running = bool(ex) and "error" not in ex
|
||||||
except Exception:
|
except Exception:
|
||||||
dpi_running = False
|
dpi_running = False
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,13 @@
|
||||||
|
secubox-mediaflow (2.0.1-1~bookworm1) bookworm; urgency=medium
|
||||||
|
|
||||||
|
* Fix: consume the DPI engine's public /exfil view (category-tagged) instead of
|
||||||
|
the dead netifyd /flows + auth-gated /status. mediaflow's internal DPI calls
|
||||||
|
were unauthenticated -> 401 -> empty/error dashboard. Now /status /services
|
||||||
|
/clients /get_active_streams /get_service_details /summary + the monitor task
|
||||||
|
read /exfil and filter category=="media". Dashboard works again.
|
||||||
|
|
||||||
|
-- Gerald KERMA <devel@cybermind.fr> Wed, 24 Jun 2026 17:00:00 +0000
|
||||||
|
|
||||||
secubox-mediaflow (1.0.4-1~bookworm1) bookworm; urgency=medium
|
secubox-mediaflow (1.0.4-1~bookworm1) bookworm; urgency=medium
|
||||||
|
|
||||||
* Add dynamic menu system with menu.d JSON definitions
|
* Add dynamic menu system with menu.d JSON definitions
|
||||||
|
|
|
||||||
58
packages/secubox-podcaster/README.md
Normal file
58
packages/secubox-podcaster/README.md
Normal file
|
|
@ -0,0 +1,58 @@
|
||||||
|
# secubox-podcaster
|
||||||
|
|
||||||
|
Modern podcast manager for SecuBox — subscribe, download locally, and relay a
|
||||||
|
shareable RSS feed.
|
||||||
|
|
||||||
|
## What it does (v1, #726)
|
||||||
|
|
||||||
|
- **Subscribe** by RSS URL or **OPML import** (pure-stdlib feed parsing — no
|
||||||
|
`feedparser` dependency).
|
||||||
|
- **Download locally** into `media_path` via an asyncio + httpx queue with
|
||||||
|
per-episode progress.
|
||||||
|
- **Relay / share**: a generated RSS of the local library at
|
||||||
|
`/api/v1/podcaster/share/feed.xml`. LAN by default; set `public_base` to your
|
||||||
|
exposed vhost to publish externally.
|
||||||
|
- **In-UI service status + TOML config**; modern C3BOX WebUI with inline player.
|
||||||
|
|
||||||
|
Lyrion integration is deferred to a follow-up (standalone first).
|
||||||
|
|
||||||
|
## Layout
|
||||||
|
|
||||||
|
| Path | Role |
|
||||||
|
|------|------|
|
||||||
|
| `/usr/share/secubox/podcaster/api` | FastAPI app (uvicorn WorkingDirectory) |
|
||||||
|
| `/run/secubox/podcaster.sock` | Unix socket |
|
||||||
|
| `/var/lib/secubox/podcaster/podcaster.db` | SQLite store |
|
||||||
|
| `/var/lib/secubox/podcaster/media/<feed_id>/` | downloaded episodes |
|
||||||
|
| `/etc/secubox/podcaster.toml` | config |
|
||||||
|
| `/etc/nginx/secubox-routes.d/podcaster.conf` | nginx route (active include) |
|
||||||
|
|
||||||
|
## API
|
||||||
|
|
||||||
|
| Method | Path | Auth | Description |
|
||||||
|
|--------|------|------|-------------|
|
||||||
|
| GET | `/health`, `/status` | public | service + counts |
|
||||||
|
| GET | `/share/feed.xml` | public | shareable RSS of local library |
|
||||||
|
| GET | `/media/{id}` | public | stream a downloaded episode |
|
||||||
|
| GET/POST/DELETE | `/feeds*` | JWT | manage feeds (+ `/feeds/import-opml`) |
|
||||||
|
| GET | `/episodes` | JWT | list (optional `feed_id`, `state`) |
|
||||||
|
| POST | `/episodes/{id}/download` | JWT | enqueue download |
|
||||||
|
| GET/POST | `/config` | JWT | TOML config |
|
||||||
|
|
||||||
|
## Public exposure (relay to the web)
|
||||||
|
|
||||||
|
Publish the share feed externally via HAProxy TLS → mitmproxy (never bypass the
|
||||||
|
WAF):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
haproxyctl vhost add podcast.gk2.secubox.in # backend = mitmproxy_inspector
|
||||||
|
# add the route to BOTH mitmproxy routes files -> 127.0.0.1:<nginx>
|
||||||
|
systemctl restart mitmproxy
|
||||||
|
```
|
||||||
|
|
||||||
|
Then set `public_base = "https://podcast.gk2.secubox.in"` in
|
||||||
|
`/etc/secubox/podcaster.toml` and restart `secubox-podcaster` so the generated
|
||||||
|
feed's enclosure URLs are absolute.
|
||||||
|
|
||||||
|
---
|
||||||
|
*CyberMind — Gérald Kerma. LicenseRef-CMSD-1.0.*
|
||||||
0
packages/secubox-podcaster/api/__init__.py
Normal file
0
packages/secubox-podcaster/api/__init__.py
Normal file
547
packages/secubox-podcaster/api/main.py
Normal file
547
packages/secubox-podcaster/api/main.py
Normal file
|
|
@ -0,0 +1,547 @@
|
||||||
|
# SPDX-License-Identifier: LicenseRef-CMSD-1.0
|
||||||
|
# Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
|
||||||
|
# Source-Disclosed License — All rights reserved except as expressly granted.
|
||||||
|
# See LICENCE-CMSD-1.0.md for terms.
|
||||||
|
|
||||||
|
"""
|
||||||
|
SecuBox-Deb :: Podcaster
|
||||||
|
CyberMind — https://cybermind.fr
|
||||||
|
|
||||||
|
Modern podcast manager: subscribe to feeds, download episodes locally, and
|
||||||
|
re-publish a shareable RSS. FastAPI on a Unix socket; SQLite store; an asyncio
|
||||||
|
download worker (background task, fire-and-forget queue). Feed parsing is pure
|
||||||
|
stdlib (xml.etree) to keep Debian deps minimal — only httpx is required.
|
||||||
|
"""
|
||||||
|
import asyncio
|
||||||
|
import html
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
import tomllib
|
||||||
|
import zipfile
|
||||||
|
from email.utils import parsedate_to_datetime, format_datetime
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Optional
|
||||||
|
from xml.etree import ElementTree as ET
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
from fastapi import FastAPI, APIRouter, BackgroundTasks, Depends, HTTPException, Request
|
||||||
|
from fastapi.responses import Response, FileResponse, JSONResponse
|
||||||
|
from pydantic import BaseModel
|
||||||
|
|
||||||
|
from secubox_core.auth import router as auth_router, require_jwt
|
||||||
|
from secubox_core.logger import get_logger
|
||||||
|
|
||||||
|
from . import store
|
||||||
|
|
||||||
|
log = get_logger("podcaster")
|
||||||
|
|
||||||
|
CONFIG_FILE = Path("/etc/secubox/podcaster.toml")
|
||||||
|
DEFAULT_CONFIG = {
|
||||||
|
"media_path": "/var/lib/secubox/podcaster/media",
|
||||||
|
"max_parallel": 2,
|
||||||
|
"refresh_minutes": 60,
|
||||||
|
"public_base": "", # e.g. https://podcast.gk2.secubox.in (exposure); empty → LAN/relative
|
||||||
|
"share_title": "SecuBox Podcaster",
|
||||||
|
"keep_per_feed": 0, # 0 = unlimited
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def load_config() -> dict:
|
||||||
|
cfg = dict(DEFAULT_CONFIG)
|
||||||
|
try:
|
||||||
|
if CONFIG_FILE.exists():
|
||||||
|
cfg.update(tomllib.loads(CONFIG_FILE.read_text()))
|
||||||
|
except Exception as e: # pragma: no cover - defensive
|
||||||
|
log.error(f"config load failed: {e}")
|
||||||
|
return cfg
|
||||||
|
|
||||||
|
|
||||||
|
CFG = load_config()
|
||||||
|
MEDIA = Path(CFG["media_path"])
|
||||||
|
|
||||||
|
app = FastAPI(title="secubox-podcaster", version="1.0.0", root_path="/api/v1/podcaster")
|
||||||
|
app.include_router(auth_router, prefix="/auth")
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
# ── download queue (in-process, persisted state in SQLite) ──────────
|
||||||
|
_queue: "asyncio.Queue[int]" = asyncio.Queue()
|
||||||
|
_worker_started = False
|
||||||
|
|
||||||
|
|
||||||
|
# ════════════════════════════════════════════════════════════════════
|
||||||
|
# Models
|
||||||
|
# ════════════════════════════════════════════════════════════════════
|
||||||
|
class FeedIn(BaseModel):
|
||||||
|
url: str
|
||||||
|
auto_dl: bool = True
|
||||||
|
|
||||||
|
|
||||||
|
class OPMLIn(BaseModel):
|
||||||
|
opml: str
|
||||||
|
|
||||||
|
|
||||||
|
class ConfigIn(BaseModel):
|
||||||
|
media_path: Optional[str] = None
|
||||||
|
max_parallel: Optional[int] = None
|
||||||
|
refresh_minutes: Optional[int] = None
|
||||||
|
public_base: Optional[str] = None
|
||||||
|
share_title: Optional[str] = None
|
||||||
|
keep_per_feed: Optional[int] = None
|
||||||
|
|
||||||
|
|
||||||
|
# ════════════════════════════════════════════════════════════════════
|
||||||
|
# Feed parsing (stdlib)
|
||||||
|
# ════════════════════════════════════════════════════════════════════
|
||||||
|
_NS = {"itunes": "http://www.itunes.com/dtds/podcast-1.0.dtd",
|
||||||
|
"atom": "http://www.w3.org/2005/Atom"}
|
||||||
|
|
||||||
|
|
||||||
|
def _ts(s: Optional[str]) -> int:
|
||||||
|
if not s:
|
||||||
|
return 0
|
||||||
|
try:
|
||||||
|
return int(parsedate_to_datetime(s).timestamp())
|
||||||
|
except Exception:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def parse_feed(xml_bytes: bytes) -> tuple[dict, list[dict]]:
|
||||||
|
"""Return (feed_meta, [episode, ...]) from RSS 2.0 bytes."""
|
||||||
|
root = ET.fromstring(xml_bytes)
|
||||||
|
chan = root.find("channel")
|
||||||
|
if chan is None: # not RSS we understand
|
||||||
|
return {}, []
|
||||||
|
|
||||||
|
def t(el, tag):
|
||||||
|
x = el.find(tag)
|
||||||
|
return x.text.strip() if x is not None and x.text else None
|
||||||
|
|
||||||
|
img = None
|
||||||
|
ic = chan.find("itunes:image", _NS)
|
||||||
|
if ic is not None:
|
||||||
|
img = ic.get("href")
|
||||||
|
if not img:
|
||||||
|
im = chan.find("image/url")
|
||||||
|
img = im.text.strip() if im is not None and im.text else None
|
||||||
|
|
||||||
|
meta = {
|
||||||
|
"title": t(chan, "title"),
|
||||||
|
"description": t(chan, "description"),
|
||||||
|
"site": t(chan, "link"),
|
||||||
|
"image": img,
|
||||||
|
}
|
||||||
|
|
||||||
|
episodes = []
|
||||||
|
for it in chan.findall("item"):
|
||||||
|
enc = it.find("enclosure")
|
||||||
|
if enc is None or not enc.get("url"):
|
||||||
|
continue
|
||||||
|
guid = t(it, "guid") or enc.get("url")
|
||||||
|
dur = it.find("itunes:duration", _NS)
|
||||||
|
episodes.append({
|
||||||
|
"guid": guid,
|
||||||
|
"title": t(it, "title"),
|
||||||
|
"description": t(it, "description"),
|
||||||
|
"pubdate": _ts(t(it, "pubDate")),
|
||||||
|
"enclosure": enc.get("url"),
|
||||||
|
"mime": enc.get("type") or "audio/mpeg",
|
||||||
|
"bytes": int(enc.get("length") or 0),
|
||||||
|
"duration": dur.text.strip() if dur is not None and dur.text else None,
|
||||||
|
})
|
||||||
|
return meta, episodes
|
||||||
|
|
||||||
|
|
||||||
|
async def fetch_and_store(url: str, auto_dl: Optional[bool] = None) -> dict:
|
||||||
|
async with httpx.AsyncClient(follow_redirects=True, timeout=30) as cli:
|
||||||
|
r = await cli.get(url, headers={"User-Agent": "SecuBox-Podcaster/1.0"})
|
||||||
|
r.raise_for_status()
|
||||||
|
meta, eps = parse_feed(r.content)
|
||||||
|
fid = store.add_feed(url, meta, auto_dl=1 if auto_dl else 0)
|
||||||
|
store.update_feed_meta(fid, meta)
|
||||||
|
if auto_dl is not None: # explicit add/toggle; None = refresh (keep current)
|
||||||
|
store.set_feed_autodl(fid, 1 if auto_dl else 0)
|
||||||
|
for ep in eps:
|
||||||
|
store.upsert_episode(fid, ep)
|
||||||
|
queued = await _autoqueue(fid)
|
||||||
|
return {"feed_id": fid, "title": meta.get("title"),
|
||||||
|
"episodes": len(eps), "queued": queued}
|
||||||
|
|
||||||
|
|
||||||
|
async def _autoqueue(fid: int) -> int:
|
||||||
|
"""If a feed has auto_dl, enqueue its not-yet-downloaded episodes (newest
|
||||||
|
first, capped by keep_per_feed to avoid unbounded disk use)."""
|
||||||
|
if not store.feed_autodl(fid):
|
||||||
|
return 0
|
||||||
|
cap = int(CFG.get("keep_per_feed", 0) or 0)
|
||||||
|
n = 0
|
||||||
|
for ep_id in store.pending_episode_ids(fid, cap):
|
||||||
|
store.set_episode(ep_id, state="queued", progress=0, error=None)
|
||||||
|
await _queue.put(ep_id)
|
||||||
|
n += 1
|
||||||
|
return n
|
||||||
|
|
||||||
|
|
||||||
|
# ════════════════════════════════════════════════════════════════════
|
||||||
|
# Download worker
|
||||||
|
# ════════════════════════════════════════════════════════════════════
|
||||||
|
def _safe_name(s: str, ext: str) -> str:
|
||||||
|
base = re.sub(r"[^A-Za-z0-9._-]+", "_", (s or "episode"))[:80].strip("_")
|
||||||
|
return f"{base or 'episode'}{ext}"
|
||||||
|
|
||||||
|
|
||||||
|
async def _download_one(ep_id: int) -> None:
|
||||||
|
ep = store.get_episode(ep_id)
|
||||||
|
if not ep or not ep.get("enclosure"):
|
||||||
|
return
|
||||||
|
store.set_episode(ep_id, state="downloading", progress=0, error=None)
|
||||||
|
ext = os.path.splitext(ep["enclosure"].split("?")[0])[1][:5] or ".mp3"
|
||||||
|
fdir = MEDIA / str(ep["feed_id"])
|
||||||
|
fdir.mkdir(parents=True, exist_ok=True)
|
||||||
|
dest = fdir / _safe_name(f"{ep_id}_{ep.get('title','')}", ext)
|
||||||
|
tmp = dest.with_suffix(dest.suffix + ".part")
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(follow_redirects=True, timeout=None) as cli:
|
||||||
|
async with cli.stream("GET", ep["enclosure"],
|
||||||
|
headers={"User-Agent": "SecuBox-Podcaster/1.0"}) as r:
|
||||||
|
r.raise_for_status()
|
||||||
|
total = int(r.headers.get("content-length") or ep.get("bytes") or 0)
|
||||||
|
got = 0
|
||||||
|
with open(tmp, "wb") as fh:
|
||||||
|
async for chunk in r.aiter_bytes(65536):
|
||||||
|
fh.write(chunk)
|
||||||
|
got += len(chunk)
|
||||||
|
if total:
|
||||||
|
store.set_episode(ep_id, progress=min(99, got * 100 // total))
|
||||||
|
tmp.rename(dest)
|
||||||
|
store.set_episode(ep_id, state="done", progress=100,
|
||||||
|
local_path=str(dest), bytes=dest.stat().st_size)
|
||||||
|
log.info(f"downloaded ep {ep_id} -> {dest}")
|
||||||
|
except Exception as e:
|
||||||
|
tmp.unlink(missing_ok=True)
|
||||||
|
store.set_episode(ep_id, state="error", error=str(e)[:200])
|
||||||
|
log.error(f"download ep {ep_id} failed: {e}")
|
||||||
|
|
||||||
|
|
||||||
|
async def _worker() -> None:
|
||||||
|
sem = asyncio.Semaphore(max(1, int(CFG.get("max_parallel", 2))))
|
||||||
|
|
||||||
|
async def run(ep_id):
|
||||||
|
async with sem:
|
||||||
|
await _download_one(ep_id)
|
||||||
|
|
||||||
|
while True:
|
||||||
|
ep_id = await _queue.get()
|
||||||
|
asyncio.create_task(run(ep_id))
|
||||||
|
|
||||||
|
|
||||||
|
async def _refresher() -> None:
|
||||||
|
while True:
|
||||||
|
await asyncio.sleep(max(5, int(CFG.get("refresh_minutes", 60))) * 60)
|
||||||
|
for fid, url in store.all_feed_urls():
|
||||||
|
try:
|
||||||
|
await fetch_and_store(url)
|
||||||
|
except Exception as e:
|
||||||
|
log.error(f"refresh feed {fid} failed: {e}")
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_worker() -> None:
|
||||||
|
"""Lazy-start background tasks (sub-app lifespans don't fire under the aggregator)."""
|
||||||
|
global _worker_started
|
||||||
|
if _worker_started:
|
||||||
|
return
|
||||||
|
_worker_started = True
|
||||||
|
store.init()
|
||||||
|
MEDIA.mkdir(parents=True, exist_ok=True)
|
||||||
|
asyncio.create_task(_worker())
|
||||||
|
asyncio.create_task(_refresher())
|
||||||
|
|
||||||
|
|
||||||
|
@app.on_event("startup")
|
||||||
|
async def _startup():
|
||||||
|
_ensure_worker()
|
||||||
|
|
||||||
|
|
||||||
|
# ════════════════════════════════════════════════════════════════════
|
||||||
|
# Public endpoints (status / health / share)
|
||||||
|
# ════════════════════════════════════════════════════════════════════
|
||||||
|
@app.get("/health")
|
||||||
|
async def health():
|
||||||
|
return {"status": "ok", "module": "deb"}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/status")
|
||||||
|
async def status():
|
||||||
|
_ensure_worker()
|
||||||
|
return {
|
||||||
|
"service": "active",
|
||||||
|
"worker": _worker_started,
|
||||||
|
"media_path": str(MEDIA),
|
||||||
|
"public_base": CFG.get("public_base", ""),
|
||||||
|
**store.counts(),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _xml_esc(s: Optional[str]) -> str:
|
||||||
|
return html.escape(s or "", quote=True)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/share/feed.xml")
|
||||||
|
async def share_feed(request: Request):
|
||||||
|
"""A generated RSS of the locally downloaded episodes (the relay/share feed)."""
|
||||||
|
_ensure_worker()
|
||||||
|
base = (CFG.get("public_base") or "").rstrip("/")
|
||||||
|
if not base:
|
||||||
|
base = str(request.base_url).rstrip("/")
|
||||||
|
eps = store.downloaded_episodes()
|
||||||
|
items = []
|
||||||
|
for e in eps:
|
||||||
|
url = f"{base}/api/v1/podcaster/media/{e['id']}"
|
||||||
|
pub = format_datetime(datetime.fromtimestamp(e["pubdate"] or time.time(),
|
||||||
|
tz=timezone.utc))
|
||||||
|
items.append(
|
||||||
|
f"<item><title>{_xml_esc(e.get('title'))}</title>"
|
||||||
|
f"<description>{_xml_esc(e.get('description'))}</description>"
|
||||||
|
f"<pubDate>{pub}</pubDate>"
|
||||||
|
f"<guid isPermaLink=\"false\">sbx-{e['id']}</guid>"
|
||||||
|
f"<enclosure url=\"{_xml_esc(url)}\" type=\"{_xml_esc(e.get('mime') or 'audio/mpeg')}\" "
|
||||||
|
f"length=\"{e.get('bytes') or 0}\"/></item>"
|
||||||
|
)
|
||||||
|
title = _xml_esc(CFG.get("share_title", "SecuBox Podcaster"))
|
||||||
|
rss = (
|
||||||
|
'<?xml version="1.0" encoding="UTF-8"?>'
|
||||||
|
'<rss version="2.0"><channel>'
|
||||||
|
f"<title>{title}</title><link>{_xml_esc(base)}</link>"
|
||||||
|
f"<description>Locally relayed by SecuBox Podcaster</description>"
|
||||||
|
+ "".join(items) + "</channel></rss>"
|
||||||
|
)
|
||||||
|
return Response(content=rss, media_type="application/rss+xml")
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/public/library")
|
||||||
|
async def public_library():
|
||||||
|
"""Public listener library — downloaded/shared episodes only (no auth).
|
||||||
|
Feeds the external portal frontend; exposes nothing un-shared."""
|
||||||
|
_ensure_worker()
|
||||||
|
eps = store.downloaded_episodes()
|
||||||
|
feeds: dict = {}
|
||||||
|
for e in eps:
|
||||||
|
k = e.get("feed_title") or "Podcast"
|
||||||
|
feeds[k] = feeds.get(k, 0) + 1
|
||||||
|
return {
|
||||||
|
"title": CFG.get("share_title", "SecuBox Podcaster"),
|
||||||
|
"episodes": [{
|
||||||
|
"id": e["id"], "title": e.get("title"), "feed": e.get("feed_title"),
|
||||||
|
"feed_id": e.get("feed_id"),
|
||||||
|
"pubdate": e.get("pubdate"), "duration": e.get("duration"),
|
||||||
|
"mime": e.get("mime"), "bytes": e.get("bytes"),
|
||||||
|
"media": f"/api/v1/podcaster/media/{e['id']}",
|
||||||
|
} for e in eps],
|
||||||
|
"feeds": feeds,
|
||||||
|
"share": "/api/v1/podcaster/share/feed.xml",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/public/feed/{fid}/zip")
|
||||||
|
async def public_feed_zip(fid: int, background: BackgroundTasks):
|
||||||
|
"""Public: download all downloaded episodes of a feed as one ZIP.
|
||||||
|
mp3/audio are already compressed → STORED (no recompress). Built to a temp
|
||||||
|
file then streamed; cleaned up after send."""
|
||||||
|
_ensure_worker()
|
||||||
|
eps = [e for e in store.downloaded_episodes(limit=2000)
|
||||||
|
if e.get("feed_id") == fid and e.get("local_path")
|
||||||
|
and Path(e["local_path"]).exists()]
|
||||||
|
if not eps:
|
||||||
|
raise HTTPException(404, "no downloaded episodes for this feed")
|
||||||
|
name = re.sub(r"[^A-Za-z0-9._-]+", "_", (eps[0].get("feed_title") or f"feed{fid}"))[:60] or f"feed{fid}"
|
||||||
|
tmp = Path(tempfile.mkstemp(suffix=".zip", dir="/var/lib/secubox/podcaster")[1])
|
||||||
|
eps.sort(key=lambda e: e.get("pubdate") or 0)
|
||||||
|
with zipfile.ZipFile(tmp, "w", compression=zipfile.ZIP_STORED) as z:
|
||||||
|
for i, e in enumerate(eps):
|
||||||
|
p = Path(e["local_path"])
|
||||||
|
z.write(p, arcname=_safe_name(f"{i+1:03d}_{e.get('title') or p.stem}", p.suffix))
|
||||||
|
background.add_task(lambda: tmp.unlink(missing_ok=True))
|
||||||
|
return FileResponse(tmp, media_type="application/zip", filename=f"{name}.zip",
|
||||||
|
background=background)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/media/{ep_id}")
|
||||||
|
async def media(ep_id: int):
|
||||||
|
ep = store.get_episode(ep_id)
|
||||||
|
if not ep or ep.get("state") != "done" or not ep.get("local_path"):
|
||||||
|
raise HTTPException(404, "not downloaded")
|
||||||
|
p = Path(ep["local_path"])
|
||||||
|
if not p.exists():
|
||||||
|
raise HTTPException(404, "file missing")
|
||||||
|
return FileResponse(p, media_type=ep.get("mime") or "audio/mpeg",
|
||||||
|
filename=p.name)
|
||||||
|
|
||||||
|
|
||||||
|
# ════════════════════════════════════════════════════════════════════
|
||||||
|
# Authenticated endpoints (manage)
|
||||||
|
# ════════════════════════════════════════════════════════════════════
|
||||||
|
@router.get("/feeds", dependencies=[Depends(require_jwt)])
|
||||||
|
async def feeds():
|
||||||
|
_ensure_worker()
|
||||||
|
return {"feeds": store.list_feeds()}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/feeds", dependencies=[Depends(require_jwt)])
|
||||||
|
async def add_feed(body: FeedIn):
|
||||||
|
_ensure_worker()
|
||||||
|
try:
|
||||||
|
return await fetch_and_store(body.url.strip(), auto_dl=body.auto_dl)
|
||||||
|
except Exception as e:
|
||||||
|
raise HTTPException(400, f"feed add failed: {e}")
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/feeds/{fid}/autodl", dependencies=[Depends(require_jwt)])
|
||||||
|
async def toggle_autodl(fid: int, on: bool = True):
|
||||||
|
"""Toggle auto-download for a feed. Turning it on also queues any episodes
|
||||||
|
not yet downloaded (so the portal/share feed sync immediately)."""
|
||||||
|
_ensure_worker()
|
||||||
|
store.set_feed_autodl(fid, 1 if on else 0)
|
||||||
|
queued = await _autoqueue(fid) if on else 0
|
||||||
|
return {"ok": True, "auto_dl": on, "queued": queued}
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/feeds/{fid}", dependencies=[Depends(require_jwt)])
|
||||||
|
async def del_feed(fid: int):
|
||||||
|
store.delete_feed(fid)
|
||||||
|
return {"ok": True}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/feeds/import-opml", dependencies=[Depends(require_jwt)])
|
||||||
|
async def import_opml(body: OPMLIn):
|
||||||
|
_ensure_worker()
|
||||||
|
try:
|
||||||
|
root = ET.fromstring(body.opml)
|
||||||
|
except Exception as e:
|
||||||
|
raise HTTPException(400, f"bad OPML: {e}")
|
||||||
|
urls = [o.get("xmlUrl") for o in root.iter("outline") if o.get("xmlUrl")]
|
||||||
|
added = 0
|
||||||
|
for u in urls:
|
||||||
|
try:
|
||||||
|
await fetch_and_store(u)
|
||||||
|
added += 1
|
||||||
|
except Exception as e:
|
||||||
|
log.error(f"opml feed {u}: {e}")
|
||||||
|
return {"added": added, "total": len(urls)}
|
||||||
|
|
||||||
|
|
||||||
|
_AUDIO_EXT = {".mp3", ".m4a", ".m4b", ".aac", ".ogg", ".opus", ".flac", ".wav", ".mp4"}
|
||||||
|
_AUDIO_MIME = {".mp3": "audio/mpeg", ".m4a": "audio/mp4", ".m4b": "audio/mp4",
|
||||||
|
".aac": "audio/aac", ".ogg": "audio/ogg", ".opus": "audio/opus",
|
||||||
|
".flac": "audio/flac", ".wav": "audio/wav", ".mp4": "audio/mp4"}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/audiobook/upload", dependencies=[Depends(require_jwt)])
|
||||||
|
async def audiobook_upload(request: Request, title: str = "Audiobook"):
|
||||||
|
"""Stream an uploaded ZIP to a temp file, extract its audio tracks, and
|
||||||
|
publish them as a self-contained (synthetic) feed — already 'done' so they
|
||||||
|
show in the library + share feed. Raw body (no python-multipart dep)."""
|
||||||
|
_ensure_worker()
|
||||||
|
title = (title or "Audiobook").strip() or "Audiobook"
|
||||||
|
tmp = Path(tempfile.mkstemp(suffix=".zip", dir="/var/lib/secubox/podcaster")[1])
|
||||||
|
try:
|
||||||
|
with open(tmp, "wb") as fh:
|
||||||
|
async for chunk in request.stream():
|
||||||
|
fh.write(chunk)
|
||||||
|
if not zipfile.is_zipfile(tmp):
|
||||||
|
raise HTTPException(400, "not a valid ZIP")
|
||||||
|
# synthetic feed
|
||||||
|
slug = re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-") or "audiobook"
|
||||||
|
fid = store.add_feed(f"audiobook:{slug}:{int(time.time())}",
|
||||||
|
{"title": title, "description": f"Audiobook: {title}"})
|
||||||
|
fdir = MEDIA / str(fid)
|
||||||
|
fdir.mkdir(parents=True, exist_ok=True)
|
||||||
|
tracks = 0
|
||||||
|
with zipfile.ZipFile(tmp) as z:
|
||||||
|
names = [n for n in z.namelist()
|
||||||
|
if os.path.splitext(n)[1].lower() in _AUDIO_EXT
|
||||||
|
and not n.endswith("/")]
|
||||||
|
names.sort() # chapter order
|
||||||
|
base = int(time.time())
|
||||||
|
for i, n in enumerate(names):
|
||||||
|
ext = os.path.splitext(n)[1].lower()
|
||||||
|
tname = _safe_name(f"{i+1:03d}_{os.path.basename(n)}", ext)
|
||||||
|
dest = fdir / tname
|
||||||
|
with z.open(n) as src, open(dest, "wb") as out:
|
||||||
|
shutil.copyfileobj(src, out, 1024 * 256)
|
||||||
|
store.upsert_episode(fid, {
|
||||||
|
"guid": f"ab:{fid}:{i}", "title": f"{i+1:02d} · {os.path.splitext(os.path.basename(n))[0]}",
|
||||||
|
"description": title, "pubdate": base + i,
|
||||||
|
"enclosure": f"local:{dest}", "mime": _AUDIO_MIME.get(ext, "audio/mpeg"),
|
||||||
|
"bytes": dest.stat().st_size,
|
||||||
|
})
|
||||||
|
# mark immediately downloaded (local file already present)
|
||||||
|
ep = store.list_episodes(feed_id=fid)
|
||||||
|
for e in ep:
|
||||||
|
if e["guid"] == f"ab:{fid}:{i}":
|
||||||
|
store.set_episode(e["id"], state="done", progress=100,
|
||||||
|
local_path=str(dest))
|
||||||
|
break
|
||||||
|
tracks += 1
|
||||||
|
if not tracks:
|
||||||
|
store.delete_feed(fid)
|
||||||
|
raise HTTPException(400, "no audio files found in ZIP")
|
||||||
|
log.info(f"audiobook '{title}' -> feed {fid}, {tracks} tracks")
|
||||||
|
return {"title": title, "feed_id": fid, "tracks": tracks}
|
||||||
|
finally:
|
||||||
|
tmp.unlink(missing_ok=True)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/episodes", dependencies=[Depends(require_jwt)])
|
||||||
|
async def episodes(feed_id: Optional[int] = None, state: Optional[str] = None):
|
||||||
|
_ensure_worker()
|
||||||
|
return {"episodes": store.list_episodes(feed_id, state)}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/episodes/{ep_id}/download", dependencies=[Depends(require_jwt)])
|
||||||
|
async def download(ep_id: int):
|
||||||
|
_ensure_worker()
|
||||||
|
ep = store.get_episode(ep_id)
|
||||||
|
if not ep:
|
||||||
|
raise HTTPException(404, "no such episode")
|
||||||
|
store.set_episode(ep_id, state="queued", progress=0, error=None)
|
||||||
|
await _queue.put(ep_id)
|
||||||
|
return {"ok": True, "queued": ep_id}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/downloads", dependencies=[Depends(require_jwt)])
|
||||||
|
async def downloads():
|
||||||
|
_ensure_worker()
|
||||||
|
return {"downloads": store.list_episodes(state="downloading")
|
||||||
|
+ store.list_episodes(state="queued")}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/config", dependencies=[Depends(require_jwt)])
|
||||||
|
async def get_config():
|
||||||
|
return {"config": load_config()}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/config", dependencies=[Depends(require_jwt)])
|
||||||
|
async def set_config(body: ConfigIn):
|
||||||
|
cfg = load_config()
|
||||||
|
for k, v in body.model_dump(exclude_none=True).items():
|
||||||
|
cfg[k] = v
|
||||||
|
lines = []
|
||||||
|
for k, v in cfg.items():
|
||||||
|
if isinstance(v, str):
|
||||||
|
lines.append(f'{k} = "{v}"')
|
||||||
|
elif isinstance(v, bool):
|
||||||
|
lines.append(f"{k} = {'true' if v else 'false'}")
|
||||||
|
else:
|
||||||
|
lines.append(f"{k} = {v}")
|
||||||
|
try:
|
||||||
|
CONFIG_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
CONFIG_FILE.write_text("\n".join(lines) + "\n")
|
||||||
|
except Exception as e:
|
||||||
|
raise HTTPException(500, f"write failed: {e}")
|
||||||
|
return {"ok": True, "config": cfg, "note": "restart secubox-podcaster to apply"}
|
||||||
|
|
||||||
|
|
||||||
|
app.include_router(router)
|
||||||
198
packages/secubox-podcaster/api/store.py
Normal file
198
packages/secubox-podcaster/api/store.py
Normal file
|
|
@ -0,0 +1,198 @@
|
||||||
|
# SPDX-License-Identifier: LicenseRef-CMSD-1.0
|
||||||
|
# Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
|
||||||
|
# Source-Disclosed License — All rights reserved except as expressly granted.
|
||||||
|
# See LICENCE-CMSD-1.0.md for terms.
|
||||||
|
|
||||||
|
"""
|
||||||
|
SecuBox-Deb :: Podcaster store
|
||||||
|
CyberMind — https://cybermind.fr
|
||||||
|
|
||||||
|
SQLite persistence for feeds + episodes. Pure stdlib (sqlite3); no ORM.
|
||||||
|
"""
|
||||||
|
import sqlite3
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Optional
|
||||||
|
|
||||||
|
DB_PATH = Path("/var/lib/secubox/podcaster/podcaster.db")
|
||||||
|
|
||||||
|
_SCHEMA = """
|
||||||
|
CREATE TABLE IF NOT EXISTS feeds (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
url TEXT UNIQUE NOT NULL,
|
||||||
|
title TEXT,
|
||||||
|
description TEXT,
|
||||||
|
image TEXT,
|
||||||
|
site TEXT,
|
||||||
|
added_ts INTEGER NOT NULL,
|
||||||
|
last_fetch INTEGER DEFAULT 0,
|
||||||
|
auto_dl INTEGER DEFAULT 0
|
||||||
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS episodes (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
feed_id INTEGER NOT NULL REFERENCES feeds(id) ON DELETE CASCADE,
|
||||||
|
guid TEXT NOT NULL,
|
||||||
|
title TEXT,
|
||||||
|
description TEXT,
|
||||||
|
pubdate INTEGER DEFAULT 0,
|
||||||
|
enclosure TEXT,
|
||||||
|
mime TEXT,
|
||||||
|
bytes INTEGER DEFAULT 0,
|
||||||
|
duration TEXT,
|
||||||
|
local_path TEXT,
|
||||||
|
state TEXT DEFAULT 'new', -- new | queued | downloading | done | error
|
||||||
|
progress INTEGER DEFAULT 0, -- 0..100
|
||||||
|
error TEXT,
|
||||||
|
UNIQUE(feed_id, guid)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_ep_feed ON episodes(feed_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_ep_state ON episodes(state);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_ep_pub ON episodes(pubdate DESC);
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def _conn() -> sqlite3.Connection:
|
||||||
|
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
c = sqlite3.connect(DB_PATH, timeout=10)
|
||||||
|
c.row_factory = sqlite3.Row
|
||||||
|
c.execute("PRAGMA foreign_keys=ON")
|
||||||
|
c.execute("PRAGMA journal_mode=WAL")
|
||||||
|
return c
|
||||||
|
|
||||||
|
|
||||||
|
def init() -> None:
|
||||||
|
with _conn() as c:
|
||||||
|
c.executescript(_SCHEMA)
|
||||||
|
|
||||||
|
|
||||||
|
# ── feeds ──────────────────────────────────────────────────────────
|
||||||
|
def add_feed(url: str, meta: dict, auto_dl: int = 0) -> int:
|
||||||
|
with _conn() as c:
|
||||||
|
cur = c.execute(
|
||||||
|
"INSERT OR IGNORE INTO feeds(url,title,description,image,site,added_ts,auto_dl) "
|
||||||
|
"VALUES(?,?,?,?,?,?,?)",
|
||||||
|
(url, meta.get("title"), meta.get("description"),
|
||||||
|
meta.get("image"), meta.get("site"), int(time.time()), int(auto_dl)),
|
||||||
|
)
|
||||||
|
if cur.lastrowid:
|
||||||
|
return cur.lastrowid
|
||||||
|
row = c.execute("SELECT id FROM feeds WHERE url=?", (url,)).fetchone()
|
||||||
|
return row["id"] if row else 0
|
||||||
|
|
||||||
|
|
||||||
|
def set_feed_autodl(feed_id: int, on: int) -> None:
|
||||||
|
with _conn() as c:
|
||||||
|
c.execute("UPDATE feeds SET auto_dl=? WHERE id=?", (int(on), feed_id))
|
||||||
|
|
||||||
|
|
||||||
|
def feed_autodl(feed_id: int) -> int:
|
||||||
|
with _conn() as c:
|
||||||
|
r = c.execute("SELECT auto_dl FROM feeds WHERE id=?", (feed_id,)).fetchone()
|
||||||
|
return int(r["auto_dl"]) if r else 0
|
||||||
|
|
||||||
|
|
||||||
|
def pending_episode_ids(feed_id: int, limit: int = 0) -> list[int]:
|
||||||
|
"""Episode ids in state 'new' for a feed (newest first); for auto-download."""
|
||||||
|
q = "SELECT id FROM episodes WHERE feed_id=? AND state='new' ORDER BY pubdate DESC"
|
||||||
|
if limit and limit > 0:
|
||||||
|
q += f" LIMIT {int(limit)}"
|
||||||
|
with _conn() as c:
|
||||||
|
return [r["id"] for r in c.execute(q, (feed_id,)).fetchall()]
|
||||||
|
|
||||||
|
|
||||||
|
def update_feed_meta(feed_id: int, meta: dict) -> None:
|
||||||
|
with _conn() as c:
|
||||||
|
c.execute(
|
||||||
|
"UPDATE feeds SET title=COALESCE(?,title), description=COALESCE(?,description), "
|
||||||
|
"image=COALESCE(?,image), site=COALESCE(?,site), last_fetch=? WHERE id=?",
|
||||||
|
(meta.get("title"), meta.get("description"), meta.get("image"),
|
||||||
|
meta.get("site"), int(time.time()), feed_id),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def list_feeds() -> list[dict]:
|
||||||
|
with _conn() as c:
|
||||||
|
rows = c.execute(
|
||||||
|
"SELECT f.*, "
|
||||||
|
"(SELECT COUNT(*) FROM episodes e WHERE e.feed_id=f.id) AS episodes, "
|
||||||
|
"(SELECT COUNT(*) FROM episodes e WHERE e.feed_id=f.id AND e.state='done') AS downloaded "
|
||||||
|
"FROM feeds f ORDER BY f.title COLLATE NOCASE"
|
||||||
|
).fetchall()
|
||||||
|
return [dict(r) for r in rows]
|
||||||
|
|
||||||
|
|
||||||
|
def delete_feed(feed_id: int) -> None:
|
||||||
|
with _conn() as c:
|
||||||
|
c.execute("DELETE FROM feeds WHERE id=?", (feed_id,))
|
||||||
|
|
||||||
|
|
||||||
|
def all_feed_urls() -> list[tuple[int, str]]:
|
||||||
|
with _conn() as c:
|
||||||
|
return [(r["id"], r["url"]) for r in c.execute("SELECT id,url FROM feeds")]
|
||||||
|
|
||||||
|
|
||||||
|
# ── episodes ───────────────────────────────────────────────────────
|
||||||
|
def upsert_episode(feed_id: int, ep: dict) -> None:
|
||||||
|
with _conn() as c:
|
||||||
|
c.execute(
|
||||||
|
"INSERT OR IGNORE INTO episodes"
|
||||||
|
"(feed_id,guid,title,description,pubdate,enclosure,mime,bytes,duration) "
|
||||||
|
"VALUES(?,?,?,?,?,?,?,?,?)",
|
||||||
|
(feed_id, ep["guid"], ep.get("title"), ep.get("description"),
|
||||||
|
ep.get("pubdate", 0), ep.get("enclosure"), ep.get("mime"),
|
||||||
|
ep.get("bytes", 0), ep.get("duration")),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def list_episodes(feed_id: Optional[int] = None, state: Optional[str] = None,
|
||||||
|
limit: int = 200) -> list[dict]:
|
||||||
|
q = ("SELECT e.*, f.title AS feed_title, f.image AS feed_image "
|
||||||
|
"FROM episodes e JOIN feeds f ON f.id=e.feed_id")
|
||||||
|
where, args = [], []
|
||||||
|
if feed_id is not None:
|
||||||
|
where.append("e.feed_id=?"); args.append(feed_id)
|
||||||
|
if state:
|
||||||
|
where.append("e.state=?"); args.append(state)
|
||||||
|
if where:
|
||||||
|
q += " WHERE " + " AND ".join(where)
|
||||||
|
q += " ORDER BY e.pubdate DESC LIMIT ?"; args.append(limit)
|
||||||
|
with _conn() as c:
|
||||||
|
return [dict(r) for r in c.execute(q, args).fetchall()]
|
||||||
|
|
||||||
|
|
||||||
|
def get_episode(ep_id: int) -> Optional[dict]:
|
||||||
|
with _conn() as c:
|
||||||
|
r = c.execute("SELECT * FROM episodes WHERE id=?", (ep_id,)).fetchone()
|
||||||
|
return dict(r) if r else None
|
||||||
|
|
||||||
|
|
||||||
|
def set_episode(ep_id: int, **fields: Any) -> None:
|
||||||
|
if not fields:
|
||||||
|
return
|
||||||
|
cols = ", ".join(f"{k}=?" for k in fields)
|
||||||
|
with _conn() as c:
|
||||||
|
c.execute(f"UPDATE episodes SET {cols} WHERE id=?", (*fields.values(), ep_id))
|
||||||
|
|
||||||
|
|
||||||
|
def downloaded_episodes(limit: int = 500) -> list[dict]:
|
||||||
|
"""Episodes with a local file — the shareable library."""
|
||||||
|
with _conn() as c:
|
||||||
|
rows = c.execute(
|
||||||
|
"SELECT e.*, f.title AS feed_title FROM episodes e JOIN feeds f ON f.id=e.feed_id "
|
||||||
|
"WHERE e.state='done' AND e.local_path IS NOT NULL "
|
||||||
|
"ORDER BY e.pubdate DESC LIMIT ?", (limit,)
|
||||||
|
).fetchall()
|
||||||
|
return [dict(r) for r in rows]
|
||||||
|
|
||||||
|
|
||||||
|
def counts() -> dict:
|
||||||
|
with _conn() as c:
|
||||||
|
return {
|
||||||
|
"feeds": c.execute("SELECT COUNT(*) FROM feeds").fetchone()[0],
|
||||||
|
"episodes": c.execute("SELECT COUNT(*) FROM episodes").fetchone()[0],
|
||||||
|
"downloaded": c.execute(
|
||||||
|
"SELECT COUNT(*) FROM episodes WHERE state='done'").fetchone()[0],
|
||||||
|
"queued": c.execute(
|
||||||
|
"SELECT COUNT(*) FROM episodes WHERE state IN('queued','downloading')"
|
||||||
|
).fetchone()[0],
|
||||||
|
}
|
||||||
23
packages/secubox-podcaster/conf/podcaster.toml
Normal file
23
packages/secubox-podcaster/conf/podcaster.toml
Normal file
|
|
@ -0,0 +1,23 @@
|
||||||
|
# /etc/secubox/podcaster.toml — SecuBox Podcaster
|
||||||
|
# Restart secubox-podcaster after editing.
|
||||||
|
|
||||||
|
# Where downloaded episodes are stored.
|
||||||
|
media_path = "/var/lib/secubox/podcaster/media"
|
||||||
|
|
||||||
|
# Concurrent downloads.
|
||||||
|
max_parallel = 2
|
||||||
|
|
||||||
|
# Auto-refresh feeds every N minutes (background).
|
||||||
|
refresh_minutes = 60
|
||||||
|
|
||||||
|
# Public base URL for the shareable RSS enclosure links. Leave empty for
|
||||||
|
# LAN/relative URLs. Set to your exposed vhost to relay publicly, e.g.
|
||||||
|
# public_base = "https://podcast.gk2.secubox.in"
|
||||||
|
# (route it through HAProxy TLS -> mitmproxy per WAF policy; see README).
|
||||||
|
public_base = ""
|
||||||
|
|
||||||
|
# Title of the generated share feed.
|
||||||
|
share_title = "SecuBox Podcaster"
|
||||||
|
|
||||||
|
# Keep at most N downloaded episodes per feed (0 = unlimited).
|
||||||
|
keep_per_feed = 0
|
||||||
46
packages/secubox-podcaster/debian/changelog
Normal file
46
packages/secubox-podcaster/debian/changelog
Normal file
|
|
@ -0,0 +1,46 @@
|
||||||
|
secubox-podcaster (1.0.3-1~bookworm1) bookworm; urgency=medium
|
||||||
|
|
||||||
|
* Auto-download: feeds can auto-queue new episodes so the portal + share feed
|
||||||
|
stay synced without manual downloads. New feeds default auto_dl=on (UI
|
||||||
|
checkbox); per-feed toggle (POST /feeds/{id}/autodl, ⏬ in admin). The
|
||||||
|
periodic refresher auto-queues newly published episodes; bounded by
|
||||||
|
keep_per_feed. (#726)
|
||||||
|
|
||||||
|
-- Gerald KERMA <devel@cybermind.fr> Wed, 24 Jun 2026 14:20:00 +0000
|
||||||
|
|
||||||
|
secubox-podcaster (1.0.2-1~bookworm1) bookworm; urgency=medium
|
||||||
|
|
||||||
|
* Public portal: per-episode download (⬇, mp3/audio) + per-feed
|
||||||
|
"Download all (ZIP)" via new public GET /public/feed/{id}/zip (STORED, no
|
||||||
|
recompress; built to temp then streamed + cleaned up). public/library now
|
||||||
|
carries feed_id so the portal can zip by feed.
|
||||||
|
|
||||||
|
-- Gerald KERMA <devel@cybermind.fr> Wed, 24 Jun 2026 13:40:00 +0000
|
||||||
|
|
||||||
|
secubox-podcaster (1.0.1-1~bookworm1) bookworm; urgency=medium
|
||||||
|
|
||||||
|
* Admin WebUI: integrate the shared Hub sidebar (/shared/sidebar.js) + correct
|
||||||
|
auth token (sbx_token, 401 -> /login.html) — was missing the navbar and
|
||||||
|
falsely prompting to log in.
|
||||||
|
* Public listener PORTAL (no auth) at /podcaster/portal/ — standalone
|
||||||
|
C3BOX page driven by a new public GET /public/library endpoint; subscribe
|
||||||
|
(RSS) + per-feed filter + inline players. Meant to be exposed externally.
|
||||||
|
* Audiobook ZIP import: POST /audiobook/upload (raw body, no python-multipart)
|
||||||
|
streams the ZIP to a temp file, extracts audio tracks, and publishes them as
|
||||||
|
a self-contained synthetic feed (already 'done' -> in library + share feed).
|
||||||
|
|
||||||
|
-- Gerald KERMA <devel@cybermind.fr> Wed, 24 Jun 2026 13:00:00 +0000
|
||||||
|
|
||||||
|
secubox-podcaster (1.0.0-1~bookworm1) bookworm; urgency=medium
|
||||||
|
|
||||||
|
* Initial release (#726). Modern podcast manager:
|
||||||
|
- subscribe by RSS URL / OPML import; pure-stdlib feed parsing
|
||||||
|
- local download queue (asyncio + httpx) with per-episode progress
|
||||||
|
- generated shareable RSS of the local library (/share/feed.xml) — LAN by
|
||||||
|
default, public via HAProxy / secubox-exposure (public_base)
|
||||||
|
- in-UI service status + TOML config; C3BOX WebUI with inline player
|
||||||
|
- FastAPI on /run/secubox/podcaster.sock; SQLite store; nginx route shipped
|
||||||
|
to the active secubox-routes.d/ include
|
||||||
|
- Lyrion integration deferred to a follow-up (standalone first)
|
||||||
|
|
||||||
|
-- Gerald KERMA <devel@cybermind.fr> Wed, 24 Jun 2026 12:00:00 +0000
|
||||||
22
packages/secubox-podcaster/debian/control
Normal file
22
packages/secubox-podcaster/debian/control
Normal file
|
|
@ -0,0 +1,22 @@
|
||||||
|
Source: secubox-podcaster
|
||||||
|
Section: sound
|
||||||
|
Priority: optional
|
||||||
|
Maintainer: Gerald KERMA <devel@cybermind.fr>
|
||||||
|
Build-Depends: debhelper-compat (= 13)
|
||||||
|
Standards-Version: 4.6.2
|
||||||
|
|
||||||
|
Package: secubox-podcaster
|
||||||
|
Architecture: all
|
||||||
|
Depends: ${misc:Depends}, secubox-core (>= 1.0), python3, python3-uvicorn | python3-pip, python3-httpx | python3-pip
|
||||||
|
Description: Modern podcast manager for SecuBox
|
||||||
|
Subscribe to podcast feeds, download episodes locally, and re-publish a
|
||||||
|
shareable RSS feed (LAN by default, public via HAProxy / secubox-exposure).
|
||||||
|
FastAPI backend on /api/v1/podcaster/ over a Unix socket; SQLite store; an
|
||||||
|
asyncio download queue. Feed parsing is pure stdlib (no feedparser dep).
|
||||||
|
.
|
||||||
|
Features:
|
||||||
|
- Subscribe by RSS URL or OPML import
|
||||||
|
- Local download queue with per-episode progress
|
||||||
|
- Generated shareable RSS of the local library (relay/share to the web)
|
||||||
|
- In-UI service status + TOML config
|
||||||
|
- Modern C3BOX cyberpunk WebUI with inline player
|
||||||
33
packages/secubox-podcaster/debian/postinst
Normal file
33
packages/secubox-podcaster/debian/postinst
Normal file
|
|
@ -0,0 +1,33 @@
|
||||||
|
#!/bin/bash
|
||||||
|
set -e
|
||||||
|
|
||||||
|
case "$1" in
|
||||||
|
configure)
|
||||||
|
# Data dirs (module-owned; never touch the shared /run/secubox parent, #494).
|
||||||
|
install -d -o secubox -g secubox -m 0750 /var/lib/secubox/podcaster
|
||||||
|
install -d -o secubox -g secubox -m 0750 /var/lib/secubox/podcaster/media
|
||||||
|
|
||||||
|
# Config on first install only (preserve operator edits on upgrade).
|
||||||
|
if [ ! -f /etc/secubox/podcaster.toml ]; then
|
||||||
|
if [ -f /usr/share/secubox/podcaster/podcaster.toml ]; then
|
||||||
|
install -o secubox -g secubox -m 0640 \
|
||||||
|
/usr/share/secubox/podcaster/podcaster.toml /etc/secubox/podcaster.toml
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
systemctl daemon-reload 2>/dev/null || true
|
||||||
|
# Respect operator masking (set -e would abort on a masked enable).
|
||||||
|
if [ "$(systemctl is-enabled secubox-podcaster.service 2>/dev/null)" != "masked" ]; then
|
||||||
|
systemctl enable secubox-podcaster.service 2>/dev/null || true
|
||||||
|
systemctl restart secubox-podcaster.service 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Reload nginx only if the resulting config is valid.
|
||||||
|
if command -v nginx >/dev/null 2>&1 && nginx -t 2>/dev/null; then
|
||||||
|
systemctl reload nginx 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
#DEBHELPER#
|
||||||
|
exit 0
|
||||||
12
packages/secubox-podcaster/debian/prerm
Normal file
12
packages/secubox-podcaster/debian/prerm
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
#!/bin/bash
|
||||||
|
set -e
|
||||||
|
|
||||||
|
case "$1" in
|
||||||
|
remove|deconfigure)
|
||||||
|
systemctl stop secubox-podcaster.service 2>/dev/null || true
|
||||||
|
systemctl disable secubox-podcaster.service 2>/dev/null || true
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
#DEBHELPER#
|
||||||
|
exit 0
|
||||||
30
packages/secubox-podcaster/debian/rules
Executable file
30
packages/secubox-podcaster/debian/rules
Executable file
|
|
@ -0,0 +1,30 @@
|
||||||
|
#!/usr/bin/make -f
|
||||||
|
%:
|
||||||
|
dh $@
|
||||||
|
|
||||||
|
override_dh_auto_install:
|
||||||
|
# API files (service WorkingDirectory = /usr/share/secubox/podcaster)
|
||||||
|
install -d debian/secubox-podcaster/usr/share/secubox/podcaster/
|
||||||
|
cp -r api debian/secubox-podcaster/usr/share/secubox/podcaster/
|
||||||
|
# Static www files
|
||||||
|
install -d debian/secubox-podcaster/usr/share/secubox/www
|
||||||
|
[ -d www ] && cp -r www/. debian/secubox-podcaster/usr/share/secubox/www/ || true
|
||||||
|
# Menu definitions
|
||||||
|
install -d debian/secubox-podcaster/usr/share/secubox/menu.d
|
||||||
|
[ -d menu.d ] && cp -r menu.d/. debian/secubox-podcaster/usr/share/secubox/menu.d/ || true
|
||||||
|
# Config template (installed by postinst on first install only)
|
||||||
|
install -d debian/secubox-podcaster/usr/share/secubox/podcaster
|
||||||
|
[ -f conf/podcaster.toml ] && cp conf/podcaster.toml debian/secubox-podcaster/usr/share/secubox/podcaster/ || true
|
||||||
|
# nginx route — ship to the ACTIVE include (secubox-routes.d) AND legacy
|
||||||
|
# secubox.d for back-compat (mirrors grafana/peertube; see #65).
|
||||||
|
install -d debian/secubox-podcaster/etc/nginx/secubox-routes.d
|
||||||
|
install -d debian/secubox-podcaster/etc/nginx/secubox.d
|
||||||
|
[ -f nginx/podcaster.conf ] && cp nginx/podcaster.conf debian/secubox-podcaster/etc/nginx/secubox-routes.d/ || true
|
||||||
|
[ -f nginx/podcaster.conf ] && cp nginx/podcaster.conf debian/secubox-podcaster/etc/nginx/secubox.d/ || true
|
||||||
|
# Systemd unit
|
||||||
|
install -d debian/secubox-podcaster/usr/lib/systemd/system
|
||||||
|
cp systemd/secubox-podcaster.service debian/secubox-podcaster/usr/lib/systemd/system/
|
||||||
|
|
||||||
|
override_dh_installsystemd:
|
||||||
|
# Handled manually in postinst (respect operator masking).
|
||||||
|
true
|
||||||
1
packages/secubox-podcaster/debian/source/format
Normal file
1
packages/secubox-podcaster/debian/source/format
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
3.0 (native)
|
||||||
9
packages/secubox-podcaster/menu.d/612-podcaster.json
Normal file
9
packages/secubox-podcaster/menu.d/612-podcaster.json
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
{
|
||||||
|
"id": "podcaster",
|
||||||
|
"name": "Podcaster",
|
||||||
|
"icon": "🎙️",
|
||||||
|
"path": "/podcaster/",
|
||||||
|
"category": "mesh",
|
||||||
|
"order": 612,
|
||||||
|
"description": "Subscribe, download and relay podcasts"
|
||||||
|
}
|
||||||
18
packages/secubox-podcaster/nginx/podcaster.conf
Normal file
18
packages/secubox-podcaster/nginx/podcaster.conf
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
# /etc/nginx/secubox-routes.d/podcaster.conf
|
||||||
|
# Installed by secubox-podcaster — auto-registered on install, removed on purge.
|
||||||
|
|
||||||
|
# Static frontend
|
||||||
|
location /podcaster/ {
|
||||||
|
alias /usr/share/secubox/www/podcaster/;
|
||||||
|
index index.html;
|
||||||
|
try_files $uri $uri/ /podcaster/index.html;
|
||||||
|
}
|
||||||
|
|
||||||
|
# API backend (Unix socket). Episode media + share feed stream through here too.
|
||||||
|
location /api/v1/podcaster/ {
|
||||||
|
proxy_pass http://unix:/run/secubox/podcaster.sock:/;
|
||||||
|
# secubox-proxy.conf already sets proxy_buffering off + proxy_read_timeout;
|
||||||
|
# only add what it doesn't (large episode uploads/streams).
|
||||||
|
include /etc/nginx/snippets/secubox-proxy.conf;
|
||||||
|
client_max_body_size 0;
|
||||||
|
}
|
||||||
27
packages/secubox-podcaster/systemd/secubox-podcaster.service
Normal file
27
packages/secubox-podcaster/systemd/secubox-podcaster.service
Normal file
|
|
@ -0,0 +1,27 @@
|
||||||
|
[Unit]
|
||||||
|
Description=SecuBox Podcaster API
|
||||||
|
After=network.target secubox-runtime.service
|
||||||
|
Requires=secubox-runtime.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=root
|
||||||
|
Group=root
|
||||||
|
WorkingDirectory=/usr/share/secubox/podcaster
|
||||||
|
ExecStart=/usr/bin/python3 -m uvicorn api.main:app --uds /run/secubox/podcaster.sock --workers 1
|
||||||
|
# #494: only chmod/chown our OWN socket — never the shared /run/secubox parent.
|
||||||
|
ExecStartPost=-/bin/sleep 1
|
||||||
|
ExecStartPost=-/bin/chmod 660 /run/secubox/podcaster.sock
|
||||||
|
ExecStartPost=-/bin/chown secubox:secubox /run/secubox/podcaster.sock
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=5
|
||||||
|
|
||||||
|
# Security hardening
|
||||||
|
NoNewPrivileges=no
|
||||||
|
ProtectSystem=full
|
||||||
|
ProtectHome=read-only
|
||||||
|
PrivateTmp=true
|
||||||
|
ReadWritePaths=/run/secubox /var/lib/secubox /etc/secubox
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
191
packages/secubox-podcaster/www/podcaster/index.html
Normal file
191
packages/secubox-podcaster/www/podcaster/index.html
Normal file
|
|
@ -0,0 +1,191 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<!-- SPDX-License-Identifier: LicenseRef-CMSD-1.0 -->
|
||||||
|
<!-- SecuBox-Deb :: Podcaster admin WebUI — CyberMind https://cybermind.fr -->
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>SecuBox · Podcaster</title>
|
||||||
|
<link rel="stylesheet" href="/shared/crt-light.css">
|
||||||
|
<link rel="stylesheet" href="/shared/sidebar-light.css">
|
||||||
|
<style>
|
||||||
|
:root{
|
||||||
|
--cosmos-black:#0a0a0f; --gold-hermetic:#c9a84c; --cinnabar:#e63946;
|
||||||
|
--matrix-green:#00ff41; --void-purple:#6e40c9; --cyber-cyan:#00d4ff;
|
||||||
|
--text-primary:#e8e6d9; --text-muted:#6b6b7a; --panel:#13131c; --line:#23232f;
|
||||||
|
}
|
||||||
|
body{margin:0;background:var(--cosmos-black);color:var(--text-primary);
|
||||||
|
font-family:'JetBrains Mono',ui-monospace,Menlo,monospace;font-size:14px;display:flex}
|
||||||
|
.main{flex:1;margin-left:220px;min-height:100vh}
|
||||||
|
@media(max-width:900px){.sidebar{display:none}.main{margin-left:0}}
|
||||||
|
.topbar{display:flex;align-items:center;gap:14px;padding:14px 20px;
|
||||||
|
border-bottom:1px solid var(--line);background:linear-gradient(90deg,#0a0a0f,#13131c)}
|
||||||
|
.topbar h1{font-size:18px;margin:0;letter-spacing:1px;color:var(--gold-hermetic)}
|
||||||
|
.topbar .em{font-size:22px}
|
||||||
|
.stats{margin-left:auto;display:flex;gap:16px;color:var(--text-muted);font-size:12px}
|
||||||
|
.stats b{color:var(--cyber-cyan)}
|
||||||
|
.wrap{display:grid;grid-template-columns:300px 1fr;gap:0;min-height:calc(100vh - 58px)}
|
||||||
|
aside.feeds{border-right:1px solid var(--line);background:var(--panel);padding:14px;overflow:auto}
|
||||||
|
section.eps{padding:18px 22px;overflow:auto}
|
||||||
|
@media(max-width:760px){.wrap{grid-template-columns:1fr}aside.feeds{border-right:0;border-bottom:1px solid var(--line)}}
|
||||||
|
.row{display:flex;gap:8px;margin-bottom:12px}
|
||||||
|
input,button,select{font-family:inherit;font-size:13px;border-radius:8px;border:1px solid var(--line)}
|
||||||
|
input{flex:1;background:#0e0e16;color:var(--text-primary);padding:9px 11px}
|
||||||
|
button{background:#1a1a26;color:var(--text-primary);padding:9px 13px;cursor:pointer;transition:.15s}
|
||||||
|
button:hover{border-color:var(--cyber-cyan);color:var(--cyber-cyan)}
|
||||||
|
button.go{background:var(--void-purple);border-color:var(--void-purple);color:#fff}
|
||||||
|
a.portal{color:var(--cyber-cyan);text-decoration:none;font-size:12px;border:1px solid var(--cyber-cyan);
|
||||||
|
padding:7px 11px;border-radius:8px}
|
||||||
|
.feed{display:flex;gap:10px;align-items:center;padding:9px;border-radius:8px;cursor:pointer;border:1px solid transparent}
|
||||||
|
.feed:hover{background:#0e0e16}.feed.sel{background:#0e0e16;border-color:var(--void-purple)}
|
||||||
|
.feed img{width:40px;height:40px;border-radius:6px;object-fit:cover;background:#222}
|
||||||
|
.feed .t{flex:1;min-width:0}.feed .t b{display:block;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;font-size:13px}
|
||||||
|
.feed .t span{color:var(--text-muted);font-size:11px}
|
||||||
|
.feed .x{color:var(--text-muted);opacity:0;font-size:16px}.feed:hover .x{opacity:1}
|
||||||
|
.feed .ad{font-size:14px;cursor:pointer;opacity:.5}.feed .ad.on{opacity:1;color:var(--matrix-green)}
|
||||||
|
label.autodl{display:flex;gap:7px;align-items:center;color:var(--text-muted);font-size:12px;margin:-4px 0 12px;cursor:pointer}
|
||||||
|
h2{font-size:15px;color:var(--gold-hermetic);border-bottom:1px solid var(--line);padding-bottom:8px;margin:0 0 14px}
|
||||||
|
.ep{display:flex;gap:12px;align-items:center;padding:12px;border:1px solid var(--line);border-radius:10px;margin-bottom:10px;background:var(--panel)}
|
||||||
|
.ep .meta{flex:1;min-width:0}.ep .meta b{display:block;white-space:nowrap;overflow:hidden;text-overflow:ellipsis}
|
||||||
|
.ep .meta span{color:var(--text-muted);font-size:11px}
|
||||||
|
.ep .act{display:flex;gap:6px;align-items:center}
|
||||||
|
.badge{font-size:10px;padding:2px 7px;border-radius:20px;border:1px solid var(--line);color:var(--text-muted)}
|
||||||
|
.badge.downloading{color:var(--cyber-cyan);border-color:var(--cyber-cyan)}
|
||||||
|
.badge.error{color:var(--cinnabar);border-color:var(--cinnabar)}
|
||||||
|
.bar{height:4px;border-radius:3px;background:#0e0e16;overflow:hidden;width:90px}.bar i{display:block;height:100%;background:var(--cyber-cyan);width:0}
|
||||||
|
audio{height:34px}
|
||||||
|
.share{margin-top:6px;padding:12px;border:1px dashed var(--void-purple);border-radius:10px;color:var(--text-muted);font-size:12px}
|
||||||
|
.share a{color:var(--cyber-cyan)}
|
||||||
|
.muted{color:var(--text-muted)}.empty{color:var(--text-muted);padding:40px;text-align:center}
|
||||||
|
dialog{background:var(--panel);color:var(--text-primary);border:1px solid var(--void-purple);border-radius:12px;padding:20px;width:min(520px,92vw)}
|
||||||
|
dialog label{display:block;margin:10px 0 4px;color:var(--text-muted);font-size:12px}dialog input{width:100%}
|
||||||
|
.toast{position:fixed;bottom:18px;right:18px;background:#13131c;border:1px solid var(--cyber-cyan);color:var(--cyber-cyan);padding:10px 14px;border-radius:8px;opacity:0;transition:.2s;font-size:12px}.toast.on{opacity:1}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<nav class="sidebar" id="sidebar"></nav>
|
||||||
|
<main class="main">
|
||||||
|
<div class="topbar">
|
||||||
|
<span class="em">🎙️</span><h1>PODCASTER</h1>
|
||||||
|
<div class="stats" id="stats"></div>
|
||||||
|
<a class="portal" href="/podcaster/portal/" target="_blank">🌐 Public portal</a>
|
||||||
|
<button onclick="openCfg()" title="Config & status">⚙️</button>
|
||||||
|
</div>
|
||||||
|
<div class="wrap">
|
||||||
|
<aside class="feeds">
|
||||||
|
<div class="row">
|
||||||
|
<input id="feedUrl" placeholder="RSS feed URL…" onkeydown="if(event.key==='Enter')addFeed()">
|
||||||
|
<button class="go" onclick="addFeed()">+</button>
|
||||||
|
</div>
|
||||||
|
<label class="autodl"><input type="checkbox" id="autodl" checked> auto-download new episodes</label>
|
||||||
|
<div class="row">
|
||||||
|
<button onclick="document.getElementById('opml').click()">Import OPML</button>
|
||||||
|
<button onclick="document.getElementById('abzip').click()" title="Upload an audiobook ZIP">📚 Audiobook ZIP</button>
|
||||||
|
<button onclick="loadAll()">↻</button>
|
||||||
|
</div>
|
||||||
|
<input type="file" id="opml" accept=".opml,.xml" style="display:none" onchange="importOpml(this)">
|
||||||
|
<input type="file" id="abzip" accept=".zip" style="display:none" onchange="uploadZip(this)">
|
||||||
|
<div id="feeds"></div>
|
||||||
|
<div class="share" id="share"></div>
|
||||||
|
</aside>
|
||||||
|
<section class="eps">
|
||||||
|
<h2 id="title">Latest episodes</h2>
|
||||||
|
<div id="eps"><div class="empty">Loading…</div></div>
|
||||||
|
</section>
|
||||||
|
</div>
|
||||||
|
</main>
|
||||||
|
|
||||||
|
<dialog id="cfg">
|
||||||
|
<h2>Config & status</h2>
|
||||||
|
<div id="svc" class="muted"></div>
|
||||||
|
<label>Media path</label><input id="c_media">
|
||||||
|
<label>Max parallel downloads</label><input id="c_par" type="number">
|
||||||
|
<label>Refresh minutes</label><input id="c_ref" type="number">
|
||||||
|
<label>Public base URL (for shareable feed — leave empty for LAN)</label><input id="c_pub">
|
||||||
|
<label>Share feed title</label><input id="c_title">
|
||||||
|
<div class="row" style="margin-top:16px">
|
||||||
|
<button class="go" onclick="saveCfg()">Save</button>
|
||||||
|
<button onclick="document.getElementById('cfg').close()">Close</button>
|
||||||
|
</div>
|
||||||
|
</dialog>
|
||||||
|
<div class="toast" id="toast"></div>
|
||||||
|
|
||||||
|
<script src="/shared/sidebar.js"></script>
|
||||||
|
<script>
|
||||||
|
const API="/api/v1/podcaster";
|
||||||
|
let SEL=null;
|
||||||
|
function tok(){return localStorage.getItem("sbx_token")||""}
|
||||||
|
async function api(p,opt={}){
|
||||||
|
opt.headers=Object.assign({},opt.headers||{});
|
||||||
|
if(!(opt.body instanceof FormData) && !opt.headers["Content-Type"]) opt.headers["Content-Type"]="application/json";
|
||||||
|
const t=tok(); if(t) opt.headers["Authorization"]="Bearer "+t;
|
||||||
|
const r=await fetch(API+p,opt);
|
||||||
|
if(r.status===401){window.location="/login.html";throw new Error("401")}
|
||||||
|
if(!r.ok) throw new Error(await r.text());
|
||||||
|
const ct=r.headers.get("content-type")||""; return ct.includes("json")?r.json():r.text();
|
||||||
|
}
|
||||||
|
function toast(m){const t=document.getElementById("toast");t.textContent=m;t.classList.add("on");clearTimeout(t._);t._=setTimeout(()=>t.classList.remove("on"),3000)}
|
||||||
|
function esc(s){return (s||"").replace(/[&<>"]/g,c=>({"&":"&","<":"<",">":">",'"':"""}[c]))}
|
||||||
|
async function loadStatus(){
|
||||||
|
try{const s=await fetch(API+"/status").then(r=>r.json());
|
||||||
|
document.getElementById("stats").innerHTML=`feeds <b>${s.feeds}</b> · eps <b>${s.episodes}</b> · local <b>${s.downloaded}</b> · queue <b>${s.queued}</b>`;
|
||||||
|
const base=(s.public_base||location.origin);
|
||||||
|
document.getElementById("share").innerHTML=`🔗 Share feed:<br><a href="${API}/share/feed.xml" target="_blank">${esc(base)}/api/v1/podcaster/share/feed.xml</a>`;
|
||||||
|
}catch(e){}
|
||||||
|
}
|
||||||
|
async function loadFeeds(){
|
||||||
|
let d; try{d=await api("/feeds")}catch(e){return}
|
||||||
|
const el=document.getElementById("feeds");
|
||||||
|
if(!d.feeds.length){el.innerHTML='<div class="muted" style="padding:12px">No feeds yet — add one above.</div>';return}
|
||||||
|
el.innerHTML=d.feeds.map(f=>`<div class="feed ${SEL===f.id?'sel':''}" onclick="selFeed(${f.id})">
|
||||||
|
<img src="${esc(f.image||'')}" onerror="this.style.visibility='hidden'">
|
||||||
|
<div class="t"><b>${esc(f.title||f.url)}</b><span>${f.downloaded}/${f.episodes} local</span></div>
|
||||||
|
<span class="ad ${f.auto_dl?'on':''}" title="auto-download ${f.auto_dl?'on':'off'}" onclick="event.stopPropagation();toggleAd(${f.id},${f.auto_dl?0:1})">⏬</span>
|
||||||
|
<span class="x" onclick="event.stopPropagation();delFeed(${f.id})">✕</span></div>`).join("");
|
||||||
|
}
|
||||||
|
async function selFeed(id){SEL=id;loadFeeds();loadEps()}
|
||||||
|
async function loadEps(){
|
||||||
|
const el=document.getElementById("eps");const q=SEL?`?feed_id=${SEL}`:"";
|
||||||
|
document.getElementById("title").textContent=SEL?"Episodes":"Latest episodes";
|
||||||
|
let d; try{d=await api("/episodes"+q)}catch(e){return}
|
||||||
|
if(!d.episodes.length){el.innerHTML='<div class="empty">No episodes.</div>';return}
|
||||||
|
el.innerHTML=d.episodes.map(e=>{
|
||||||
|
const date=e.pubdate?new Date(e.pubdate*1000).toLocaleDateString():"";
|
||||||
|
let act;
|
||||||
|
if(e.state==="done") act=`<audio controls preload="none" src="${API}/media/${e.id}"></audio>`;
|
||||||
|
else if(e.state==="downloading"||e.state==="queued") act=`<span class="badge downloading">${e.state}</span><div class="bar"><i style="width:${e.progress||0}%"></i></div>`;
|
||||||
|
else if(e.state==="error") act=`<span class="badge error" title="${esc(e.error)}">error</span><button onclick="dl(${e.id})">retry</button>`;
|
||||||
|
else act=`<button onclick="dl(${e.id})">⬇ download</button>`;
|
||||||
|
return `<div class="ep"><div class="meta"><b>${esc(e.title)}</b><span>${esc(e.feed_title)} · ${date} ${e.duration?'· '+esc(e.duration):''}</span></div><div class="act">${act}</div></div>`;
|
||||||
|
}).join("");
|
||||||
|
}
|
||||||
|
async function addFeed(){const u=document.getElementById("feedUrl").value.trim();if(!u)return;
|
||||||
|
const ad=document.getElementById("autodl").checked;
|
||||||
|
try{const r=await api("/feeds",{method:"POST",body:JSON.stringify({url:u,auto_dl:ad})});
|
||||||
|
document.getElementById("feedUrl").value="";
|
||||||
|
toast(`Added ${r.title||u} (${r.episodes} eps${r.queued?', '+r.queued+' queued':''})`);loadAll()}
|
||||||
|
catch(e){toast("Add failed: "+e.message.slice(0,60))}}
|
||||||
|
async function toggleAd(id,on){try{const r=await api(`/feeds/${id}/autodl?on=${on?'true':'false'}`,{method:"POST"});
|
||||||
|
toast(on?`auto-download on${r.queued?' · '+r.queued+' queued':''}`:"auto-download off");loadAll()}catch(e){toast("toggle failed")}}
|
||||||
|
async function delFeed(id){if(!confirm("Remove this feed?"))return;await api("/feeds/"+id,{method:"DELETE"});if(SEL===id)SEL=null;loadAll()}
|
||||||
|
async function dl(id){try{await api(`/episodes/${id}/download`,{method:"POST"});toast("Queued");loadEps()}catch(e){toast("Queue failed")}}
|
||||||
|
async function importOpml(inp){const f=inp.files[0];if(!f)return;const opml=await f.text();
|
||||||
|
try{const r=await api("/feeds/import-opml",{method:"POST",body:JSON.stringify({opml})});toast(`Imported ${r.added}/${r.total}`);loadAll()}catch(e){toast("OPML import failed")}inp.value=""}
|
||||||
|
async function uploadZip(inp){const f=inp.files[0];if(!f)return;
|
||||||
|
const title=prompt("Audiobook title:",f.name.replace(/\.zip$/i,""));if(title===null){inp.value="";return}
|
||||||
|
toast("Uploading "+f.name+" …");
|
||||||
|
try{const r=await api("/audiobook/upload?title="+encodeURIComponent(title),
|
||||||
|
{method:"POST",headers:{"Content-Type":"application/zip"},body:f});
|
||||||
|
toast(`Published "${r.title}" — ${r.tracks} tracks`);loadAll()}
|
||||||
|
catch(e){toast("Upload failed: "+e.message.slice(0,60))}inp.value=""}
|
||||||
|
async function openCfg(){const d=await api("/config").then(r=>r.config).catch(()=>null);
|
||||||
|
const s=await fetch(API+"/status").then(r=>r.json()).catch(()=>({}));
|
||||||
|
document.getElementById("svc").innerHTML=`service <b style="color:var(--matrix-green)">${s.service||'?'}</b> · worker ${s.worker?'on':'off'} · ${s.downloaded||0} local`;
|
||||||
|
if(d){c_media.value=d.media_path||"";c_par.value=d.max_parallel||2;c_ref.value=d.refresh_minutes||60;c_pub.value=d.public_base||"";c_title.value=d.share_title||""}
|
||||||
|
document.getElementById("cfg").showModal()}
|
||||||
|
async function saveCfg(){try{await api("/config",{method:"POST",body:JSON.stringify({media_path:c_media.value,max_parallel:+c_par.value,refresh_minutes:+c_ref.value,public_base:c_pub.value,share_title:c_title.value})});toast("Saved — restart to apply");document.getElementById("cfg").close()}catch(e){toast("Save failed")}}
|
||||||
|
function loadAll(){loadStatus();loadFeeds();loadEps()}
|
||||||
|
loadAll();setInterval(()=>{loadStatus();loadEps()},4000);
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
107
packages/secubox-podcaster/www/podcaster/portal/index.html
Normal file
107
packages/secubox-podcaster/www/podcaster/portal/index.html
Normal file
|
|
@ -0,0 +1,107 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<!-- SPDX-License-Identifier: LicenseRef-CMSD-1.0 -->
|
||||||
|
<!-- SecuBox-Deb :: Podcaster PUBLIC portal (no auth) — CyberMind https://cybermind.fr -->
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>Podcasts · SecuBox</title>
|
||||||
|
<style>
|
||||||
|
:root{
|
||||||
|
--cosmos-black:#0a0a0f; --gold-hermetic:#c9a84c; --void-purple:#6e40c9;
|
||||||
|
--cyber-cyan:#00d4ff; --matrix-green:#00ff41; --text-primary:#e8e6d9;
|
||||||
|
--text-muted:#6b6b7a; --panel:#13131c; --line:#23232f;
|
||||||
|
}
|
||||||
|
*{box-sizing:border-box}
|
||||||
|
body{margin:0;background:radial-gradient(1200px 600px at 50% -10%,#15101f,#0a0a0f 60%);
|
||||||
|
color:var(--text-primary);font-family:'JetBrains Mono',ui-monospace,Menlo,monospace}
|
||||||
|
header{max-width:1000px;margin:0 auto;padding:38px 20px 18px;text-align:center}
|
||||||
|
header .em{font-size:42px}
|
||||||
|
header h1{margin:8px 0 4px;font-size:30px;letter-spacing:2px;color:var(--gold-hermetic)}
|
||||||
|
header p{color:var(--text-muted);margin:0}
|
||||||
|
.subs{display:inline-flex;gap:10px;margin-top:16px;flex-wrap:wrap;justify-content:center}
|
||||||
|
.subs a{color:var(--cyber-cyan);text-decoration:none;border:1px solid var(--cyber-cyan);
|
||||||
|
padding:8px 14px;border-radius:24px;font-size:13px}
|
||||||
|
.subs a:hover{background:rgba(0,212,255,.1)}
|
||||||
|
.filters{max-width:1000px;margin:18px auto 0;padding:0 20px;display:flex;gap:8px;flex-wrap:wrap}
|
||||||
|
.chip{font-size:12px;padding:6px 12px;border-radius:20px;border:1px solid var(--line);
|
||||||
|
color:var(--text-muted);cursor:pointer}
|
||||||
|
.chip.on{color:var(--void-purple);border-color:var(--void-purple);background:rgba(110,64,201,.12)}
|
||||||
|
main{max-width:1000px;margin:18px auto 60px;padding:0 20px}
|
||||||
|
.ep{display:flex;gap:14px;align-items:center;padding:14px;border:1px solid var(--line);
|
||||||
|
border-radius:12px;margin-bottom:12px;background:var(--panel)}
|
||||||
|
.ep .n{font-size:20px;color:var(--void-purple);width:30px;text-align:center}
|
||||||
|
.ep .meta{flex:1;min-width:0}
|
||||||
|
.ep .meta b{display:block;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;font-size:15px}
|
||||||
|
.ep .meta span{color:var(--text-muted);font-size:12px}
|
||||||
|
audio{height:38px;max-width:300px}
|
||||||
|
.ep a.dl{color:var(--matrix-green);text-decoration:none;border:1px solid var(--matrix-green);
|
||||||
|
border-radius:8px;padding:7px 10px;font-size:13px;white-space:nowrap}
|
||||||
|
.ep a.dl:hover{background:rgba(0,255,65,.1)}
|
||||||
|
.zipbar{max-width:1000px;margin:14px auto 0;padding:0 20px}
|
||||||
|
.zipbar a{color:var(--gold-hermetic);text-decoration:none;border:1px solid var(--gold-hermetic);
|
||||||
|
border-radius:24px;padding:8px 16px;font-size:13px}
|
||||||
|
.zipbar a:hover{background:rgba(201,168,76,.12)}
|
||||||
|
.empty{text-align:center;color:var(--text-muted);padding:60px}
|
||||||
|
footer{text-align:center;color:var(--text-muted);font-size:11px;padding:24px}
|
||||||
|
footer a{color:var(--text-muted)}
|
||||||
|
@media(max-width:680px){.ep{flex-wrap:wrap}audio{max-width:100%;width:100%}}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<header>
|
||||||
|
<div class="em">🎙️</div>
|
||||||
|
<h1 id="title">Podcasts</h1>
|
||||||
|
<p id="sub">Relayed locally by SecuBox · listen freely</p>
|
||||||
|
<div class="subs">
|
||||||
|
<a id="rss" href="/api/v1/podcaster/share/feed.xml" target="_blank">📡 Subscribe (RSS)</a>
|
||||||
|
<a id="copy" href="#" onclick="copyRss(event)">🔗 Copy feed URL</a>
|
||||||
|
</div>
|
||||||
|
</header>
|
||||||
|
<div class="filters" id="filters"></div>
|
||||||
|
<div class="zipbar" id="zipbar"></div>
|
||||||
|
<main><div id="list" class="empty">Loading…</div></main>
|
||||||
|
<footer>SecuBox Podcaster · <a href="https://cybermind.fr" target="_blank">CyberMind</a></footer>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
const API="/api/v1/podcaster";
|
||||||
|
let DATA={episodes:[],feeds:{}}, FILT=null;
|
||||||
|
function esc(s){return (s||"").replace(/[&<>"]/g,c=>({"&":"&","<":"<",">":">",'"':"""}[c]))}
|
||||||
|
function rssUrl(){return location.origin+API+"/share/feed.xml"}
|
||||||
|
function copyRss(e){e.preventDefault();navigator.clipboard.writeText(rssUrl()).then(()=>{
|
||||||
|
const a=document.getElementById("copy");a.textContent="✓ Copied";setTimeout(()=>a.textContent="🔗 Copy feed URL",1800)})}
|
||||||
|
async function load(){
|
||||||
|
let d; try{d=await fetch(API+"/public/library").then(r=>r.json())}
|
||||||
|
catch(e){document.getElementById("list").innerHTML='<div class="empty">Portal unavailable.</div>';return}
|
||||||
|
DATA=d;
|
||||||
|
document.getElementById("title").textContent=d.title||"Podcasts";
|
||||||
|
document.getElementById("rss").href=API+"/share/feed.xml";
|
||||||
|
const fl=document.getElementById("filters");
|
||||||
|
const feeds=Object.keys(d.feeds||{});
|
||||||
|
fl.innerHTML=(feeds.length>1?[`<span class="chip ${FILT===null?'on':''}" onclick="setF(null)">All</span>`]:[])
|
||||||
|
.concat(feeds.map(f=>`<span class="chip ${FILT===f?'on':''}" onclick="setF('${esc(f).replace(/'/g,"")}')">${esc(f)} · ${d.feeds[f]}</span>`)).join("");
|
||||||
|
render();
|
||||||
|
}
|
||||||
|
function setF(f){FILT=f;load._render?render():render()}
|
||||||
|
function render(){
|
||||||
|
const list=document.getElementById("list");
|
||||||
|
let eps=DATA.episodes||[];
|
||||||
|
if(FILT) eps=eps.filter(e=>e.feed===FILT);
|
||||||
|
// ZIP-all button when a single feed is in view
|
||||||
|
const zb=document.getElementById("zipbar");
|
||||||
|
const fid=eps.length?eps[0].feed_id:null;
|
||||||
|
zb.innerHTML=(FILT&&fid!=null)
|
||||||
|
? `<a href="${API}/public/feed/${fid}/zip">⬇ Download all (ZIP) · ${esc(FILT)}</a>` : "";
|
||||||
|
if(!eps.length){list.innerHTML='<div class="empty">No episodes published yet.</div>';return}
|
||||||
|
list.innerHTML=eps.map((e,i)=>{
|
||||||
|
const date=e.pubdate?new Date(e.pubdate*1000).toLocaleDateString():"";
|
||||||
|
return `<div class="ep"><div class="n">${i+1}</div>
|
||||||
|
<div class="meta"><b>${esc(e.title)}</b><span>${esc(e.feed)} · ${date} ${e.duration?'· '+esc(e.duration):''}</span></div>
|
||||||
|
<audio controls preload="none" src="${API}/media/${e.id}"></audio>
|
||||||
|
<a class="dl" href="${API}/media/${e.id}" download title="Download">⬇</a></div>`;
|
||||||
|
}).join("");
|
||||||
|
}
|
||||||
|
load();setInterval(load,15000);
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
0
packages/secubox-threatmesh/api/__init__.py
Normal file
0
packages/secubox-threatmesh/api/__init__.py
Normal file
267
packages/secubox-threatmesh/api/main.py
Normal file
267
packages/secubox-threatmesh/api/main.py
Normal file
|
|
@ -0,0 +1,267 @@
|
||||||
|
# SPDX-License-Identifier: LicenseRef-CMSD-1.0
|
||||||
|
# Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
|
||||||
|
# Source-Disclosed License — All rights reserved except as expressly granted.
|
||||||
|
# See LICENCE-CMSD-1.0.md for terms.
|
||||||
|
"""
|
||||||
|
SecuBox-Deb :: secubox-threatmesh (#728)
|
||||||
|
CyberMind — https://cybermind.fr
|
||||||
|
|
||||||
|
Sovereign threat-intel control plane that replaces CrowdSec CAPI:
|
||||||
|
|
||||||
|
Phase 1 free public feeds (secubox-threatfeed timer) -> threat_intel
|
||||||
|
Phase 2 MESH distribution of LOCAL decisions over the SecuBox P2P mesh
|
||||||
|
(gossip to peers from secubox-p2p; ingest peer decisions) -> threat_intel
|
||||||
|
Phase 3 status / aggregate / CrowdSec-bouncer-compatible decisions API + UI
|
||||||
|
|
||||||
|
All IOCs land in the shared `threat_intel` table (toolbox.db); secubox-blacklist-sync
|
||||||
|
drains it to the nft blacklist. No central account, no CAPI, no paywall.
|
||||||
|
"""
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import sqlite3
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
import urllib.request
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from fastapi import FastAPI, APIRouter, Depends, Request
|
||||||
|
from fastapi.responses import JSONResponse
|
||||||
|
|
||||||
|
from secubox_core.auth import require_jwt
|
||||||
|
from secubox_core.logger import get_logger
|
||||||
|
|
||||||
|
log = get_logger("threatmesh")
|
||||||
|
|
||||||
|
TI_DB = Path("/var/lib/secubox/toolbox/toolbox.db")
|
||||||
|
P2P_PEERS = Path("/var/lib/secubox/p2p/peers.json")
|
||||||
|
P2P_NODE = Path("/var/lib/secubox/p2p/node_id")
|
||||||
|
MESH_INTERVAL = 600 # gossip every 10 min
|
||||||
|
LOCAL_ORIGINS = {"crowdsec", "cscli", "manual", "secubox-waf", "secubox"}
|
||||||
|
IPV4 = re.compile(r"^\d{1,3}(\.\d{1,3}){3}(/\d{1,2})?$")
|
||||||
|
|
||||||
|
app = FastAPI(title="secubox-threatmesh", version="1.0.0", root_path="/api/v1/threatmesh")
|
||||||
|
router = APIRouter()
|
||||||
|
_node_id = None
|
||||||
|
|
||||||
|
|
||||||
|
# ── store ───────────────────────────────────────────────────────────
|
||||||
|
def _conn():
|
||||||
|
c = sqlite3.connect(TI_DB, timeout=20)
|
||||||
|
c.row_factory = sqlite3.Row
|
||||||
|
return c
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure(c):
|
||||||
|
c.execute("""CREATE TABLE IF NOT EXISTS threat_intel (
|
||||||
|
ioc TEXT NOT NULL, ioc_type TEXT NOT NULL, source TEXT NOT NULL,
|
||||||
|
weight INTEGER NOT NULL DEFAULT 50, label TEXT,
|
||||||
|
first_seen INTEGER, last_seen INTEGER,
|
||||||
|
PRIMARY KEY (ioc, ioc_type, source))""")
|
||||||
|
|
||||||
|
|
||||||
|
def node_id() -> str:
|
||||||
|
global _node_id
|
||||||
|
if _node_id:
|
||||||
|
return _node_id
|
||||||
|
try:
|
||||||
|
_node_id = P2P_NODE.read_text().strip()
|
||||||
|
except Exception:
|
||||||
|
try:
|
||||||
|
_node_id = json.loads(P2P_PEERS.read_text())["peers"][0]["id"]
|
||||||
|
except Exception:
|
||||||
|
_node_id = "sb-local"
|
||||||
|
return _node_id
|
||||||
|
|
||||||
|
|
||||||
|
def upsert_iocs(rows, source, weight=60, label=None):
|
||||||
|
"""rows: iterable of (ioc, ioc_type). Returns count."""
|
||||||
|
now = int(time.time())
|
||||||
|
n = 0
|
||||||
|
with _conn() as c:
|
||||||
|
_ensure(c)
|
||||||
|
for ioc, t in rows:
|
||||||
|
c.execute(
|
||||||
|
"INSERT INTO threat_intel(ioc,ioc_type,source,weight,label,first_seen,last_seen) "
|
||||||
|
"VALUES(?,?,?,?,?,?,?) ON CONFLICT(ioc,ioc_type,source) DO UPDATE SET last_seen=excluded.last_seen",
|
||||||
|
(ioc, t, source, weight, label, now, now))
|
||||||
|
n += 1
|
||||||
|
c.commit()
|
||||||
|
return n
|
||||||
|
|
||||||
|
|
||||||
|
def counts_by_source():
|
||||||
|
with _conn() as c:
|
||||||
|
_ensure(c)
|
||||||
|
return {r["source"]: r["n"] for r in c.execute(
|
||||||
|
"SELECT source, COUNT(*) n FROM threat_intel GROUP BY source ORDER BY n DESC")}
|
||||||
|
|
||||||
|
|
||||||
|
def aggregate_ips(limit=50000):
|
||||||
|
"""Distinct bad IPs across ALL sources, with a consensus count."""
|
||||||
|
with _conn() as c:
|
||||||
|
_ensure(c)
|
||||||
|
return [dict(r) for r in c.execute(
|
||||||
|
"SELECT ioc, COUNT(DISTINCT source) consensus, MAX(weight) weight, "
|
||||||
|
"GROUP_CONCAT(DISTINCT source) sources FROM threat_intel "
|
||||||
|
"WHERE ioc_type='ip' GROUP BY ioc ORDER BY consensus DESC LIMIT ?", (limit,))]
|
||||||
|
|
||||||
|
|
||||||
|
# ── mesh (Phase 2) ──────────────────────────────────────────────────
|
||||||
|
def mesh_peers():
|
||||||
|
"""Other SecuBox nodes from secubox-p2p (skip self)."""
|
||||||
|
me = node_id()
|
||||||
|
out = []
|
||||||
|
try:
|
||||||
|
d = json.loads(P2P_PEERS.read_text())
|
||||||
|
for p in d.get("peers", []):
|
||||||
|
if p.get("id") == me:
|
||||||
|
continue
|
||||||
|
addr = None
|
||||||
|
for a in p.get("addresses", []):
|
||||||
|
if a.get("type") in ("wg", "wireguard", "mesh"):
|
||||||
|
addr = a.get("address"); break
|
||||||
|
addr = addr or (p.get("wg_addresses") or [None])[0] or p.get("address")
|
||||||
|
if addr:
|
||||||
|
out.append({"id": p.get("id"), "name": p.get("name"), "address": addr})
|
||||||
|
except Exception as e:
|
||||||
|
log.debug(f"peers read: {e}")
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def local_decisions():
|
||||||
|
"""Our OWN locally-detected bad IPs (CrowdSec LAPI bans of local origin)."""
|
||||||
|
try:
|
||||||
|
out = subprocess.run(["cscli", "decisions", "list", "-o", "json", "-a"],
|
||||||
|
capture_output=True, text=True, timeout=20)
|
||||||
|
data = json.loads(out.stdout or "[]") or []
|
||||||
|
except Exception as e:
|
||||||
|
log.debug(f"cscli decisions: {e}")
|
||||||
|
return []
|
||||||
|
ips = []
|
||||||
|
for alert in data:
|
||||||
|
for dec in (alert.get("decisions") or []):
|
||||||
|
if dec.get("type") == "ban" and dec.get("scope", "").lower() == "ip":
|
||||||
|
origin = (dec.get("origin") or "").lower()
|
||||||
|
val = dec.get("value", "")
|
||||||
|
if IPV4.match(val) and not origin.startswith(("feed:", "mesh:", "lists:")):
|
||||||
|
ips.append(val)
|
||||||
|
return sorted(set(ips))
|
||||||
|
|
||||||
|
|
||||||
|
async def _post_json(url, payload, timeout=15):
|
||||||
|
def _do():
|
||||||
|
req = urllib.request.Request(
|
||||||
|
url, data=json.dumps(payload).encode(),
|
||||||
|
headers={"Content-Type": "application/json", "User-Agent": "SecuBox-ThreatMesh/1.0"},
|
||||||
|
method="POST")
|
||||||
|
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||||
|
return r.status
|
||||||
|
return await asyncio.to_thread(_do)
|
||||||
|
|
||||||
|
|
||||||
|
async def mesh_gossip_once():
|
||||||
|
peers = mesh_peers()
|
||||||
|
decisions = local_decisions()
|
||||||
|
if not peers or not decisions:
|
||||||
|
return {"peers": len(peers), "shared": len(decisions), "pushed": 0}
|
||||||
|
payload = {"node": node_id(), "ts": int(time.time()), "ips": decisions}
|
||||||
|
pushed = 0
|
||||||
|
for p in peers:
|
||||||
|
url = f"http://{p['address']}:8780/api/v1/threatmesh/mesh/ingest"
|
||||||
|
try:
|
||||||
|
await _post_json(url, payload)
|
||||||
|
pushed += 1
|
||||||
|
except Exception as e:
|
||||||
|
log.debug(f"gossip to {p['id']} failed: {e}")
|
||||||
|
return {"peers": len(peers), "shared": len(decisions), "pushed": pushed}
|
||||||
|
|
||||||
|
|
||||||
|
async def _mesh_loop():
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
r = await mesh_gossip_once()
|
||||||
|
log.info(f"mesh gossip: {r}")
|
||||||
|
except Exception as e:
|
||||||
|
log.error(f"mesh loop: {e}")
|
||||||
|
await asyncio.sleep(MESH_INTERVAL)
|
||||||
|
|
||||||
|
|
||||||
|
@app.on_event("startup")
|
||||||
|
async def _startup():
|
||||||
|
asyncio.create_task(_mesh_loop())
|
||||||
|
|
||||||
|
|
||||||
|
# ── endpoints ───────────────────────────────────────────────────────
|
||||||
|
@app.get("/health")
|
||||||
|
async def health():
|
||||||
|
return {"status": "ok", "module": "deb"}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/status")
|
||||||
|
async def status():
|
||||||
|
src = counts_by_source()
|
||||||
|
feeds = {k: v for k, v in src.items() if k.startswith("feed:")}
|
||||||
|
mesh = {k: v for k, v in src.items() if k.startswith("mesh:")}
|
||||||
|
return {
|
||||||
|
"mode": "sovereign", # CAPI dropped (#728)
|
||||||
|
"node": node_id(),
|
||||||
|
"capi": False,
|
||||||
|
"feeds": feeds, "feed_total": sum(feeds.values()),
|
||||||
|
"mesh": mesh, "mesh_total": sum(mesh.values()),
|
||||||
|
"peers": len(mesh_peers()),
|
||||||
|
"sources": src,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/peers")
|
||||||
|
async def peers():
|
||||||
|
return {"node": node_id(), "peers": mesh_peers()}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/decisions")
|
||||||
|
async def decisions(min_consensus: int = 1, limit: int = 50000):
|
||||||
|
"""Aggregated sovereign blocklist (feeds + mesh + local). CrowdSec-bouncer
|
||||||
|
friendly shape so external consumers can poll OUR server, not crowdsec.net."""
|
||||||
|
rows = [r for r in aggregate_ips(limit) if r["consensus"] >= min_consensus]
|
||||||
|
return [{
|
||||||
|
"id": i, "origin": "secubox-threatmesh", "type": "ban", "scope": "Ip",
|
||||||
|
"value": r["ioc"], "duration": "24h",
|
||||||
|
"consensus": r["consensus"], "sources": r["sources"],
|
||||||
|
} for i, r in enumerate(rows)]
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/mesh/ingest")
|
||||||
|
async def mesh_ingest(request: Request):
|
||||||
|
"""Receive a peer's locally-detected decisions over the mesh -> threat_intel
|
||||||
|
tagged mesh:<node>. Trust boundary = the WireGuard mesh transport."""
|
||||||
|
try:
|
||||||
|
body = await request.json()
|
||||||
|
except Exception:
|
||||||
|
return JSONResponse({"ok": False, "error": "bad json"}, status_code=400)
|
||||||
|
node = str(body.get("node", "")).strip()[:64] or "unknown"
|
||||||
|
if not re.match(r"^[A-Za-z0-9._:-]+$", node):
|
||||||
|
return JSONResponse({"ok": False, "error": "bad node"}, status_code=400)
|
||||||
|
ips = [ip for ip in (body.get("ips") or []) if isinstance(ip, str) and IPV4.match(ip)]
|
||||||
|
n = upsert_iocs(((ip, "ip") for ip in ips[:20000]), f"mesh:{node}", weight=70, label="mesh")
|
||||||
|
log.info(f"mesh ingest from {node}: {n} ips")
|
||||||
|
return {"ok": True, "ingested": n}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/feeds/refresh", dependencies=[Depends(require_jwt)])
|
||||||
|
async def feeds_refresh():
|
||||||
|
"""Trigger the feed fetcher now (normally on a timer)."""
|
||||||
|
try:
|
||||||
|
subprocess.Popen(["/usr/sbin/secubox-threatfeed"])
|
||||||
|
return {"ok": True, "started": True}
|
||||||
|
except Exception as e:
|
||||||
|
return JSONResponse({"ok": False, "error": str(e)}, status_code=500)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/mesh/gossip", dependencies=[Depends(require_jwt)])
|
||||||
|
async def mesh_gossip_now():
|
||||||
|
return await mesh_gossip_once()
|
||||||
|
|
||||||
|
|
||||||
|
app.include_router(router)
|
||||||
13
packages/secubox-threatmesh/debian/changelog
Normal file
13
packages/secubox-threatmesh/debian/changelog
Normal file
|
|
@ -0,0 +1,13 @@
|
||||||
|
secubox-threatmesh (1.0.0-1~bookworm1) bookworm; urgency=medium
|
||||||
|
|
||||||
|
* Initial release (#728) — sovereign threat-intel, CrowdSec CAPI replacement.
|
||||||
|
- Phase 1: secubox-threatfeed timer pulls free public blocklists
|
||||||
|
(feodo/sslbl/firehol/spamhaus-drop/blocklist.de/cins/et/dshield) into the
|
||||||
|
shared threat_intel table -> secubox-blacklist-sync -> nft.
|
||||||
|
- Phase 2: MESH distribution — gossip locally-detected CrowdSec decisions to
|
||||||
|
SecuBox P2P peers over WireGuard; ingest peer decisions (tagged mesh:<node>),
|
||||||
|
consensus-counted. :8780 locked to wg*/loopback by an nft drop-in.
|
||||||
|
- Phase 3: /status, /peers, /decisions (bouncer-compatible aggregate),
|
||||||
|
/mesh/ingest, dashboard. CrowdSec LAPI kept; CAPI dropped.
|
||||||
|
|
||||||
|
-- Gerald KERMA <devel@cybermind.fr> Wed, 24 Jun 2026 16:00:00 +0000
|
||||||
18
packages/secubox-threatmesh/debian/control
Normal file
18
packages/secubox-threatmesh/debian/control
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
Source: secubox-threatmesh
|
||||||
|
Section: net
|
||||||
|
Priority: optional
|
||||||
|
Maintainer: Gerald KERMA <devel@cybermind.fr>
|
||||||
|
Build-Depends: debhelper-compat (= 13)
|
||||||
|
Standards-Version: 4.6.2
|
||||||
|
|
||||||
|
Package: secubox-threatmesh
|
||||||
|
Architecture: all
|
||||||
|
Depends: ${misc:Depends}, secubox-core (>= 1.0), python3, python3-uvicorn | python3-pip, secubox-vortex-firewall | secubox-toolbox
|
||||||
|
Recommends: secubox-p2p, crowdsec
|
||||||
|
Description: Sovereign threat-intel mesh for SecuBox (CrowdSec CAPI replacement)
|
||||||
|
Replaces the CrowdSec Central API with self-sourced free public blocklists
|
||||||
|
(Phase 1) and peer-to-peer MESH distribution of locally-detected decisions over
|
||||||
|
the SecuBox WireGuard mesh (Phase 2), plus a status/aggregate/bouncer-compatible
|
||||||
|
API and dashboard (Phase 3). All IOCs flow into the shared threat_intel table
|
||||||
|
that secubox-blacklist-sync drains into the nft blacklist. No central account,
|
||||||
|
no enrollment, no paywall, no IP-blocklisting by a third party.
|
||||||
23
packages/secubox-threatmesh/debian/postinst
Normal file
23
packages/secubox-threatmesh/debian/postinst
Normal file
|
|
@ -0,0 +1,23 @@
|
||||||
|
#!/bin/bash
|
||||||
|
set -e
|
||||||
|
case "$1" in
|
||||||
|
configure)
|
||||||
|
install -d -m 0755 /var/lib/secubox/threatmesh
|
||||||
|
systemctl daemon-reload 2>/dev/null || true
|
||||||
|
# mesh-port firewall (lock :8780 to wg*/lo under DEFAULT DROP)
|
||||||
|
if [ -f /usr/share/secubox/threatmesh/nftables.d/secubox-threatmesh.nft ]; then
|
||||||
|
nft -f /usr/share/secubox/threatmesh/nftables.d/secubox-threatmesh.nft 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
if [ "$(systemctl is-enabled secubox-threatmesh.service 2>/dev/null)" != "masked" ]; then
|
||||||
|
systemctl enable --now secubox-threatmesh.service 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
systemctl enable --now secubox-threatfeed.timer 2>/dev/null || true
|
||||||
|
# first feed population (background; logs to /tmp to avoid /var/log perms)
|
||||||
|
/usr/sbin/secubox-threatfeed >/tmp/secubox-threatfeed-firstrun.log 2>&1 &
|
||||||
|
if command -v nginx >/dev/null 2>&1 && nginx -t 2>/dev/null; then
|
||||||
|
systemctl reload nginx 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
#DEBHELPER#
|
||||||
|
exit 0
|
||||||
11
packages/secubox-threatmesh/debian/prerm
Normal file
11
packages/secubox-threatmesh/debian/prerm
Normal file
|
|
@ -0,0 +1,11 @@
|
||||||
|
#!/bin/bash
|
||||||
|
set -e
|
||||||
|
case "$1" in
|
||||||
|
remove|deconfigure)
|
||||||
|
systemctl stop secubox-threatmesh.service 2>/dev/null || true
|
||||||
|
systemctl disable secubox-threatmesh.service secubox-threatfeed.timer 2>/dev/null || true
|
||||||
|
nft delete table inet secubox_threatmesh 2>/dev/null || true
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
#DEBHELPER#
|
||||||
|
exit 0
|
||||||
23
packages/secubox-threatmesh/debian/rules
Executable file
23
packages/secubox-threatmesh/debian/rules
Executable file
|
|
@ -0,0 +1,23 @@
|
||||||
|
#!/usr/bin/make -f
|
||||||
|
%:
|
||||||
|
dh $@
|
||||||
|
|
||||||
|
override_dh_auto_install:
|
||||||
|
install -d debian/secubox-threatmesh/usr/lib/secubox/threatmesh/api
|
||||||
|
cp -r api/. debian/secubox-threatmesh/usr/lib/secubox/threatmesh/api/
|
||||||
|
install -d debian/secubox-threatmesh/usr/sbin
|
||||||
|
install -m 755 sbin/secubox-threatfeed debian/secubox-threatmesh/usr/sbin/
|
||||||
|
install -d debian/secubox-threatmesh/usr/lib/systemd/system
|
||||||
|
cp systemd/*.service systemd/*.timer debian/secubox-threatmesh/usr/lib/systemd/system/
|
||||||
|
install -d debian/secubox-threatmesh/usr/share/secubox/www
|
||||||
|
cp -r www/. debian/secubox-threatmesh/usr/share/secubox/www/
|
||||||
|
install -d debian/secubox-threatmesh/usr/share/secubox/menu.d
|
||||||
|
cp menu.d/. -r debian/secubox-threatmesh/usr/share/secubox/menu.d/
|
||||||
|
install -d debian/secubox-threatmesh/etc/nginx/secubox-routes.d debian/secubox-threatmesh/etc/nginx/secubox.d
|
||||||
|
cp nginx/threatmesh.conf debian/secubox-threatmesh/etc/nginx/secubox-routes.d/
|
||||||
|
cp nginx/threatmesh.conf debian/secubox-threatmesh/etc/nginx/secubox.d/
|
||||||
|
install -d debian/secubox-threatmesh/usr/share/secubox/threatmesh/nftables.d
|
||||||
|
cp nftables.d/secubox-threatmesh.nft debian/secubox-threatmesh/usr/share/secubox/threatmesh/nftables.d/
|
||||||
|
|
||||||
|
override_dh_installsystemd:
|
||||||
|
true
|
||||||
1
packages/secubox-threatmesh/debian/source/format
Normal file
1
packages/secubox-threatmesh/debian/source/format
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
3.0 (native)
|
||||||
9
packages/secubox-threatmesh/menu.d/37-threatmesh.json
Normal file
9
packages/secubox-threatmesh/menu.d/37-threatmesh.json
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
{
|
||||||
|
"id": "threatmesh",
|
||||||
|
"name": "ThreatMesh",
|
||||||
|
"icon": "🛰️",
|
||||||
|
"path": "/threatmesh/",
|
||||||
|
"category": "security",
|
||||||
|
"order": 37,
|
||||||
|
"description": "Sovereign threat-intel (feeds + mesh, no CAPI)"
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,10 @@
|
||||||
|
# ThreatMesh peer port (#728): only reachable over the WireGuard mesh + loopback.
|
||||||
|
# The base inet filter INPUT chain is DEFAULT DROP, so this only OPENS 8780 to wg.
|
||||||
|
table inet secubox_threatmesh {
|
||||||
|
chain input {
|
||||||
|
type filter hook input priority 0; policy accept;
|
||||||
|
iifname "lo" tcp dport 8780 accept
|
||||||
|
iifname "wg*" tcp dport 8780 accept
|
||||||
|
tcp dport 8780 drop
|
||||||
|
}
|
||||||
|
}
|
||||||
10
packages/secubox-threatmesh/nginx/threatmesh.conf
Normal file
10
packages/secubox-threatmesh/nginx/threatmesh.conf
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
# /etc/nginx/secubox-routes.d/threatmesh.conf — secubox-threatmesh (#728)
|
||||||
|
location /threatmesh/ {
|
||||||
|
alias /usr/share/secubox/www/threatmesh/;
|
||||||
|
index index.html;
|
||||||
|
try_files $uri $uri/ /threatmesh/index.html;
|
||||||
|
}
|
||||||
|
location /api/v1/threatmesh/ {
|
||||||
|
proxy_pass http://127.0.0.1:8780/api/v1/threatmesh/;
|
||||||
|
include /etc/nginx/snippets/secubox-proxy.conf;
|
||||||
|
}
|
||||||
121
packages/secubox-threatmesh/sbin/secubox-threatfeed
Normal file
121
packages/secubox-threatmesh/sbin/secubox-threatfeed
Normal file
|
|
@ -0,0 +1,121 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# SPDX-License-Identifier: LicenseRef-CMSD-1.0
|
||||||
|
# Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
|
||||||
|
# Source-Disclosed License — All rights reserved except as expressly granted.
|
||||||
|
# See LICENCE-CMSD-1.0.md for terms.
|
||||||
|
"""
|
||||||
|
SecuBox-Deb :: secubox-threatfeed (Phase 1 — sovereign threat-intel #728)
|
||||||
|
CyberMind — https://cybermind.fr
|
||||||
|
|
||||||
|
Pulls FREE, no-enrollment public IP/CIDR/domain blocklists and writes them into
|
||||||
|
the shared `threat_intel` table (toolbox.db) that secubox-blacklist-sync already
|
||||||
|
drains into the nft blacklist. This REPLACES the CrowdSec CAPI community feed
|
||||||
|
with self-sourced lists — no central account, no paywall, no IP blocklisting.
|
||||||
|
|
||||||
|
Pure stdlib (urllib) — no extra Debian deps. Idempotent, safe on a timer.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import urllib.request
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
DB = Path(os.environ.get("SECUBOX_TI_DB", "/var/lib/secubox/toolbox/toolbox.db"))
|
||||||
|
TIMEOUT = int(os.environ.get("SECUBOX_TF_TIMEOUT", "30"))
|
||||||
|
UA = "SecuBox-ThreatFeed/1.0 (+https://secubox.in)"
|
||||||
|
TTL_DAYS = int(os.environ.get("SECUBOX_TF_TTL_DAYS", "10"))
|
||||||
|
|
||||||
|
# (name, url, ioc_type, weight). All free + no enrollment. CIDRs allowed for ip.
|
||||||
|
FEEDS = [
|
||||||
|
("feodo", "https://feodotracker.abuse.ch/downloads/ipblocklist.txt", "ip", 90),
|
||||||
|
("sslbl", "https://sslbl.abuse.ch/blacklist/sslipblacklist.txt", "ip", 85),
|
||||||
|
("firehol1", "https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset", "ip", 80),
|
||||||
|
("spamhaus-drop", "https://www.spamhaus.org/drop/drop.txt", "ip", 85),
|
||||||
|
("blocklist-de", "https://lists.blocklist.de/lists/all.txt", "ip", 60),
|
||||||
|
("cins-army", "https://cinsscore.com/list/ci-badguys.txt", "ip", 70),
|
||||||
|
("et-compromised", "https://rules.emergingthreats.net/blockrules/compromised-ips.txt", "ip", 75),
|
||||||
|
("dshield", "https://feeds.dshield.org/block.txt", "ip", 65),
|
||||||
|
]
|
||||||
|
# Optional (set SECUBOX_TF_TOR=1 to also pull Tor exit nodes)
|
||||||
|
if os.environ.get("SECUBOX_TF_TOR") == "1":
|
||||||
|
FEEDS.append(("tor-exit", "https://check.torproject.org/torbulkexitlist", "ip", 40))
|
||||||
|
|
||||||
|
_IPV4 = re.compile(r"^\s*(\d{1,3}(?:\.\d{1,3}){3}(?:/\d{1,2})?)")
|
||||||
|
_IPV6 = re.compile(r"^\s*([0-9A-Fa-f:]{2,}(?:/\d{1,3})?)\s*$")
|
||||||
|
|
||||||
|
|
||||||
|
def log(msg):
|
||||||
|
print(f"[threatfeed] {msg}", file=sys.stderr, flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
def fetch(url):
|
||||||
|
req = urllib.request.Request(url, headers={"User-Agent": UA})
|
||||||
|
with urllib.request.urlopen(req, timeout=TIMEOUT) as r:
|
||||||
|
return r.read().decode("utf-8", "replace")
|
||||||
|
|
||||||
|
|
||||||
|
def parse_ips(text):
|
||||||
|
out = []
|
||||||
|
for line in text.splitlines():
|
||||||
|
s = line.strip()
|
||||||
|
if not s or s.startswith("#") or s.startswith(";"):
|
||||||
|
continue
|
||||||
|
# spamhaus DROP: "1.2.3.0/24 ; SBL..." -> take the first field
|
||||||
|
token = s.split(";")[0].split()[0].strip() if (";" in s or " " in s) else s
|
||||||
|
m = _IPV4.match(token)
|
||||||
|
if m:
|
||||||
|
out.append((m.group(1), "ip"))
|
||||||
|
continue
|
||||||
|
m = _IPV6.match(token)
|
||||||
|
if m and ":" in m.group(1):
|
||||||
|
out.append((m.group(1), "ip"))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_schema(c):
|
||||||
|
c.execute("""CREATE TABLE IF NOT EXISTS threat_intel (
|
||||||
|
ioc TEXT NOT NULL, ioc_type TEXT NOT NULL, source TEXT NOT NULL,
|
||||||
|
weight INTEGER NOT NULL DEFAULT 50, label TEXT,
|
||||||
|
first_seen INTEGER, last_seen INTEGER,
|
||||||
|
PRIMARY KEY (ioc, ioc_type, source))""")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
if not DB.exists():
|
||||||
|
log(f"db missing: {DB}")
|
||||||
|
return 1
|
||||||
|
now = int(time.time())
|
||||||
|
total = 0
|
||||||
|
with sqlite3.connect(DB, timeout=20) as c:
|
||||||
|
ensure_schema(c)
|
||||||
|
for name, url, ioc_type, weight in FEEDS:
|
||||||
|
src = f"feed:{name}"
|
||||||
|
try:
|
||||||
|
iocs = parse_ips(fetch(url))
|
||||||
|
except Exception as e:
|
||||||
|
log(f"{name}: fetch failed: {e}")
|
||||||
|
continue
|
||||||
|
n = 0
|
||||||
|
for ioc, t in iocs:
|
||||||
|
c.execute(
|
||||||
|
"INSERT INTO threat_intel(ioc,ioc_type,source,weight,label,first_seen,last_seen) "
|
||||||
|
"VALUES(?,?,?,?,?,?,?) "
|
||||||
|
"ON CONFLICT(ioc,ioc_type,source) DO UPDATE SET last_seen=excluded.last_seen, weight=excluded.weight",
|
||||||
|
(ioc, t, src, weight, name, now, now))
|
||||||
|
n += 1
|
||||||
|
log(f"{name}: {n} iocs")
|
||||||
|
total += n
|
||||||
|
# prune stale feed entries (older than TTL) — keeps the set fresh.
|
||||||
|
cutoff = now - TTL_DAYS * 86400
|
||||||
|
pruned = c.execute(
|
||||||
|
"DELETE FROM threat_intel WHERE source LIKE 'feed:%' AND last_seen < ?",
|
||||||
|
(cutoff,)).rowcount
|
||||||
|
c.commit()
|
||||||
|
log(f"done: {total} iocs across {len(FEEDS)} feeds, pruned {pruned} stale")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
[Unit]
|
||||||
|
Description=SecuBox ThreatFeed — pull free public blocklists into threat_intel (#728)
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
User=root
|
||||||
|
ExecStart=/usr/sbin/secubox-threatfeed
|
||||||
|
Nice=10
|
||||||
|
IOSchedulingClass=idle
|
||||||
|
TimeoutStartSec=600
|
||||||
11
packages/secubox-threatmesh/systemd/secubox-threatfeed.timer
Normal file
11
packages/secubox-threatmesh/systemd/secubox-threatfeed.timer
Normal file
|
|
@ -0,0 +1,11 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Refresh sovereign threat-intel feeds (#728)
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnBootSec=5min
|
||||||
|
OnUnitActiveSec=6h
|
||||||
|
RandomizedDelaySec=15min
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
|
|
@ -0,0 +1,22 @@
|
||||||
|
[Unit]
|
||||||
|
Description=SecuBox ThreatMesh — sovereign threat-intel mesh API (#728)
|
||||||
|
After=network.target secubox-runtime.service secubox-p2p.service
|
||||||
|
Wants=secubox-p2p.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=root
|
||||||
|
WorkingDirectory=/usr/lib/secubox/threatmesh
|
||||||
|
# Listens on :8780 — nginx proxies the dashboard; peers POST mesh decisions
|
||||||
|
# over the WireGuard mesh. nft drop-in restricts :8780 to wg* + loopback.
|
||||||
|
ExecStartPre=-/usr/sbin/nft -f /usr/share/secubox/threatmesh/nftables.d/secubox-threatmesh.nft
|
||||||
|
ExecStart=/usr/bin/python3 -m uvicorn api.main:app --host 0.0.0.0 --port 8780 --workers 1
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=5
|
||||||
|
NoNewPrivileges=no
|
||||||
|
ProtectSystem=full
|
||||||
|
PrivateTmp=true
|
||||||
|
ReadWritePaths=/run/secubox /var/lib/secubox /etc/secubox
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
87
packages/secubox-threatmesh/www/threatmesh/index.html
Normal file
87
packages/secubox-threatmesh/www/threatmesh/index.html
Normal file
|
|
@ -0,0 +1,87 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<!-- SPDX-License-Identifier: LicenseRef-CMSD-1.0 -->
|
||||||
|
<!-- SecuBox-Deb :: ThreatMesh — sovereign threat-intel (#728) — CyberMind -->
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>SecuBox · ThreatMesh</title>
|
||||||
|
<link rel="stylesheet" href="/shared/crt-light.css">
|
||||||
|
<link rel="stylesheet" href="/shared/sidebar-light.css">
|
||||||
|
<style>
|
||||||
|
:root{--cosmos-black:#0a0a0f;--gold:#c9a84c;--cinnabar:#e63946;--matrix:#00ff41;
|
||||||
|
--void:#6e40c9;--cyan:#00d4ff;--text:#e8e6d9;--muted:#6b6b7a;--panel:#13131c;--line:#23232f}
|
||||||
|
body{margin:0;background:var(--cosmos-black);color:var(--text);font-family:'JetBrains Mono',ui-monospace,monospace;display:flex}
|
||||||
|
.main{flex:1;margin-left:220px;min-height:100vh;padding:0 0 40px}
|
||||||
|
@media(max-width:900px){.sidebar{display:none}.main{margin-left:0}}
|
||||||
|
.topbar{display:flex;align-items:center;gap:14px;padding:14px 22px;border-bottom:1px solid var(--line);
|
||||||
|
background:linear-gradient(90deg,#0a0a0f,#13131c)}
|
||||||
|
.topbar h1{font-size:18px;margin:0;letter-spacing:1px;color:var(--gold)} .topbar .em{font-size:22px}
|
||||||
|
.pill{margin-left:auto;font-size:12px;padding:5px 12px;border-radius:20px;border:1px solid var(--matrix);color:var(--matrix)}
|
||||||
|
.wrap{padding:20px 22px;display:grid;gap:16px;grid-template-columns:repeat(auto-fit,minmax(260px,1fr))}
|
||||||
|
.card{background:var(--panel);border:1px solid var(--line);border-radius:12px;padding:16px}
|
||||||
|
.card h2{font-size:13px;color:var(--gold);margin:0 0 10px;letter-spacing:.5px}
|
||||||
|
.big{font-size:34px;color:var(--cyan);font-weight:700} .sub{color:var(--muted);font-size:12px}
|
||||||
|
.src{display:flex;justify-content:space-between;padding:4px 0;border-bottom:1px dashed var(--line);font-size:12px}
|
||||||
|
.src b{color:var(--cyan)} .src.mesh b{color:var(--void)}
|
||||||
|
.bar{display:flex;gap:8px;padding:0 22px 16px}
|
||||||
|
button{font-family:inherit;font-size:13px;background:#1a1a26;color:var(--text);border:1px solid var(--line);
|
||||||
|
border-radius:8px;padding:9px 14px;cursor:pointer} button:hover{border-color:var(--cyan);color:var(--cyan)}
|
||||||
|
button.go{background:var(--void);border-color:var(--void);color:#fff}
|
||||||
|
table{width:100%;border-collapse:collapse;font-size:12px} td,th{text-align:left;padding:5px 8px;border-bottom:1px solid var(--line)}
|
||||||
|
th{color:var(--muted)} .toast{position:fixed;bottom:18px;right:18px;background:#13131c;border:1px solid var(--cyan);
|
||||||
|
color:var(--cyan);padding:10px 14px;border-radius:8px;opacity:0;transition:.2s;font-size:12px}.toast.on{opacity:1}
|
||||||
|
.full{grid-column:1/-1}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<nav class="sidebar" id="sidebar"></nav>
|
||||||
|
<main class="main">
|
||||||
|
<div class="topbar"><span class="em">🛰️</span><h1>THREATMESH</h1>
|
||||||
|
<span class="pill" id="mode">sovereign · CAPI off</span></div>
|
||||||
|
<div class="bar">
|
||||||
|
<button class="go" onclick="refreshFeeds()">↻ Refresh feeds</button>
|
||||||
|
<button onclick="gossip()">🛰️ Gossip now</button>
|
||||||
|
<button onclick="load()">⟳</button>
|
||||||
|
</div>
|
||||||
|
<div class="wrap">
|
||||||
|
<div class="card"><h2>BLOCKED IPs (aggregate)</h2><div class="big" id="total">…</div><div class="sub">feeds + mesh + local → nft</div></div>
|
||||||
|
<div class="card"><h2>FREE FEEDS</h2><div class="big" id="feedtotal">…</div><div class="sub" id="feedn"></div></div>
|
||||||
|
<div class="card"><h2>MESH</h2><div class="big" id="meshtotal">…</div><div class="sub" id="peern"></div></div>
|
||||||
|
<div class="card"><h2>SOURCES</h2><div id="srclist"></div></div>
|
||||||
|
<div class="card full"><h2>MESH PEERS</h2><div id="peers" class="sub">…</div></div>
|
||||||
|
<div class="card full"><h2>TOP CONSENSUS IPs</h2><table id="agg"><thead><tr><th>IP</th><th>consensus</th><th>sources</th></tr></thead><tbody></tbody></table></div>
|
||||||
|
</div>
|
||||||
|
</main>
|
||||||
|
<div class="toast" id="toast"></div>
|
||||||
|
<script src="/shared/sidebar.js"></script>
|
||||||
|
<script>
|
||||||
|
const API="/api/v1/threatmesh";
|
||||||
|
function tok(){return localStorage.getItem("sbx_token")||""}
|
||||||
|
async function api(p,opt={}){opt.headers=Object.assign({},opt.headers||{});const t=tok();if(t)opt.headers.Authorization="Bearer "+t;
|
||||||
|
const r=await fetch(API+p,opt);if(r.status===401){toast("login via Hub");throw 0}if(!r.ok)throw 0;
|
||||||
|
const ct=r.headers.get("content-type")||"";return ct.includes("json")?r.json():r.text()}
|
||||||
|
function toast(m){const t=document.getElementById("toast");t.textContent=m;t.classList.add("on");clearTimeout(t._);t._=setTimeout(()=>t.classList.remove("on"),2600)}
|
||||||
|
function esc(s){return (s||"").replace(/[&<>"]/g,c=>({"&":"&","<":"<",">":">",'"':"""}[c]))}
|
||||||
|
async function load(){
|
||||||
|
let s;try{s=await fetch(API+"/status").then(r=>r.json())}catch(e){return}
|
||||||
|
document.getElementById("mode").textContent=(s.mode||"sovereign")+" · CAPI "+(s.capi?"on":"off")+" · node "+(s.node||"?");
|
||||||
|
document.getElementById("feedtotal").textContent=s.feed_total??0;
|
||||||
|
document.getElementById("feedn").textContent=Object.keys(s.feeds||{}).length+" feeds";
|
||||||
|
document.getElementById("meshtotal").textContent=s.mesh_total??0;
|
||||||
|
document.getElementById("peern").textContent=(s.peers||0)+" peers";
|
||||||
|
const all=s.sources||{};document.getElementById("total").textContent=Object.values(all).reduce((a,b)=>a+b,0);
|
||||||
|
document.getElementById("srclist").innerHTML=Object.entries(all).sort((a,b)=>b[1]-a[1]).slice(0,12)
|
||||||
|
.map(([k,v])=>`<div class="src ${k.startsWith('mesh:')?'mesh':''}"><span>${esc(k)}</span><b>${v}</b></div>`).join("")||'<div class="sub">no sources yet</div>';
|
||||||
|
try{const p=await fetch(API+"/peers").then(r=>r.json());
|
||||||
|
document.getElementById("peers").innerHTML=(p.peers||[]).length?p.peers.map(x=>`${esc(x.name||x.id)} <span style="color:var(--muted)">(${esc(x.address)})</span>`).join(" · "):"no mesh peers yet (single-node) — decisions will gossip when peers join";}catch(e){}
|
||||||
|
try{const d=await fetch(API+"/decisions?min_consensus=1&limit=40").then(r=>r.json());
|
||||||
|
d.sort((a,b)=>b.consensus-a.consensus);
|
||||||
|
document.querySelector("#agg tbody").innerHTML=d.slice(0,25).map(r=>`<tr><td>${esc(r.value)}</td><td>${r.consensus}</td><td style="color:var(--muted)">${esc((r.sources||'').slice(0,60))}</td></tr>`).join("");}catch(e){}
|
||||||
|
}
|
||||||
|
async function refreshFeeds(){try{await api("/feeds/refresh",{method:"POST"});toast("feed refresh started (~1 min)")}catch(e){toast("need Hub login")}}
|
||||||
|
async function gossip(){try{const r=await api("/mesh/gossip",{method:"POST"});toast(`gossip: ${r.pushed}/${r.peers} peers, ${r.shared} shared`)}catch(e){toast("need Hub login")}}
|
||||||
|
load();setInterval(load,8000);
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
|
@ -1,3 +1,14 @@
|
||||||
|
secubox-toolbox (2.7.19-1~bookworm1) bookworm; urgency=medium
|
||||||
|
|
||||||
|
* #728 blacklist-sync confidence gate: enforce a threat_intel IP only when
|
||||||
|
corroborated by >= SECUBOX_BL_MIN_CONSENSUS sources (default 2) OR carried by
|
||||||
|
a curated high-trust feed (weight >= SECUBOX_BL_MIN_WEIGHT, default 80).
|
||||||
|
Arming all ~45k aggregated-feed IPs (mostly noisy single-source) risked
|
||||||
|
blocking legit traffic; this enforces the high-confidence ~2k set. CrowdSec
|
||||||
|
local decisions + DNS-guard stay always-enforced. Tunable via env.
|
||||||
|
|
||||||
|
-- Gerald KERMA <devel@cybermind.fr> Wed, 24 Jun 2026 16:30:00 +0000
|
||||||
|
|
||||||
secubox-toolbox (2.7.18-1~bookworm1) bookworm; urgency=medium
|
secubox-toolbox (2.7.18-1~bookworm1) bookworm; urgency=medium
|
||||||
|
|
||||||
* #519/#522 fix(blacklist-sync): the DNS-guard domain loop aborted the whole
|
* #519/#522 fix(blacklist-sync): the DNS-guard domain loop aborted the whole
|
||||||
|
|
|
||||||
|
|
@ -37,11 +37,18 @@ fi
|
||||||
TMP4=$(mktemp); TMP6=$(mktemp)
|
TMP4=$(mktemp); TMP6=$(mktemp)
|
||||||
trap 'rm -f "$TMP4" "$TMP6"' EXIT
|
trap 'rm -f "$TMP4" "$TMP6"' EXIT
|
||||||
|
|
||||||
# Source 1 : threat-intel C2 IPs (feodo / threatfox / sslbl) from the
|
# Source 1 : threat-intel IPs from the toolbox SQLite (ioc_type='ip').
|
||||||
# toolbox SQLite. ioc_type='ip'.
|
# Confidence gate (#728): aggregated public feeds carry lots of noisy
|
||||||
|
# single-source entries (e.g. blocklist.de) — arming all of them risks blocking
|
||||||
|
# legitimate traffic. Enforce an IP only when corroborated by >= MIN_CONSENSUS
|
||||||
|
# distinct sources OR carried by a curated high-trust feed (weight >= MIN_WEIGHT).
|
||||||
|
# CrowdSec local decisions (Source 2) and DNS-guard domains are always enforced.
|
||||||
|
TI_MIN_CONSENSUS="${SECUBOX_BL_MIN_CONSENSUS:-2}"
|
||||||
|
TI_MIN_WEIGHT="${SECUBOX_BL_MIN_WEIGHT:-80}"
|
||||||
if [ -r "$TOOLBOX_DB" ] && command -v sqlite3 >/dev/null 2>&1; then
|
if [ -r "$TOOLBOX_DB" ] && command -v sqlite3 >/dev/null 2>&1; then
|
||||||
sqlite3 "$TOOLBOX_DB" \
|
sqlite3 "$TOOLBOX_DB" \
|
||||||
"SELECT DISTINCT ioc FROM threat_intel WHERE ioc_type='ip';" \
|
"SELECT ioc FROM threat_intel WHERE ioc_type='ip' GROUP BY ioc \
|
||||||
|
HAVING COUNT(DISTINCT source) >= $TI_MIN_CONSENSUS OR MAX(weight) >= $TI_MIN_WEIGHT;" \
|
||||||
2>/dev/null >> "$TMP4.raw" || true
|
2>/dev/null >> "$TMP4.raw" || true
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@ SecuBox is an appliance and network model - distributed peer applications.
|
||||||
import subprocess
|
import subprocess
|
||||||
import os
|
import os
|
||||||
import json
|
import json
|
||||||
|
import re
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Optional, List, Dict, Any
|
from typing import Optional, List, Dict, Any
|
||||||
from fastapi import FastAPI, Depends, HTTPException, BackgroundTasks
|
from fastapi import FastAPI, Depends, HTTPException, BackgroundTasks
|
||||||
|
|
@ -235,24 +236,62 @@ class VHostUpdate(BaseModel):
|
||||||
enabled: Optional[bool] = None
|
enabled: Optional[bool] = None
|
||||||
|
|
||||||
|
|
||||||
|
HAPROXY_ROUTES_FILE = Path("/srv/mitmproxy/haproxy-routes.json")
|
||||||
|
HAPROXY_CERTS_DIR = Path("/data/haproxy/certs")
|
||||||
|
|
||||||
|
|
||||||
|
def _is_public_fqdn(name: str) -> bool:
|
||||||
|
name = (name or "").strip().rstrip(";")
|
||||||
|
if not name or name in ("_", "localhost") or name.endswith(".local"):
|
||||||
|
return False
|
||||||
|
if re.match(r"^\d{1,3}(\.\d{1,3}){3}$", name):
|
||||||
|
return False
|
||||||
|
return "." in name
|
||||||
|
|
||||||
|
|
||||||
|
def _server_name_fqdn(content: str):
|
||||||
|
"""First public FQDN from any server_name directive (skips _, localhost, IPs)."""
|
||||||
|
for line in content.split("\n"):
|
||||||
|
s = line.strip()
|
||||||
|
if s.startswith("server_name"):
|
||||||
|
for tok in s[len("server_name"):].strip().rstrip(";").split():
|
||||||
|
if _is_public_fqdn(tok):
|
||||||
|
return tok
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _load_haproxy_routes() -> dict:
|
||||||
|
"""Public FQDN -> backend target from the HAProxy/mitmproxy route map."""
|
||||||
|
try:
|
||||||
|
d = json.loads(HAPROXY_ROUTES_FILE.read_text())
|
||||||
|
return {k: v for k, v in d.items() if _is_public_fqdn(k)}
|
||||||
|
except Exception:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
@app.get("/vhosts", dependencies=[Depends(require_jwt)])
|
@app.get("/vhosts", dependencies=[Depends(require_jwt)])
|
||||||
async def list_vhosts():
|
async def list_vhosts():
|
||||||
"""List all virtual hosts"""
|
"""List all virtual hosts with full, clickable public URLs.
|
||||||
|
|
||||||
|
Public vhosts are HAProxy-fronted (TLS terminated there), so the real domain
|
||||||
|
is the nginx server_name FQDN — NOT the config filename. We fall back to the
|
||||||
|
HAProxy route map for backends whose nginx config only has local names, and
|
||||||
|
append HAProxy public routes that have no nginx config, so the list is the
|
||||||
|
COMPLETE set of reachable vhosts with working https:// links.
|
||||||
|
"""
|
||||||
vhosts = []
|
vhosts = []
|
||||||
|
routes = _load_haproxy_routes()
|
||||||
|
seen = set()
|
||||||
|
|
||||||
if NGINX_VHOST_DIR.exists():
|
if NGINX_VHOST_DIR.exists():
|
||||||
for conf_file in NGINX_VHOST_DIR.glob("*.conf"):
|
for conf_file in sorted(NGINX_VHOST_DIR.glob("*.conf")):
|
||||||
if conf_file.name.startswith("_") or conf_file.name == "default.conf":
|
if conf_file.name.startswith("_") or conf_file.name == "default.conf":
|
||||||
continue
|
continue
|
||||||
domain = conf_file.stem
|
stem = conf_file.stem
|
||||||
enabled = (NGINX_ENABLED_DIR / conf_file.name).exists()
|
enabled = (NGINX_ENABLED_DIR / conf_file.name).exists()
|
||||||
|
|
||||||
# Parse config
|
|
||||||
backend = ""
|
backend = ""
|
||||||
tls_mode = "off"
|
|
||||||
websocket = False
|
websocket = False
|
||||||
ssl = False
|
content = ""
|
||||||
|
|
||||||
try:
|
try:
|
||||||
content = conf_file.read_text()
|
content = conf_file.read_text()
|
||||||
for line in content.split("\n"):
|
for line in content.split("\n"):
|
||||||
|
|
@ -260,27 +299,28 @@ async def list_vhosts():
|
||||||
backend = line.split()[-1].rstrip(";")
|
backend = line.split()[-1].rstrip(";")
|
||||||
if "Upgrade" in line:
|
if "Upgrade" in line:
|
||||||
websocket = True
|
websocket = True
|
||||||
ssl = "listen 443" in content or "ssl_certificate" in content
|
except Exception:
|
||||||
|
|
||||||
if ssl:
|
|
||||||
if "/etc/acme/" in content:
|
|
||||||
tls_mode = "acme"
|
|
||||||
else:
|
|
||||||
tls_mode = "manual"
|
|
||||||
except:
|
|
||||||
pass
|
pass
|
||||||
|
|
||||||
# Check certificate
|
domain = _server_name_fqdn(content)
|
||||||
cert_expires = None
|
if not domain:
|
||||||
if ssl:
|
domain = next((k for k in routes if k.split(".")[0] == stem), None) or stem
|
||||||
cert_path = ACME_DIR / domain / "fullchain.cer"
|
|
||||||
if cert_path.exists():
|
|
||||||
success, out, _ = run_cmd([
|
|
||||||
"openssl", "x509", "-in", str(cert_path), "-noout", "-enddate"
|
|
||||||
])
|
|
||||||
if success:
|
|
||||||
cert_expires = out.split("=")[-1]
|
|
||||||
|
|
||||||
|
is_public = _is_public_fqdn(domain)
|
||||||
|
ssl = is_public # HAProxy terminates TLS for every public vhost
|
||||||
|
url = f"https://{domain}" if is_public else None
|
||||||
|
tls_mode = "haproxy" if is_public else "off"
|
||||||
|
|
||||||
|
cert_expires = None
|
||||||
|
if is_public:
|
||||||
|
cert_path = HAPROXY_CERTS_DIR / f"{domain}.pem"
|
||||||
|
if cert_path.exists():
|
||||||
|
success, out, _ = run_cmd(
|
||||||
|
["openssl", "x509", "-in", str(cert_path), "-noout", "-enddate"])
|
||||||
|
if success:
|
||||||
|
cert_expires = out.split("=")[-1].strip()
|
||||||
|
|
||||||
|
seen.add(domain)
|
||||||
vhosts.append({
|
vhosts.append({
|
||||||
"domain": domain,
|
"domain": domain,
|
||||||
"backend": backend,
|
"backend": backend,
|
||||||
|
|
@ -289,9 +329,34 @@ async def list_vhosts():
|
||||||
"websocket": websocket,
|
"websocket": websocket,
|
||||||
"enabled": enabled,
|
"enabled": enabled,
|
||||||
"cert_expires": cert_expires,
|
"cert_expires": cert_expires,
|
||||||
|
"url": url,
|
||||||
|
"source": "nginx",
|
||||||
"config_file": str(conf_file),
|
"config_file": str(conf_file),
|
||||||
})
|
})
|
||||||
|
|
||||||
|
# Append HAProxy public vhosts with no nginx config (complete the list).
|
||||||
|
for domain, target in routes.items():
|
||||||
|
if domain in seen:
|
||||||
|
continue
|
||||||
|
seen.add(domain)
|
||||||
|
try:
|
||||||
|
backend = f"{target[0]}:{target[1]}" if isinstance(target, (list, tuple)) else str(target)
|
||||||
|
except Exception:
|
||||||
|
backend = ""
|
||||||
|
vhosts.append({
|
||||||
|
"domain": domain,
|
||||||
|
"backend": backend,
|
||||||
|
"tls_mode": "haproxy",
|
||||||
|
"ssl": True,
|
||||||
|
"websocket": False,
|
||||||
|
"enabled": True,
|
||||||
|
"cert_expires": None,
|
||||||
|
"url": f"https://{domain}",
|
||||||
|
"source": "haproxy",
|
||||||
|
"config_file": None,
|
||||||
|
})
|
||||||
|
|
||||||
|
vhosts.sort(key=lambda v: v["domain"])
|
||||||
return {"vhosts": vhosts, "count": len(vhosts)}
|
return {"vhosts": vhosts, "count": len(vhosts)}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,14 @@
|
||||||
|
secubox-vhost (1.1.1-1~bookworm1) bookworm; urgency=medium
|
||||||
|
|
||||||
|
* Fix vhost list: use the real public FQDN (nginx server_name, HAProxy route
|
||||||
|
map) instead of the config filename stem, set full https:// URLs + correct
|
||||||
|
ssl (HAProxy terminates TLS), and append HAProxy public routes that have no
|
||||||
|
nginx config. The list is now the COMPLETE set of reachable vhosts with
|
||||||
|
working clickable links (was showing short names like "arm"/"lyrion" with
|
||||||
|
url=None -> broken links).
|
||||||
|
|
||||||
|
-- Gerald KERMA <devel@cybermind.fr> Wed, 24 Jun 2026 14:00:00 +0000
|
||||||
|
|
||||||
secubox-vhost (1.1.0-1~bookworm1) bookworm; urgency=medium
|
secubox-vhost (1.1.0-1~bookworm1) bookworm; urgency=medium
|
||||||
|
|
||||||
* Add three-fold architecture (components, status, access)
|
* Add three-fold architecture (components, status, access)
|
||||||
|
|
|
||||||
103
wiki/ThreatMesh-FR.md
Normal file
103
wiki/ThreatMesh-FR.md
Normal file
|
|
@ -0,0 +1,103 @@
|
||||||
|
# ThreatMesh 🛰️
|
||||||
|
|
||||||
|
[EN](ThreatMesh) | **[FR](ThreatMesh-FR)** | **🔴 BOOT · 🛡️ SÉCURITÉ** | renseignement de menace souverain
|
||||||
|
|
||||||
|
> Votre réseau de vigilance de quartier pour l'internet — *listes gratuites + astuces des voisins, sans chef central, sans paywall, impossible à bannir.*
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
|
ThreatMesh est la couche SecuBox qui **bloque automatiquement les adresses
|
||||||
|
internet malveillantes connues** — née après que l'API centrale de CrowdSec
|
||||||
|
(CAPI) a blacklisté l'IP de notre box puis fait payer le déblocage. Elle
|
||||||
|
remplace cette dépendance centrale par des **listes publiques auto-sourcées**
|
||||||
|
plus un **partage d'astuces pair-à-pair** entre vos propres box. Vous possédez
|
||||||
|
toute la chaîne, de bout en bout.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🏘️ L'idée simple
|
||||||
|
|
||||||
|
Voyez votre SecuBox comme une **maison avec un concierge malin**. Le concierge
|
||||||
|
tient une seule liste « à ne pas laisser entrer », alimentée par deux flux, et
|
||||||
|
refuse tout ce qui y figure.
|
||||||
|
|
||||||
|
```
|
||||||
|
LISTES « RECHERCHÉ » GRATUITES VOS AUTRES BOX (mesh)
|
||||||
|
(bulletins publics) (voisins qui s'échangent des astuces)
|
||||||
|
\ /
|
||||||
|
\ /
|
||||||
|
▼ ▼
|
||||||
|
┌──────────────────────────────────┐
|
||||||
|
│ LE CONCIERGE — une liste de │
|
||||||
|
│ blocage, ne croit que les bonnes │
|
||||||
|
│ pistes │
|
||||||
|
└──────────────────────────────────┘
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
🚪 une mauvaise adresse frappe → REFUSÉE
|
||||||
|
```
|
||||||
|
|
||||||
|
1. **📋 Listes de surveillance gratuites** — toutes les 6 h la box récupère des
|
||||||
|
listes publiques « ces IP sont dangereuses » (C2 de malware, réseaux
|
||||||
|
détournés, attaquants connus). Gratuit, sans inscription, sans compte.
|
||||||
|
2. **🤝 Astuces des voisins (mesh)** — quand *votre* box attrape un attaquant,
|
||||||
|
elle prévient vos *autres* box via le mesh chiffré SecuBox (WireGuard). Sans
|
||||||
|
intermédiaire.
|
||||||
|
3. **🛡️ Le concierge agit** — chaque astuce atterrit dans une seule liste de
|
||||||
|
blocage et la box refuse le trafic vers/depuis ces adresses au pare-feu
|
||||||
|
(nftables).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🆚 Pourquoi souverain
|
||||||
|
|
||||||
|
| Avant (CrowdSec CAPI) | Maintenant (ThreatMesh) |
|
||||||
|
|------------------------|-------------------------|
|
||||||
|
| La liste centrale d'une entreprise | **La vôtre**, depuis des sources ouvertes |
|
||||||
|
| Ils peuvent **bannir votre IP** | **Personne ne peut vous exclure** |
|
||||||
|
| **Payer** pour être débanni | **Gratuit, pour toujours** |
|
||||||
|
| Vous dépendez d'eux | **Vous possédez toute la chaîne** |
|
||||||
|
|
||||||
|
Le moteur de détection hors-ligne de CrowdSec (LAPI) est conservé — seul le flux
|
||||||
|
central toxique (CAPI) est abandonné.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🔍 Sous le capot
|
||||||
|
|
||||||
|
| Étape | Composant | Rôle |
|
||||||
|
|-------|-----------|------|
|
||||||
|
| **Feeds** | `secubox-threatfeed` (timer, 6 h) | tire des listes gratuites — feodo, sslbl, FireHOL, Spamhaus DROP, blocklist.de, CINS, ET-compromised, DShield — dans la table partagée `threat_intel` |
|
||||||
|
| **Mesh** | `secubox-threatmesh` (service) | diffuse les décisions détectées localement aux pairs du mesh via WireGuard ; ingère les décisions des pairs (`mesh:<node>`), comptées par consensus ; port `:8780` verrouillé au mesh par nftables |
|
||||||
|
| **Application** | `secubox-blacklist-sync` | vide `threat_intel` → ensembles de drop nft `blacklist_v4/v6` |
|
||||||
|
| **Visualiser** | tableau de bord `/threatmesh/` + `/api/v1/threatmesh/decisions` (compatible bouncer CrowdSec) | statut, sources, pairs, IP à plus fort consensus |
|
||||||
|
|
||||||
|
### 🎯 La porte de confiance (zéro carpet-bomb de faux positifs)
|
||||||
|
|
||||||
|
Les feeds publics agrégés contiennent beaucoup d'entrées bruyantes à source
|
||||||
|
unique. ThreatMesh **n'applique le blocage** que si une IP est **corroborée par
|
||||||
|
≥ 2 sources** *ou* provient d'un **feed curé de haute confiance** (poids ≥ 80).
|
||||||
|
Le reste reste *visible mais non bloqué*. Les décisions locales de CrowdSec et le
|
||||||
|
DNS-guard sont toujours appliqués.
|
||||||
|
|
||||||
|
Réglage via variables d'env sur `secubox-blacklist-sync` :
|
||||||
|
|
||||||
|
```
|
||||||
|
SECUBOX_BL_MIN_CONSENSUS=2 # sources qui doivent concorder (plus bas = plus de couverture)
|
||||||
|
SECUBOX_BL_MIN_WEIGHT=80 # niveau de confiance qui contourne la règle de consensus
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 📊 En un coup d'œil
|
||||||
|
|
||||||
|
- **~45 000** IP dangereuses connues (rafraîchies toutes les 6 h)
|
||||||
|
- **~3 000** IP de haute confiance bloquées activement au pare-feu
|
||||||
|
- Le partage mesh s'active tout seul dès qu'une seconde SecuBox rejoint le mesh
|
||||||
|
- **0** compte externe · **0** paywall · **0** moyen pour un tiers de vous couper
|
||||||
|
|
||||||
|
> *Ils nous ont bloqués et ont demandé de l'argent pour débloquer. Alors on a construit le nôtre — et maintenant personne ne peut nous couper.* 🔓
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
*Voir aussi : [[Anti-Track]] · [[Architecture]] · `secubox-threatmesh` (#728)*
|
||||||
97
wiki/ThreatMesh.md
Normal file
97
wiki/ThreatMesh.md
Normal file
|
|
@ -0,0 +1,97 @@
|
||||||
|
# ThreatMesh 🛰️
|
||||||
|
|
||||||
|
**[EN](ThreatMesh)** | [FR](ThreatMesh-FR) | **🔴 BOOT · 🛡️ SECURITY** | sovereign threat-intel
|
||||||
|
|
||||||
|
> Your own neighborhood watch for the internet — *free feeds + neighbor tips, no central boss, no paywall, can't be banned.*
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
|
ThreatMesh is the SecuBox layer that automatically **blocks known-bad internet
|
||||||
|
addresses** on its own — built after CrowdSec's central API (CAPI) IP-blocklisted
|
||||||
|
our box and paywalled the un-blocking. It replaces that central dependency with
|
||||||
|
**self-sourced public lists** plus **peer-to-peer tip sharing** between your own
|
||||||
|
boxes. You own the whole thing end to end.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🏘️ The simple idea
|
||||||
|
|
||||||
|
Think of your SecuBox as a **house with a smart doorman**. The doorman keeps one
|
||||||
|
"do not let in" list, fed by two streams, and turns away anything on it.
|
||||||
|
|
||||||
|
```
|
||||||
|
FREE "WANTED" LISTS YOUR OTHER BOXES (mesh)
|
||||||
|
(public bulletins) (neighbors swapping tips)
|
||||||
|
\ /
|
||||||
|
\ /
|
||||||
|
▼ ▼
|
||||||
|
┌──────────────────────────────────┐
|
||||||
|
│ THE DOORMAN — one block list, │
|
||||||
|
│ only trusts solid tips │
|
||||||
|
└──────────────────────────────────┘
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
🚪 bad address knocks → DROPPED
|
||||||
|
```
|
||||||
|
|
||||||
|
1. **📋 Free watch-lists** — every 6 h the box pulls public "these IPs are
|
||||||
|
dangerous" lists (malware C2, hijacked networks, known attackers). Free, no
|
||||||
|
sign-up, no account.
|
||||||
|
2. **🤝 Neighbor tips (mesh)** — when *your* box catches an attacker it tells your
|
||||||
|
*other* boxes over the encrypted SecuBox mesh (WireGuard). No middleman.
|
||||||
|
3. **🛡️ The doorman acts** — every tip lands in one block-list and the box
|
||||||
|
refuses traffic to/from those addresses at the firewall (nftables).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🆚 Why sovereign
|
||||||
|
|
||||||
|
| Before (CrowdSec CAPI) | Now (ThreatMesh) |
|
||||||
|
|------------------------|------------------|
|
||||||
|
| One company's central list | **Your own**, from open sources |
|
||||||
|
| They can **ban your IP** | **No one can lock you out** |
|
||||||
|
| **Pay** to get un-banned | **Free, forever** |
|
||||||
|
| You depend on them | **You own the whole pipeline** |
|
||||||
|
|
||||||
|
CrowdSec's offline detection engine (LAPI) is kept — only the toxic central feed
|
||||||
|
(CAPI) is dropped.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🔍 Under the hood
|
||||||
|
|
||||||
|
| Stage | Component | What it does |
|
||||||
|
|-------|-----------|--------------|
|
||||||
|
| **Feeds** | `secubox-threatfeed` (timer, 6 h) | pulls free lists — feodo, sslbl, FireHOL, Spamhaus DROP, blocklist.de, CINS, ET-compromised, DShield — into the shared `threat_intel` table |
|
||||||
|
| **Mesh** | `secubox-threatmesh` (service) | gossips locally-detected decisions to mesh peers over WireGuard; ingests peer decisions (`mesh:<node>`), consensus-counted; port `:8780` locked to the mesh by nftables |
|
||||||
|
| **Enforce** | `secubox-blacklist-sync` | drains `threat_intel` → nft `blacklist_v4/v6` drop sets |
|
||||||
|
| **See it** | `/threatmesh/` dashboard + `/api/v1/threatmesh/decisions` (CrowdSec-bouncer-compatible) | status, sources, peers, top-consensus IPs |
|
||||||
|
|
||||||
|
### 🎯 The confidence gate (no false-positive carpet-bomb)
|
||||||
|
|
||||||
|
Aggregated public feeds carry many noisy single-source entries. ThreatMesh
|
||||||
|
**only enforces** an IP that is **corroborated by ≥ 2 sources** *or* comes from a
|
||||||
|
**curated high-trust feed** (weight ≥ 80). The rest stay *visible but not
|
||||||
|
blocked*. CrowdSec local decisions + DNS-guard are always enforced.
|
||||||
|
|
||||||
|
Tune via env on `secubox-blacklist-sync`:
|
||||||
|
|
||||||
|
```
|
||||||
|
SECUBOX_BL_MIN_CONSENSUS=2 # sources that must agree (lower = more coverage)
|
||||||
|
SECUBOX_BL_MIN_WEIGHT=80 # trust level that bypasses the consensus rule
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 📊 At a glance
|
||||||
|
|
||||||
|
- **~45 000** dangerous IPs known (refreshed every 6 h)
|
||||||
|
- **~3 000** high-confidence IPs actively dropped at the firewall
|
||||||
|
- Mesh sharing lights up automatically when a second SecuBox joins the mesh
|
||||||
|
- **0** external accounts · **0** paywall · **0** ways for a third party to switch you off
|
||||||
|
|
||||||
|
> *They blocked us and asked for money to unblock. So we built our own — and now nobody can switch us off.* 🔓
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
*See also: [[Anti-Track]] · [[Architecture]] · `secubox-threatmesh` (#728)*
|
||||||
|
|
@ -41,6 +41,7 @@
|
||||||
### 🟣 MIND — Modules
|
### 🟣 MIND — Modules
|
||||||
|
|
||||||
* [[Anti-Track]] 🛡️ bloque · empoisonne · anonymise
|
* [[Anti-Track]] 🛡️ bloque · empoisonne · anonymise
|
||||||
|
* [[ThreatMesh]] 🛰️ blocklist souveraine (feeds + mesh, sans CAPI) | [FR](ThreatMesh-FR)
|
||||||
* [[MODULES-EN|Modules]] 🇬🇧
|
* [[MODULES-EN|Modules]] 🇬🇧
|
||||||
* [[MODULES-FR]] 🇫🇷
|
* [[MODULES-FR]] 🇫🇷
|
||||||
* [[MODULES-DE]] 🇩🇪
|
* [[MODULES-DE]] 🇩🇪
|
||||||
|
|
|
||||||
|
|
@ -7,3 +7,5 @@ Local image assets referenced by wiki pages.
|
||||||
| File | Used by | Notes |
|
| File | Used by | Notes |
|
||||||
|------|---------|-------|
|
|------|---------|-------|
|
||||||
| `anti-track-v2-poster.png` | [[Anti-Track]] | Comic-style hero poster (Bloque · Empoisonne · Anonymise). Portrait, ~1024×1536. |
|
| `anti-track-v2-poster.png` | [[Anti-Track]] | Comic-style hero poster (Bloque · Empoisonne · Anonymise). Portrait, ~1024×1536. |
|
||||||
|
| `threatmesh-poster.png` | [[ThreatMesh]] | Neighborhood-watch hero poster (sovereign threat-intel — free feeds + mesh, no CAPI). Portrait, ~1024×1536. |
|
||||||
|
| `threatmesh-poster-fr.png` | [[ThreatMesh-FR]] | Version FR de l'affiche ThreatMesh (vigilance de quartier). Portrait, ~1024×1536. |
|
||||||
|
|
|
||||||
BIN
wiki/images/threatmesh-poster-fr.png
Normal file
BIN
wiki/images/threatmesh-poster-fr.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 2.0 MiB |
BIN
wiki/images/threatmesh-poster.png
Normal file
BIN
wiki/images/threatmesh-poster.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 1.9 MiB |
Loading…
Reference in New Issue
Block a user