mirror of
https://github.com/CyberMind-FR/secubox-deb.git
synced 2026-06-29 10:08:36 +00:00
Compare commits
16 Commits
39d7002b7a
...
4f8eb711f3
| Author | SHA1 | Date | |
|---|---|---|---|
| 4f8eb711f3 | |||
| 4cf7c85191 | |||
| 6286b83bda | |||
| 6907b95d11 | |||
| d6eaf52ce1 | |||
| 0a05bed028 | |||
| fdfc404818 | |||
| 0fc5871169 | |||
| 36cfb72e41 | |||
| 6e62c0166d | |||
| ff6fd7632f | |||
| 0566672615 | |||
| 9f5bec6a87 | |||
| 560b8d8213 | |||
| 7da61e8fd5 | |||
| f839c9260e |
|
|
@ -1,3 +1,14 @@
|
|||
secubox-aggregator (0.2.3-1~bookworm1) bookworm; urgency=medium
|
||||
|
||||
* #727 auto-heal watchdog: ship secubox-aggregator-watchdog.{sh,service,timer}.
|
||||
The in-process aggregator is the hub/auth/menu SPOF; under a host load spike
|
||||
its event loop can wedge and the socket stops answering (board-wide 502/000:
|
||||
sparse navbar, login errors). The timer probes aggregator.sock every 2 min
|
||||
and restarts the service after 2 consecutive failures. Enabled in postinst
|
||||
(respects operator masking). Packages the live fix from the 2026-06-24 incident.
|
||||
|
||||
-- Gerald KERMA <devel@cybermind.fr> Wed, 24 Jun 2026 15:10:00 +0000
|
||||
|
||||
secubox-aggregator (0.2.1-1~bookworm1) bookworm; urgency=medium
|
||||
|
||||
* Phase 7 follow-up (#498) — relax hardening for module sudoers :
|
||||
|
|
|
|||
|
|
@ -17,6 +17,12 @@ case "$1" in
|
|||
systemctl enable secubox-aggregator.service
|
||||
systemctl start secubox-aggregator.service || true
|
||||
|
||||
# Auto-heal watchdog (#727): restart the aggregator if its socket wedges
|
||||
# under load (the hub/auth/menu SPOF). Respect operator masking.
|
||||
if [ "$(systemctl is-enabled secubox-aggregator-watchdog.timer 2>/dev/null)" != "masked" ]; then
|
||||
systemctl enable --now secubox-aggregator-watchdog.timer 2>/dev/null || true
|
||||
fi
|
||||
|
||||
echo "secubox-aggregator: to migrate all installed SecuBox modules into"
|
||||
echo " the aggregator (replaces per-module uvicorn processes) run :"
|
||||
echo " sudo /usr/sbin/secubox-aggregator-migrate"
|
||||
|
|
|
|||
6
packages/secubox-aggregator/debian/rules
Normal file → Executable file
6
packages/secubox-aggregator/debian/rules
Normal file → Executable file
|
|
@ -14,6 +14,12 @@ override_dh_auto_install:
|
|||
install -d $(CURDIR)/debian/secubox-aggregator/lib/systemd/system
|
||||
install -m 644 systemd/secubox-aggregator.service \
|
||||
$(CURDIR)/debian/secubox-aggregator/lib/systemd/system/
|
||||
install -m 644 systemd/secubox-aggregator-watchdog.service \
|
||||
$(CURDIR)/debian/secubox-aggregator/lib/systemd/system/
|
||||
install -m 644 systemd/secubox-aggregator-watchdog.timer \
|
||||
$(CURDIR)/debian/secubox-aggregator/lib/systemd/system/
|
||||
install -d $(CURDIR)/debian/secubox-aggregator/usr/sbin
|
||||
install -m 755 sbin/secubox-aggregator-migrate \
|
||||
$(CURDIR)/debian/secubox-aggregator/usr/sbin/
|
||||
install -m 755 sbin/secubox-aggregator-watchdog.sh \
|
||||
$(CURDIR)/debian/secubox-aggregator/usr/sbin/
|
||||
|
|
|
|||
|
|
@ -0,0 +1,10 @@
|
|||
|
||||
# Automatically added by dh_python3
|
||||
if command -v py3compile >/dev/null 2>&1; then
|
||||
py3compile -p secubox-aggregator
|
||||
fi
|
||||
if command -v pypy3compile >/dev/null 2>&1; then
|
||||
pypy3compile -p secubox-aggregator || true
|
||||
fi
|
||||
|
||||
# End automatically added section
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
|
||||
# Automatically added by dh_python3
|
||||
if command -v py3clean >/dev/null 2>&1; then
|
||||
py3clean -p secubox-aggregator
|
||||
else
|
||||
dpkg -L secubox-aggregator | sed -En -e '/^(.*)\/(.+)\.py$/s,,rm "\1/__pycache__/\2".*,e'
|
||||
find /usr/lib/python3/dist-packages/ -type d -name __pycache__ -empty -print0 | xargs --null --no-run-if-empty rmdir
|
||||
fi
|
||||
|
||||
# End automatically added section
|
||||
48
packages/secubox-aggregator/sbin/secubox-aggregator-watchdog.sh
Executable file
48
packages/secubox-aggregator/sbin/secubox-aggregator-watchdog.sh
Executable file
|
|
@ -0,0 +1,48 @@
|
|||
#!/bin/bash
|
||||
# SPDX-License-Identifier: LicenseRef-CMSD-1.0
|
||||
# Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
|
||||
# Source-Disclosed License — All rights reserved except as expressly granted.
|
||||
# See LICENCE-CMSD-1.0.md for terms.
|
||||
#
|
||||
# SecuBox-Deb :: secubox-aggregator-watchdog
|
||||
#
|
||||
# Auto-heal the in-process aggregator — the hub/auth/menu single point of
|
||||
# failure. Under a host load spike its shared event loop can wedge and its
|
||||
# socket stops answering, taking down the navbar, login and service status
|
||||
# board-wide (incident 2026-06-24). Probe the socket; if /api/v1/hub/public/menu
|
||||
# stops answering for N consecutive checks, restart the service. Idempotent,
|
||||
# safe to run on a timer.
|
||||
set -uo pipefail
|
||||
readonly MODULE="secubox-aggregator-watchdog"
|
||||
readonly VERSION="1.0"
|
||||
|
||||
SOCK="/run/secubox/aggregator.sock"
|
||||
# State lives in /run (root-owned), NOT the shared sticky /run/secubox: that dir
|
||||
# is 1777 and a stale secubox-owned file there can't be overwritten by this
|
||||
# (CSPN-hardened, CAP_DAC_OVERRIDE-less) root — which would silently freeze the
|
||||
# streak counter and stop the watchdog ever triggering.
|
||||
STATE="/run/secubox-aggregator-watchdog.fails"
|
||||
FAIL_THRESHOLD="${SECUBOX_AGG_WD_THRESHOLD:-2}"
|
||||
TIMEOUT="${SECUBOX_AGG_WD_TIMEOUT:-12}"
|
||||
|
||||
# No socket yet (service still starting / not migrated) → nothing to heal.
|
||||
[ -S "$SOCK" ] || exit 0
|
||||
|
||||
code=$(curl -s -o /dev/null -w "%{http_code}" --max-time "$TIMEOUT" \
|
||||
--unix-socket "$SOCK" http://localhost/api/v1/hub/public/menu 2>/dev/null || echo 000)
|
||||
|
||||
if [ "$code" = "200" ]; then
|
||||
echo 0 > "$STATE" 2>/dev/null || true
|
||||
exit 0
|
||||
fi
|
||||
|
||||
n=$(( $(cat "$STATE" 2>/dev/null || echo 0) + 1 ))
|
||||
echo "$n" > "$STATE" 2>/dev/null || true
|
||||
logger -t "$MODULE" "aggregator probe failed (code=$code, streak=$n/$FAIL_THRESHOLD)"
|
||||
|
||||
if [ "$n" -ge "$FAIL_THRESHOLD" ]; then
|
||||
logger -t "$MODULE" "restarting secubox-aggregator (auto-heal)"
|
||||
systemctl restart secubox-aggregator.service
|
||||
echo 0 > "$STATE" 2>/dev/null || true
|
||||
fi
|
||||
exit 0
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
[Unit]
|
||||
Description=SecuBox aggregator auto-heal watchdog
|
||||
Documentation=https://github.com/CyberMind-FR/secubox-deb/issues/727
|
||||
After=secubox-aggregator.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/sbin/secubox-aggregator-watchdog.sh
|
||||
Nice=10
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
[Unit]
|
||||
Description=Probe + auto-heal secubox-aggregator every 2 min
|
||||
|
||||
[Timer]
|
||||
OnBootSec=2min
|
||||
OnUnitActiveSec=2min
|
||||
AccuracySec=20s
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
|
|
@ -268,27 +268,20 @@ async def _monitor_streams():
|
|||
try:
|
||||
settings = _load_settings()
|
||||
if settings.get("detection_enabled", True):
|
||||
flows = await _dpi("/flows")
|
||||
ex = await _dpi("/exfil")
|
||||
media_stats: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
for f in flows.get("flows", []):
|
||||
app_name = f.get("app_name", "Unknown")
|
||||
if app_name in MEDIA_APPS:
|
||||
if app_name not in media_stats:
|
||||
media_stats[app_name] = {"name": app_name, "flows": 0, "bytes": 0}
|
||||
media_stats[app_name]["flows"] += 1
|
||||
media_stats[app_name]["bytes"] += f.get("bytes", 0)
|
||||
for f in _exfil_media_flows(ex):
|
||||
name = f.get("service") or f.get("dst") or "Unknown"
|
||||
if name not in media_stats:
|
||||
media_stats[name] = {"name": name, "flows": 0, "bytes": 0}
|
||||
media_stats[name]["flows"] += int(f.get("flows", 1) or 1)
|
||||
media_stats[name]["bytes"] += int(f.get("up_bytes", 0) or 0) + int(f.get("down_bytes", 0) or 0)
|
||||
|
||||
# Check for new services
|
||||
if settings.get("alert_on_new_service") and app_name not in seen_services:
|
||||
seen_services.add(app_name)
|
||||
await _notify_webhooks("new_service", {
|
||||
"service": app_name,
|
||||
"category": next(
|
||||
(cat for cat, apps in STREAMING_CATEGORIES.items() if app_name in apps),
|
||||
"other"
|
||||
)
|
||||
})
|
||||
if settings.get("alert_on_new_service") and name not in seen_services:
|
||||
seen_services.add(name)
|
||||
await _notify_webhooks("new_service", {"service": name, "category": "media"})
|
||||
|
||||
# Check alerts
|
||||
await _check_alerts(media_stats)
|
||||
|
|
@ -323,16 +316,40 @@ async def health():
|
|||
return {"status": "ok", "module": "mediaflow", "version": "2.0.0"}
|
||||
|
||||
|
||||
# The DPI engine now exposes a public, category-tagged exfil view (the netifyd
|
||||
# /flows path is dead). Media = the exfil classifier's "media" category.
|
||||
MEDIA_CATEGORIES = {"media"}
|
||||
|
||||
|
||||
def _exfil_media_flows(exfil: Dict[str, Any]):
|
||||
"""Flatten exfil devices->services + active_flows to category=='media' rows."""
|
||||
rows = []
|
||||
for dev in exfil.get("devices", []) or []:
|
||||
for s in dev.get("services", []) or []:
|
||||
if s.get("category") in MEDIA_CATEGORIES:
|
||||
rows.append(s)
|
||||
for f in exfil.get("active_flows", []) or []:
|
||||
if f.get("category") in MEDIA_CATEGORIES:
|
||||
rows.append(f)
|
||||
return rows
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
async def status(user=Depends(require_jwt)):
|
||||
try:
|
||||
s = await _dpi("/status")
|
||||
ex = await _dpi("/exfil")
|
||||
settings = _load_settings()
|
||||
media = _exfil_media_flows(ex)
|
||||
return {
|
||||
**s,
|
||||
"running": bool(ex) and "error" not in ex,
|
||||
"source": "dpi-exfil",
|
||||
"devices": len(ex.get("devices", []) or []),
|
||||
"active_flows": len(ex.get("active_flows", []) or []),
|
||||
"media_flows": len(media),
|
||||
"media_detection": settings.get("detection_enabled", True),
|
||||
"monitored_apps": len(MEDIA_APPS),
|
||||
"timestamp": datetime.now().isoformat()
|
||||
"monitored_categories": sorted(MEDIA_CATEGORIES),
|
||||
"generated_at": ex.get("generated_at"),
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
}
|
||||
except Exception as e:
|
||||
return {"running": False, "error": str(e)}
|
||||
|
|
@ -340,42 +357,28 @@ async def status(user=Depends(require_jwt)):
|
|||
|
||||
@router.get("/services")
|
||||
async def services(user=Depends(require_jwt)):
|
||||
"""Get active media services with statistics."""
|
||||
"""Active media services (from the DPI exfil view), grouped by service/host."""
|
||||
cached = stats_cache.get("services")
|
||||
if cached:
|
||||
return cached
|
||||
|
||||
try:
|
||||
flows = await _dpi("/flows")
|
||||
ex = await _dpi("/exfil")
|
||||
media: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
for f in flows.get("flows", []):
|
||||
app_name = f.get("app_name", "Unknown")
|
||||
if app_name in MEDIA_APPS:
|
||||
if app_name not in media:
|
||||
category = next(
|
||||
(cat for cat, apps in STREAMING_CATEGORIES.items() if app_name in apps),
|
||||
"other"
|
||||
)
|
||||
media[app_name] = {
|
||||
"name": app_name,
|
||||
"category": category,
|
||||
"flows": 0,
|
||||
"bytes": 0,
|
||||
"clients": set()
|
||||
}
|
||||
media[app_name]["flows"] += 1
|
||||
media[app_name]["bytes"] += f.get("bytes", 0)
|
||||
if f.get("src_ip"):
|
||||
media[app_name]["clients"].add(f.get("src_ip"))
|
||||
|
||||
# Convert sets to counts
|
||||
for f in _exfil_media_flows(ex):
|
||||
name = f.get("service") or f.get("dst") or "Unknown"
|
||||
if name not in media:
|
||||
media[name] = {"name": name, "category": "media",
|
||||
"host": f.get("dst"), "cloud": f.get("cloud"),
|
||||
"flows": 0, "bytes": 0, "clients": set()}
|
||||
media[name]["flows"] += int(f.get("flows", 1) or 1)
|
||||
media[name]["bytes"] += int(f.get("up_bytes", 0) or 0) + int(f.get("down_bytes", 0) or 0)
|
||||
if f.get("device"):
|
||||
media[name]["clients"].add(f.get("device"))
|
||||
result = []
|
||||
for name, data in media.items():
|
||||
for data in media.values():
|
||||
data["clients"] = len(data["clients"])
|
||||
data["bytes_human"] = _format_bytes(data["bytes"])
|
||||
result.append(data)
|
||||
|
||||
result.sort(key=lambda x: x["bytes"], reverse=True)
|
||||
stats_cache.set("services", result)
|
||||
return result
|
||||
|
|
@ -413,24 +416,38 @@ async def services_by_category(user=Depends(require_jwt)):
|
|||
|
||||
@router.get("/clients")
|
||||
async def clients(user=Depends(require_jwt)):
|
||||
"""Get clients using media services."""
|
||||
"""Get clients (devices) seen by the DPI exfil view, with totals."""
|
||||
try:
|
||||
devices = await _dpi("/devices")
|
||||
return devices if isinstance(devices, list) else []
|
||||
ex = await _dpi("/exfil")
|
||||
out = []
|
||||
for d in ex.get("devices", []) or []:
|
||||
tot = int(d.get("up_bytes", 0) or 0) + int(d.get("down_bytes", 0) or 0)
|
||||
out.append({
|
||||
"device": d.get("device"),
|
||||
"flows": d.get("flows", 0),
|
||||
"up_bytes": d.get("up_bytes", 0),
|
||||
"down_bytes": d.get("down_bytes", 0),
|
||||
"bytes": tot,
|
||||
"bytes_human": _format_bytes(tot),
|
||||
"media_flows": sum(1 for s in (d.get("services") or [])
|
||||
if s.get("category") in MEDIA_CATEGORIES),
|
||||
})
|
||||
out.sort(key=lambda x: x["bytes"], reverse=True)
|
||||
return out
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
@router.get("/get_active_streams")
|
||||
async def get_active_streams(user=Depends(require_jwt)):
|
||||
"""Get active media streams."""
|
||||
"""Active media streams (category=='media') from the DPI exfil view."""
|
||||
try:
|
||||
flows = await _dpi("/flows")
|
||||
ex = await _dpi("/exfil")
|
||||
streams = []
|
||||
for f in flows.get("flows", []):
|
||||
if f.get("app_name") in MEDIA_APPS:
|
||||
f["bytes_human"] = _format_bytes(f.get("bytes", 0))
|
||||
streams.append(f)
|
||||
for f in _exfil_media_flows(ex):
|
||||
b = int(f.get("up_bytes", 0) or 0) + int(f.get("down_bytes", 0) or 0)
|
||||
streams.append({**f, "bytes": b, "bytes_human": _format_bytes(b)})
|
||||
streams.sort(key=lambda x: x["bytes"], reverse=True)
|
||||
return streams
|
||||
except Exception:
|
||||
return []
|
||||
|
|
@ -440,23 +457,20 @@ async def get_active_streams(user=Depends(require_jwt)):
|
|||
async def get_service_details(service: str, user=Depends(require_jwt)):
|
||||
"""Get detailed info for a specific media service."""
|
||||
try:
|
||||
flows = await _dpi("/flows")
|
||||
service_flows = [f for f in flows.get("flows", []) if f.get("app_name") == service]
|
||||
|
||||
total_bytes = sum(f.get("bytes", 0) for f in service_flows)
|
||||
clients = set(f.get("src_ip") for f in service_flows if f.get("src_ip"))
|
||||
|
||||
ex = await _dpi("/exfil")
|
||||
service_flows = [f for f in _exfil_media_flows(ex)
|
||||
if (f.get("service") or f.get("dst")) == service]
|
||||
total_bytes = sum(int(f.get("up_bytes", 0) or 0) + int(f.get("down_bytes", 0) or 0)
|
||||
for f in service_flows)
|
||||
clients = set(f.get("device") for f in service_flows if f.get("device"))
|
||||
return {
|
||||
"service": service,
|
||||
"category": next(
|
||||
(cat for cat, apps in STREAMING_CATEGORIES.items() if service in apps),
|
||||
"other"
|
||||
),
|
||||
"category": "media",
|
||||
"active_flows": len(service_flows),
|
||||
"total_bytes": total_bytes,
|
||||
"total_bytes_human": _format_bytes(total_bytes),
|
||||
"unique_clients": len(clients),
|
||||
"flows": service_flows[:50] # Limit to 50 flows
|
||||
"flows": service_flows[:50],
|
||||
}
|
||||
except Exception:
|
||||
return {"service": service, "error": "Failed to fetch details"}
|
||||
|
|
@ -592,8 +606,8 @@ async def stop_ndpid(user=Depends(require_jwt)):
|
|||
async def summary(user=Depends(require_jwt)):
|
||||
"""Get mediaflow summary."""
|
||||
try:
|
||||
dpi_status = await _dpi("/status")
|
||||
dpi_running = dpi_status.get("running", False)
|
||||
ex = await _dpi("/exfil")
|
||||
dpi_running = bool(ex) and "error" not in ex
|
||||
except Exception:
|
||||
dpi_running = False
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,13 @@
|
|||
secubox-mediaflow (2.0.1-1~bookworm1) bookworm; urgency=medium
|
||||
|
||||
* Fix: consume the DPI engine's public /exfil view (category-tagged) instead of
|
||||
the dead netifyd /flows + auth-gated /status. mediaflow's internal DPI calls
|
||||
were unauthenticated -> 401 -> empty/error dashboard. Now /status /services
|
||||
/clients /get_active_streams /get_service_details /summary + the monitor task
|
||||
read /exfil and filter category=="media". Dashboard works again.
|
||||
|
||||
-- Gerald KERMA <devel@cybermind.fr> Wed, 24 Jun 2026 17:00:00 +0000
|
||||
|
||||
secubox-mediaflow (1.0.4-1~bookworm1) bookworm; urgency=medium
|
||||
|
||||
* Add dynamic menu system with menu.d JSON definitions
|
||||
|
|
|
|||
58
packages/secubox-podcaster/README.md
Normal file
58
packages/secubox-podcaster/README.md
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
# secubox-podcaster
|
||||
|
||||
Modern podcast manager for SecuBox — subscribe, download locally, and relay a
|
||||
shareable RSS feed.
|
||||
|
||||
## What it does (v1, #726)
|
||||
|
||||
- **Subscribe** by RSS URL or **OPML import** (pure-stdlib feed parsing — no
|
||||
`feedparser` dependency).
|
||||
- **Download locally** into `media_path` via an asyncio + httpx queue with
|
||||
per-episode progress.
|
||||
- **Relay / share**: a generated RSS of the local library at
|
||||
`/api/v1/podcaster/share/feed.xml`. LAN by default; set `public_base` to your
|
||||
exposed vhost to publish externally.
|
||||
- **In-UI service status + TOML config**; modern C3BOX WebUI with inline player.
|
||||
|
||||
Lyrion integration is deferred to a follow-up (standalone first).
|
||||
|
||||
## Layout
|
||||
|
||||
| Path | Role |
|
||||
|------|------|
|
||||
| `/usr/share/secubox/podcaster/api` | FastAPI app (uvicorn WorkingDirectory) |
|
||||
| `/run/secubox/podcaster.sock` | Unix socket |
|
||||
| `/var/lib/secubox/podcaster/podcaster.db` | SQLite store |
|
||||
| `/var/lib/secubox/podcaster/media/<feed_id>/` | downloaded episodes |
|
||||
| `/etc/secubox/podcaster.toml` | config |
|
||||
| `/etc/nginx/secubox-routes.d/podcaster.conf` | nginx route (active include) |
|
||||
|
||||
## API
|
||||
|
||||
| Method | Path | Auth | Description |
|
||||
|--------|------|------|-------------|
|
||||
| GET | `/health`, `/status` | public | service + counts |
|
||||
| GET | `/share/feed.xml` | public | shareable RSS of local library |
|
||||
| GET | `/media/{id}` | public | stream a downloaded episode |
|
||||
| GET/POST/DELETE | `/feeds*` | JWT | manage feeds (+ `/feeds/import-opml`) |
|
||||
| GET | `/episodes` | JWT | list (optional `feed_id`, `state`) |
|
||||
| POST | `/episodes/{id}/download` | JWT | enqueue download |
|
||||
| GET/POST | `/config` | JWT | TOML config |
|
||||
|
||||
## Public exposure (relay to the web)
|
||||
|
||||
Publish the share feed externally via HAProxy TLS → mitmproxy (never bypass the
|
||||
WAF):
|
||||
|
||||
```bash
|
||||
haproxyctl vhost add podcast.gk2.secubox.in # backend = mitmproxy_inspector
|
||||
# add the route to BOTH mitmproxy routes files -> 127.0.0.1:<nginx>
|
||||
systemctl restart mitmproxy
|
||||
```
|
||||
|
||||
Then set `public_base = "https://podcast.gk2.secubox.in"` in
|
||||
`/etc/secubox/podcaster.toml` and restart `secubox-podcaster` so the generated
|
||||
feed's enclosure URLs are absolute.
|
||||
|
||||
---
|
||||
*CyberMind — Gérald Kerma. LicenseRef-CMSD-1.0.*
|
||||
0
packages/secubox-podcaster/api/__init__.py
Normal file
0
packages/secubox-podcaster/api/__init__.py
Normal file
547
packages/secubox-podcaster/api/main.py
Normal file
547
packages/secubox-podcaster/api/main.py
Normal file
|
|
@ -0,0 +1,547 @@
|
|||
# SPDX-License-Identifier: LicenseRef-CMSD-1.0
|
||||
# Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
|
||||
# Source-Disclosed License — All rights reserved except as expressly granted.
|
||||
# See LICENCE-CMSD-1.0.md for terms.
|
||||
|
||||
"""
|
||||
SecuBox-Deb :: Podcaster
|
||||
CyberMind — https://cybermind.fr
|
||||
|
||||
Modern podcast manager: subscribe to feeds, download episodes locally, and
|
||||
re-publish a shareable RSS. FastAPI on a Unix socket; SQLite store; an asyncio
|
||||
download worker (background task, fire-and-forget queue). Feed parsing is pure
|
||||
stdlib (xml.etree) to keep Debian deps minimal — only httpx is required.
|
||||
"""
|
||||
import asyncio
|
||||
import html
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import tempfile
|
||||
import time
|
||||
import tomllib
|
||||
import zipfile
|
||||
from email.utils import parsedate_to_datetime, format_datetime
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from xml.etree import ElementTree as ET
|
||||
|
||||
import httpx
|
||||
from fastapi import FastAPI, APIRouter, BackgroundTasks, Depends, HTTPException, Request
|
||||
from fastapi.responses import Response, FileResponse, JSONResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
from secubox_core.auth import router as auth_router, require_jwt
|
||||
from secubox_core.logger import get_logger
|
||||
|
||||
from . import store
|
||||
|
||||
log = get_logger("podcaster")
|
||||
|
||||
CONFIG_FILE = Path("/etc/secubox/podcaster.toml")
|
||||
DEFAULT_CONFIG = {
|
||||
"media_path": "/var/lib/secubox/podcaster/media",
|
||||
"max_parallel": 2,
|
||||
"refresh_minutes": 60,
|
||||
"public_base": "", # e.g. https://podcast.gk2.secubox.in (exposure); empty → LAN/relative
|
||||
"share_title": "SecuBox Podcaster",
|
||||
"keep_per_feed": 0, # 0 = unlimited
|
||||
}
|
||||
|
||||
|
||||
def load_config() -> dict:
|
||||
cfg = dict(DEFAULT_CONFIG)
|
||||
try:
|
||||
if CONFIG_FILE.exists():
|
||||
cfg.update(tomllib.loads(CONFIG_FILE.read_text()))
|
||||
except Exception as e: # pragma: no cover - defensive
|
||||
log.error(f"config load failed: {e}")
|
||||
return cfg
|
||||
|
||||
|
||||
CFG = load_config()
|
||||
MEDIA = Path(CFG["media_path"])
|
||||
|
||||
app = FastAPI(title="secubox-podcaster", version="1.0.0", root_path="/api/v1/podcaster")
|
||||
app.include_router(auth_router, prefix="/auth")
|
||||
router = APIRouter()
|
||||
|
||||
# ── download queue (in-process, persisted state in SQLite) ──────────
|
||||
_queue: "asyncio.Queue[int]" = asyncio.Queue()
|
||||
_worker_started = False
|
||||
|
||||
|
||||
# ════════════════════════════════════════════════════════════════════
|
||||
# Models
|
||||
# ════════════════════════════════════════════════════════════════════
|
||||
class FeedIn(BaseModel):
|
||||
url: str
|
||||
auto_dl: bool = True
|
||||
|
||||
|
||||
class OPMLIn(BaseModel):
|
||||
opml: str
|
||||
|
||||
|
||||
class ConfigIn(BaseModel):
|
||||
media_path: Optional[str] = None
|
||||
max_parallel: Optional[int] = None
|
||||
refresh_minutes: Optional[int] = None
|
||||
public_base: Optional[str] = None
|
||||
share_title: Optional[str] = None
|
||||
keep_per_feed: Optional[int] = None
|
||||
|
||||
|
||||
# ════════════════════════════════════════════════════════════════════
|
||||
# Feed parsing (stdlib)
|
||||
# ════════════════════════════════════════════════════════════════════
|
||||
_NS = {"itunes": "http://www.itunes.com/dtds/podcast-1.0.dtd",
|
||||
"atom": "http://www.w3.org/2005/Atom"}
|
||||
|
||||
|
||||
def _ts(s: Optional[str]) -> int:
|
||||
if not s:
|
||||
return 0
|
||||
try:
|
||||
return int(parsedate_to_datetime(s).timestamp())
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
|
||||
def parse_feed(xml_bytes: bytes) -> tuple[dict, list[dict]]:
|
||||
"""Return (feed_meta, [episode, ...]) from RSS 2.0 bytes."""
|
||||
root = ET.fromstring(xml_bytes)
|
||||
chan = root.find("channel")
|
||||
if chan is None: # not RSS we understand
|
||||
return {}, []
|
||||
|
||||
def t(el, tag):
|
||||
x = el.find(tag)
|
||||
return x.text.strip() if x is not None and x.text else None
|
||||
|
||||
img = None
|
||||
ic = chan.find("itunes:image", _NS)
|
||||
if ic is not None:
|
||||
img = ic.get("href")
|
||||
if not img:
|
||||
im = chan.find("image/url")
|
||||
img = im.text.strip() if im is not None and im.text else None
|
||||
|
||||
meta = {
|
||||
"title": t(chan, "title"),
|
||||
"description": t(chan, "description"),
|
||||
"site": t(chan, "link"),
|
||||
"image": img,
|
||||
}
|
||||
|
||||
episodes = []
|
||||
for it in chan.findall("item"):
|
||||
enc = it.find("enclosure")
|
||||
if enc is None or not enc.get("url"):
|
||||
continue
|
||||
guid = t(it, "guid") or enc.get("url")
|
||||
dur = it.find("itunes:duration", _NS)
|
||||
episodes.append({
|
||||
"guid": guid,
|
||||
"title": t(it, "title"),
|
||||
"description": t(it, "description"),
|
||||
"pubdate": _ts(t(it, "pubDate")),
|
||||
"enclosure": enc.get("url"),
|
||||
"mime": enc.get("type") or "audio/mpeg",
|
||||
"bytes": int(enc.get("length") or 0),
|
||||
"duration": dur.text.strip() if dur is not None and dur.text else None,
|
||||
})
|
||||
return meta, episodes
|
||||
|
||||
|
||||
async def fetch_and_store(url: str, auto_dl: Optional[bool] = None) -> dict:
|
||||
async with httpx.AsyncClient(follow_redirects=True, timeout=30) as cli:
|
||||
r = await cli.get(url, headers={"User-Agent": "SecuBox-Podcaster/1.0"})
|
||||
r.raise_for_status()
|
||||
meta, eps = parse_feed(r.content)
|
||||
fid = store.add_feed(url, meta, auto_dl=1 if auto_dl else 0)
|
||||
store.update_feed_meta(fid, meta)
|
||||
if auto_dl is not None: # explicit add/toggle; None = refresh (keep current)
|
||||
store.set_feed_autodl(fid, 1 if auto_dl else 0)
|
||||
for ep in eps:
|
||||
store.upsert_episode(fid, ep)
|
||||
queued = await _autoqueue(fid)
|
||||
return {"feed_id": fid, "title": meta.get("title"),
|
||||
"episodes": len(eps), "queued": queued}
|
||||
|
||||
|
||||
async def _autoqueue(fid: int) -> int:
|
||||
"""If a feed has auto_dl, enqueue its not-yet-downloaded episodes (newest
|
||||
first, capped by keep_per_feed to avoid unbounded disk use)."""
|
||||
if not store.feed_autodl(fid):
|
||||
return 0
|
||||
cap = int(CFG.get("keep_per_feed", 0) or 0)
|
||||
n = 0
|
||||
for ep_id in store.pending_episode_ids(fid, cap):
|
||||
store.set_episode(ep_id, state="queued", progress=0, error=None)
|
||||
await _queue.put(ep_id)
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
# ════════════════════════════════════════════════════════════════════
|
||||
# Download worker
|
||||
# ════════════════════════════════════════════════════════════════════
|
||||
def _safe_name(s: str, ext: str) -> str:
|
||||
base = re.sub(r"[^A-Za-z0-9._-]+", "_", (s or "episode"))[:80].strip("_")
|
||||
return f"{base or 'episode'}{ext}"
|
||||
|
||||
|
||||
async def _download_one(ep_id: int) -> None:
|
||||
ep = store.get_episode(ep_id)
|
||||
if not ep or not ep.get("enclosure"):
|
||||
return
|
||||
store.set_episode(ep_id, state="downloading", progress=0, error=None)
|
||||
ext = os.path.splitext(ep["enclosure"].split("?")[0])[1][:5] or ".mp3"
|
||||
fdir = MEDIA / str(ep["feed_id"])
|
||||
fdir.mkdir(parents=True, exist_ok=True)
|
||||
dest = fdir / _safe_name(f"{ep_id}_{ep.get('title','')}", ext)
|
||||
tmp = dest.with_suffix(dest.suffix + ".part")
|
||||
try:
|
||||
async with httpx.AsyncClient(follow_redirects=True, timeout=None) as cli:
|
||||
async with cli.stream("GET", ep["enclosure"],
|
||||
headers={"User-Agent": "SecuBox-Podcaster/1.0"}) as r:
|
||||
r.raise_for_status()
|
||||
total = int(r.headers.get("content-length") or ep.get("bytes") or 0)
|
||||
got = 0
|
||||
with open(tmp, "wb") as fh:
|
||||
async for chunk in r.aiter_bytes(65536):
|
||||
fh.write(chunk)
|
||||
got += len(chunk)
|
||||
if total:
|
||||
store.set_episode(ep_id, progress=min(99, got * 100 // total))
|
||||
tmp.rename(dest)
|
||||
store.set_episode(ep_id, state="done", progress=100,
|
||||
local_path=str(dest), bytes=dest.stat().st_size)
|
||||
log.info(f"downloaded ep {ep_id} -> {dest}")
|
||||
except Exception as e:
|
||||
tmp.unlink(missing_ok=True)
|
||||
store.set_episode(ep_id, state="error", error=str(e)[:200])
|
||||
log.error(f"download ep {ep_id} failed: {e}")
|
||||
|
||||
|
||||
async def _worker() -> None:
|
||||
sem = asyncio.Semaphore(max(1, int(CFG.get("max_parallel", 2))))
|
||||
|
||||
async def run(ep_id):
|
||||
async with sem:
|
||||
await _download_one(ep_id)
|
||||
|
||||
while True:
|
||||
ep_id = await _queue.get()
|
||||
asyncio.create_task(run(ep_id))
|
||||
|
||||
|
||||
async def _refresher() -> None:
|
||||
while True:
|
||||
await asyncio.sleep(max(5, int(CFG.get("refresh_minutes", 60))) * 60)
|
||||
for fid, url in store.all_feed_urls():
|
||||
try:
|
||||
await fetch_and_store(url)
|
||||
except Exception as e:
|
||||
log.error(f"refresh feed {fid} failed: {e}")
|
||||
|
||||
|
||||
def _ensure_worker() -> None:
|
||||
"""Lazy-start background tasks (sub-app lifespans don't fire under the aggregator)."""
|
||||
global _worker_started
|
||||
if _worker_started:
|
||||
return
|
||||
_worker_started = True
|
||||
store.init()
|
||||
MEDIA.mkdir(parents=True, exist_ok=True)
|
||||
asyncio.create_task(_worker())
|
||||
asyncio.create_task(_refresher())
|
||||
|
||||
|
||||
@app.on_event("startup")
|
||||
async def _startup():
|
||||
_ensure_worker()
|
||||
|
||||
|
||||
# ════════════════════════════════════════════════════════════════════
|
||||
# Public endpoints (status / health / share)
|
||||
# ════════════════════════════════════════════════════════════════════
|
||||
@app.get("/health")
|
||||
async def health():
|
||||
return {"status": "ok", "module": "deb"}
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
async def status():
|
||||
_ensure_worker()
|
||||
return {
|
||||
"service": "active",
|
||||
"worker": _worker_started,
|
||||
"media_path": str(MEDIA),
|
||||
"public_base": CFG.get("public_base", ""),
|
||||
**store.counts(),
|
||||
}
|
||||
|
||||
|
||||
def _xml_esc(s: Optional[str]) -> str:
|
||||
return html.escape(s or "", quote=True)
|
||||
|
||||
|
||||
@router.get("/share/feed.xml")
|
||||
async def share_feed(request: Request):
|
||||
"""A generated RSS of the locally downloaded episodes (the relay/share feed)."""
|
||||
_ensure_worker()
|
||||
base = (CFG.get("public_base") or "").rstrip("/")
|
||||
if not base:
|
||||
base = str(request.base_url).rstrip("/")
|
||||
eps = store.downloaded_episodes()
|
||||
items = []
|
||||
for e in eps:
|
||||
url = f"{base}/api/v1/podcaster/media/{e['id']}"
|
||||
pub = format_datetime(datetime.fromtimestamp(e["pubdate"] or time.time(),
|
||||
tz=timezone.utc))
|
||||
items.append(
|
||||
f"<item><title>{_xml_esc(e.get('title'))}</title>"
|
||||
f"<description>{_xml_esc(e.get('description'))}</description>"
|
||||
f"<pubDate>{pub}</pubDate>"
|
||||
f"<guid isPermaLink=\"false\">sbx-{e['id']}</guid>"
|
||||
f"<enclosure url=\"{_xml_esc(url)}\" type=\"{_xml_esc(e.get('mime') or 'audio/mpeg')}\" "
|
||||
f"length=\"{e.get('bytes') or 0}\"/></item>"
|
||||
)
|
||||
title = _xml_esc(CFG.get("share_title", "SecuBox Podcaster"))
|
||||
rss = (
|
||||
'<?xml version="1.0" encoding="UTF-8"?>'
|
||||
'<rss version="2.0"><channel>'
|
||||
f"<title>{title}</title><link>{_xml_esc(base)}</link>"
|
||||
f"<description>Locally relayed by SecuBox Podcaster</description>"
|
||||
+ "".join(items) + "</channel></rss>"
|
||||
)
|
||||
return Response(content=rss, media_type="application/rss+xml")
|
||||
|
||||
|
||||
@router.get("/public/library")
|
||||
async def public_library():
|
||||
"""Public listener library — downloaded/shared episodes only (no auth).
|
||||
Feeds the external portal frontend; exposes nothing un-shared."""
|
||||
_ensure_worker()
|
||||
eps = store.downloaded_episodes()
|
||||
feeds: dict = {}
|
||||
for e in eps:
|
||||
k = e.get("feed_title") or "Podcast"
|
||||
feeds[k] = feeds.get(k, 0) + 1
|
||||
return {
|
||||
"title": CFG.get("share_title", "SecuBox Podcaster"),
|
||||
"episodes": [{
|
||||
"id": e["id"], "title": e.get("title"), "feed": e.get("feed_title"),
|
||||
"feed_id": e.get("feed_id"),
|
||||
"pubdate": e.get("pubdate"), "duration": e.get("duration"),
|
||||
"mime": e.get("mime"), "bytes": e.get("bytes"),
|
||||
"media": f"/api/v1/podcaster/media/{e['id']}",
|
||||
} for e in eps],
|
||||
"feeds": feeds,
|
||||
"share": "/api/v1/podcaster/share/feed.xml",
|
||||
}
|
||||
|
||||
|
||||
@router.get("/public/feed/{fid}/zip")
|
||||
async def public_feed_zip(fid: int, background: BackgroundTasks):
|
||||
"""Public: download all downloaded episodes of a feed as one ZIP.
|
||||
mp3/audio are already compressed → STORED (no recompress). Built to a temp
|
||||
file then streamed; cleaned up after send."""
|
||||
_ensure_worker()
|
||||
eps = [e for e in store.downloaded_episodes(limit=2000)
|
||||
if e.get("feed_id") == fid and e.get("local_path")
|
||||
and Path(e["local_path"]).exists()]
|
||||
if not eps:
|
||||
raise HTTPException(404, "no downloaded episodes for this feed")
|
||||
name = re.sub(r"[^A-Za-z0-9._-]+", "_", (eps[0].get("feed_title") or f"feed{fid}"))[:60] or f"feed{fid}"
|
||||
tmp = Path(tempfile.mkstemp(suffix=".zip", dir="/var/lib/secubox/podcaster")[1])
|
||||
eps.sort(key=lambda e: e.get("pubdate") or 0)
|
||||
with zipfile.ZipFile(tmp, "w", compression=zipfile.ZIP_STORED) as z:
|
||||
for i, e in enumerate(eps):
|
||||
p = Path(e["local_path"])
|
||||
z.write(p, arcname=_safe_name(f"{i+1:03d}_{e.get('title') or p.stem}", p.suffix))
|
||||
background.add_task(lambda: tmp.unlink(missing_ok=True))
|
||||
return FileResponse(tmp, media_type="application/zip", filename=f"{name}.zip",
|
||||
background=background)
|
||||
|
||||
|
||||
@router.get("/media/{ep_id}")
|
||||
async def media(ep_id: int):
|
||||
ep = store.get_episode(ep_id)
|
||||
if not ep or ep.get("state") != "done" or not ep.get("local_path"):
|
||||
raise HTTPException(404, "not downloaded")
|
||||
p = Path(ep["local_path"])
|
||||
if not p.exists():
|
||||
raise HTTPException(404, "file missing")
|
||||
return FileResponse(p, media_type=ep.get("mime") or "audio/mpeg",
|
||||
filename=p.name)
|
||||
|
||||
|
||||
# ════════════════════════════════════════════════════════════════════
|
||||
# Authenticated endpoints (manage)
|
||||
# ════════════════════════════════════════════════════════════════════
|
||||
@router.get("/feeds", dependencies=[Depends(require_jwt)])
|
||||
async def feeds():
|
||||
_ensure_worker()
|
||||
return {"feeds": store.list_feeds()}
|
||||
|
||||
|
||||
@router.post("/feeds", dependencies=[Depends(require_jwt)])
|
||||
async def add_feed(body: FeedIn):
|
||||
_ensure_worker()
|
||||
try:
|
||||
return await fetch_and_store(body.url.strip(), auto_dl=body.auto_dl)
|
||||
except Exception as e:
|
||||
raise HTTPException(400, f"feed add failed: {e}")
|
||||
|
||||
|
||||
@router.post("/feeds/{fid}/autodl", dependencies=[Depends(require_jwt)])
|
||||
async def toggle_autodl(fid: int, on: bool = True):
|
||||
"""Toggle auto-download for a feed. Turning it on also queues any episodes
|
||||
not yet downloaded (so the portal/share feed sync immediately)."""
|
||||
_ensure_worker()
|
||||
store.set_feed_autodl(fid, 1 if on else 0)
|
||||
queued = await _autoqueue(fid) if on else 0
|
||||
return {"ok": True, "auto_dl": on, "queued": queued}
|
||||
|
||||
|
||||
@router.delete("/feeds/{fid}", dependencies=[Depends(require_jwt)])
|
||||
async def del_feed(fid: int):
|
||||
store.delete_feed(fid)
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.post("/feeds/import-opml", dependencies=[Depends(require_jwt)])
|
||||
async def import_opml(body: OPMLIn):
|
||||
_ensure_worker()
|
||||
try:
|
||||
root = ET.fromstring(body.opml)
|
||||
except Exception as e:
|
||||
raise HTTPException(400, f"bad OPML: {e}")
|
||||
urls = [o.get("xmlUrl") for o in root.iter("outline") if o.get("xmlUrl")]
|
||||
added = 0
|
||||
for u in urls:
|
||||
try:
|
||||
await fetch_and_store(u)
|
||||
added += 1
|
||||
except Exception as e:
|
||||
log.error(f"opml feed {u}: {e}")
|
||||
return {"added": added, "total": len(urls)}
|
||||
|
||||
|
||||
_AUDIO_EXT = {".mp3", ".m4a", ".m4b", ".aac", ".ogg", ".opus", ".flac", ".wav", ".mp4"}
|
||||
_AUDIO_MIME = {".mp3": "audio/mpeg", ".m4a": "audio/mp4", ".m4b": "audio/mp4",
|
||||
".aac": "audio/aac", ".ogg": "audio/ogg", ".opus": "audio/opus",
|
||||
".flac": "audio/flac", ".wav": "audio/wav", ".mp4": "audio/mp4"}
|
||||
|
||||
|
||||
@router.post("/audiobook/upload", dependencies=[Depends(require_jwt)])
|
||||
async def audiobook_upload(request: Request, title: str = "Audiobook"):
|
||||
"""Stream an uploaded ZIP to a temp file, extract its audio tracks, and
|
||||
publish them as a self-contained (synthetic) feed — already 'done' so they
|
||||
show in the library + share feed. Raw body (no python-multipart dep)."""
|
||||
_ensure_worker()
|
||||
title = (title or "Audiobook").strip() or "Audiobook"
|
||||
tmp = Path(tempfile.mkstemp(suffix=".zip", dir="/var/lib/secubox/podcaster")[1])
|
||||
try:
|
||||
with open(tmp, "wb") as fh:
|
||||
async for chunk in request.stream():
|
||||
fh.write(chunk)
|
||||
if not zipfile.is_zipfile(tmp):
|
||||
raise HTTPException(400, "not a valid ZIP")
|
||||
# synthetic feed
|
||||
slug = re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-") or "audiobook"
|
||||
fid = store.add_feed(f"audiobook:{slug}:{int(time.time())}",
|
||||
{"title": title, "description": f"Audiobook: {title}"})
|
||||
fdir = MEDIA / str(fid)
|
||||
fdir.mkdir(parents=True, exist_ok=True)
|
||||
tracks = 0
|
||||
with zipfile.ZipFile(tmp) as z:
|
||||
names = [n for n in z.namelist()
|
||||
if os.path.splitext(n)[1].lower() in _AUDIO_EXT
|
||||
and not n.endswith("/")]
|
||||
names.sort() # chapter order
|
||||
base = int(time.time())
|
||||
for i, n in enumerate(names):
|
||||
ext = os.path.splitext(n)[1].lower()
|
||||
tname = _safe_name(f"{i+1:03d}_{os.path.basename(n)}", ext)
|
||||
dest = fdir / tname
|
||||
with z.open(n) as src, open(dest, "wb") as out:
|
||||
shutil.copyfileobj(src, out, 1024 * 256)
|
||||
store.upsert_episode(fid, {
|
||||
"guid": f"ab:{fid}:{i}", "title": f"{i+1:02d} · {os.path.splitext(os.path.basename(n))[0]}",
|
||||
"description": title, "pubdate": base + i,
|
||||
"enclosure": f"local:{dest}", "mime": _AUDIO_MIME.get(ext, "audio/mpeg"),
|
||||
"bytes": dest.stat().st_size,
|
||||
})
|
||||
# mark immediately downloaded (local file already present)
|
||||
ep = store.list_episodes(feed_id=fid)
|
||||
for e in ep:
|
||||
if e["guid"] == f"ab:{fid}:{i}":
|
||||
store.set_episode(e["id"], state="done", progress=100,
|
||||
local_path=str(dest))
|
||||
break
|
||||
tracks += 1
|
||||
if not tracks:
|
||||
store.delete_feed(fid)
|
||||
raise HTTPException(400, "no audio files found in ZIP")
|
||||
log.info(f"audiobook '{title}' -> feed {fid}, {tracks} tracks")
|
||||
return {"title": title, "feed_id": fid, "tracks": tracks}
|
||||
finally:
|
||||
tmp.unlink(missing_ok=True)
|
||||
|
||||
|
||||
@router.get("/episodes", dependencies=[Depends(require_jwt)])
|
||||
async def episodes(feed_id: Optional[int] = None, state: Optional[str] = None):
|
||||
_ensure_worker()
|
||||
return {"episodes": store.list_episodes(feed_id, state)}
|
||||
|
||||
|
||||
@router.post("/episodes/{ep_id}/download", dependencies=[Depends(require_jwt)])
|
||||
async def download(ep_id: int):
|
||||
_ensure_worker()
|
||||
ep = store.get_episode(ep_id)
|
||||
if not ep:
|
||||
raise HTTPException(404, "no such episode")
|
||||
store.set_episode(ep_id, state="queued", progress=0, error=None)
|
||||
await _queue.put(ep_id)
|
||||
return {"ok": True, "queued": ep_id}
|
||||
|
||||
|
||||
@router.get("/downloads", dependencies=[Depends(require_jwt)])
|
||||
async def downloads():
|
||||
_ensure_worker()
|
||||
return {"downloads": store.list_episodes(state="downloading")
|
||||
+ store.list_episodes(state="queued")}
|
||||
|
||||
|
||||
@router.get("/config", dependencies=[Depends(require_jwt)])
|
||||
async def get_config():
|
||||
return {"config": load_config()}
|
||||
|
||||
|
||||
@router.post("/config", dependencies=[Depends(require_jwt)])
|
||||
async def set_config(body: ConfigIn):
|
||||
cfg = load_config()
|
||||
for k, v in body.model_dump(exclude_none=True).items():
|
||||
cfg[k] = v
|
||||
lines = []
|
||||
for k, v in cfg.items():
|
||||
if isinstance(v, str):
|
||||
lines.append(f'{k} = "{v}"')
|
||||
elif isinstance(v, bool):
|
||||
lines.append(f"{k} = {'true' if v else 'false'}")
|
||||
else:
|
||||
lines.append(f"{k} = {v}")
|
||||
try:
|
||||
CONFIG_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
CONFIG_FILE.write_text("\n".join(lines) + "\n")
|
||||
except Exception as e:
|
||||
raise HTTPException(500, f"write failed: {e}")
|
||||
return {"ok": True, "config": cfg, "note": "restart secubox-podcaster to apply"}
|
||||
|
||||
|
||||
app.include_router(router)
|
||||
198
packages/secubox-podcaster/api/store.py
Normal file
198
packages/secubox-podcaster/api/store.py
Normal file
|
|
@ -0,0 +1,198 @@
|
|||
# SPDX-License-Identifier: LicenseRef-CMSD-1.0
|
||||
# Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
|
||||
# Source-Disclosed License — All rights reserved except as expressly granted.
|
||||
# See LICENCE-CMSD-1.0.md for terms.
|
||||
|
||||
"""
|
||||
SecuBox-Deb :: Podcaster store
|
||||
CyberMind — https://cybermind.fr
|
||||
|
||||
SQLite persistence for feeds + episodes. Pure stdlib (sqlite3); no ORM.
|
||||
"""
|
||||
import sqlite3
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
DB_PATH = Path("/var/lib/secubox/podcaster/podcaster.db")
|
||||
|
||||
_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS feeds (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
url TEXT UNIQUE NOT NULL,
|
||||
title TEXT,
|
||||
description TEXT,
|
||||
image TEXT,
|
||||
site TEXT,
|
||||
added_ts INTEGER NOT NULL,
|
||||
last_fetch INTEGER DEFAULT 0,
|
||||
auto_dl INTEGER DEFAULT 0
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS episodes (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
feed_id INTEGER NOT NULL REFERENCES feeds(id) ON DELETE CASCADE,
|
||||
guid TEXT NOT NULL,
|
||||
title TEXT,
|
||||
description TEXT,
|
||||
pubdate INTEGER DEFAULT 0,
|
||||
enclosure TEXT,
|
||||
mime TEXT,
|
||||
bytes INTEGER DEFAULT 0,
|
||||
duration TEXT,
|
||||
local_path TEXT,
|
||||
state TEXT DEFAULT 'new', -- new | queued | downloading | done | error
|
||||
progress INTEGER DEFAULT 0, -- 0..100
|
||||
error TEXT,
|
||||
UNIQUE(feed_id, guid)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_ep_feed ON episodes(feed_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_ep_state ON episodes(state);
|
||||
CREATE INDEX IF NOT EXISTS idx_ep_pub ON episodes(pubdate DESC);
|
||||
"""
|
||||
|
||||
|
||||
def _conn() -> sqlite3.Connection:
|
||||
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
c = sqlite3.connect(DB_PATH, timeout=10)
|
||||
c.row_factory = sqlite3.Row
|
||||
c.execute("PRAGMA foreign_keys=ON")
|
||||
c.execute("PRAGMA journal_mode=WAL")
|
||||
return c
|
||||
|
||||
|
||||
def init() -> None:
|
||||
with _conn() as c:
|
||||
c.executescript(_SCHEMA)
|
||||
|
||||
|
||||
# ── feeds ──────────────────────────────────────────────────────────
|
||||
def add_feed(url: str, meta: dict, auto_dl: int = 0) -> int:
|
||||
with _conn() as c:
|
||||
cur = c.execute(
|
||||
"INSERT OR IGNORE INTO feeds(url,title,description,image,site,added_ts,auto_dl) "
|
||||
"VALUES(?,?,?,?,?,?,?)",
|
||||
(url, meta.get("title"), meta.get("description"),
|
||||
meta.get("image"), meta.get("site"), int(time.time()), int(auto_dl)),
|
||||
)
|
||||
if cur.lastrowid:
|
||||
return cur.lastrowid
|
||||
row = c.execute("SELECT id FROM feeds WHERE url=?", (url,)).fetchone()
|
||||
return row["id"] if row else 0
|
||||
|
||||
|
||||
def set_feed_autodl(feed_id: int, on: int) -> None:
|
||||
with _conn() as c:
|
||||
c.execute("UPDATE feeds SET auto_dl=? WHERE id=?", (int(on), feed_id))
|
||||
|
||||
|
||||
def feed_autodl(feed_id: int) -> int:
|
||||
with _conn() as c:
|
||||
r = c.execute("SELECT auto_dl FROM feeds WHERE id=?", (feed_id,)).fetchone()
|
||||
return int(r["auto_dl"]) if r else 0
|
||||
|
||||
|
||||
def pending_episode_ids(feed_id: int, limit: int = 0) -> list[int]:
|
||||
"""Episode ids in state 'new' for a feed (newest first); for auto-download."""
|
||||
q = "SELECT id FROM episodes WHERE feed_id=? AND state='new' ORDER BY pubdate DESC"
|
||||
if limit and limit > 0:
|
||||
q += f" LIMIT {int(limit)}"
|
||||
with _conn() as c:
|
||||
return [r["id"] for r in c.execute(q, (feed_id,)).fetchall()]
|
||||
|
||||
|
||||
def update_feed_meta(feed_id: int, meta: dict) -> None:
|
||||
with _conn() as c:
|
||||
c.execute(
|
||||
"UPDATE feeds SET title=COALESCE(?,title), description=COALESCE(?,description), "
|
||||
"image=COALESCE(?,image), site=COALESCE(?,site), last_fetch=? WHERE id=?",
|
||||
(meta.get("title"), meta.get("description"), meta.get("image"),
|
||||
meta.get("site"), int(time.time()), feed_id),
|
||||
)
|
||||
|
||||
|
||||
def list_feeds() -> list[dict]:
|
||||
with _conn() as c:
|
||||
rows = c.execute(
|
||||
"SELECT f.*, "
|
||||
"(SELECT COUNT(*) FROM episodes e WHERE e.feed_id=f.id) AS episodes, "
|
||||
"(SELECT COUNT(*) FROM episodes e WHERE e.feed_id=f.id AND e.state='done') AS downloaded "
|
||||
"FROM feeds f ORDER BY f.title COLLATE NOCASE"
|
||||
).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
def delete_feed(feed_id: int) -> None:
|
||||
with _conn() as c:
|
||||
c.execute("DELETE FROM feeds WHERE id=?", (feed_id,))
|
||||
|
||||
|
||||
def all_feed_urls() -> list[tuple[int, str]]:
|
||||
with _conn() as c:
|
||||
return [(r["id"], r["url"]) for r in c.execute("SELECT id,url FROM feeds")]
|
||||
|
||||
|
||||
# ── episodes ───────────────────────────────────────────────────────
|
||||
def upsert_episode(feed_id: int, ep: dict) -> None:
|
||||
with _conn() as c:
|
||||
c.execute(
|
||||
"INSERT OR IGNORE INTO episodes"
|
||||
"(feed_id,guid,title,description,pubdate,enclosure,mime,bytes,duration) "
|
||||
"VALUES(?,?,?,?,?,?,?,?,?)",
|
||||
(feed_id, ep["guid"], ep.get("title"), ep.get("description"),
|
||||
ep.get("pubdate", 0), ep.get("enclosure"), ep.get("mime"),
|
||||
ep.get("bytes", 0), ep.get("duration")),
|
||||
)
|
||||
|
||||
|
||||
def list_episodes(feed_id: Optional[int] = None, state: Optional[str] = None,
|
||||
limit: int = 200) -> list[dict]:
|
||||
q = ("SELECT e.*, f.title AS feed_title, f.image AS feed_image "
|
||||
"FROM episodes e JOIN feeds f ON f.id=e.feed_id")
|
||||
where, args = [], []
|
||||
if feed_id is not None:
|
||||
where.append("e.feed_id=?"); args.append(feed_id)
|
||||
if state:
|
||||
where.append("e.state=?"); args.append(state)
|
||||
if where:
|
||||
q += " WHERE " + " AND ".join(where)
|
||||
q += " ORDER BY e.pubdate DESC LIMIT ?"; args.append(limit)
|
||||
with _conn() as c:
|
||||
return [dict(r) for r in c.execute(q, args).fetchall()]
|
||||
|
||||
|
||||
def get_episode(ep_id: int) -> Optional[dict]:
|
||||
with _conn() as c:
|
||||
r = c.execute("SELECT * FROM episodes WHERE id=?", (ep_id,)).fetchone()
|
||||
return dict(r) if r else None
|
||||
|
||||
|
||||
def set_episode(ep_id: int, **fields: Any) -> None:
|
||||
if not fields:
|
||||
return
|
||||
cols = ", ".join(f"{k}=?" for k in fields)
|
||||
with _conn() as c:
|
||||
c.execute(f"UPDATE episodes SET {cols} WHERE id=?", (*fields.values(), ep_id))
|
||||
|
||||
|
||||
def downloaded_episodes(limit: int = 500) -> list[dict]:
|
||||
"""Episodes with a local file — the shareable library."""
|
||||
with _conn() as c:
|
||||
rows = c.execute(
|
||||
"SELECT e.*, f.title AS feed_title FROM episodes e JOIN feeds f ON f.id=e.feed_id "
|
||||
"WHERE e.state='done' AND e.local_path IS NOT NULL "
|
||||
"ORDER BY e.pubdate DESC LIMIT ?", (limit,)
|
||||
).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
def counts() -> dict:
|
||||
with _conn() as c:
|
||||
return {
|
||||
"feeds": c.execute("SELECT COUNT(*) FROM feeds").fetchone()[0],
|
||||
"episodes": c.execute("SELECT COUNT(*) FROM episodes").fetchone()[0],
|
||||
"downloaded": c.execute(
|
||||
"SELECT COUNT(*) FROM episodes WHERE state='done'").fetchone()[0],
|
||||
"queued": c.execute(
|
||||
"SELECT COUNT(*) FROM episodes WHERE state IN('queued','downloading')"
|
||||
).fetchone()[0],
|
||||
}
|
||||
23
packages/secubox-podcaster/conf/podcaster.toml
Normal file
23
packages/secubox-podcaster/conf/podcaster.toml
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
# /etc/secubox/podcaster.toml — SecuBox Podcaster
|
||||
# Restart secubox-podcaster after editing.
|
||||
|
||||
# Where downloaded episodes are stored.
|
||||
media_path = "/var/lib/secubox/podcaster/media"
|
||||
|
||||
# Concurrent downloads.
|
||||
max_parallel = 2
|
||||
|
||||
# Auto-refresh feeds every N minutes (background).
|
||||
refresh_minutes = 60
|
||||
|
||||
# Public base URL for the shareable RSS enclosure links. Leave empty for
|
||||
# LAN/relative URLs. Set to your exposed vhost to relay publicly, e.g.
|
||||
# public_base = "https://podcast.gk2.secubox.in"
|
||||
# (route it through HAProxy TLS -> mitmproxy per WAF policy; see README).
|
||||
public_base = ""
|
||||
|
||||
# Title of the generated share feed.
|
||||
share_title = "SecuBox Podcaster"
|
||||
|
||||
# Keep at most N downloaded episodes per feed (0 = unlimited).
|
||||
keep_per_feed = 0
|
||||
46
packages/secubox-podcaster/debian/changelog
Normal file
46
packages/secubox-podcaster/debian/changelog
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
secubox-podcaster (1.0.3-1~bookworm1) bookworm; urgency=medium
|
||||
|
||||
* Auto-download: feeds can auto-queue new episodes so the portal + share feed
|
||||
stay synced without manual downloads. New feeds default auto_dl=on (UI
|
||||
checkbox); per-feed toggle (POST /feeds/{id}/autodl, ⏬ in admin). The
|
||||
periodic refresher auto-queues newly published episodes; bounded by
|
||||
keep_per_feed. (#726)
|
||||
|
||||
-- Gerald KERMA <devel@cybermind.fr> Wed, 24 Jun 2026 14:20:00 +0000
|
||||
|
||||
secubox-podcaster (1.0.2-1~bookworm1) bookworm; urgency=medium
|
||||
|
||||
* Public portal: per-episode download (⬇, mp3/audio) + per-feed
|
||||
"Download all (ZIP)" via new public GET /public/feed/{id}/zip (STORED, no
|
||||
recompress; built to temp then streamed + cleaned up). public/library now
|
||||
carries feed_id so the portal can zip by feed.
|
||||
|
||||
-- Gerald KERMA <devel@cybermind.fr> Wed, 24 Jun 2026 13:40:00 +0000
|
||||
|
||||
secubox-podcaster (1.0.1-1~bookworm1) bookworm; urgency=medium
|
||||
|
||||
* Admin WebUI: integrate the shared Hub sidebar (/shared/sidebar.js) + correct
|
||||
auth token (sbx_token, 401 -> /login.html) — was missing the navbar and
|
||||
falsely prompting to log in.
|
||||
* Public listener PORTAL (no auth) at /podcaster/portal/ — standalone
|
||||
C3BOX page driven by a new public GET /public/library endpoint; subscribe
|
||||
(RSS) + per-feed filter + inline players. Meant to be exposed externally.
|
||||
* Audiobook ZIP import: POST /audiobook/upload (raw body, no python-multipart)
|
||||
streams the ZIP to a temp file, extracts audio tracks, and publishes them as
|
||||
a self-contained synthetic feed (already 'done' -> in library + share feed).
|
||||
|
||||
-- Gerald KERMA <devel@cybermind.fr> Wed, 24 Jun 2026 13:00:00 +0000
|
||||
|
||||
secubox-podcaster (1.0.0-1~bookworm1) bookworm; urgency=medium
|
||||
|
||||
* Initial release (#726). Modern podcast manager:
|
||||
- subscribe by RSS URL / OPML import; pure-stdlib feed parsing
|
||||
- local download queue (asyncio + httpx) with per-episode progress
|
||||
- generated shareable RSS of the local library (/share/feed.xml) — LAN by
|
||||
default, public via HAProxy / secubox-exposure (public_base)
|
||||
- in-UI service status + TOML config; C3BOX WebUI with inline player
|
||||
- FastAPI on /run/secubox/podcaster.sock; SQLite store; nginx route shipped
|
||||
to the active secubox-routes.d/ include
|
||||
- Lyrion integration deferred to a follow-up (standalone first)
|
||||
|
||||
-- Gerald KERMA <devel@cybermind.fr> Wed, 24 Jun 2026 12:00:00 +0000
|
||||
22
packages/secubox-podcaster/debian/control
Normal file
22
packages/secubox-podcaster/debian/control
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
Source: secubox-podcaster
|
||||
Section: sound
|
||||
Priority: optional
|
||||
Maintainer: Gerald KERMA <devel@cybermind.fr>
|
||||
Build-Depends: debhelper-compat (= 13)
|
||||
Standards-Version: 4.6.2
|
||||
|
||||
Package: secubox-podcaster
|
||||
Architecture: all
|
||||
Depends: ${misc:Depends}, secubox-core (>= 1.0), python3, python3-uvicorn | python3-pip, python3-httpx | python3-pip
|
||||
Description: Modern podcast manager for SecuBox
|
||||
Subscribe to podcast feeds, download episodes locally, and re-publish a
|
||||
shareable RSS feed (LAN by default, public via HAProxy / secubox-exposure).
|
||||
FastAPI backend on /api/v1/podcaster/ over a Unix socket; SQLite store; an
|
||||
asyncio download queue. Feed parsing is pure stdlib (no feedparser dep).
|
||||
.
|
||||
Features:
|
||||
- Subscribe by RSS URL or OPML import
|
||||
- Local download queue with per-episode progress
|
||||
- Generated shareable RSS of the local library (relay/share to the web)
|
||||
- In-UI service status + TOML config
|
||||
- Modern C3BOX cyberpunk WebUI with inline player
|
||||
33
packages/secubox-podcaster/debian/postinst
Normal file
33
packages/secubox-podcaster/debian/postinst
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
case "$1" in
|
||||
configure)
|
||||
# Data dirs (module-owned; never touch the shared /run/secubox parent, #494).
|
||||
install -d -o secubox -g secubox -m 0750 /var/lib/secubox/podcaster
|
||||
install -d -o secubox -g secubox -m 0750 /var/lib/secubox/podcaster/media
|
||||
|
||||
# Config on first install only (preserve operator edits on upgrade).
|
||||
if [ ! -f /etc/secubox/podcaster.toml ]; then
|
||||
if [ -f /usr/share/secubox/podcaster/podcaster.toml ]; then
|
||||
install -o secubox -g secubox -m 0640 \
|
||||
/usr/share/secubox/podcaster/podcaster.toml /etc/secubox/podcaster.toml
|
||||
fi
|
||||
fi
|
||||
|
||||
systemctl daemon-reload 2>/dev/null || true
|
||||
# Respect operator masking (set -e would abort on a masked enable).
|
||||
if [ "$(systemctl is-enabled secubox-podcaster.service 2>/dev/null)" != "masked" ]; then
|
||||
systemctl enable secubox-podcaster.service 2>/dev/null || true
|
||||
systemctl restart secubox-podcaster.service 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# Reload nginx only if the resulting config is valid.
|
||||
if command -v nginx >/dev/null 2>&1 && nginx -t 2>/dev/null; then
|
||||
systemctl reload nginx 2>/dev/null || true
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
#DEBHELPER#
|
||||
exit 0
|
||||
12
packages/secubox-podcaster/debian/prerm
Normal file
12
packages/secubox-podcaster/debian/prerm
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
case "$1" in
|
||||
remove|deconfigure)
|
||||
systemctl stop secubox-podcaster.service 2>/dev/null || true
|
||||
systemctl disable secubox-podcaster.service 2>/dev/null || true
|
||||
;;
|
||||
esac
|
||||
|
||||
#DEBHELPER#
|
||||
exit 0
|
||||
30
packages/secubox-podcaster/debian/rules
Executable file
30
packages/secubox-podcaster/debian/rules
Executable file
|
|
@ -0,0 +1,30 @@
|
|||
#!/usr/bin/make -f
|
||||
%:
|
||||
dh $@
|
||||
|
||||
override_dh_auto_install:
|
||||
# API files (service WorkingDirectory = /usr/share/secubox/podcaster)
|
||||
install -d debian/secubox-podcaster/usr/share/secubox/podcaster/
|
||||
cp -r api debian/secubox-podcaster/usr/share/secubox/podcaster/
|
||||
# Static www files
|
||||
install -d debian/secubox-podcaster/usr/share/secubox/www
|
||||
[ -d www ] && cp -r www/. debian/secubox-podcaster/usr/share/secubox/www/ || true
|
||||
# Menu definitions
|
||||
install -d debian/secubox-podcaster/usr/share/secubox/menu.d
|
||||
[ -d menu.d ] && cp -r menu.d/. debian/secubox-podcaster/usr/share/secubox/menu.d/ || true
|
||||
# Config template (installed by postinst on first install only)
|
||||
install -d debian/secubox-podcaster/usr/share/secubox/podcaster
|
||||
[ -f conf/podcaster.toml ] && cp conf/podcaster.toml debian/secubox-podcaster/usr/share/secubox/podcaster/ || true
|
||||
# nginx route — ship to the ACTIVE include (secubox-routes.d) AND legacy
|
||||
# secubox.d for back-compat (mirrors grafana/peertube; see #65).
|
||||
install -d debian/secubox-podcaster/etc/nginx/secubox-routes.d
|
||||
install -d debian/secubox-podcaster/etc/nginx/secubox.d
|
||||
[ -f nginx/podcaster.conf ] && cp nginx/podcaster.conf debian/secubox-podcaster/etc/nginx/secubox-routes.d/ || true
|
||||
[ -f nginx/podcaster.conf ] && cp nginx/podcaster.conf debian/secubox-podcaster/etc/nginx/secubox.d/ || true
|
||||
# Systemd unit
|
||||
install -d debian/secubox-podcaster/usr/lib/systemd/system
|
||||
cp systemd/secubox-podcaster.service debian/secubox-podcaster/usr/lib/systemd/system/
|
||||
|
||||
override_dh_installsystemd:
|
||||
# Handled manually in postinst (respect operator masking).
|
||||
true
|
||||
1
packages/secubox-podcaster/debian/source/format
Normal file
1
packages/secubox-podcaster/debian/source/format
Normal file
|
|
@ -0,0 +1 @@
|
|||
3.0 (native)
|
||||
9
packages/secubox-podcaster/menu.d/612-podcaster.json
Normal file
9
packages/secubox-podcaster/menu.d/612-podcaster.json
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"id": "podcaster",
|
||||
"name": "Podcaster",
|
||||
"icon": "🎙️",
|
||||
"path": "/podcaster/",
|
||||
"category": "mesh",
|
||||
"order": 612,
|
||||
"description": "Subscribe, download and relay podcasts"
|
||||
}
|
||||
18
packages/secubox-podcaster/nginx/podcaster.conf
Normal file
18
packages/secubox-podcaster/nginx/podcaster.conf
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
# /etc/nginx/secubox-routes.d/podcaster.conf
|
||||
# Installed by secubox-podcaster — auto-registered on install, removed on purge.
|
||||
|
||||
# Static frontend
|
||||
location /podcaster/ {
|
||||
alias /usr/share/secubox/www/podcaster/;
|
||||
index index.html;
|
||||
try_files $uri $uri/ /podcaster/index.html;
|
||||
}
|
||||
|
||||
# API backend (Unix socket). Episode media + share feed stream through here too.
|
||||
location /api/v1/podcaster/ {
|
||||
proxy_pass http://unix:/run/secubox/podcaster.sock:/;
|
||||
# secubox-proxy.conf already sets proxy_buffering off + proxy_read_timeout;
|
||||
# only add what it doesn't (large episode uploads/streams).
|
||||
include /etc/nginx/snippets/secubox-proxy.conf;
|
||||
client_max_body_size 0;
|
||||
}
|
||||
27
packages/secubox-podcaster/systemd/secubox-podcaster.service
Normal file
27
packages/secubox-podcaster/systemd/secubox-podcaster.service
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
[Unit]
|
||||
Description=SecuBox Podcaster API
|
||||
After=network.target secubox-runtime.service
|
||||
Requires=secubox-runtime.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=root
|
||||
Group=root
|
||||
WorkingDirectory=/usr/share/secubox/podcaster
|
||||
ExecStart=/usr/bin/python3 -m uvicorn api.main:app --uds /run/secubox/podcaster.sock --workers 1
|
||||
# #494: only chmod/chown our OWN socket — never the shared /run/secubox parent.
|
||||
ExecStartPost=-/bin/sleep 1
|
||||
ExecStartPost=-/bin/chmod 660 /run/secubox/podcaster.sock
|
||||
ExecStartPost=-/bin/chown secubox:secubox /run/secubox/podcaster.sock
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
|
||||
# Security hardening
|
||||
NoNewPrivileges=no
|
||||
ProtectSystem=full
|
||||
ProtectHome=read-only
|
||||
PrivateTmp=true
|
||||
ReadWritePaths=/run/secubox /var/lib/secubox /etc/secubox
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
191
packages/secubox-podcaster/www/podcaster/index.html
Normal file
191
packages/secubox-podcaster/www/podcaster/index.html
Normal file
|
|
@ -0,0 +1,191 @@
|
|||
<!DOCTYPE html>
|
||||
<!-- SPDX-License-Identifier: LicenseRef-CMSD-1.0 -->
|
||||
<!-- SecuBox-Deb :: Podcaster admin WebUI — CyberMind https://cybermind.fr -->
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>SecuBox · Podcaster</title>
|
||||
<link rel="stylesheet" href="/shared/crt-light.css">
|
||||
<link rel="stylesheet" href="/shared/sidebar-light.css">
|
||||
<style>
|
||||
:root{
|
||||
--cosmos-black:#0a0a0f; --gold-hermetic:#c9a84c; --cinnabar:#e63946;
|
||||
--matrix-green:#00ff41; --void-purple:#6e40c9; --cyber-cyan:#00d4ff;
|
||||
--text-primary:#e8e6d9; --text-muted:#6b6b7a; --panel:#13131c; --line:#23232f;
|
||||
}
|
||||
body{margin:0;background:var(--cosmos-black);color:var(--text-primary);
|
||||
font-family:'JetBrains Mono',ui-monospace,Menlo,monospace;font-size:14px;display:flex}
|
||||
.main{flex:1;margin-left:220px;min-height:100vh}
|
||||
@media(max-width:900px){.sidebar{display:none}.main{margin-left:0}}
|
||||
.topbar{display:flex;align-items:center;gap:14px;padding:14px 20px;
|
||||
border-bottom:1px solid var(--line);background:linear-gradient(90deg,#0a0a0f,#13131c)}
|
||||
.topbar h1{font-size:18px;margin:0;letter-spacing:1px;color:var(--gold-hermetic)}
|
||||
.topbar .em{font-size:22px}
|
||||
.stats{margin-left:auto;display:flex;gap:16px;color:var(--text-muted);font-size:12px}
|
||||
.stats b{color:var(--cyber-cyan)}
|
||||
.wrap{display:grid;grid-template-columns:300px 1fr;gap:0;min-height:calc(100vh - 58px)}
|
||||
aside.feeds{border-right:1px solid var(--line);background:var(--panel);padding:14px;overflow:auto}
|
||||
section.eps{padding:18px 22px;overflow:auto}
|
||||
@media(max-width:760px){.wrap{grid-template-columns:1fr}aside.feeds{border-right:0;border-bottom:1px solid var(--line)}}
|
||||
.row{display:flex;gap:8px;margin-bottom:12px}
|
||||
input,button,select{font-family:inherit;font-size:13px;border-radius:8px;border:1px solid var(--line)}
|
||||
input{flex:1;background:#0e0e16;color:var(--text-primary);padding:9px 11px}
|
||||
button{background:#1a1a26;color:var(--text-primary);padding:9px 13px;cursor:pointer;transition:.15s}
|
||||
button:hover{border-color:var(--cyber-cyan);color:var(--cyber-cyan)}
|
||||
button.go{background:var(--void-purple);border-color:var(--void-purple);color:#fff}
|
||||
a.portal{color:var(--cyber-cyan);text-decoration:none;font-size:12px;border:1px solid var(--cyber-cyan);
|
||||
padding:7px 11px;border-radius:8px}
|
||||
.feed{display:flex;gap:10px;align-items:center;padding:9px;border-radius:8px;cursor:pointer;border:1px solid transparent}
|
||||
.feed:hover{background:#0e0e16}.feed.sel{background:#0e0e16;border-color:var(--void-purple)}
|
||||
.feed img{width:40px;height:40px;border-radius:6px;object-fit:cover;background:#222}
|
||||
.feed .t{flex:1;min-width:0}.feed .t b{display:block;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;font-size:13px}
|
||||
.feed .t span{color:var(--text-muted);font-size:11px}
|
||||
.feed .x{color:var(--text-muted);opacity:0;font-size:16px}.feed:hover .x{opacity:1}
|
||||
.feed .ad{font-size:14px;cursor:pointer;opacity:.5}.feed .ad.on{opacity:1;color:var(--matrix-green)}
|
||||
label.autodl{display:flex;gap:7px;align-items:center;color:var(--text-muted);font-size:12px;margin:-4px 0 12px;cursor:pointer}
|
||||
h2{font-size:15px;color:var(--gold-hermetic);border-bottom:1px solid var(--line);padding-bottom:8px;margin:0 0 14px}
|
||||
.ep{display:flex;gap:12px;align-items:center;padding:12px;border:1px solid var(--line);border-radius:10px;margin-bottom:10px;background:var(--panel)}
|
||||
.ep .meta{flex:1;min-width:0}.ep .meta b{display:block;white-space:nowrap;overflow:hidden;text-overflow:ellipsis}
|
||||
.ep .meta span{color:var(--text-muted);font-size:11px}
|
||||
.ep .act{display:flex;gap:6px;align-items:center}
|
||||
.badge{font-size:10px;padding:2px 7px;border-radius:20px;border:1px solid var(--line);color:var(--text-muted)}
|
||||
.badge.downloading{color:var(--cyber-cyan);border-color:var(--cyber-cyan)}
|
||||
.badge.error{color:var(--cinnabar);border-color:var(--cinnabar)}
|
||||
.bar{height:4px;border-radius:3px;background:#0e0e16;overflow:hidden;width:90px}.bar i{display:block;height:100%;background:var(--cyber-cyan);width:0}
|
||||
audio{height:34px}
|
||||
.share{margin-top:6px;padding:12px;border:1px dashed var(--void-purple);border-radius:10px;color:var(--text-muted);font-size:12px}
|
||||
.share a{color:var(--cyber-cyan)}
|
||||
.muted{color:var(--text-muted)}.empty{color:var(--text-muted);padding:40px;text-align:center}
|
||||
dialog{background:var(--panel);color:var(--text-primary);border:1px solid var(--void-purple);border-radius:12px;padding:20px;width:min(520px,92vw)}
|
||||
dialog label{display:block;margin:10px 0 4px;color:var(--text-muted);font-size:12px}dialog input{width:100%}
|
||||
.toast{position:fixed;bottom:18px;right:18px;background:#13131c;border:1px solid var(--cyber-cyan);color:var(--cyber-cyan);padding:10px 14px;border-radius:8px;opacity:0;transition:.2s;font-size:12px}.toast.on{opacity:1}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<nav class="sidebar" id="sidebar"></nav>
|
||||
<main class="main">
|
||||
<div class="topbar">
|
||||
<span class="em">🎙️</span><h1>PODCASTER</h1>
|
||||
<div class="stats" id="stats"></div>
|
||||
<a class="portal" href="/podcaster/portal/" target="_blank">🌐 Public portal</a>
|
||||
<button onclick="openCfg()" title="Config & status">⚙️</button>
|
||||
</div>
|
||||
<div class="wrap">
|
||||
<aside class="feeds">
|
||||
<div class="row">
|
||||
<input id="feedUrl" placeholder="RSS feed URL…" onkeydown="if(event.key==='Enter')addFeed()">
|
||||
<button class="go" onclick="addFeed()">+</button>
|
||||
</div>
|
||||
<label class="autodl"><input type="checkbox" id="autodl" checked> auto-download new episodes</label>
|
||||
<div class="row">
|
||||
<button onclick="document.getElementById('opml').click()">Import OPML</button>
|
||||
<button onclick="document.getElementById('abzip').click()" title="Upload an audiobook ZIP">📚 Audiobook ZIP</button>
|
||||
<button onclick="loadAll()">↻</button>
|
||||
</div>
|
||||
<input type="file" id="opml" accept=".opml,.xml" style="display:none" onchange="importOpml(this)">
|
||||
<input type="file" id="abzip" accept=".zip" style="display:none" onchange="uploadZip(this)">
|
||||
<div id="feeds"></div>
|
||||
<div class="share" id="share"></div>
|
||||
</aside>
|
||||
<section class="eps">
|
||||
<h2 id="title">Latest episodes</h2>
|
||||
<div id="eps"><div class="empty">Loading…</div></div>
|
||||
</section>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
<dialog id="cfg">
|
||||
<h2>Config & status</h2>
|
||||
<div id="svc" class="muted"></div>
|
||||
<label>Media path</label><input id="c_media">
|
||||
<label>Max parallel downloads</label><input id="c_par" type="number">
|
||||
<label>Refresh minutes</label><input id="c_ref" type="number">
|
||||
<label>Public base URL (for shareable feed — leave empty for LAN)</label><input id="c_pub">
|
||||
<label>Share feed title</label><input id="c_title">
|
||||
<div class="row" style="margin-top:16px">
|
||||
<button class="go" onclick="saveCfg()">Save</button>
|
||||
<button onclick="document.getElementById('cfg').close()">Close</button>
|
||||
</div>
|
||||
</dialog>
|
||||
<div class="toast" id="toast"></div>
|
||||
|
||||
<script src="/shared/sidebar.js"></script>
|
||||
<script>
|
||||
const API="/api/v1/podcaster";
|
||||
let SEL=null;
|
||||
function tok(){return localStorage.getItem("sbx_token")||""}
|
||||
async function api(p,opt={}){
|
||||
opt.headers=Object.assign({},opt.headers||{});
|
||||
if(!(opt.body instanceof FormData) && !opt.headers["Content-Type"]) opt.headers["Content-Type"]="application/json";
|
||||
const t=tok(); if(t) opt.headers["Authorization"]="Bearer "+t;
|
||||
const r=await fetch(API+p,opt);
|
||||
if(r.status===401){window.location="/login.html";throw new Error("401")}
|
||||
if(!r.ok) throw new Error(await r.text());
|
||||
const ct=r.headers.get("content-type")||""; return ct.includes("json")?r.json():r.text();
|
||||
}
|
||||
function toast(m){const t=document.getElementById("toast");t.textContent=m;t.classList.add("on");clearTimeout(t._);t._=setTimeout(()=>t.classList.remove("on"),3000)}
|
||||
function esc(s){return (s||"").replace(/[&<>"]/g,c=>({"&":"&","<":"<",">":">",'"':"""}[c]))}
|
||||
async function loadStatus(){
|
||||
try{const s=await fetch(API+"/status").then(r=>r.json());
|
||||
document.getElementById("stats").innerHTML=`feeds <b>${s.feeds}</b> · eps <b>${s.episodes}</b> · local <b>${s.downloaded}</b> · queue <b>${s.queued}</b>`;
|
||||
const base=(s.public_base||location.origin);
|
||||
document.getElementById("share").innerHTML=`🔗 Share feed:<br><a href="${API}/share/feed.xml" target="_blank">${esc(base)}/api/v1/podcaster/share/feed.xml</a>`;
|
||||
}catch(e){}
|
||||
}
|
||||
async function loadFeeds(){
|
||||
let d; try{d=await api("/feeds")}catch(e){return}
|
||||
const el=document.getElementById("feeds");
|
||||
if(!d.feeds.length){el.innerHTML='<div class="muted" style="padding:12px">No feeds yet — add one above.</div>';return}
|
||||
el.innerHTML=d.feeds.map(f=>`<div class="feed ${SEL===f.id?'sel':''}" onclick="selFeed(${f.id})">
|
||||
<img src="${esc(f.image||'')}" onerror="this.style.visibility='hidden'">
|
||||
<div class="t"><b>${esc(f.title||f.url)}</b><span>${f.downloaded}/${f.episodes} local</span></div>
|
||||
<span class="ad ${f.auto_dl?'on':''}" title="auto-download ${f.auto_dl?'on':'off'}" onclick="event.stopPropagation();toggleAd(${f.id},${f.auto_dl?0:1})">⏬</span>
|
||||
<span class="x" onclick="event.stopPropagation();delFeed(${f.id})">✕</span></div>`).join("");
|
||||
}
|
||||
async function selFeed(id){SEL=id;loadFeeds();loadEps()}
|
||||
async function loadEps(){
|
||||
const el=document.getElementById("eps");const q=SEL?`?feed_id=${SEL}`:"";
|
||||
document.getElementById("title").textContent=SEL?"Episodes":"Latest episodes";
|
||||
let d; try{d=await api("/episodes"+q)}catch(e){return}
|
||||
if(!d.episodes.length){el.innerHTML='<div class="empty">No episodes.</div>';return}
|
||||
el.innerHTML=d.episodes.map(e=>{
|
||||
const date=e.pubdate?new Date(e.pubdate*1000).toLocaleDateString():"";
|
||||
let act;
|
||||
if(e.state==="done") act=`<audio controls preload="none" src="${API}/media/${e.id}"></audio>`;
|
||||
else if(e.state==="downloading"||e.state==="queued") act=`<span class="badge downloading">${e.state}</span><div class="bar"><i style="width:${e.progress||0}%"></i></div>`;
|
||||
else if(e.state==="error") act=`<span class="badge error" title="${esc(e.error)}">error</span><button onclick="dl(${e.id})">retry</button>`;
|
||||
else act=`<button onclick="dl(${e.id})">⬇ download</button>`;
|
||||
return `<div class="ep"><div class="meta"><b>${esc(e.title)}</b><span>${esc(e.feed_title)} · ${date} ${e.duration?'· '+esc(e.duration):''}</span></div><div class="act">${act}</div></div>`;
|
||||
}).join("");
|
||||
}
|
||||
async function addFeed(){const u=document.getElementById("feedUrl").value.trim();if(!u)return;
|
||||
const ad=document.getElementById("autodl").checked;
|
||||
try{const r=await api("/feeds",{method:"POST",body:JSON.stringify({url:u,auto_dl:ad})});
|
||||
document.getElementById("feedUrl").value="";
|
||||
toast(`Added ${r.title||u} (${r.episodes} eps${r.queued?', '+r.queued+' queued':''})`);loadAll()}
|
||||
catch(e){toast("Add failed: "+e.message.slice(0,60))}}
|
||||
async function toggleAd(id,on){try{const r=await api(`/feeds/${id}/autodl?on=${on?'true':'false'}`,{method:"POST"});
|
||||
toast(on?`auto-download on${r.queued?' · '+r.queued+' queued':''}`:"auto-download off");loadAll()}catch(e){toast("toggle failed")}}
|
||||
async function delFeed(id){if(!confirm("Remove this feed?"))return;await api("/feeds/"+id,{method:"DELETE"});if(SEL===id)SEL=null;loadAll()}
|
||||
async function dl(id){try{await api(`/episodes/${id}/download`,{method:"POST"});toast("Queued");loadEps()}catch(e){toast("Queue failed")}}
|
||||
async function importOpml(inp){const f=inp.files[0];if(!f)return;const opml=await f.text();
|
||||
try{const r=await api("/feeds/import-opml",{method:"POST",body:JSON.stringify({opml})});toast(`Imported ${r.added}/${r.total}`);loadAll()}catch(e){toast("OPML import failed")}inp.value=""}
|
||||
async function uploadZip(inp){const f=inp.files[0];if(!f)return;
|
||||
const title=prompt("Audiobook title:",f.name.replace(/\.zip$/i,""));if(title===null){inp.value="";return}
|
||||
toast("Uploading "+f.name+" …");
|
||||
try{const r=await api("/audiobook/upload?title="+encodeURIComponent(title),
|
||||
{method:"POST",headers:{"Content-Type":"application/zip"},body:f});
|
||||
toast(`Published "${r.title}" — ${r.tracks} tracks`);loadAll()}
|
||||
catch(e){toast("Upload failed: "+e.message.slice(0,60))}inp.value=""}
|
||||
async function openCfg(){const d=await api("/config").then(r=>r.config).catch(()=>null);
|
||||
const s=await fetch(API+"/status").then(r=>r.json()).catch(()=>({}));
|
||||
document.getElementById("svc").innerHTML=`service <b style="color:var(--matrix-green)">${s.service||'?'}</b> · worker ${s.worker?'on':'off'} · ${s.downloaded||0} local`;
|
||||
if(d){c_media.value=d.media_path||"";c_par.value=d.max_parallel||2;c_ref.value=d.refresh_minutes||60;c_pub.value=d.public_base||"";c_title.value=d.share_title||""}
|
||||
document.getElementById("cfg").showModal()}
|
||||
async function saveCfg(){try{await api("/config",{method:"POST",body:JSON.stringify({media_path:c_media.value,max_parallel:+c_par.value,refresh_minutes:+c_ref.value,public_base:c_pub.value,share_title:c_title.value})});toast("Saved — restart to apply");document.getElementById("cfg").close()}catch(e){toast("Save failed")}}
|
||||
function loadAll(){loadStatus();loadFeeds();loadEps()}
|
||||
loadAll();setInterval(()=>{loadStatus();loadEps()},4000);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
107
packages/secubox-podcaster/www/podcaster/portal/index.html
Normal file
107
packages/secubox-podcaster/www/podcaster/portal/index.html
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
<!DOCTYPE html>
|
||||
<!-- SPDX-License-Identifier: LicenseRef-CMSD-1.0 -->
|
||||
<!-- SecuBox-Deb :: Podcaster PUBLIC portal (no auth) — CyberMind https://cybermind.fr -->
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Podcasts · SecuBox</title>
|
||||
<style>
|
||||
:root{
|
||||
--cosmos-black:#0a0a0f; --gold-hermetic:#c9a84c; --void-purple:#6e40c9;
|
||||
--cyber-cyan:#00d4ff; --matrix-green:#00ff41; --text-primary:#e8e6d9;
|
||||
--text-muted:#6b6b7a; --panel:#13131c; --line:#23232f;
|
||||
}
|
||||
*{box-sizing:border-box}
|
||||
body{margin:0;background:radial-gradient(1200px 600px at 50% -10%,#15101f,#0a0a0f 60%);
|
||||
color:var(--text-primary);font-family:'JetBrains Mono',ui-monospace,Menlo,monospace}
|
||||
header{max-width:1000px;margin:0 auto;padding:38px 20px 18px;text-align:center}
|
||||
header .em{font-size:42px}
|
||||
header h1{margin:8px 0 4px;font-size:30px;letter-spacing:2px;color:var(--gold-hermetic)}
|
||||
header p{color:var(--text-muted);margin:0}
|
||||
.subs{display:inline-flex;gap:10px;margin-top:16px;flex-wrap:wrap;justify-content:center}
|
||||
.subs a{color:var(--cyber-cyan);text-decoration:none;border:1px solid var(--cyber-cyan);
|
||||
padding:8px 14px;border-radius:24px;font-size:13px}
|
||||
.subs a:hover{background:rgba(0,212,255,.1)}
|
||||
.filters{max-width:1000px;margin:18px auto 0;padding:0 20px;display:flex;gap:8px;flex-wrap:wrap}
|
||||
.chip{font-size:12px;padding:6px 12px;border-radius:20px;border:1px solid var(--line);
|
||||
color:var(--text-muted);cursor:pointer}
|
||||
.chip.on{color:var(--void-purple);border-color:var(--void-purple);background:rgba(110,64,201,.12)}
|
||||
main{max-width:1000px;margin:18px auto 60px;padding:0 20px}
|
||||
.ep{display:flex;gap:14px;align-items:center;padding:14px;border:1px solid var(--line);
|
||||
border-radius:12px;margin-bottom:12px;background:var(--panel)}
|
||||
.ep .n{font-size:20px;color:var(--void-purple);width:30px;text-align:center}
|
||||
.ep .meta{flex:1;min-width:0}
|
||||
.ep .meta b{display:block;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;font-size:15px}
|
||||
.ep .meta span{color:var(--text-muted);font-size:12px}
|
||||
audio{height:38px;max-width:300px}
|
||||
.ep a.dl{color:var(--matrix-green);text-decoration:none;border:1px solid var(--matrix-green);
|
||||
border-radius:8px;padding:7px 10px;font-size:13px;white-space:nowrap}
|
||||
.ep a.dl:hover{background:rgba(0,255,65,.1)}
|
||||
.zipbar{max-width:1000px;margin:14px auto 0;padding:0 20px}
|
||||
.zipbar a{color:var(--gold-hermetic);text-decoration:none;border:1px solid var(--gold-hermetic);
|
||||
border-radius:24px;padding:8px 16px;font-size:13px}
|
||||
.zipbar a:hover{background:rgba(201,168,76,.12)}
|
||||
.empty{text-align:center;color:var(--text-muted);padding:60px}
|
||||
footer{text-align:center;color:var(--text-muted);font-size:11px;padding:24px}
|
||||
footer a{color:var(--text-muted)}
|
||||
@media(max-width:680px){.ep{flex-wrap:wrap}audio{max-width:100%;width:100%}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<div class="em">🎙️</div>
|
||||
<h1 id="title">Podcasts</h1>
|
||||
<p id="sub">Relayed locally by SecuBox · listen freely</p>
|
||||
<div class="subs">
|
||||
<a id="rss" href="/api/v1/podcaster/share/feed.xml" target="_blank">📡 Subscribe (RSS)</a>
|
||||
<a id="copy" href="#" onclick="copyRss(event)">🔗 Copy feed URL</a>
|
||||
</div>
|
||||
</header>
|
||||
<div class="filters" id="filters"></div>
|
||||
<div class="zipbar" id="zipbar"></div>
|
||||
<main><div id="list" class="empty">Loading…</div></main>
|
||||
<footer>SecuBox Podcaster · <a href="https://cybermind.fr" target="_blank">CyberMind</a></footer>
|
||||
|
||||
<script>
|
||||
const API="/api/v1/podcaster";
|
||||
let DATA={episodes:[],feeds:{}}, FILT=null;
|
||||
function esc(s){return (s||"").replace(/[&<>"]/g,c=>({"&":"&","<":"<",">":">",'"':"""}[c]))}
|
||||
function rssUrl(){return location.origin+API+"/share/feed.xml"}
|
||||
function copyRss(e){e.preventDefault();navigator.clipboard.writeText(rssUrl()).then(()=>{
|
||||
const a=document.getElementById("copy");a.textContent="✓ Copied";setTimeout(()=>a.textContent="🔗 Copy feed URL",1800)})}
|
||||
async function load(){
|
||||
let d; try{d=await fetch(API+"/public/library").then(r=>r.json())}
|
||||
catch(e){document.getElementById("list").innerHTML='<div class="empty">Portal unavailable.</div>';return}
|
||||
DATA=d;
|
||||
document.getElementById("title").textContent=d.title||"Podcasts";
|
||||
document.getElementById("rss").href=API+"/share/feed.xml";
|
||||
const fl=document.getElementById("filters");
|
||||
const feeds=Object.keys(d.feeds||{});
|
||||
fl.innerHTML=(feeds.length>1?[`<span class="chip ${FILT===null?'on':''}" onclick="setF(null)">All</span>`]:[])
|
||||
.concat(feeds.map(f=>`<span class="chip ${FILT===f?'on':''}" onclick="setF('${esc(f).replace(/'/g,"")}')">${esc(f)} · ${d.feeds[f]}</span>`)).join("");
|
||||
render();
|
||||
}
|
||||
function setF(f){FILT=f;load._render?render():render()}
|
||||
function render(){
|
||||
const list=document.getElementById("list");
|
||||
let eps=DATA.episodes||[];
|
||||
if(FILT) eps=eps.filter(e=>e.feed===FILT);
|
||||
// ZIP-all button when a single feed is in view
|
||||
const zb=document.getElementById("zipbar");
|
||||
const fid=eps.length?eps[0].feed_id:null;
|
||||
zb.innerHTML=(FILT&&fid!=null)
|
||||
? `<a href="${API}/public/feed/${fid}/zip">⬇ Download all (ZIP) · ${esc(FILT)}</a>` : "";
|
||||
if(!eps.length){list.innerHTML='<div class="empty">No episodes published yet.</div>';return}
|
||||
list.innerHTML=eps.map((e,i)=>{
|
||||
const date=e.pubdate?new Date(e.pubdate*1000).toLocaleDateString():"";
|
||||
return `<div class="ep"><div class="n">${i+1}</div>
|
||||
<div class="meta"><b>${esc(e.title)}</b><span>${esc(e.feed)} · ${date} ${e.duration?'· '+esc(e.duration):''}</span></div>
|
||||
<audio controls preload="none" src="${API}/media/${e.id}"></audio>
|
||||
<a class="dl" href="${API}/media/${e.id}" download title="Download">⬇</a></div>`;
|
||||
}).join("");
|
||||
}
|
||||
load();setInterval(load,15000);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
0
packages/secubox-threatmesh/api/__init__.py
Normal file
0
packages/secubox-threatmesh/api/__init__.py
Normal file
267
packages/secubox-threatmesh/api/main.py
Normal file
267
packages/secubox-threatmesh/api/main.py
Normal file
|
|
@ -0,0 +1,267 @@
|
|||
# SPDX-License-Identifier: LicenseRef-CMSD-1.0
|
||||
# Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
|
||||
# Source-Disclosed License — All rights reserved except as expressly granted.
|
||||
# See LICENCE-CMSD-1.0.md for terms.
|
||||
"""
|
||||
SecuBox-Deb :: secubox-threatmesh (#728)
|
||||
CyberMind — https://cybermind.fr
|
||||
|
||||
Sovereign threat-intel control plane that replaces CrowdSec CAPI:
|
||||
|
||||
Phase 1 free public feeds (secubox-threatfeed timer) -> threat_intel
|
||||
Phase 2 MESH distribution of LOCAL decisions over the SecuBox P2P mesh
|
||||
(gossip to peers from secubox-p2p; ingest peer decisions) -> threat_intel
|
||||
Phase 3 status / aggregate / CrowdSec-bouncer-compatible decisions API + UI
|
||||
|
||||
All IOCs land in the shared `threat_intel` table (toolbox.db); secubox-blacklist-sync
|
||||
drains it to the nft blacklist. No central account, no CAPI, no paywall.
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import FastAPI, APIRouter, Depends, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from secubox_core.auth import require_jwt
|
||||
from secubox_core.logger import get_logger
|
||||
|
||||
log = get_logger("threatmesh")
|
||||
|
||||
TI_DB = Path("/var/lib/secubox/toolbox/toolbox.db")
|
||||
P2P_PEERS = Path("/var/lib/secubox/p2p/peers.json")
|
||||
P2P_NODE = Path("/var/lib/secubox/p2p/node_id")
|
||||
MESH_INTERVAL = 600 # gossip every 10 min
|
||||
LOCAL_ORIGINS = {"crowdsec", "cscli", "manual", "secubox-waf", "secubox"}
|
||||
IPV4 = re.compile(r"^\d{1,3}(\.\d{1,3}){3}(/\d{1,2})?$")
|
||||
|
||||
app = FastAPI(title="secubox-threatmesh", version="1.0.0", root_path="/api/v1/threatmesh")
|
||||
router = APIRouter()
|
||||
_node_id = None
|
||||
|
||||
|
||||
# ── store ───────────────────────────────────────────────────────────
|
||||
def _conn():
|
||||
c = sqlite3.connect(TI_DB, timeout=20)
|
||||
c.row_factory = sqlite3.Row
|
||||
return c
|
||||
|
||||
|
||||
def _ensure(c):
|
||||
c.execute("""CREATE TABLE IF NOT EXISTS threat_intel (
|
||||
ioc TEXT NOT NULL, ioc_type TEXT NOT NULL, source TEXT NOT NULL,
|
||||
weight INTEGER NOT NULL DEFAULT 50, label TEXT,
|
||||
first_seen INTEGER, last_seen INTEGER,
|
||||
PRIMARY KEY (ioc, ioc_type, source))""")
|
||||
|
||||
|
||||
def node_id() -> str:
|
||||
global _node_id
|
||||
if _node_id:
|
||||
return _node_id
|
||||
try:
|
||||
_node_id = P2P_NODE.read_text().strip()
|
||||
except Exception:
|
||||
try:
|
||||
_node_id = json.loads(P2P_PEERS.read_text())["peers"][0]["id"]
|
||||
except Exception:
|
||||
_node_id = "sb-local"
|
||||
return _node_id
|
||||
|
||||
|
||||
def upsert_iocs(rows, source, weight=60, label=None):
|
||||
"""rows: iterable of (ioc, ioc_type). Returns count."""
|
||||
now = int(time.time())
|
||||
n = 0
|
||||
with _conn() as c:
|
||||
_ensure(c)
|
||||
for ioc, t in rows:
|
||||
c.execute(
|
||||
"INSERT INTO threat_intel(ioc,ioc_type,source,weight,label,first_seen,last_seen) "
|
||||
"VALUES(?,?,?,?,?,?,?) ON CONFLICT(ioc,ioc_type,source) DO UPDATE SET last_seen=excluded.last_seen",
|
||||
(ioc, t, source, weight, label, now, now))
|
||||
n += 1
|
||||
c.commit()
|
||||
return n
|
||||
|
||||
|
||||
def counts_by_source():
|
||||
with _conn() as c:
|
||||
_ensure(c)
|
||||
return {r["source"]: r["n"] for r in c.execute(
|
||||
"SELECT source, COUNT(*) n FROM threat_intel GROUP BY source ORDER BY n DESC")}
|
||||
|
||||
|
||||
def aggregate_ips(limit=50000):
|
||||
"""Distinct bad IPs across ALL sources, with a consensus count."""
|
||||
with _conn() as c:
|
||||
_ensure(c)
|
||||
return [dict(r) for r in c.execute(
|
||||
"SELECT ioc, COUNT(DISTINCT source) consensus, MAX(weight) weight, "
|
||||
"GROUP_CONCAT(DISTINCT source) sources FROM threat_intel "
|
||||
"WHERE ioc_type='ip' GROUP BY ioc ORDER BY consensus DESC LIMIT ?", (limit,))]
|
||||
|
||||
|
||||
# ── mesh (Phase 2) ──────────────────────────────────────────────────
|
||||
def mesh_peers():
|
||||
"""Other SecuBox nodes from secubox-p2p (skip self)."""
|
||||
me = node_id()
|
||||
out = []
|
||||
try:
|
||||
d = json.loads(P2P_PEERS.read_text())
|
||||
for p in d.get("peers", []):
|
||||
if p.get("id") == me:
|
||||
continue
|
||||
addr = None
|
||||
for a in p.get("addresses", []):
|
||||
if a.get("type") in ("wg", "wireguard", "mesh"):
|
||||
addr = a.get("address"); break
|
||||
addr = addr or (p.get("wg_addresses") or [None])[0] or p.get("address")
|
||||
if addr:
|
||||
out.append({"id": p.get("id"), "name": p.get("name"), "address": addr})
|
||||
except Exception as e:
|
||||
log.debug(f"peers read: {e}")
|
||||
return out
|
||||
|
||||
|
||||
def local_decisions():
|
||||
"""Our OWN locally-detected bad IPs (CrowdSec LAPI bans of local origin)."""
|
||||
try:
|
||||
out = subprocess.run(["cscli", "decisions", "list", "-o", "json", "-a"],
|
||||
capture_output=True, text=True, timeout=20)
|
||||
data = json.loads(out.stdout or "[]") or []
|
||||
except Exception as e:
|
||||
log.debug(f"cscli decisions: {e}")
|
||||
return []
|
||||
ips = []
|
||||
for alert in data:
|
||||
for dec in (alert.get("decisions") or []):
|
||||
if dec.get("type") == "ban" and dec.get("scope", "").lower() == "ip":
|
||||
origin = (dec.get("origin") or "").lower()
|
||||
val = dec.get("value", "")
|
||||
if IPV4.match(val) and not origin.startswith(("feed:", "mesh:", "lists:")):
|
||||
ips.append(val)
|
||||
return sorted(set(ips))
|
||||
|
||||
|
||||
async def _post_json(url, payload, timeout=15):
|
||||
def _do():
|
||||
req = urllib.request.Request(
|
||||
url, data=json.dumps(payload).encode(),
|
||||
headers={"Content-Type": "application/json", "User-Agent": "SecuBox-ThreatMesh/1.0"},
|
||||
method="POST")
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return r.status
|
||||
return await asyncio.to_thread(_do)
|
||||
|
||||
|
||||
async def mesh_gossip_once():
|
||||
peers = mesh_peers()
|
||||
decisions = local_decisions()
|
||||
if not peers or not decisions:
|
||||
return {"peers": len(peers), "shared": len(decisions), "pushed": 0}
|
||||
payload = {"node": node_id(), "ts": int(time.time()), "ips": decisions}
|
||||
pushed = 0
|
||||
for p in peers:
|
||||
url = f"http://{p['address']}:8780/api/v1/threatmesh/mesh/ingest"
|
||||
try:
|
||||
await _post_json(url, payload)
|
||||
pushed += 1
|
||||
except Exception as e:
|
||||
log.debug(f"gossip to {p['id']} failed: {e}")
|
||||
return {"peers": len(peers), "shared": len(decisions), "pushed": pushed}
|
||||
|
||||
|
||||
async def _mesh_loop():
|
||||
while True:
|
||||
try:
|
||||
r = await mesh_gossip_once()
|
||||
log.info(f"mesh gossip: {r}")
|
||||
except Exception as e:
|
||||
log.error(f"mesh loop: {e}")
|
||||
await asyncio.sleep(MESH_INTERVAL)
|
||||
|
||||
|
||||
@app.on_event("startup")
|
||||
async def _startup():
|
||||
asyncio.create_task(_mesh_loop())
|
||||
|
||||
|
||||
# ── endpoints ───────────────────────────────────────────────────────
|
||||
@app.get("/health")
|
||||
async def health():
|
||||
return {"status": "ok", "module": "deb"}
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
async def status():
|
||||
src = counts_by_source()
|
||||
feeds = {k: v for k, v in src.items() if k.startswith("feed:")}
|
||||
mesh = {k: v for k, v in src.items() if k.startswith("mesh:")}
|
||||
return {
|
||||
"mode": "sovereign", # CAPI dropped (#728)
|
||||
"node": node_id(),
|
||||
"capi": False,
|
||||
"feeds": feeds, "feed_total": sum(feeds.values()),
|
||||
"mesh": mesh, "mesh_total": sum(mesh.values()),
|
||||
"peers": len(mesh_peers()),
|
||||
"sources": src,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/peers")
|
||||
async def peers():
|
||||
return {"node": node_id(), "peers": mesh_peers()}
|
||||
|
||||
|
||||
@router.get("/decisions")
|
||||
async def decisions(min_consensus: int = 1, limit: int = 50000):
|
||||
"""Aggregated sovereign blocklist (feeds + mesh + local). CrowdSec-bouncer
|
||||
friendly shape so external consumers can poll OUR server, not crowdsec.net."""
|
||||
rows = [r for r in aggregate_ips(limit) if r["consensus"] >= min_consensus]
|
||||
return [{
|
||||
"id": i, "origin": "secubox-threatmesh", "type": "ban", "scope": "Ip",
|
||||
"value": r["ioc"], "duration": "24h",
|
||||
"consensus": r["consensus"], "sources": r["sources"],
|
||||
} for i, r in enumerate(rows)]
|
||||
|
||||
|
||||
@router.post("/mesh/ingest")
|
||||
async def mesh_ingest(request: Request):
|
||||
"""Receive a peer's locally-detected decisions over the mesh -> threat_intel
|
||||
tagged mesh:<node>. Trust boundary = the WireGuard mesh transport."""
|
||||
try:
|
||||
body = await request.json()
|
||||
except Exception:
|
||||
return JSONResponse({"ok": False, "error": "bad json"}, status_code=400)
|
||||
node = str(body.get("node", "")).strip()[:64] or "unknown"
|
||||
if not re.match(r"^[A-Za-z0-9._:-]+$", node):
|
||||
return JSONResponse({"ok": False, "error": "bad node"}, status_code=400)
|
||||
ips = [ip for ip in (body.get("ips") or []) if isinstance(ip, str) and IPV4.match(ip)]
|
||||
n = upsert_iocs(((ip, "ip") for ip in ips[:20000]), f"mesh:{node}", weight=70, label="mesh")
|
||||
log.info(f"mesh ingest from {node}: {n} ips")
|
||||
return {"ok": True, "ingested": n}
|
||||
|
||||
|
||||
@router.post("/feeds/refresh", dependencies=[Depends(require_jwt)])
|
||||
async def feeds_refresh():
|
||||
"""Trigger the feed fetcher now (normally on a timer)."""
|
||||
try:
|
||||
subprocess.Popen(["/usr/sbin/secubox-threatfeed"])
|
||||
return {"ok": True, "started": True}
|
||||
except Exception as e:
|
||||
return JSONResponse({"ok": False, "error": str(e)}, status_code=500)
|
||||
|
||||
|
||||
@router.post("/mesh/gossip", dependencies=[Depends(require_jwt)])
|
||||
async def mesh_gossip_now():
|
||||
return await mesh_gossip_once()
|
||||
|
||||
|
||||
app.include_router(router)
|
||||
13
packages/secubox-threatmesh/debian/changelog
Normal file
13
packages/secubox-threatmesh/debian/changelog
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
secubox-threatmesh (1.0.0-1~bookworm1) bookworm; urgency=medium
|
||||
|
||||
* Initial release (#728) — sovereign threat-intel, CrowdSec CAPI replacement.
|
||||
- Phase 1: secubox-threatfeed timer pulls free public blocklists
|
||||
(feodo/sslbl/firehol/spamhaus-drop/blocklist.de/cins/et/dshield) into the
|
||||
shared threat_intel table -> secubox-blacklist-sync -> nft.
|
||||
- Phase 2: MESH distribution — gossip locally-detected CrowdSec decisions to
|
||||
SecuBox P2P peers over WireGuard; ingest peer decisions (tagged mesh:<node>),
|
||||
consensus-counted. :8780 locked to wg*/loopback by an nft drop-in.
|
||||
- Phase 3: /status, /peers, /decisions (bouncer-compatible aggregate),
|
||||
/mesh/ingest, dashboard. CrowdSec LAPI kept; CAPI dropped.
|
||||
|
||||
-- Gerald KERMA <devel@cybermind.fr> Wed, 24 Jun 2026 16:00:00 +0000
|
||||
18
packages/secubox-threatmesh/debian/control
Normal file
18
packages/secubox-threatmesh/debian/control
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
Source: secubox-threatmesh
|
||||
Section: net
|
||||
Priority: optional
|
||||
Maintainer: Gerald KERMA <devel@cybermind.fr>
|
||||
Build-Depends: debhelper-compat (= 13)
|
||||
Standards-Version: 4.6.2
|
||||
|
||||
Package: secubox-threatmesh
|
||||
Architecture: all
|
||||
Depends: ${misc:Depends}, secubox-core (>= 1.0), python3, python3-uvicorn | python3-pip, secubox-vortex-firewall | secubox-toolbox
|
||||
Recommends: secubox-p2p, crowdsec
|
||||
Description: Sovereign threat-intel mesh for SecuBox (CrowdSec CAPI replacement)
|
||||
Replaces the CrowdSec Central API with self-sourced free public blocklists
|
||||
(Phase 1) and peer-to-peer MESH distribution of locally-detected decisions over
|
||||
the SecuBox WireGuard mesh (Phase 2), plus a status/aggregate/bouncer-compatible
|
||||
API and dashboard (Phase 3). All IOCs flow into the shared threat_intel table
|
||||
that secubox-blacklist-sync drains into the nft blacklist. No central account,
|
||||
no enrollment, no paywall, no IP-blocklisting by a third party.
|
||||
23
packages/secubox-threatmesh/debian/postinst
Normal file
23
packages/secubox-threatmesh/debian/postinst
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
case "$1" in
|
||||
configure)
|
||||
install -d -m 0755 /var/lib/secubox/threatmesh
|
||||
systemctl daemon-reload 2>/dev/null || true
|
||||
# mesh-port firewall (lock :8780 to wg*/lo under DEFAULT DROP)
|
||||
if [ -f /usr/share/secubox/threatmesh/nftables.d/secubox-threatmesh.nft ]; then
|
||||
nft -f /usr/share/secubox/threatmesh/nftables.d/secubox-threatmesh.nft 2>/dev/null || true
|
||||
fi
|
||||
if [ "$(systemctl is-enabled secubox-threatmesh.service 2>/dev/null)" != "masked" ]; then
|
||||
systemctl enable --now secubox-threatmesh.service 2>/dev/null || true
|
||||
fi
|
||||
systemctl enable --now secubox-threatfeed.timer 2>/dev/null || true
|
||||
# first feed population (background; logs to /tmp to avoid /var/log perms)
|
||||
/usr/sbin/secubox-threatfeed >/tmp/secubox-threatfeed-firstrun.log 2>&1 &
|
||||
if command -v nginx >/dev/null 2>&1 && nginx -t 2>/dev/null; then
|
||||
systemctl reload nginx 2>/dev/null || true
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
#DEBHELPER#
|
||||
exit 0
|
||||
11
packages/secubox-threatmesh/debian/prerm
Normal file
11
packages/secubox-threatmesh/debian/prerm
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
case "$1" in
|
||||
remove|deconfigure)
|
||||
systemctl stop secubox-threatmesh.service 2>/dev/null || true
|
||||
systemctl disable secubox-threatmesh.service secubox-threatfeed.timer 2>/dev/null || true
|
||||
nft delete table inet secubox_threatmesh 2>/dev/null || true
|
||||
;;
|
||||
esac
|
||||
#DEBHELPER#
|
||||
exit 0
|
||||
23
packages/secubox-threatmesh/debian/rules
Executable file
23
packages/secubox-threatmesh/debian/rules
Executable file
|
|
@ -0,0 +1,23 @@
|
|||
#!/usr/bin/make -f
|
||||
%:
|
||||
dh $@
|
||||
|
||||
override_dh_auto_install:
|
||||
install -d debian/secubox-threatmesh/usr/lib/secubox/threatmesh/api
|
||||
cp -r api/. debian/secubox-threatmesh/usr/lib/secubox/threatmesh/api/
|
||||
install -d debian/secubox-threatmesh/usr/sbin
|
||||
install -m 755 sbin/secubox-threatfeed debian/secubox-threatmesh/usr/sbin/
|
||||
install -d debian/secubox-threatmesh/usr/lib/systemd/system
|
||||
cp systemd/*.service systemd/*.timer debian/secubox-threatmesh/usr/lib/systemd/system/
|
||||
install -d debian/secubox-threatmesh/usr/share/secubox/www
|
||||
cp -r www/. debian/secubox-threatmesh/usr/share/secubox/www/
|
||||
install -d debian/secubox-threatmesh/usr/share/secubox/menu.d
|
||||
cp menu.d/. -r debian/secubox-threatmesh/usr/share/secubox/menu.d/
|
||||
install -d debian/secubox-threatmesh/etc/nginx/secubox-routes.d debian/secubox-threatmesh/etc/nginx/secubox.d
|
||||
cp nginx/threatmesh.conf debian/secubox-threatmesh/etc/nginx/secubox-routes.d/
|
||||
cp nginx/threatmesh.conf debian/secubox-threatmesh/etc/nginx/secubox.d/
|
||||
install -d debian/secubox-threatmesh/usr/share/secubox/threatmesh/nftables.d
|
||||
cp nftables.d/secubox-threatmesh.nft debian/secubox-threatmesh/usr/share/secubox/threatmesh/nftables.d/
|
||||
|
||||
override_dh_installsystemd:
|
||||
true
|
||||
1
packages/secubox-threatmesh/debian/source/format
Normal file
1
packages/secubox-threatmesh/debian/source/format
Normal file
|
|
@ -0,0 +1 @@
|
|||
3.0 (native)
|
||||
9
packages/secubox-threatmesh/menu.d/37-threatmesh.json
Normal file
9
packages/secubox-threatmesh/menu.d/37-threatmesh.json
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"id": "threatmesh",
|
||||
"name": "ThreatMesh",
|
||||
"icon": "🛰️",
|
||||
"path": "/threatmesh/",
|
||||
"category": "security",
|
||||
"order": 37,
|
||||
"description": "Sovereign threat-intel (feeds + mesh, no CAPI)"
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
# ThreatMesh peer port (#728): only reachable over the WireGuard mesh + loopback.
|
||||
# The base inet filter INPUT chain is DEFAULT DROP, so this only OPENS 8780 to wg.
|
||||
table inet secubox_threatmesh {
|
||||
chain input {
|
||||
type filter hook input priority 0; policy accept;
|
||||
iifname "lo" tcp dport 8780 accept
|
||||
iifname "wg*" tcp dport 8780 accept
|
||||
tcp dport 8780 drop
|
||||
}
|
||||
}
|
||||
10
packages/secubox-threatmesh/nginx/threatmesh.conf
Normal file
10
packages/secubox-threatmesh/nginx/threatmesh.conf
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
# /etc/nginx/secubox-routes.d/threatmesh.conf — secubox-threatmesh (#728)
|
||||
location /threatmesh/ {
|
||||
alias /usr/share/secubox/www/threatmesh/;
|
||||
index index.html;
|
||||
try_files $uri $uri/ /threatmesh/index.html;
|
||||
}
|
||||
location /api/v1/threatmesh/ {
|
||||
proxy_pass http://127.0.0.1:8780/api/v1/threatmesh/;
|
||||
include /etc/nginx/snippets/secubox-proxy.conf;
|
||||
}
|
||||
121
packages/secubox-threatmesh/sbin/secubox-threatfeed
Normal file
121
packages/secubox-threatmesh/sbin/secubox-threatfeed
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: LicenseRef-CMSD-1.0
|
||||
# Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
|
||||
# Source-Disclosed License — All rights reserved except as expressly granted.
|
||||
# See LICENCE-CMSD-1.0.md for terms.
|
||||
"""
|
||||
SecuBox-Deb :: secubox-threatfeed (Phase 1 — sovereign threat-intel #728)
|
||||
CyberMind — https://cybermind.fr
|
||||
|
||||
Pulls FREE, no-enrollment public IP/CIDR/domain blocklists and writes them into
|
||||
the shared `threat_intel` table (toolbox.db) that secubox-blacklist-sync already
|
||||
drains into the nft blacklist. This REPLACES the CrowdSec CAPI community feed
|
||||
with self-sourced lists — no central account, no paywall, no IP blocklisting.
|
||||
|
||||
Pure stdlib (urllib) — no extra Debian deps. Idempotent, safe on a timer.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
DB = Path(os.environ.get("SECUBOX_TI_DB", "/var/lib/secubox/toolbox/toolbox.db"))
|
||||
TIMEOUT = int(os.environ.get("SECUBOX_TF_TIMEOUT", "30"))
|
||||
UA = "SecuBox-ThreatFeed/1.0 (+https://secubox.in)"
|
||||
TTL_DAYS = int(os.environ.get("SECUBOX_TF_TTL_DAYS", "10"))
|
||||
|
||||
# (name, url, ioc_type, weight). All free + no enrollment. CIDRs allowed for ip.
|
||||
FEEDS = [
|
||||
("feodo", "https://feodotracker.abuse.ch/downloads/ipblocklist.txt", "ip", 90),
|
||||
("sslbl", "https://sslbl.abuse.ch/blacklist/sslipblacklist.txt", "ip", 85),
|
||||
("firehol1", "https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset", "ip", 80),
|
||||
("spamhaus-drop", "https://www.spamhaus.org/drop/drop.txt", "ip", 85),
|
||||
("blocklist-de", "https://lists.blocklist.de/lists/all.txt", "ip", 60),
|
||||
("cins-army", "https://cinsscore.com/list/ci-badguys.txt", "ip", 70),
|
||||
("et-compromised", "https://rules.emergingthreats.net/blockrules/compromised-ips.txt", "ip", 75),
|
||||
("dshield", "https://feeds.dshield.org/block.txt", "ip", 65),
|
||||
]
|
||||
# Optional (set SECUBOX_TF_TOR=1 to also pull Tor exit nodes)
|
||||
if os.environ.get("SECUBOX_TF_TOR") == "1":
|
||||
FEEDS.append(("tor-exit", "https://check.torproject.org/torbulkexitlist", "ip", 40))
|
||||
|
||||
_IPV4 = re.compile(r"^\s*(\d{1,3}(?:\.\d{1,3}){3}(?:/\d{1,2})?)")
|
||||
_IPV6 = re.compile(r"^\s*([0-9A-Fa-f:]{2,}(?:/\d{1,3})?)\s*$")
|
||||
|
||||
|
||||
def log(msg):
|
||||
print(f"[threatfeed] {msg}", file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
def fetch(url):
|
||||
req = urllib.request.Request(url, headers={"User-Agent": UA})
|
||||
with urllib.request.urlopen(req, timeout=TIMEOUT) as r:
|
||||
return r.read().decode("utf-8", "replace")
|
||||
|
||||
|
||||
def parse_ips(text):
|
||||
out = []
|
||||
for line in text.splitlines():
|
||||
s = line.strip()
|
||||
if not s or s.startswith("#") or s.startswith(";"):
|
||||
continue
|
||||
# spamhaus DROP: "1.2.3.0/24 ; SBL..." -> take the first field
|
||||
token = s.split(";")[0].split()[0].strip() if (";" in s or " " in s) else s
|
||||
m = _IPV4.match(token)
|
||||
if m:
|
||||
out.append((m.group(1), "ip"))
|
||||
continue
|
||||
m = _IPV6.match(token)
|
||||
if m and ":" in m.group(1):
|
||||
out.append((m.group(1), "ip"))
|
||||
return out
|
||||
|
||||
|
||||
def ensure_schema(c):
|
||||
c.execute("""CREATE TABLE IF NOT EXISTS threat_intel (
|
||||
ioc TEXT NOT NULL, ioc_type TEXT NOT NULL, source TEXT NOT NULL,
|
||||
weight INTEGER NOT NULL DEFAULT 50, label TEXT,
|
||||
first_seen INTEGER, last_seen INTEGER,
|
||||
PRIMARY KEY (ioc, ioc_type, source))""")
|
||||
|
||||
|
||||
def main():
|
||||
if not DB.exists():
|
||||
log(f"db missing: {DB}")
|
||||
return 1
|
||||
now = int(time.time())
|
||||
total = 0
|
||||
with sqlite3.connect(DB, timeout=20) as c:
|
||||
ensure_schema(c)
|
||||
for name, url, ioc_type, weight in FEEDS:
|
||||
src = f"feed:{name}"
|
||||
try:
|
||||
iocs = parse_ips(fetch(url))
|
||||
except Exception as e:
|
||||
log(f"{name}: fetch failed: {e}")
|
||||
continue
|
||||
n = 0
|
||||
for ioc, t in iocs:
|
||||
c.execute(
|
||||
"INSERT INTO threat_intel(ioc,ioc_type,source,weight,label,first_seen,last_seen) "
|
||||
"VALUES(?,?,?,?,?,?,?) "
|
||||
"ON CONFLICT(ioc,ioc_type,source) DO UPDATE SET last_seen=excluded.last_seen, weight=excluded.weight",
|
||||
(ioc, t, src, weight, name, now, now))
|
||||
n += 1
|
||||
log(f"{name}: {n} iocs")
|
||||
total += n
|
||||
# prune stale feed entries (older than TTL) — keeps the set fresh.
|
||||
cutoff = now - TTL_DAYS * 86400
|
||||
pruned = c.execute(
|
||||
"DELETE FROM threat_intel WHERE source LIKE 'feed:%' AND last_seen < ?",
|
||||
(cutoff,)).rowcount
|
||||
c.commit()
|
||||
log(f"done: {total} iocs across {len(FEEDS)} feeds, pruned {pruned} stale")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
[Unit]
|
||||
Description=SecuBox ThreatFeed — pull free public blocklists into threat_intel (#728)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=root
|
||||
ExecStart=/usr/sbin/secubox-threatfeed
|
||||
Nice=10
|
||||
IOSchedulingClass=idle
|
||||
TimeoutStartSec=600
|
||||
11
packages/secubox-threatmesh/systemd/secubox-threatfeed.timer
Normal file
11
packages/secubox-threatmesh/systemd/secubox-threatfeed.timer
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
[Unit]
|
||||
Description=Refresh sovereign threat-intel feeds (#728)
|
||||
|
||||
[Timer]
|
||||
OnBootSec=5min
|
||||
OnUnitActiveSec=6h
|
||||
RandomizedDelaySec=15min
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
[Unit]
|
||||
Description=SecuBox ThreatMesh — sovereign threat-intel mesh API (#728)
|
||||
After=network.target secubox-runtime.service secubox-p2p.service
|
||||
Wants=secubox-p2p.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=root
|
||||
WorkingDirectory=/usr/lib/secubox/threatmesh
|
||||
# Listens on :8780 — nginx proxies the dashboard; peers POST mesh decisions
|
||||
# over the WireGuard mesh. nft drop-in restricts :8780 to wg* + loopback.
|
||||
ExecStartPre=-/usr/sbin/nft -f /usr/share/secubox/threatmesh/nftables.d/secubox-threatmesh.nft
|
||||
ExecStart=/usr/bin/python3 -m uvicorn api.main:app --host 0.0.0.0 --port 8780 --workers 1
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
NoNewPrivileges=no
|
||||
ProtectSystem=full
|
||||
PrivateTmp=true
|
||||
ReadWritePaths=/run/secubox /var/lib/secubox /etc/secubox
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
87
packages/secubox-threatmesh/www/threatmesh/index.html
Normal file
87
packages/secubox-threatmesh/www/threatmesh/index.html
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
<!DOCTYPE html>
|
||||
<!-- SPDX-License-Identifier: LicenseRef-CMSD-1.0 -->
|
||||
<!-- SecuBox-Deb :: ThreatMesh — sovereign threat-intel (#728) — CyberMind -->
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>SecuBox · ThreatMesh</title>
|
||||
<link rel="stylesheet" href="/shared/crt-light.css">
|
||||
<link rel="stylesheet" href="/shared/sidebar-light.css">
|
||||
<style>
|
||||
:root{--cosmos-black:#0a0a0f;--gold:#c9a84c;--cinnabar:#e63946;--matrix:#00ff41;
|
||||
--void:#6e40c9;--cyan:#00d4ff;--text:#e8e6d9;--muted:#6b6b7a;--panel:#13131c;--line:#23232f}
|
||||
body{margin:0;background:var(--cosmos-black);color:var(--text);font-family:'JetBrains Mono',ui-monospace,monospace;display:flex}
|
||||
.main{flex:1;margin-left:220px;min-height:100vh;padding:0 0 40px}
|
||||
@media(max-width:900px){.sidebar{display:none}.main{margin-left:0}}
|
||||
.topbar{display:flex;align-items:center;gap:14px;padding:14px 22px;border-bottom:1px solid var(--line);
|
||||
background:linear-gradient(90deg,#0a0a0f,#13131c)}
|
||||
.topbar h1{font-size:18px;margin:0;letter-spacing:1px;color:var(--gold)} .topbar .em{font-size:22px}
|
||||
.pill{margin-left:auto;font-size:12px;padding:5px 12px;border-radius:20px;border:1px solid var(--matrix);color:var(--matrix)}
|
||||
.wrap{padding:20px 22px;display:grid;gap:16px;grid-template-columns:repeat(auto-fit,minmax(260px,1fr))}
|
||||
.card{background:var(--panel);border:1px solid var(--line);border-radius:12px;padding:16px}
|
||||
.card h2{font-size:13px;color:var(--gold);margin:0 0 10px;letter-spacing:.5px}
|
||||
.big{font-size:34px;color:var(--cyan);font-weight:700} .sub{color:var(--muted);font-size:12px}
|
||||
.src{display:flex;justify-content:space-between;padding:4px 0;border-bottom:1px dashed var(--line);font-size:12px}
|
||||
.src b{color:var(--cyan)} .src.mesh b{color:var(--void)}
|
||||
.bar{display:flex;gap:8px;padding:0 22px 16px}
|
||||
button{font-family:inherit;font-size:13px;background:#1a1a26;color:var(--text);border:1px solid var(--line);
|
||||
border-radius:8px;padding:9px 14px;cursor:pointer} button:hover{border-color:var(--cyan);color:var(--cyan)}
|
||||
button.go{background:var(--void);border-color:var(--void);color:#fff}
|
||||
table{width:100%;border-collapse:collapse;font-size:12px} td,th{text-align:left;padding:5px 8px;border-bottom:1px solid var(--line)}
|
||||
th{color:var(--muted)} .toast{position:fixed;bottom:18px;right:18px;background:#13131c;border:1px solid var(--cyan);
|
||||
color:var(--cyan);padding:10px 14px;border-radius:8px;opacity:0;transition:.2s;font-size:12px}.toast.on{opacity:1}
|
||||
.full{grid-column:1/-1}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<nav class="sidebar" id="sidebar"></nav>
|
||||
<main class="main">
|
||||
<div class="topbar"><span class="em">🛰️</span><h1>THREATMESH</h1>
|
||||
<span class="pill" id="mode">sovereign · CAPI off</span></div>
|
||||
<div class="bar">
|
||||
<button class="go" onclick="refreshFeeds()">↻ Refresh feeds</button>
|
||||
<button onclick="gossip()">🛰️ Gossip now</button>
|
||||
<button onclick="load()">⟳</button>
|
||||
</div>
|
||||
<div class="wrap">
|
||||
<div class="card"><h2>BLOCKED IPs (aggregate)</h2><div class="big" id="total">…</div><div class="sub">feeds + mesh + local → nft</div></div>
|
||||
<div class="card"><h2>FREE FEEDS</h2><div class="big" id="feedtotal">…</div><div class="sub" id="feedn"></div></div>
|
||||
<div class="card"><h2>MESH</h2><div class="big" id="meshtotal">…</div><div class="sub" id="peern"></div></div>
|
||||
<div class="card"><h2>SOURCES</h2><div id="srclist"></div></div>
|
||||
<div class="card full"><h2>MESH PEERS</h2><div id="peers" class="sub">…</div></div>
|
||||
<div class="card full"><h2>TOP CONSENSUS IPs</h2><table id="agg"><thead><tr><th>IP</th><th>consensus</th><th>sources</th></tr></thead><tbody></tbody></table></div>
|
||||
</div>
|
||||
</main>
|
||||
<div class="toast" id="toast"></div>
|
||||
<script src="/shared/sidebar.js"></script>
|
||||
<script>
|
||||
const API="/api/v1/threatmesh";
|
||||
function tok(){return localStorage.getItem("sbx_token")||""}
|
||||
async function api(p,opt={}){opt.headers=Object.assign({},opt.headers||{});const t=tok();if(t)opt.headers.Authorization="Bearer "+t;
|
||||
const r=await fetch(API+p,opt);if(r.status===401){toast("login via Hub");throw 0}if(!r.ok)throw 0;
|
||||
const ct=r.headers.get("content-type")||"";return ct.includes("json")?r.json():r.text()}
|
||||
function toast(m){const t=document.getElementById("toast");t.textContent=m;t.classList.add("on");clearTimeout(t._);t._=setTimeout(()=>t.classList.remove("on"),2600)}
|
||||
function esc(s){return (s||"").replace(/[&<>"]/g,c=>({"&":"&","<":"<",">":">",'"':"""}[c]))}
|
||||
async function load(){
|
||||
let s;try{s=await fetch(API+"/status").then(r=>r.json())}catch(e){return}
|
||||
document.getElementById("mode").textContent=(s.mode||"sovereign")+" · CAPI "+(s.capi?"on":"off")+" · node "+(s.node||"?");
|
||||
document.getElementById("feedtotal").textContent=s.feed_total??0;
|
||||
document.getElementById("feedn").textContent=Object.keys(s.feeds||{}).length+" feeds";
|
||||
document.getElementById("meshtotal").textContent=s.mesh_total??0;
|
||||
document.getElementById("peern").textContent=(s.peers||0)+" peers";
|
||||
const all=s.sources||{};document.getElementById("total").textContent=Object.values(all).reduce((a,b)=>a+b,0);
|
||||
document.getElementById("srclist").innerHTML=Object.entries(all).sort((a,b)=>b[1]-a[1]).slice(0,12)
|
||||
.map(([k,v])=>`<div class="src ${k.startsWith('mesh:')?'mesh':''}"><span>${esc(k)}</span><b>${v}</b></div>`).join("")||'<div class="sub">no sources yet</div>';
|
||||
try{const p=await fetch(API+"/peers").then(r=>r.json());
|
||||
document.getElementById("peers").innerHTML=(p.peers||[]).length?p.peers.map(x=>`${esc(x.name||x.id)} <span style="color:var(--muted)">(${esc(x.address)})</span>`).join(" · "):"no mesh peers yet (single-node) — decisions will gossip when peers join";}catch(e){}
|
||||
try{const d=await fetch(API+"/decisions?min_consensus=1&limit=40").then(r=>r.json());
|
||||
d.sort((a,b)=>b.consensus-a.consensus);
|
||||
document.querySelector("#agg tbody").innerHTML=d.slice(0,25).map(r=>`<tr><td>${esc(r.value)}</td><td>${r.consensus}</td><td style="color:var(--muted)">${esc((r.sources||'').slice(0,60))}</td></tr>`).join("");}catch(e){}
|
||||
}
|
||||
async function refreshFeeds(){try{await api("/feeds/refresh",{method:"POST"});toast("feed refresh started (~1 min)")}catch(e){toast("need Hub login")}}
|
||||
async function gossip(){try{const r=await api("/mesh/gossip",{method:"POST"});toast(`gossip: ${r.pushed}/${r.peers} peers, ${r.shared} shared`)}catch(e){toast("need Hub login")}}
|
||||
load();setInterval(load,8000);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -1,3 +1,14 @@
|
|||
secubox-toolbox (2.7.19-1~bookworm1) bookworm; urgency=medium
|
||||
|
||||
* #728 blacklist-sync confidence gate: enforce a threat_intel IP only when
|
||||
corroborated by >= SECUBOX_BL_MIN_CONSENSUS sources (default 2) OR carried by
|
||||
a curated high-trust feed (weight >= SECUBOX_BL_MIN_WEIGHT, default 80).
|
||||
Arming all ~45k aggregated-feed IPs (mostly noisy single-source) risked
|
||||
blocking legit traffic; this enforces the high-confidence ~2k set. CrowdSec
|
||||
local decisions + DNS-guard stay always-enforced. Tunable via env.
|
||||
|
||||
-- Gerald KERMA <devel@cybermind.fr> Wed, 24 Jun 2026 16:30:00 +0000
|
||||
|
||||
secubox-toolbox (2.7.18-1~bookworm1) bookworm; urgency=medium
|
||||
|
||||
* #519/#522 fix(blacklist-sync): the DNS-guard domain loop aborted the whole
|
||||
|
|
|
|||
|
|
@ -37,11 +37,18 @@ fi
|
|||
TMP4=$(mktemp); TMP6=$(mktemp)
|
||||
trap 'rm -f "$TMP4" "$TMP6"' EXIT
|
||||
|
||||
# Source 1 : threat-intel C2 IPs (feodo / threatfox / sslbl) from the
|
||||
# toolbox SQLite. ioc_type='ip'.
|
||||
# Source 1 : threat-intel IPs from the toolbox SQLite (ioc_type='ip').
|
||||
# Confidence gate (#728): aggregated public feeds carry lots of noisy
|
||||
# single-source entries (e.g. blocklist.de) — arming all of them risks blocking
|
||||
# legitimate traffic. Enforce an IP only when corroborated by >= MIN_CONSENSUS
|
||||
# distinct sources OR carried by a curated high-trust feed (weight >= MIN_WEIGHT).
|
||||
# CrowdSec local decisions (Source 2) and DNS-guard domains are always enforced.
|
||||
TI_MIN_CONSENSUS="${SECUBOX_BL_MIN_CONSENSUS:-2}"
|
||||
TI_MIN_WEIGHT="${SECUBOX_BL_MIN_WEIGHT:-80}"
|
||||
if [ -r "$TOOLBOX_DB" ] && command -v sqlite3 >/dev/null 2>&1; then
|
||||
sqlite3 "$TOOLBOX_DB" \
|
||||
"SELECT DISTINCT ioc FROM threat_intel WHERE ioc_type='ip';" \
|
||||
"SELECT ioc FROM threat_intel WHERE ioc_type='ip' GROUP BY ioc \
|
||||
HAVING COUNT(DISTINCT source) >= $TI_MIN_CONSENSUS OR MAX(weight) >= $TI_MIN_WEIGHT;" \
|
||||
2>/dev/null >> "$TMP4.raw" || true
|
||||
fi
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ SecuBox is an appliance and network model - distributed peer applications.
|
|||
import subprocess
|
||||
import os
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Optional, List, Dict, Any
|
||||
from fastapi import FastAPI, Depends, HTTPException, BackgroundTasks
|
||||
|
|
@ -235,24 +236,62 @@ class VHostUpdate(BaseModel):
|
|||
enabled: Optional[bool] = None
|
||||
|
||||
|
||||
HAPROXY_ROUTES_FILE = Path("/srv/mitmproxy/haproxy-routes.json")
|
||||
HAPROXY_CERTS_DIR = Path("/data/haproxy/certs")
|
||||
|
||||
|
||||
def _is_public_fqdn(name: str) -> bool:
|
||||
name = (name or "").strip().rstrip(";")
|
||||
if not name or name in ("_", "localhost") or name.endswith(".local"):
|
||||
return False
|
||||
if re.match(r"^\d{1,3}(\.\d{1,3}){3}$", name):
|
||||
return False
|
||||
return "." in name
|
||||
|
||||
|
||||
def _server_name_fqdn(content: str):
|
||||
"""First public FQDN from any server_name directive (skips _, localhost, IPs)."""
|
||||
for line in content.split("\n"):
|
||||
s = line.strip()
|
||||
if s.startswith("server_name"):
|
||||
for tok in s[len("server_name"):].strip().rstrip(";").split():
|
||||
if _is_public_fqdn(tok):
|
||||
return tok
|
||||
return None
|
||||
|
||||
|
||||
def _load_haproxy_routes() -> dict:
|
||||
"""Public FQDN -> backend target from the HAProxy/mitmproxy route map."""
|
||||
try:
|
||||
d = json.loads(HAPROXY_ROUTES_FILE.read_text())
|
||||
return {k: v for k, v in d.items() if _is_public_fqdn(k)}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
@app.get("/vhosts", dependencies=[Depends(require_jwt)])
|
||||
async def list_vhosts():
|
||||
"""List all virtual hosts"""
|
||||
"""List all virtual hosts with full, clickable public URLs.
|
||||
|
||||
Public vhosts are HAProxy-fronted (TLS terminated there), so the real domain
|
||||
is the nginx server_name FQDN — NOT the config filename. We fall back to the
|
||||
HAProxy route map for backends whose nginx config only has local names, and
|
||||
append HAProxy public routes that have no nginx config, so the list is the
|
||||
COMPLETE set of reachable vhosts with working https:// links.
|
||||
"""
|
||||
vhosts = []
|
||||
routes = _load_haproxy_routes()
|
||||
seen = set()
|
||||
|
||||
if NGINX_VHOST_DIR.exists():
|
||||
for conf_file in NGINX_VHOST_DIR.glob("*.conf"):
|
||||
for conf_file in sorted(NGINX_VHOST_DIR.glob("*.conf")):
|
||||
if conf_file.name.startswith("_") or conf_file.name == "default.conf":
|
||||
continue
|
||||
domain = conf_file.stem
|
||||
stem = conf_file.stem
|
||||
enabled = (NGINX_ENABLED_DIR / conf_file.name).exists()
|
||||
|
||||
# Parse config
|
||||
backend = ""
|
||||
tls_mode = "off"
|
||||
websocket = False
|
||||
ssl = False
|
||||
|
||||
content = ""
|
||||
try:
|
||||
content = conf_file.read_text()
|
||||
for line in content.split("\n"):
|
||||
|
|
@ -260,27 +299,28 @@ async def list_vhosts():
|
|||
backend = line.split()[-1].rstrip(";")
|
||||
if "Upgrade" in line:
|
||||
websocket = True
|
||||
ssl = "listen 443" in content or "ssl_certificate" in content
|
||||
|
||||
if ssl:
|
||||
if "/etc/acme/" in content:
|
||||
tls_mode = "acme"
|
||||
else:
|
||||
tls_mode = "manual"
|
||||
except:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Check certificate
|
||||
cert_expires = None
|
||||
if ssl:
|
||||
cert_path = ACME_DIR / domain / "fullchain.cer"
|
||||
if cert_path.exists():
|
||||
success, out, _ = run_cmd([
|
||||
"openssl", "x509", "-in", str(cert_path), "-noout", "-enddate"
|
||||
])
|
||||
if success:
|
||||
cert_expires = out.split("=")[-1]
|
||||
domain = _server_name_fqdn(content)
|
||||
if not domain:
|
||||
domain = next((k for k in routes if k.split(".")[0] == stem), None) or stem
|
||||
|
||||
is_public = _is_public_fqdn(domain)
|
||||
ssl = is_public # HAProxy terminates TLS for every public vhost
|
||||
url = f"https://{domain}" if is_public else None
|
||||
tls_mode = "haproxy" if is_public else "off"
|
||||
|
||||
cert_expires = None
|
||||
if is_public:
|
||||
cert_path = HAPROXY_CERTS_DIR / f"{domain}.pem"
|
||||
if cert_path.exists():
|
||||
success, out, _ = run_cmd(
|
||||
["openssl", "x509", "-in", str(cert_path), "-noout", "-enddate"])
|
||||
if success:
|
||||
cert_expires = out.split("=")[-1].strip()
|
||||
|
||||
seen.add(domain)
|
||||
vhosts.append({
|
||||
"domain": domain,
|
||||
"backend": backend,
|
||||
|
|
@ -289,9 +329,34 @@ async def list_vhosts():
|
|||
"websocket": websocket,
|
||||
"enabled": enabled,
|
||||
"cert_expires": cert_expires,
|
||||
"url": url,
|
||||
"source": "nginx",
|
||||
"config_file": str(conf_file),
|
||||
})
|
||||
|
||||
# Append HAProxy public vhosts with no nginx config (complete the list).
|
||||
for domain, target in routes.items():
|
||||
if domain in seen:
|
||||
continue
|
||||
seen.add(domain)
|
||||
try:
|
||||
backend = f"{target[0]}:{target[1]}" if isinstance(target, (list, tuple)) else str(target)
|
||||
except Exception:
|
||||
backend = ""
|
||||
vhosts.append({
|
||||
"domain": domain,
|
||||
"backend": backend,
|
||||
"tls_mode": "haproxy",
|
||||
"ssl": True,
|
||||
"websocket": False,
|
||||
"enabled": True,
|
||||
"cert_expires": None,
|
||||
"url": f"https://{domain}",
|
||||
"source": "haproxy",
|
||||
"config_file": None,
|
||||
})
|
||||
|
||||
vhosts.sort(key=lambda v: v["domain"])
|
||||
return {"vhosts": vhosts, "count": len(vhosts)}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,14 @@
|
|||
secubox-vhost (1.1.1-1~bookworm1) bookworm; urgency=medium
|
||||
|
||||
* Fix vhost list: use the real public FQDN (nginx server_name, HAProxy route
|
||||
map) instead of the config filename stem, set full https:// URLs + correct
|
||||
ssl (HAProxy terminates TLS), and append HAProxy public routes that have no
|
||||
nginx config. The list is now the COMPLETE set of reachable vhosts with
|
||||
working clickable links (was showing short names like "arm"/"lyrion" with
|
||||
url=None -> broken links).
|
||||
|
||||
-- Gerald KERMA <devel@cybermind.fr> Wed, 24 Jun 2026 14:00:00 +0000
|
||||
|
||||
secubox-vhost (1.1.0-1~bookworm1) bookworm; urgency=medium
|
||||
|
||||
* Add three-fold architecture (components, status, access)
|
||||
|
|
|
|||
103
wiki/ThreatMesh-FR.md
Normal file
103
wiki/ThreatMesh-FR.md
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
# ThreatMesh 🛰️
|
||||
|
||||
[EN](ThreatMesh) | **[FR](ThreatMesh-FR)** | **🔴 BOOT · 🛡️ SÉCURITÉ** | renseignement de menace souverain
|
||||
|
||||
> Votre réseau de vigilance de quartier pour l'internet — *listes gratuites + astuces des voisins, sans chef central, sans paywall, impossible à bannir.*
|
||||
|
||||

|
||||
|
||||
ThreatMesh est la couche SecuBox qui **bloque automatiquement les adresses
|
||||
internet malveillantes connues** — née après que l'API centrale de CrowdSec
|
||||
(CAPI) a blacklisté l'IP de notre box puis fait payer le déblocage. Elle
|
||||
remplace cette dépendance centrale par des **listes publiques auto-sourcées**
|
||||
plus un **partage d'astuces pair-à-pair** entre vos propres box. Vous possédez
|
||||
toute la chaîne, de bout en bout.
|
||||
|
||||
---
|
||||
|
||||
## 🏘️ L'idée simple
|
||||
|
||||
Voyez votre SecuBox comme une **maison avec un concierge malin**. Le concierge
|
||||
tient une seule liste « à ne pas laisser entrer », alimentée par deux flux, et
|
||||
refuse tout ce qui y figure.
|
||||
|
||||
```
|
||||
LISTES « RECHERCHÉ » GRATUITES VOS AUTRES BOX (mesh)
|
||||
(bulletins publics) (voisins qui s'échangent des astuces)
|
||||
\ /
|
||||
\ /
|
||||
▼ ▼
|
||||
┌──────────────────────────────────┐
|
||||
│ LE CONCIERGE — une liste de │
|
||||
│ blocage, ne croit que les bonnes │
|
||||
│ pistes │
|
||||
└──────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
🚪 une mauvaise adresse frappe → REFUSÉE
|
||||
```
|
||||
|
||||
1. **📋 Listes de surveillance gratuites** — toutes les 6 h la box récupère des
|
||||
listes publiques « ces IP sont dangereuses » (C2 de malware, réseaux
|
||||
détournés, attaquants connus). Gratuit, sans inscription, sans compte.
|
||||
2. **🤝 Astuces des voisins (mesh)** — quand *votre* box attrape un attaquant,
|
||||
elle prévient vos *autres* box via le mesh chiffré SecuBox (WireGuard). Sans
|
||||
intermédiaire.
|
||||
3. **🛡️ Le concierge agit** — chaque astuce atterrit dans une seule liste de
|
||||
blocage et la box refuse le trafic vers/depuis ces adresses au pare-feu
|
||||
(nftables).
|
||||
|
||||
---
|
||||
|
||||
## 🆚 Pourquoi souverain
|
||||
|
||||
| Avant (CrowdSec CAPI) | Maintenant (ThreatMesh) |
|
||||
|------------------------|-------------------------|
|
||||
| La liste centrale d'une entreprise | **La vôtre**, depuis des sources ouvertes |
|
||||
| Ils peuvent **bannir votre IP** | **Personne ne peut vous exclure** |
|
||||
| **Payer** pour être débanni | **Gratuit, pour toujours** |
|
||||
| Vous dépendez d'eux | **Vous possédez toute la chaîne** |
|
||||
|
||||
Le moteur de détection hors-ligne de CrowdSec (LAPI) est conservé — seul le flux
|
||||
central toxique (CAPI) est abandonné.
|
||||
|
||||
---
|
||||
|
||||
## 🔍 Sous le capot
|
||||
|
||||
| Étape | Composant | Rôle |
|
||||
|-------|-----------|------|
|
||||
| **Feeds** | `secubox-threatfeed` (timer, 6 h) | tire des listes gratuites — feodo, sslbl, FireHOL, Spamhaus DROP, blocklist.de, CINS, ET-compromised, DShield — dans la table partagée `threat_intel` |
|
||||
| **Mesh** | `secubox-threatmesh` (service) | diffuse les décisions détectées localement aux pairs du mesh via WireGuard ; ingère les décisions des pairs (`mesh:<node>`), comptées par consensus ; port `:8780` verrouillé au mesh par nftables |
|
||||
| **Application** | `secubox-blacklist-sync` | vide `threat_intel` → ensembles de drop nft `blacklist_v4/v6` |
|
||||
| **Visualiser** | tableau de bord `/threatmesh/` + `/api/v1/threatmesh/decisions` (compatible bouncer CrowdSec) | statut, sources, pairs, IP à plus fort consensus |
|
||||
|
||||
### 🎯 La porte de confiance (zéro carpet-bomb de faux positifs)
|
||||
|
||||
Les feeds publics agrégés contiennent beaucoup d'entrées bruyantes à source
|
||||
unique. ThreatMesh **n'applique le blocage** que si une IP est **corroborée par
|
||||
≥ 2 sources** *ou* provient d'un **feed curé de haute confiance** (poids ≥ 80).
|
||||
Le reste reste *visible mais non bloqué*. Les décisions locales de CrowdSec et le
|
||||
DNS-guard sont toujours appliqués.
|
||||
|
||||
Réglage via variables d'env sur `secubox-blacklist-sync` :
|
||||
|
||||
```
|
||||
SECUBOX_BL_MIN_CONSENSUS=2 # sources qui doivent concorder (plus bas = plus de couverture)
|
||||
SECUBOX_BL_MIN_WEIGHT=80 # niveau de confiance qui contourne la règle de consensus
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📊 En un coup d'œil
|
||||
|
||||
- **~45 000** IP dangereuses connues (rafraîchies toutes les 6 h)
|
||||
- **~3 000** IP de haute confiance bloquées activement au pare-feu
|
||||
- Le partage mesh s'active tout seul dès qu'une seconde SecuBox rejoint le mesh
|
||||
- **0** compte externe · **0** paywall · **0** moyen pour un tiers de vous couper
|
||||
|
||||
> *Ils nous ont bloqués et ont demandé de l'argent pour débloquer. Alors on a construit le nôtre — et maintenant personne ne peut nous couper.* 🔓
|
||||
|
||||
---
|
||||
|
||||
*Voir aussi : [[Anti-Track]] · [[Architecture]] · `secubox-threatmesh` (#728)*
|
||||
97
wiki/ThreatMesh.md
Normal file
97
wiki/ThreatMesh.md
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
# ThreatMesh 🛰️
|
||||
|
||||
**[EN](ThreatMesh)** | [FR](ThreatMesh-FR) | **🔴 BOOT · 🛡️ SECURITY** | sovereign threat-intel
|
||||
|
||||
> Your own neighborhood watch for the internet — *free feeds + neighbor tips, no central boss, no paywall, can't be banned.*
|
||||
|
||||

|
||||
|
||||
ThreatMesh is the SecuBox layer that automatically **blocks known-bad internet
|
||||
addresses** on its own — built after CrowdSec's central API (CAPI) IP-blocklisted
|
||||
our box and paywalled the un-blocking. It replaces that central dependency with
|
||||
**self-sourced public lists** plus **peer-to-peer tip sharing** between your own
|
||||
boxes. You own the whole thing end to end.
|
||||
|
||||
---
|
||||
|
||||
## 🏘️ The simple idea
|
||||
|
||||
Think of your SecuBox as a **house with a smart doorman**. The doorman keeps one
|
||||
"do not let in" list, fed by two streams, and turns away anything on it.
|
||||
|
||||
```
|
||||
FREE "WANTED" LISTS YOUR OTHER BOXES (mesh)
|
||||
(public bulletins) (neighbors swapping tips)
|
||||
\ /
|
||||
\ /
|
||||
▼ ▼
|
||||
┌──────────────────────────────────┐
|
||||
│ THE DOORMAN — one block list, │
|
||||
│ only trusts solid tips │
|
||||
└──────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
🚪 bad address knocks → DROPPED
|
||||
```
|
||||
|
||||
1. **📋 Free watch-lists** — every 6 h the box pulls public "these IPs are
|
||||
dangerous" lists (malware C2, hijacked networks, known attackers). Free, no
|
||||
sign-up, no account.
|
||||
2. **🤝 Neighbor tips (mesh)** — when *your* box catches an attacker it tells your
|
||||
*other* boxes over the encrypted SecuBox mesh (WireGuard). No middleman.
|
||||
3. **🛡️ The doorman acts** — every tip lands in one block-list and the box
|
||||
refuses traffic to/from those addresses at the firewall (nftables).
|
||||
|
||||
---
|
||||
|
||||
## 🆚 Why sovereign
|
||||
|
||||
| Before (CrowdSec CAPI) | Now (ThreatMesh) |
|
||||
|------------------------|------------------|
|
||||
| One company's central list | **Your own**, from open sources |
|
||||
| They can **ban your IP** | **No one can lock you out** |
|
||||
| **Pay** to get un-banned | **Free, forever** |
|
||||
| You depend on them | **You own the whole pipeline** |
|
||||
|
||||
CrowdSec's offline detection engine (LAPI) is kept — only the toxic central feed
|
||||
(CAPI) is dropped.
|
||||
|
||||
---
|
||||
|
||||
## 🔍 Under the hood
|
||||
|
||||
| Stage | Component | What it does |
|
||||
|-------|-----------|--------------|
|
||||
| **Feeds** | `secubox-threatfeed` (timer, 6 h) | pulls free lists — feodo, sslbl, FireHOL, Spamhaus DROP, blocklist.de, CINS, ET-compromised, DShield — into the shared `threat_intel` table |
|
||||
| **Mesh** | `secubox-threatmesh` (service) | gossips locally-detected decisions to mesh peers over WireGuard; ingests peer decisions (`mesh:<node>`), consensus-counted; port `:8780` locked to the mesh by nftables |
|
||||
| **Enforce** | `secubox-blacklist-sync` | drains `threat_intel` → nft `blacklist_v4/v6` drop sets |
|
||||
| **See it** | `/threatmesh/` dashboard + `/api/v1/threatmesh/decisions` (CrowdSec-bouncer-compatible) | status, sources, peers, top-consensus IPs |
|
||||
|
||||
### 🎯 The confidence gate (no false-positive carpet-bomb)
|
||||
|
||||
Aggregated public feeds carry many noisy single-source entries. ThreatMesh
|
||||
**only enforces** an IP that is **corroborated by ≥ 2 sources** *or* comes from a
|
||||
**curated high-trust feed** (weight ≥ 80). The rest stay *visible but not
|
||||
blocked*. CrowdSec local decisions + DNS-guard are always enforced.
|
||||
|
||||
Tune via env on `secubox-blacklist-sync`:
|
||||
|
||||
```
|
||||
SECUBOX_BL_MIN_CONSENSUS=2 # sources that must agree (lower = more coverage)
|
||||
SECUBOX_BL_MIN_WEIGHT=80 # trust level that bypasses the consensus rule
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📊 At a glance
|
||||
|
||||
- **~45 000** dangerous IPs known (refreshed every 6 h)
|
||||
- **~3 000** high-confidence IPs actively dropped at the firewall
|
||||
- Mesh sharing lights up automatically when a second SecuBox joins the mesh
|
||||
- **0** external accounts · **0** paywall · **0** ways for a third party to switch you off
|
||||
|
||||
> *They blocked us and asked for money to unblock. So we built our own — and now nobody can switch us off.* 🔓
|
||||
|
||||
---
|
||||
|
||||
*See also: [[Anti-Track]] · [[Architecture]] · `secubox-threatmesh` (#728)*
|
||||
|
|
@ -41,6 +41,7 @@
|
|||
### 🟣 MIND — Modules
|
||||
|
||||
* [[Anti-Track]] 🛡️ bloque · empoisonne · anonymise
|
||||
* [[ThreatMesh]] 🛰️ blocklist souveraine (feeds + mesh, sans CAPI) | [FR](ThreatMesh-FR)
|
||||
* [[MODULES-EN|Modules]] 🇬🇧
|
||||
* [[MODULES-FR]] 🇫🇷
|
||||
* [[MODULES-DE]] 🇩🇪
|
||||
|
|
|
|||
|
|
@ -7,3 +7,5 @@ Local image assets referenced by wiki pages.
|
|||
| File | Used by | Notes |
|
||||
|------|---------|-------|
|
||||
| `anti-track-v2-poster.png` | [[Anti-Track]] | Comic-style hero poster (Bloque · Empoisonne · Anonymise). Portrait, ~1024×1536. |
|
||||
| `threatmesh-poster.png` | [[ThreatMesh]] | Neighborhood-watch hero poster (sovereign threat-intel — free feeds + mesh, no CAPI). Portrait, ~1024×1536. |
|
||||
| `threatmesh-poster-fr.png` | [[ThreatMesh-FR]] | Version FR de l'affiche ThreatMesh (vigilance de quartier). Portrait, ~1024×1536. |
|
||||
|
|
|
|||
BIN
wiki/images/threatmesh-poster-fr.png
Normal file
BIN
wiki/images/threatmesh-poster-fr.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 2.0 MiB |
BIN
wiki/images/threatmesh-poster.png
Normal file
BIN
wiki/images/threatmesh-poster.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 1.9 MiB |
Loading…
Reference in New Issue
Block a user