fix(toolbox): content-aware binary streaming — restore banner on heavy sites

The #685 stream_large_bodies=1m streamed large HTML too (streamed bodies can't be banner-injected) → "plus de banner sur leparisien.fr". Replaced by the stream_binaries addon: streams only large NON-HTML (apk/xpi/video/octet-stream/ big downloads) verbatim so the R3 forging path doesn't corrupt them, while HTML is always buffered so inject_banner + ad_ghost work. toolbox 2.7.16. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
fix(toolbox): report reads LIVE social graph, not frozen events (ref #686 )
2026-06-29 14:31:31 +00:00 · 2026-06-20 08:24:17 +02:00 · 2026-06-20 08:20:49 +02:00
5 changed files with 143 additions and 52 deletions
--- a/packages/secubox-toolbox/conf/report-live.html.j2
+++ b/packages/secubox-toolbox/conf/report-live.html.j2
@ -75,17 +75,18 @@ ul{list-style:none;padding-left:.2rem}li{padding:.12rem 0;font-size:.82rem}li::b
 <body>
 {% set m = metrics or {} %}
-{% set sc = risk_score|default(0) %}
+{# #686 — summary + graphs come from the LIVE social graph (graph_stats), not the
-{% set rl = risk_label|default('LOW') %}
+   frozen events table. #}
 {% set gst = graph_stats or {} %}
 {% set sc = exposure_score|default(0) %}
 {% set ch = charts or {} %}
 {% set gcol = 'var(--phos-hot)' if sc < 30 else ('var(--amber)' if sc < 70 else 'var(--red)') %}
 {% set palette = ['#00dd44','#9e76ff','#ff8866','#66bbff','#ffb347','#ff4466'] %}
-{% set dpi_cls = dpi_classified or {} %}
+{% set n_trackers = gst.total_trackers|default(0) %}
-{% set cookies_p = cookies_providers or [] %}
+{% set n_sites = gst.total_sites|default(0) %}
-{% set geo_h = geo_top_hosts or [] %}
+{% set n_countries = gst.total_countries|default(0) %}
-{% set n_apps = (dpi_cls.top_apps|default([])|selectattr('app','ne','?')|list|length) %}
+{% set n_antibot = gst.antibot_sites|default(0) %}
-{% set n_trackers = (cookies_p|map(attribute='count')|sum) %}
+{% set n_opgrade = gst.opgrade_sites|default(0) %}
 {% set n_countries = (geo_h|map(attribute='country')|reject('equalto','')|list|unique|list|length) %}
 {% set _avatar = avatar_analysis or {} %}
 <h1>👁️ VILLAGE3B <span style="font-size:.8rem;color:var(--dim);font-weight:400">· mon rapport</span></h1>
@ -109,22 +110,21 @@ ul{list-style:none;padding-left:.2rem}li{padding:.12rem 0;font-size:.82rem}li::b
    </div>
  </div>
  <div class="verdict" style="color:{{ gcol }}">
-    {% if sc < 30 %}🟢 Tout va bien — {{ rl }}{% elif sc < 70 %}🟡 À surveiller — {{ rl }}{% else %}🔴 Attention — {{ rl }}{% endif %}
+    {% if sc < 30 %}🟢 Exposition faible{% elif sc < 70 %}🟡 Exposition modérée{% else %}🔴 Exposition élevée{% endif %}
  </div>
-  <p class="help">Score de risque de ton appareil. Plus il est <b>bas</b>, mieux tu es protégé.</p>
+  <p class="help">Niveau d'exposition au pistage (traceurs croisés + acteurs opérateur/anti-bot). Plus c'est <b>bas</b>, mieux c'est.</p>
  {% if risk_explanation %}<p style="font-size:.85rem;margin-top:.5rem">{{ risk_explanation }}</p>{% endif %}
 </div>
-{# ── KPI row ── #}
+{# ── KPI row (LIVE social graph) ── #}
 <div class="kpis">
-  <div class="kpi"><div class="e">🌐</div><div class="n">{{ m.connections|default(0) }}</div><div class="l">connexions</div></div>
+  <div class="kpi"><div class="e">🍪</div><div class="n">{{ n_trackers }}</div><div class="l">traceurs</div></div>
-  <div class="kpi"><div class="e">📡</div><div class="n">{{ m.unique_hosts|default(0) }}</div><div class="l">hôtes</div></div>
+  <div class="kpi"><div class="e">🌐</div><div class="n">{{ n_sites }}</div><div class="l">sites</div></div>
  <div class="kpi"><div class="e">🍪</div><div class="n">{{ n_trackers }}</div><div class="l">trackers</div></div>
  <div class="kpi"><div class="e">🌍</div><div class="n">{{ n_countries }}</div><div class="l">pays</div></div>
-  <div class="kpi"><div class="e">📺</div><div class="n">{{ n_apps }}</div><div class="l">apps</div></div>
+  <div class="kpi"><div class="e">🤖</div><div class="n">{{ n_antibot }}</div><div class="l">anti-bot</div></div>
-  <div class="kpi"><div class="e">🔒</div><div class="n">{{ m.tls_pinned|default(0) }}</div><div class="l">cert-pin</div></div>
+  <div class="kpi"><div class="e">📡</div><div class="n">{{ n_opgrade }}</div><div class="l">opérateur</div></div>
  <div class="kpi"><div class="e">🔗</div><div class="n">{{ (graph.edges|default([]))|length }}</div><div class="l">liens</div></div>
 </div>
-<p class="help" style="text-align:center;margin-bottom:1rem">Ton appareil a contacté {{ m.unique_hosts|default(0) }} serveurs dans {{ n_countries }} pays, avec {{ n_trackers }} traceurs repérés.</p>
+<p class="help" style="text-align:center;margin-bottom:1rem">{{ n_trackers }} traceurs te suivent à travers {{ n_sites }} sites, depuis {{ n_countries }} pays.{% if n_opgrade %} Dont {{ n_opgrade }} de qualité opérateur.{% endif %}</p>
 {# ── GRAPHS ── #}
 <div class="card">
@ -158,18 +158,18 @@ ul{list-style:none;padding-left:.2rem}li{padding:.12rem 0;font-size:.82rem}li::b
      {% else %}<div class="empty">Pas encore de données géo</div>{% endif %}
    </div>
-    {# apps bars #}
+    {# top tracked sites bars #}
    <div style="grid-column:1/-1">
-      <div style="font-size:.82rem;color:var(--dim);margin-bottom:.4rem">📺 Quelles apps / services</div>
+      <div style="font-size:.82rem;color:var(--dim);margin-bottom:.4rem">🌐 Où tu es le plus pisté (traceurs par site)</div>
-      {% if ch.apps %}
+      {% if ch.sites %}
-        {% for a in ch.apps %}
+        {% for a in ch.sites %}
-        <div class="bar-row"><span class="bar-lbl">{{ a.emoji }} {{ a.label[:16] }}</span><span class="bar-track"><span class="bar-fill" style="width:{{ a.pct }}%;background:linear-gradient(90deg,var(--violet),#c9b6ff)"></span></span><span class="bar-val" style="color:var(--violet)">{{ a.count }}</span></div>
+        <div class="bar-row"><span class="bar-lbl">{{ a.label[:22] }}</span><span class="bar-track"><span class="bar-fill" style="width:{{ a.pct }}%;background:linear-gradient(90deg,var(--violet),#c9b6ff)"></span></span><span class="bar-val" style="color:var(--violet)">{{ a.count }}</span></div>
        {% endfor %}
-      {% else %}<div class="empty">Aucune app classifiée</div>{% endif %}
+      {% else %}<div class="empty">Pas encore de sites pistés</div>{% endif %}
    </div>
  </div>
-  <p class="help">Les traceurs suivent ta navigation entre sites. Les apps cert-pinning (🔒) refusent l'analyse — c'est bon signe.</p>
+  <p class="help">Les traceurs suivent ta navigation entre sites. « opérateur » = traceurs de niveau opérateur télécom (les plus intrusifs).</p>
 </div>
 {# ── LEVEL SWITCHER (action) ── #}
--- a/packages/secubox-toolbox/debian/changelog
+++ b/packages/secubox-toolbox/debian/changelog
@ -1,3 +1,23 @@
 secubox-toolbox (2.7.16-1~bookworm1) bookworm; urgency=medium
  * fix: restore banner on heavy sites (leparisien.fr). The #685 stream_large_bodies=1m
    streamed large HTML too (streamed bodies cannot be banner-injected). Replaced
    by the content-aware stream_binaries addon: streams only large NON-HTML
    (APK/XPI/video/octet-stream/big downloads) verbatim, HTML always buffered so
    inject_banner + ad_ghost work.
 -- Gerald KERMA <devel@cybermind.fr>  Sat, 20 Jun 2026 13:40:00 +0200
 secubox-toolbox (2.7.15-1~bookworm1) bookworm; urgency=medium
  * fix(#686): /report/me/html reads the LIVE social graph (social.fetch_graph)
    instead of the frozen events table (#662 cutover) — report was all-zeros even
    when /social + webext showed data. Summary gauge = exposure score; KPIs
    (traceurs/sites/pays/anti-bot/opérateur/liens) + graphs (trackers donut,
    countries bars, top-pisté-sites bars) all from the live graph.
 -- Gerald KERMA <devel@cybermind.fr>  Sat, 20 Jun 2026 13:00:00 +0200
 secubox-toolbox (2.7.14-1~bookworm1) bookworm; urgency=medium
  * fix(#685): mitm-wg now streams large bodies (stream_large_bodies=1m) so big
--- a/packages/secubox-toolbox/mitmproxy_addons/stream_binaries.py
+++ b/packages/secubox-toolbox/mitmproxy_addons/stream_binaries.py
@ -0,0 +1,53 @@
 # SPDX-License-Identifier: LicenseRef-CMSD-1.0
 # Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
 # Source-Disclosed License — All rights reserved except as expressly granted.
 # See LICENCE-CMSD-1.0.md for terms.
 #
 # SecuBox-Deb :: toolbox :: stream large BINARY responses (#686)
 #
 # Replaces the content-agnostic `--set stream_large_bodies=1m`, which streamed
 # EVERY body >1MB — including large HTML (leparisien.fr) — and streamed bodies
 # can't be banner-injected → "plus de banner". Here we stream only large
 # NON-HTML responses (APK / XPI / video / octet-stream / big downloads) so they
 # pass through the HTTP/2 forging path VERBATIM (the buffer+reframe corrupted the
 # 14MB APK over the R3 tunnel), while HTML is always buffered so inject_banner /
 # ad_ghost still work.
 from mitmproxy import http
 _THRESHOLD = 1_000_000  # 1 MB
 # Always-stream binary content-types regardless of declared length (covers
 # chunked downloads with no Content-Length).
 _BIN_CT = (
    "application/vnd.android.package-archive",  # .apk
    "application/x-xpinstall",                  # .xpi
    "application/octet-stream",
    "application/zip",
    "application/pdf",
    "video/",
    "audio/",
 )
 class StreamBinaries:
    def responseheaders(self, flow: http.HTTPFlow) -> None:
        try:
            r = flow.response
            if r is None:
                return
            ct = (r.headers.get("content-type", "") or "").lower()
            if "text/html" in ct:
                return  # NEVER stream HTML — banner + ad_ghost need the body
            if any(b in ct for b in _BIN_CT):
                r.stream = True
                return
            try:
                cl = int(r.headers.get("content-length", "0") or "0")
            except (TypeError, ValueError):
                cl = 0
            if cl >= _THRESHOLD:
                r.stream = True
        except Exception:
            pass
 addons = [StreamBinaries()]
--- a/packages/secubox-toolbox/sbin/secubox-toolbox-mitm-wg-launch
+++ b/packages/secubox-toolbox/sbin/secubox-toolbox-mitm-wg-launch
@ -87,12 +87,10 @@ ARGS=(
    # upstream in mitmproxy 10.4 ; with mitmproxy 11+ we can safely
    # re-enable keep-alive.  Halves TCP handshakes towards busy CDNs.
    --set keep_host_header=true
-    # #685 — STREAM large bodies verbatim instead of buffering+reframing them.
+    # #686 — large-binary streaming is now content-aware via the stream_binaries
-    # Buffering a 14 MB APK / binary download through the HTTP/2 forging path
+    # addon (streams APK/XPI/video/large NON-HTML verbatim) instead of the blunt
-    # corrupted/truncated it (clients reported "apk corrupt", CA "empty name" —
+    # `stream_large_bodies=1m`, which also streamed large HTML and killed banner
-    # only when the R3 tunnel was up). No addon touches non-HTML bodies (all are
+    # injection on heavy sites (leparisien.fr).
    # text/html-gated), so streaming big responses is safe and byte-transparent.
    --set stream_large_bodies=1m
 )
 if [ -n "$IGNORE_REGEX" ]; then
@ -121,7 +119,7 @@ fi
 # ad_ghost (#566) runs right after protective_mode: for R3+/R4 it 204s known
 # ad/tracker hosts (bandwidth save) at request time and injects ad-hiding CSS
 # on HTML responses. Gated by the modular filter config (toolbox WebUI).
-for addon in tls_splice inject_xff utiq_defense protective_mode privacy_guard ad_ghost media_cache local_store social_graph inject_banner dpi cookies avatar ja4 soc_relay cert_pin_detect media_stats; do
+for addon in stream_binaries tls_splice inject_xff utiq_defense protective_mode privacy_guard ad_ghost media_cache local_store social_graph inject_banner dpi cookies avatar ja4 soc_relay cert_pin_detect media_stats; do
    ARGS+=(-s "$ADDON_DIR/${addon}.py")
 done
--- a/packages/secubox-toolbox/secubox_toolbox/api.py
+++ b/packages/secubox-toolbox/secubox_toolbox/api.py
@ -2342,11 +2342,12 @@ def _classify_apps(hosts: set[str]) -> list[str]:
    return apps
-def _build_report_charts(session: dict) -> dict:
+def _build_report_charts(graph: dict) -> dict:
-    """Graph-ready aggregates for the simplified report (trackers donut,
+    """Graph-ready aggregates for the report, from the LIVE social graph
-    countries bars, apps bars). Defensive / fail-empty. Each list item has
+    (social.fetch_graph). The events table froze at the #662 cutover, so the
-    {label, emoji/flag, count, pct}; trackers also carry cumulative start/end
+    report reads the SAME source as /social + the webext (was the bug: it read
-    for a CSS conic-gradient donut."""
+    the dead events → all zeros). Returns trackers donut + countries bars + sites
    bars; trackers also carry cumulative start/end for the CSS conic-gradient."""
    def _top_pct(items: list, n: int = 6) -> list:
        items = [it for it in items if it.get("count")]
        items.sort(key=lambda x: x["count"], reverse=True)
@ -2356,30 +2357,33 @@ def _build_report_charts(session: dict) -> dict:
            it["pct"] = round(100 * it["count"] / total)
        return items
-    cp = session.get("cookies_providers") or []
+    g = graph or {}
    nodes = g.get("nodes") or []
    trackers = _top_pct([
-        {"label": p.get("provider", "?"), "emoji": p.get("emoji", "🍪"),
+        {"label": (n.get("domain") or n.get("id") or "?"), "emoji": "🍪",
-         "count": int(p.get("count", 0) or 0)} for p in cp])
+         "count": int(n.get("hits", 0) or 0)} for n in nodes])
    cum = 0
    for it in trackers:
        it["start"] = cum
        cum += it["pct"]
        it["end"] = cum
    by_country: dict = {}
    for h in (session.get("geo_top_hosts") or []):
        key = (h.get("flag") or "🏴", h.get("country") or "?")
        by_country[key] = by_country.get(key, 0) + int(h.get("count", 0) or 0)
    countries = _top_pct([
-        {"flag": k[0], "label": k[1], "count": v} for k, v in by_country.items()])
+        {"flag": c.get("flag") or "🏴", "label": (c.get("country_iso") or "?"),
         "count": int(c.get("hits", 0) or 0)} for c in (g.get("by_country") or [])])
-    dc = session.get("dpi_classified") or {}
+    # top tracked sites = number of DISTINCT trackers reaching each first-party
-    apps = _top_pct([
+    # site (from each node's sites list) — "where you're tracked most".
-        {"label": a.get("app", "?"), "emoji": a.get("emoji", "📦"),
+    site_trk: dict = {}
-         "count": int(a.get("count", 0) or 0)}
+    for n in nodes:
-        for a in (dc.get("top_apps") or []) if a.get("app") not in (None, "", "?")])
+        for s in (n.get("sites") or []):
            if s:
                site_trk[s] = site_trk.get(s, 0) + 1
    sites = _top_pct([{"label": s, "emoji": "🌐", "count": c}
                      for s, c in site_trk.items()])
-    return {"trackers": trackers, "countries": countries, "apps": apps}
+    return {"trackers": trackers, "countries": countries, "sites": sites}
 # NOTE: route order matters in FastAPI — specific routes (/report/me,
@ -2408,6 +2412,21 @@ async def report_me_html(request: Request) -> HTMLResponse:
        )
    ip = _client_ip(request) or (request.client.host if request.client else "?")
    session = _aggregate_session(mac_hash)
    # #686 — the events table froze at the #662 cutover, so the report's numbers
    # came out all-zero. The LIVE per-client data is the social graph (same source
    # /social + the webext use). Pull it (7d) and drive the summary + graphs off it.
    try:
        from . import social as _social
        graph = _social.fetch_graph(mac_hash, since_seconds=7 * 86400)
    except Exception:
        graph = {"stats": {}, "nodes": [], "by_country": []}
    gs = graph.get("stats") or {}
    # Honest exposure indicator (0-100) from the live graph: tracker breadth +
    # operator-grade / anti-bot presence. Not a "compromise" score (events dead).
    exposure_score = min(100, int(
        (gs.get("total_trackers", 0) or 0) * 1.5
        + (gs.get("opgrade_sites", 0) or 0) * 12
        + (gs.get("antibot_sites", 0) or 0) * 8))
    # Phase 3 (#492) : pass query args + force no-cache so iPhone Safari
    # actually fetches the new template.
    # Phase 6 (#496) : also pass wg_enabled so dashboard R3 link renders
@ -2420,7 +2439,8 @@ async def report_me_html(request: Request) -> HTMLResponse:
        current_level=store.get_client_level(mac_hash) if mac_hash else "r1",
        wg_enabled=wg_enabled,
        cumulative=cumulative,
-        charts=_build_report_charts(session),
+        graph=graph, graph_stats=gs, exposure_score=exposure_score,
        charts=_build_report_charts(graph),
        **session,
    )
    return HTMLResponse(html, headers={