Merge pull request #698 from CyberMind-FR/feature/697-sbxmitm-truncates-responses-8mib-large-g

sbxmitm: stream non-injected responses verbatim — stop truncating >8MiB (closes #697)

packages/secubox-dpi/collector/main.go

@@ -378,15 +378,24 @@ func saveSeen(m map[string]bool) {
 func writeState(aggs map[string]*agg, alerts []alert, now int64) {
 	// per-device rollup
 	type devstat struct {
-		Device   string         `json:"device"`
+		Device    string         `json:"device"`
-		Flows    int            `json:"flows"`
+		Flows     int            `json:"flows"`
-		UpBytes  int64          `json:"up_bytes"`
+		UpBytes   int64          `json:"up_bytes"`
-		Services []*agg         `json:"services"`        // all classified egress (any category)
+		DownBytes int64          `json:"down_bytes"`
-		Clouds   []*agg         `json:"clouds"`          // back-compat: exfil-relevant subset
+		Services  []*agg         `json:"services"`        // all classified egress (any category)
-		ByCat    map[string]int `json:"by_category"`     // category → flow count
+		Clouds    []*agg         `json:"clouds"`          // back-compat: exfil-relevant subset
-		Alerts   []alert        `json:"alerts"`
+		ByCat     map[string]int `json:"by_category"`     // category → flow count
+		Alerts    []alert        `json:"alerts"`
 	}
 	devs := map[string]*devstat{}
+	// global rollups (incl. uncategorised dests) for the dashboard list cards
+	type rollup struct {
+		Name  string `json:"name"`
+		Bytes int64  `json:"bytes"`
+		Flows int    `json:"flows"`
+	}
+	apps := map[string]*rollup{}   // by service||dst
+	protos := map[string]*rollup{} // by nDPI proto
 	for _, a := range aggs {
 		d := devs[a.Device]
 		if d == nil {
@@ -395,6 +404,7 @@ func writeState(aggs map[string]*agg, alerts []alert, now int64) {
 		}
 		d.Flows += a.Flows
 		d.UpBytes += a.Up
+		d.DownBytes += a.Down
 		if a.Category != "" {
 			d.Services = append(d.Services, a)
 			d.ByCat[a.Category] += a.Flows
@@ -402,6 +412,32 @@ func writeState(aggs map[string]*agg, alerts []alert, now int64) {
 		if a.Cloud != "" {
 			d.Clouds = append(d.Clouds, a)
 		}
+		// global app rollup (service name if classified, else the dst host)
+		an := a.Service
+		if an == "" {
+			an = a.Dst
+		}
+		if an != "" {
+			r := apps[an]
+			if r == nil {
+				r = &rollup{Name: an}
+				apps[an] = r
+			}
+			r.Bytes += a.Up + a.Down
+			r.Flows += a.Flows
+		}
+		// global protocol rollup
+		pn := a.Proto
+		if pn == "" {
+			pn = "unknown"
+		}
+		r := protos[pn]
+		if r == nil {
+			r = &rollup{Name: pn}
+			protos[pn] = r
+		}
+		r.Bytes += a.Up + a.Down
+		r.Flows += a.Flows
 	}
 	for _, al := range alerts {
 		if d := devs[al.Device]; d != nil {
@@ -421,11 +457,37 @@ func writeState(aggs map[string]*agg, alerts []alert, now int64) {
 		list = append(list, d)
 	}
 	sort.Slice(list, func(i, j int) bool { return list[i].UpBytes > list[j].UpBytes })
+
+	// rank the global rollups, cap at topN
+	rank := func(m map[string]*rollup) []*rollup {
+		out := make([]*rollup, 0, len(m))
+		for _, r := range m {
+			out = append(out, r)
+		}
+		sort.Slice(out, func(i, j int) bool { return out[i].Bytes > out[j].Bytes })
+		if len(out) > topN {
+			out = out[:topN]
+		}
+		return out
+	}
+	// active flows = the individual aggs (incl. uncategorised), top by total bytes
+	flows := make([]*agg, 0, len(aggs))
+	for _, a := range aggs {
+		flows = append(flows, a)
+	}
+	sort.Slice(flows, func(i, j int) bool { return (flows[i].Up + flows[i].Down) > (flows[j].Up + flows[j].Down) })
+	if len(flows) > 20 {
+		flows = flows[:20]
+	}
+
 	out := map[string]any{
-		"generated_at": now,
+		"generated_at":  now,
-		"devices":      list,
+		"devices":       list,
-		"alerts":       alerts,
+		"alerts":        alerts,
-		"alert_count":  len(alerts),
+		"alert_count":   len(alerts),
+		"top_apps":      rank(apps),
+		"top_protocols": rank(protos),
+		"active_flows":  flows,
 	}
 	writeJSON(statePath, out)
 }

packages/secubox-dpi/debian/changelog

@@ -1,3 +1,13 @@
+secubox-dpi (1.1.2-1~bookworm1) bookworm; urgency=low
+
+  * #695 dashboard: fill the four list cards (Top Applications, Top Protocols,
+    Bandwidth by Device, Active Flows) from the real R3 exfil engine instead of
+    the inactive netifyd backend. Collector now emits global rollups in /exfil
+    (top_apps, top_protocols by nDPI proto, per-device up+down, active_flows
+    incl. uncategorised dests).
+
+ -- Gerald KERMA <devel@cybermind.fr>  Mon, 22 Jun 2026 10:45:00 +0000
+
 secubox-dpi (1.1.1-1~bookworm1) bookworm; urgency=low
 
   * #692 collector: beaconing scenario now requires a C2-plausible cadence

packages/secubox-dpi/www/dpi/index.html

@@ -609,6 +609,31 @@
             foot.textContent = data.generated_at
                 ? `Last capture: ${agoStr(data.generated_at)} · ${devices.length} device(s) · ${alerts.length} alert(s)`
                 : (data.note || 'no capture window completed yet');
+
+            // #695 fill the list cards from the same /exfil rollups (real DPI engine)
+            const fillList = (id, rows, render, empty) => {
+                const el = document.getElementById(id); if (!el) return;
+                el.innerHTML = (rows && rows.length) ? rows.map(render).join('')
+                    : `<p style="color: var(--text-dim);">${empty}</p>`;
+            };
+            fillList('top-apps', data.top_apps, a => `
+                <div class="app-item"><span>${a.name}</span>
+                    <span class="bytes">${formatBytes(a.bytes)} · ${a.flows}f</span></div>`,
+                'No data');
+            fillList('top-protocols', data.top_protocols, p => `
+                <div class="app-item"><span>${p.name}</span>
+                    <span class="bytes">${formatBytes(p.bytes)} · ${p.flows}f</span></div>`,
+                'No data');
+            fillList('bandwidth-devices', devices, d => `
+                <div class="app-item"><span style="font-family:monospace;">📟 ${shortDev(d.device)}</span>
+                    <span class="bytes">↑${formatBytes(d.up_bytes)} ↓${formatBytes(d.down_bytes||0)}</span></div>`,
+                'No data');
+            fillList('active-flows', data.active_flows, f => {
+                const m = catMeta(f.category); const name = f.service || f.dst;
+                return `<div class="app-item"><span><span title="${f.category||'other'}">${m.icon}</span> ${name}
+                    <span style="color:var(--text-dim);">${f.proto||''}</span></span>
+                    <span class="bytes">↑${formatBytes(f.up_bytes)} ↓${formatBytes(f.down_bytes)}</span></div>`;
+            }, 'No active flows');
         }
 
         async function setupMirred() { const r = await api('/setup_mirred', 'POST'); alert(r.error ? 'Error: ' + r.error : 'Mirred configured'); loadStatus(); }
@@ -631,11 +656,7 @@
 
         function refreshAll() {
             loadStatus();
-            loadExfil();
+            loadExfil();   // #695: also fills Top Apps/Protocols/Bandwidth/Active-Flows from the exfil engine
-            loadTopApps();
-            loadTopProtocols();
-            loadBandwidthDevices();
-            loadActiveFlows();
             loadBlockRules();
         }
 

packages/secubox-toolbox-ng/cmd/sbxmitm/main.go

@@ -526,57 +526,58 @@ func (px *Proxy) mitmPipeline(tconn *tls.Conn, rawClient net.Conn, host, verdict
 		}
 	}
 
-	body, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
-	// Inject the transparency-banner loader only on 2xx text/html responses
-	// (mirrors the Python addon, which skips non-200). The loader's same-origin
-	// <script src="/__toolbox/loader.js"> is served by the short-circuit above.
-	//
-	// #662 — the body may be compressed in WHATEVER codec the origin chose
-	// (Accept-Encoding is now forwarded verbatim, not pinned to gzip).
-	// injectIntoBody decodes→injects→re-encodes for gzip / br / zstd (encoding
-	// unchanged), injects directly when identity, and fails open (untouched) on a
-	// corrupt/unknown encoding. Only on a successful rewrite do we update the
-	// framing: writeResponse emits Content-Length from len(body), but a stale
-	// resp.ContentLength / Content-Encoding could mislead downstream — so we
-	// keep them consistent with the bytes we actually serve.
-	if resp.StatusCode >= 200 && resp.StatusCode < 300 &&
-		strings.Contains(resp.Header.Get("Content-Type"), "text/html") {
-		// #662 CONSENTED-DEMONSTRATION — ONLY here, on the responses we actually
-		// inject into (2xx text/html, R3/wg gate), and ONLY when the operator
-		// left the demo on, do we relax the page's CSP so the inline banner can
-		// run even on strict-CSP sites. cspBypassed is true iff there was a real
-		// CSP to bypass — it becomes csp=1 on the inline script and the banner
-		// renders a 🔓 as the visible proof. We never strip CSP on non-injected
-		// responses.
-		cspBypassed := false
-		if px.cspDemo {
-			cspBypassed = relaxCSPForLoader(resp.Header)
-		}
-		// #662 — INLINE the banner (supersedes the <script src="/__toolbox/
-		// loader.js"> tag): sites with a SERVICE WORKER (leparisien, cnn…) hijack
-		// the same-origin src + its fetch("/__toolbox/bundle") before they reach
-		// this engine, so the banner never appeared. We fetch the COMPLETE script
-		// body from the portal server-side (mh/wg/csp + bundle baked as JS
-		// literals — no same-origin request for the SW to touch) and bake it into
-		// a self-contained <script>…</script>. Fail-open: a dead/slow portal →
-		// scriptBody=="" → the banner inject is skipped and the page is served
-		// intact (the cosmetic <style>, already inline, is unaffected).
-		scriptBody, _ := fetchInlineBanner(px.portal, clientHash, wg, cspBypassed)
-		if out, ok := injectIntoBody(body, resp.Header.Get("Content-Encoding"), scriptBody, wg); ok {
-			body = out
-			// Keep the response framing consistent with the served bytes. The
-			// encoding is unchanged (gzip stays gzip, identity stays identity);
-			// only the length changed because injection grew the body. A stale
-			// Content-Length would truncate/corrupt the response.
-			resp.Header.Set("Content-Length", strconv.Itoa(len(body)))
-			resp.ContentLength = int64(len(body))
-		}
-	}
 	// #662 — strip Alt-Svc so the browser is never told this origin offers HTTP/3
 	// (h3). With h3 unadvertised it keeps using HTTP/2 over TCP, which we MITM;
 	// otherwise it caches "h3 available" and keeps trying QUIC (UDP 443) — which
 	// bypasses this TCP proxy and is only best-effort blocked by the nft reject.
 	resp.Header.Del("Alt-Svc")
+
+	// We only ever rewrite 2xx text/html (the transparency-banner inject).
+	// EVERYTHING else — JSON/protobuf APIs, images, downloads, mail bodies,
+	// video — must pass through byte-for-byte. Buffering them capped at 8 MiB was
+	// silently TRUNCATING any larger response (#697: Gmail messages/attachments
+	// over the tunnel just stopped rendering). Stream those verbatim.
+	injectEligible := resp.StatusCode >= 200 && resp.StatusCode < 300 &&
+		strings.Contains(resp.Header.Get("Content-Type"), "text/html")
+	if !injectEligible {
+		streamResponse(tconn, resp, nil)
+		return
+	}
+
+	// Inject path: buffer up to the cap (+1 so we DETECT an oversized page instead
+	// of truncating it). The body may be compressed in whatever codec the origin
+	// chose (Accept-Encoding is forwarded verbatim). injectIntoBody
+	// decodes→injects→re-encodes for gzip/br/zstd (encoding unchanged), injects
+	// directly when identity, and fails open (untouched) on a corrupt/unknown
+	// encoding.
+	const injectCap = 8 << 20
+	body, _ := io.ReadAll(io.LimitReader(resp.Body, injectCap+1))
+	if int64(len(body)) > injectCap {
+		// HTML larger than the inject buffer: never serve a truncated inject —
+		// stream the peeked bytes plus the remainder verbatim.
+		streamResponse(tconn, resp, body)
+		return
+	}
+	// #662 CONSENTED-DEMONSTRATION — ONLY here, on the responses we actually inject
+	// into, and ONLY when the operator left the demo on, do we relax the page's CSP
+	// so the inline banner runs even on strict-CSP sites. Never on non-injected
+	// responses. cspBypassed becomes csp=1 on the inline script (banner shows 🔓).
+	cspBypassed := false
+	if px.cspDemo {
+		cspBypassed = relaxCSPForLoader(resp.Header)
+	}
+	// #662 — INLINE the banner (supersedes the <script src="/__toolbox/loader.js">
+	// tag): sites with a SERVICE WORKER hijack the same-origin src before it
+	// reaches this engine. We fetch the COMPLETE script body from the portal
+	// server-side and bake it into a self-contained <script>. Fail-open: a
+	// dead/slow portal → scriptBody=="" → inject skipped, page served intact.
+	scriptBody, _ := fetchInlineBanner(px.portal, clientHash, wg, cspBypassed)
+	if out, ok := injectIntoBody(body, resp.Header.Get("Content-Encoding"), scriptBody, wg); ok {
+		body = out
+		// Keep framing consistent with the served bytes (only the length changed).
+		resp.Header.Set("Content-Length", strconv.Itoa(len(body)))
+		resp.ContentLength = int64(len(body))
+	}
 	writeResponse(tconn, resp, body)
 }
 

packages/secubox-toolbox-ng/cmd/sbxmitm/util.go

@@ -40,6 +40,37 @@ func writeResponse(c io.Writer, resp *http.Response, body []byte) {
 	}
 }
 
+// streamResponse serializes an http.Response by writing its status + headers and
+// then STREAMING resp.Body to the conn (io.Copy) — it never buffers the whole
+// body in memory. Used for every response we do NOT rewrite (anything but
+// 2xx text/html): buffering+capping those was silently truncating large bodies
+// (#697 — Gmail messages/attachments/images over the tunnel just stopped
+// rendering). The upstream Content-Length is preserved when present (the bytes
+// are unchanged); otherwise Connection: close delimits the body. `prefix` holds
+// any bytes already consumed from resp.Body by a caller peek (streamed first).
+func streamResponse(c io.Writer, resp *http.Response, prefix []byte) {
+	status := resp.Status
+	if status == "" {
+		status = fmt.Sprintf("%d", resp.StatusCode)
+	}
+	fmt.Fprintf(c, "HTTP/1.1 %s\r\n", status)
+	for k, vals := range resp.Header {
+		switch http.CanonicalHeaderKey(k) {
+		case "Transfer-Encoding", "Connection":
+			continue // body is de-chunked by ReadResponse; we close the conn ourselves
+		}
+		for _, v := range vals {
+			fmt.Fprintf(c, "%s: %s\r\n", k, v)
+		}
+	}
+	fmt.Fprintf(c, "Connection: close\r\n")
+	io.WriteString(c, "\r\n")
+	if len(prefix) > 0 {
+		c.Write(prefix)
+	}
+	io.Copy(c, resp.Body)
+}
+
 // writeRaw writes a minimal HTTP/1.1 response onto a (TLS) conn.
 func writeRaw(c io.Writer, code int, status string, headers map[string]string, body []byte) {
 	if status == "" {