.claude/HISTORY.md

@@ -3,21 +3,6 @@
 
 ---
 
-## 2026-06-19 — #662 post-cutover restore: ad-block metrics + popup CSS (PR #673)
-
-- **Found by verification**: the cutover ported the 204-block but NOT ad_ghost's
-  metrics recording (frozen since 2026-06-18 18:59) nor its cosmetic/popup-hiding CSS
-  (popups returned — they're 1st-party DOM, never touched by host-204).
-- **Metrics**: Go aggregates blocks in-memory (per ad_host/site + per mac_hash), flushes
-  every 10s to a new portal `POST /__toolbox/ad-event` (unauth R3-perimeter, body-bounded,
-  never 500s) → SQLite store → #ads dashboard live again (total_blocked rising).
-- **Popups**: Go injects `<style id="sbx-ghost-style">` on R3 HTML (wg-gated, idempotent,
-  on the gzip path with the banner) — ports `_COSMETIC` + ad-specific popup tokens
-  (interstitial/ad-overlay/popup-ad/popunder/exit-intent), conservative (no bare
-  modal/popup/overlay, regression-tested). Verified live on the R3 path.
-- toolbox-ng 0.1.5 deployed (rolling restart) + portal api.py hot-deployed (drift closed
-  at next .deb build). Portal uvicorn boot ~14s.
-
 ## 2026-06-18 — #662 Phase 7: Python R3 engine DECOMMISSIONED + nft persistence
 
 - **nft persistence** (master `eea46326`): the boot re-apply source is the drop-in

packages/secubox-toolbox-ng/cmd/sbxmitm/adstats.go

@@ -1,207 +0,0 @@
-// SPDX-License-Identifier: LicenseRef-CMSD-1.0
-// Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
-//
-// SecuBox-Deb :: toolbox-ng :: ad-block metrics aggregator + flusher (#662)
-//
-// The #662 cutover moved the BLOCK decision (204 on ad/tracker hosts) into the
-// Go engine but left the METRICS unported: the Python ad_ghost addon tallied
-// every block into SQLite (record_ad_blocks / record_ad_client_blocks) feeding
-// the #ads dashboard, which has been frozen since the cutover (the Go engine
-// blocked but recorded nothing).
-//
-// This restores the tally IN the engine: an in-memory aggregator accumulates
-// per-(adHost,site) and per-(macHash,adHost) hit counts + estimated bytes on the
-// block hot path (O(1), lock-guarded for the concurrent workers), and a
-// background flusher POSTs the snapshot every 10s to the portal's
-// /__toolbox/ad-event ingest, which writes them to the SAME SQLite tables the
-// dashboard already reads. Best-effort throughout: a dead/slow portal never
-// blocks the block path and can never grow the maps unbounded.
-//
-// Pure standard library — no external modules.
-package main
-
-import (
-	"bytes"
-	"encoding/json"
-	"log"
-	"net/http"
-	"net/url"
-	"sync"
-	"time"
-)
-
-// refererSite ports the ad_ghost _site_of logic: parse the Referer header as a
-// URL, take its hostname, and return registrable(hostname). Empty Referer or a
-// parse failure → "" (the page that issued the blocked request is unknown).
-// Kept here next to the metrics wiring it feeds.
-func refererSite(referer string) string {
-	if referer == "" {
-		return ""
-	}
-	u, err := url.Parse(referer)
-	if err != nil {
-		return ""
-	}
-	return registrable(u.Hostname())
-}
-
-// adEstBytesPerBlock matches the existing dashboard convention: each blocked
-// ad/tracker request is credited a flat 45 KB saved (mirrors the Python
-// _EST_BYTES_PER_REQ = 45000 in ad_ghost.py). The #ads "bytes saved" figure
-// stays comparable across the engine cutover.
-const adEstBytesPerBlock = 45000
-
-// adFlushInterval is how often the flusher drains the maps to the portal.
-const adFlushInterval = 10 * time.Second
-
-// adMapCap bounds each aggregator map: once a map holds this many keys, NEW keys
-// are dropped (existing keys still accumulate) until the next flush clears it.
-// Guards against a dead portal letting the maps grow without bound.
-const adMapCap = 5000
-
-// adCounter is the accumulated (hits, bytes) for one aggregation key.
-type adCounter struct {
-	hits  int64
-	bytes int64
-}
-
-// adStats is the lock-guarded in-memory aggregator. blocks is keyed by
-// (adHost,site); clients by (macHash,adHost). The keys are small structs so the
-// maps stay allocation-light and comparable without string concatenation.
-type adStats struct {
-	mu      sync.Mutex
-	blocks  map[adKey]*adCounter
-	clients map[cliKey]*adCounter
-}
-
-type adKey struct{ adHost, site string }
-type cliKey struct{ macHash, adHost string }
-
-func newAdStats() *adStats {
-	return &adStats{
-		blocks:  map[adKey]*adCounter{},
-		clients: map[cliKey]*adCounter{},
-	}
-}
-
-// recordAdBlock tallies one blocked ad/tracker request. It increments the
-// (adHost,site) global counter by 1 hit + adEstBytesPerBlock, and — only when
-// macHash is non-empty — the (macHash,adHost) per-client counter likewise. An
-// empty adHost is ignored (nothing to attribute). O(1), never blocks: the only
-// contention is the short mutex held across two map ops. The size cap drops only
-// NEW keys (existing keys keep accumulating), so a dead portal can't grow memory
-// unbounded.
-func (a *adStats) recordAdBlock(adHost, site, macHash string) {
-	if adHost == "" {
-		return
-	}
-	a.mu.Lock()
-	defer a.mu.Unlock()
-
-	bk := adKey{adHost: adHost, site: site}
-	if c := a.blocks[bk]; c != nil {
-		c.hits++
-		c.bytes += adEstBytesPerBlock
-	} else if len(a.blocks) < adMapCap {
-		a.blocks[bk] = &adCounter{hits: 1, bytes: adEstBytesPerBlock}
-	}
-
-	if macHash != "" {
-		ck := cliKey{macHash: macHash, adHost: adHost}
-		if c := a.clients[ck]; c != nil {
-			c.hits++
-			c.bytes += adEstBytesPerBlock
-		} else if len(a.clients) < adMapCap {
-			a.clients[ck] = &adCounter{hits: 1, bytes: adEstBytesPerBlock}
-		}
-	}
-}
-
-// ── wire payload (mirrors the portal /__toolbox/ad-event JSON contract) ──────
-
-type adBlockRow struct {
-	AdHost string `json:"ad_host"`
-	Site   string `json:"site"`
-	Hits   int64  `json:"hits"`
-	Bytes  int64  `json:"bytes"`
-}
-
-type adClientRow struct {
-	MacHash string `json:"mac_hash"`
-	AdHost  string `json:"ad_host"`
-	Hits    int64  `json:"hits"`
-	Bytes   int64  `json:"bytes"`
-}
-
-type adEventPayload struct {
-	Blocks  []adBlockRow  `json:"blocks"`
-	Clients []adClientRow `json:"clients"`
-}
-
-// snapshot atomically reads-and-clears both maps, returning the accumulated rows.
-// Clearing under the lock means a concurrent recordAdBlock either lands fully in
-// this snapshot or fully in the next one — never split. Returns empty slices (not
-// nil) only when there was nothing to flush; the caller checks emptiness.
-func (a *adStats) snapshot() adEventPayload {
-	a.mu.Lock()
-	defer a.mu.Unlock()
-	var p adEventPayload
-	for k, c := range a.blocks {
-		p.Blocks = append(p.Blocks, adBlockRow{AdHost: k.adHost, Site: k.site, Hits: c.hits, Bytes: c.bytes})
-	}
-	for k, c := range a.clients {
-		p.Clients = append(p.Clients, adClientRow{MacHash: k.macHash, AdHost: k.adHost, Hits: c.hits, Bytes: c.bytes})
-	}
-	// Clear by re-allocating: cheaper than ranged delete and drops the cap.
-	a.blocks = map[adKey]*adCounter{}
-	a.clients = map[cliKey]*adCounter{}
-	return p
-}
-
-// empty reports whether a payload carries no rows (nothing to POST).
-func (p adEventPayload) empty() bool { return len(p.Blocks) == 0 && len(p.Clients) == 0 }
-
-// adEventClient is a short-timeout fire-and-forget client for the ad-event POST.
-// Sibling of portalClient (banner.go): the portal is a fixed loopback base, so
-// we never follow redirects (SSRF hygiene) and keep the timeout tight so a slow
-// portal can't stall the flusher goroutine.
-var adEventClient = &http.Client{
-	Timeout:       5 * time.Second,
-	CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
-}
-
-// flushOnce snapshots the maps and, if non-empty, POSTs them to the portal's
-// /__toolbox/ad-event ingest. Best-effort: any error (marshal, portal down,
-// non-2xx) is swallowed with at most a debug log — the metrics are stats, not
-// security, and the engine must never block on the portal. Exposed (returns the
-// flushed payload) so the test can assert the snapshot/clear + payload shape.
-func (a *adStats) flushOnce(portal string) adEventPayload {
-	p := a.snapshot()
-	if p.empty() {
-		return p
-	}
-	buf, err := json.Marshal(p)
-	if err != nil {
-		log.Printf("ad-event marshal failed: %v", err)
-		return p
-	}
-	url := portalTargetURL(portal, "/__toolbox/ad-event")
-	resp, err := adEventClient.Post(url, "application/json", bytes.NewReader(buf))
-	if err != nil {
-		log.Printf("ad-event post failed for %s: %v", url, err)
-		return p
-	}
-	resp.Body.Close()
-	return p
-}
-
-// runAdStatsFlusher is the background flusher goroutine: every adFlushInterval it
-// drains the aggregator to the portal. Start it once from main() (like the
-// engine's other startup goroutines). It runs forever (the process lifetime).
-func (a *adStats) runAdStatsFlusher(portal string) {
-	t := time.NewTicker(adFlushInterval)
-	defer t.Stop()
-	for range t.C {
-		a.flushOnce(portal)
-	}
-}

packages/secubox-toolbox-ng/cmd/sbxmitm/adstats_test.go

@@ -1,194 +0,0 @@
-// SPDX-License-Identifier: LicenseRef-CMSD-1.0
-// Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
-//
-// SecuBox-Deb :: toolbox-ng :: ad-block metrics aggregator tests (#662)
-package main
-
-import (
-	"encoding/json"
-	"io"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-)
-
-func TestRecordAdBlockAggregatesGlobal(t *testing.T) {
-	a := newAdStats()
-	a.recordAdBlock("ads.example.com", "news.example", "")
-	a.recordAdBlock("ads.example.com", "news.example", "")
-	a.recordAdBlock("ads.example.com", "blog.example", "")
-
-	if got := len(a.blocks); got != 2 {
-		t.Fatalf("want 2 (adHost,site) keys, got %d", got)
-	}
-	c := a.blocks[adKey{"ads.example.com", "news.example"}]
-	if c == nil || c.hits != 2 || c.bytes != 2*adEstBytesPerBlock {
-		t.Fatalf("news.example counter wrong: %+v", c)
-	}
-	c = a.blocks[adKey{"ads.example.com", "blog.example"}]
-	if c == nil || c.hits != 1 || c.bytes != adEstBytesPerBlock {
-		t.Fatalf("blog.example counter wrong: %+v", c)
-	}
-}
-
-func TestRecordAdBlockEmptyHostIgnored(t *testing.T) {
-	a := newAdStats()
-	a.recordAdBlock("", "site", "mac")
-	if len(a.blocks) != 0 || len(a.clients) != 0 {
-		t.Fatalf("empty adHost must be ignored: blocks=%d clients=%d", len(a.blocks), len(a.clients))
-	}
-}
-
-func TestRecordAdBlockPerClientOnlyWhenMacSet(t *testing.T) {
-	a := newAdStats()
-	a.recordAdBlock("ads.example.com", "site", "")    // no mac → no client row
-	a.recordAdBlock("ads.example.com", "site", "mac1") // mac → client row
-	a.recordAdBlock("ads.example.com", "site", "mac1")
-
-	if len(a.clients) != 1 {
-		t.Fatalf("want 1 client key, got %d", len(a.clients))
-	}
-	c := a.clients[cliKey{"mac1", "ads.example.com"}]
-	if c == nil || c.hits != 2 || c.bytes != 2*adEstBytesPerBlock {
-		t.Fatalf("client counter wrong: %+v", c)
-	}
-	// Global counter accumulated all three hits regardless of mac.
-	if g := a.blocks[adKey{"ads.example.com", "site"}]; g == nil || g.hits != 3 {
-		t.Fatalf("global counter should be 3: %+v", g)
-	}
-}
-
-func TestSnapshotClearsMaps(t *testing.T) {
-	a := newAdStats()
-	a.recordAdBlock("ads.example.com", "site", "mac1")
-	p := a.snapshot()
-	if len(p.Blocks) != 1 || len(p.Clients) != 1 {
-		t.Fatalf("snapshot payload wrong: %d blocks %d clients", len(p.Blocks), len(p.Clients))
-	}
-	if len(a.blocks) != 0 || len(a.clients) != 0 {
-		t.Fatalf("snapshot must clear the maps: blocks=%d clients=%d", len(a.blocks), len(a.clients))
-	}
-	// A second snapshot with nothing recorded is empty.
-	if p2 := a.snapshot(); !p2.empty() {
-		t.Fatalf("empty snapshot must report empty: %+v", p2)
-	}
-}
-
-func TestSizeCapDropsNewKeysGracefully(t *testing.T) {
-	a := newAdStats()
-	// Fill past the cap with distinct keys.
-	for i := 0; i < adMapCap+50; i++ {
-		host := "ads" + itoa(i) + ".example.com"
-		a.recordAdBlock(host, "site", "mac"+itoa(i))
-	}
-	if len(a.blocks) > adMapCap {
-		t.Fatalf("blocks map exceeded cap: %d > %d", len(a.blocks), adMapCap)
-	}
-	if len(a.clients) > adMapCap {
-		t.Fatalf("clients map exceeded cap: %d > %d", len(a.clients), adMapCap)
-	}
-	// An EXISTING key still accumulates even when the map is at cap.
-	existing := "ads0.example.com"
-	before := a.blocks[adKey{existing, "site"}].hits
-	a.recordAdBlock(existing, "site", "mac0")
-	after := a.blocks[adKey{existing, "site"}].hits
-	if after != before+1 {
-		t.Fatalf("existing key must keep accumulating at cap: %d -> %d", before, after)
-	}
-}
-
-func TestFlushOncePayloadShapeMatchesContract(t *testing.T) {
-	a := newAdStats()
-	a.recordAdBlock("ads.example.com", "news.example", "mac1")
-
-	var got adEventPayload
-	var ct string
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		ct = r.Header.Get("Content-Type")
-		body, _ := io.ReadAll(r.Body)
-		_ = json.Unmarshal(body, &got)
-		w.WriteHeader(http.StatusNoContent)
-	}))
-	defer srv.Close()
-
-	a.flushOnce(srv.URL)
-
-	if ct != "application/json" {
-		t.Fatalf("Content-Type = %q, want application/json", ct)
-	}
-	if len(got.Blocks) != 1 {
-		t.Fatalf("blocks payload: %+v", got.Blocks)
-	}
-	b := got.Blocks[0]
-	if b.AdHost != "ads.example.com" || b.Site != "news.example" || b.Hits != 1 || b.Bytes != adEstBytesPerBlock {
-		t.Fatalf("block row shape wrong: %+v", b)
-	}
-	if len(got.Clients) != 1 {
-		t.Fatalf("clients payload: %+v", got.Clients)
-	}
-	c := got.Clients[0]
-	if c.MacHash != "mac1" || c.AdHost != "ads.example.com" || c.Hits != 1 || c.Bytes != adEstBytesPerBlock {
-		t.Fatalf("client row shape wrong: %+v", c)
-	}
-
-	// flushOnce cleared the maps.
-	if len(a.blocks) != 0 || len(a.clients) != 0 {
-		t.Fatalf("flushOnce must clear maps")
-	}
-}
-
-func TestFlushOnceEmptySkipsPost(t *testing.T) {
-	a := newAdStats()
-	posted := false
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		posted = true
-		w.WriteHeader(http.StatusNoContent)
-	}))
-	defer srv.Close()
-	a.flushOnce(srv.URL)
-	if posted {
-		t.Fatalf("flushOnce on empty aggregator must not POST")
-	}
-}
-
-func TestFlushOnceSwallowsPortalError(t *testing.T) {
-	a := newAdStats()
-	a.recordAdBlock("ads.example.com", "site", "")
-	// Unreachable portal → must not panic, must still clear the maps (snapshot
-	// happens before the POST).
-	a.flushOnce("http://127.0.0.1:1")
-	if len(a.blocks) != 0 {
-		t.Fatalf("flushOnce must clear maps even on POST failure")
-	}
-}
-
-func TestRefererSite(t *testing.T) {
-	cases := []struct{ ref, want string }{
-		{"", ""},
-		{"https://news.example.com/article/123", "example.com"},
-		{"https://www.bbc.co.uk/news", "bbc.co.uk"},
-		{"http://blog.example.com:8443/x", "example.com"},
-		{"not a url ::::", ""},
-	}
-	for _, c := range cases {
-		if got := refererSite(c.ref); got != c.want {
-			t.Errorf("refererSite(%q) = %q, want %q", c.ref, got, c.want)
-		}
-	}
-}
-
-// itoa is a tiny stdlib-free int→string for the cap test keys (avoids importing
-// strconv just for the test).
-func itoa(n int) string {
-	if n == 0 {
-		return "0"
-	}
-	var b [20]byte
-	i := len(b)
-	for n > 0 {
-		i--
-		b[i] = byte('0' + n%10)
-		n /= 10
-	}
-	return string(b[i:])
-}

packages/secubox-toolbox-ng/cmd/sbxmitm/cosmetic.go

@@ -1,131 +0,0 @@
-// SPDX-License-Identifier: LicenseRef-CMSD-1.0
-// Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
-//
-// SecuBox-Deb :: toolbox-ng :: cosmetic / popup-ad hiding CSS inject (#662)
-//
-// PORTS the ad_ghost cosmetic-hide <style> (../secubox-toolbox/mitmproxy_addons/
-// ad_ghost.py, _COSMETIC groups: ads / consent_nag / newsletter / social_widgets)
-// into the Go engine, which the #662 cutover left unported — so the engine was
-// injecting only the transparency banner loader, NOT the ad/popup-hiding style.
-// The result: newsletter / interstitial / subscribe / consent popups reappeared
-// for R3 (wg) clients.
-//
-// It also EXPANDS the popup coverage ("améliorer le blocage des pubs popup")
-// with clearly-ad-related popup / interstitial / overlay token patterns.
-//
-// CONSERVATISM (deliberate, load-bearing): we ONLY hide selectors whose token is
-// ad/popup-SPECIFIC (e.g. "popup-ad", "ad-overlay", "interstitial", "popunder",
-// "exit-intent"). We DO NOT add bare generic tokens like [class*="modal"],
-// [class*="popup"], [class*="overlay"], or [class*="lightbox"] — those routinely
-// match legitimate first-party UI (login modals, image lightboxes, nav overlays)
-// and hiding them would break the page. A false-negative ad here is far cheaper
-// than a broken site, and host-blocking (204) still saves the bandwidth.
-//
-// Pure standard library — no external modules.
-package main
-
-import "bytes"
-
-// cosmeticGuard is the id on the injected <style>. It makes injectCosmetic
-// idempotent (skip if already present) and mirrors the Python addon's _MARK
-// ("sbx-ghost-style"), so a page already styled by the Python engine path is not
-// double-injected either.
-const cosmeticGuard = "sbx-ghost-style"
-
-// cosmeticStyle is the single <style> the engine injects for R3 clients. The
-// selector list PORTS the four ad_ghost _COSMETIC groups verbatim (ads /
-// consent_nag / newsletter / social_widgets) and EXPANDS the popup/interstitial
-// coverage. Every selector targets an ad/popup-SPECIFIC token only (see the
-// CONSERVATISM note above). The rule mirrors the Python _style_for:
-// display:none + visibility:hidden, both !important, collapsing the slot.
-const cosmeticStyle = `<style id="sbx-ghost-style">` +
-	// ── ads (ported from _COSMETIC["ads"]) ──────────────────────────────────
-	`[id^="google_ads"],` +
-	`[id^="div-gpt-ad"],` +
-	`ins.adsbygoogle,` +
-	`iframe[src*="doubleclick"],` +
-	`iframe[src*="googlesyndication"],` +
-	`iframe[src*="amazon-adsystem"],` +
-	`[class*="ad-banner"],` +
-	`[class*="advert"],` +
-	`[id*="banner-ad"],` +
-	`[id*="ad-container"],` +
-	`[class*="-ads"],` +
-	`[class*="sponsored"],` +
-	`aside[aria-label*="publicit"],` +
-	// ── consent_nag (ported from _COSMETIC["consent_nag"]) ───────────────────
-	`#onetrust-banner-sdk,` +
-	`#onetrust-consent-sdk,` +
-	`#didomi-host,` +
-	`.qc-cmp2-container,` +
-	`[id^="sp_message_container"],` +
-	`[id*="cookie-consent"],` +
-	`[class*="cookie-banner"],` +
-	`[class*="cookie-notice"],` +
-	`[aria-label*="cookie"],` +
-	`.cmpbox,` +
-	// ── newsletter (ported from _COSMETIC["newsletter"]) ─────────────────────
-	`[class*="newsletter-popup"],` +
-	`[class*="signup-modal"],` +
-	`[id*="newsletter-modal"],` +
-	`[class*="subscribe-overlay"],` +
-	// ── social_widgets (ported from _COSMETIC["social_widgets"]) ─────────────
-	`.fb-like,` +
-	`.twitter-share-button,` +
-	`[class*="social-share"],` +
-	`iframe[src*="facebook.com/plugins"],` +
-	`iframe[src*="platform.twitter"],` +
-	// ── EXPANDED popup / interstitial / overlay (ad-SPECIFIC tokens only) ────
-	`[class*="interstitial"],` +
-	`[id*="interstitial"],` +
-	`[class*="ad-overlay"],` +
-	`[class*="ad-modal"],` +
-	`[class*="modal-ad"],` +
-	`[class*="popup-ad"],` +
-	`[id*="popup-ad"],` +
-	`[class*="popunder"],` +
-	`[class*="exit-intent"],` +
-	`[class*="-paywall-ad"]` +
-	`{display:none!important;visibility:hidden!important;}</style>`
-
-// injectCosmetic inserts cosmeticStyle into an HTML body once. Placement mirrors
-// injectLoader (and the Python addon, which prefers </head>):
-//   - idempotency: if the body already contains cosmeticGuard → unchanged.
-//   - insert right BEFORE the first (case-insensitive) "</head>".
-//   - else insert right AFTER the first "<head ...>"'s closing '>'.
-//   - else insert right BEFORE the first "<body".
-//   - else return the body unchanged (no inject).
-func injectCosmetic(body []byte) []byte {
-	if bytes.Contains(body, []byte(cosmeticGuard)) {
-		return body
-	}
-	style := []byte(cosmeticStyle)
-	low := bytes.ToLower(body)
-
-	// Prefer right before </head> (the Python _RE_HEAD.sub anchor).
-	if i := bytes.Index(low, []byte("</head>")); i >= 0 {
-		return spliceAt(body, style, i)
-	}
-	// Else right after the first <head ...>'s closing '>'.
-	if h := bytes.Index(low, []byte("<head")); h >= 0 {
-		if j := bytes.IndexByte(body[h:], '>'); j >= 0 {
-			return spliceAt(body, style, h+j+1)
-		}
-	}
-	// Else right before <body.
-	if b := bytes.Index(low, []byte("<body")); b >= 0 {
-		return spliceAt(body, style, b)
-	}
-	return body
-}
-
-// spliceAt returns body with ins inserted at byte offset at. Shared by the
-// cosmetic placement logic (and available to the loader path) so the two inject
-// helpers compose the same insertion semantics.
-func spliceAt(body, ins []byte, at int) []byte {
-	out := make([]byte, 0, len(body)+len(ins))
-	out = append(out, body[:at]...)
-	out = append(out, ins...)
-	out = append(out, body[at:]...)
-	return out
-}

packages/secubox-toolbox-ng/cmd/sbxmitm/cosmetic_test.go

@@ -1,177 +0,0 @@
-// SPDX-License-Identifier: LicenseRef-CMSD-1.0
-// Copyright (c) 2026 CyberMind — Gérald Kerma <devel@cybermind.fr>
-//
-// SecuBox-Deb :: toolbox-ng :: cosmetic / popup-ad hiding CSS inject tests (#662)
-package main
-
-import (
-	"strings"
-	"testing"
-)
-
-// representativeSelectors covers each ported group + an EXPANDED popup token,
-// asserting the cosmeticStyle carries the full surface (not a truncated port).
-var representativeSelectors = []string{
-	`ins.adsbygoogle`,             // ads
-	`[class*="ad-banner"]`,        // ads
-	`#onetrust-banner-sdk`,        // consent_nag
-	`[class*="cookie-banner"]`,    // consent_nag
-	`[class*="newsletter-popup"]`, // newsletter
-	`.fb-like`,                    // social_widgets
-	`[class*="interstitial"]`,     // EXPANDED popup
-	`[class*="popup-ad"]`,         // EXPANDED popup
-	`[class*="popunder"]`,         // EXPANDED popup
-	`[class*="exit-intent"]`,      // EXPANDED popup
-}
-
-// forbiddenSelectors are the bare generic tokens we DELIBERATELY do not hide
-// (they break legitimate first-party UI). Their absence is load-bearing.
-var forbiddenSelectors = []string{
-	`[class*="modal"]`,
-	`[class*="popup"]`,
-	`[class*="overlay"]`,
-	`[class*="lightbox"]`,
-}
-
-func TestCosmeticStyleHasRepresentativeSelectors(t *testing.T) {
-	for _, sel := range representativeSelectors {
-		if !strings.Contains(cosmeticStyle, sel) {
-			t.Errorf("cosmeticStyle missing representative selector %q", sel)
-		}
-	}
-	if !strings.Contains(cosmeticStyle, `display:none!important`) {
-		t.Errorf("cosmeticStyle missing display:none!important rule")
-	}
-	if !strings.Contains(cosmeticStyle, cosmeticGuard) {
-		t.Errorf("cosmeticStyle missing guard id %q", cosmeticGuard)
-	}
-}
-
-func TestCosmeticStyleNoGenericTokens(t *testing.T) {
-	// Conservatism guard: bare generic popup/modal/overlay/lightbox tokens must
-	// never appear (they would hide legitimate UI).
-	for _, sel := range forbiddenSelectors {
-		if strings.Contains(cosmeticStyle, sel) {
-			t.Errorf("cosmeticStyle contains forbidden generic selector %q", sel)
-		}
-	}
-}
-
-func TestInjectCosmeticIdempotent(t *testing.T) {
-	body := []byte(`<html><head><style id="sbx-ghost-style">x</style></head><body>hi</body></html>`)
-	out := injectCosmetic(body)
-	if string(out) != string(body) {
-		t.Fatalf("guarded body must be unchanged.\n got: %s", out)
-	}
-	// Double-inject from clean must also be a no-op the second time.
-	clean := []byte(`<html><head></head><body>hi</body></html>`)
-	once := injectCosmetic(clean)
-	twice := injectCosmetic(once)
-	if string(once) != string(twice) {
-		t.Fatalf("second injectCosmetic must be a no-op.\n once:  %s\n twice: %s", once, twice)
-	}
-	if strings.Count(string(twice), cosmeticGuard) != 1 {
-		t.Fatalf("guard must appear exactly once after double inject: %s", twice)
-	}
-}
-
-func TestInjectCosmeticBeforeHeadClose(t *testing.T) {
-	body := []byte(`<html><head><title>x</title></head><body>hi</body></html>`)
-	out := string(injectCosmetic(body))
-	// The <style> must land right BEFORE </head>.
-	if !strings.Contains(out, `</style></head>`) {
-		t.Fatalf("cosmetic style not placed before </head>: %s", out)
-	}
-	if !strings.Contains(out, `<title>x</title><style id="sbx-ghost-style">`) {
-		t.Fatalf("original head content displaced: %s", out)
-	}
-}
-
-func TestInjectCosmeticAfterHeadOpenNoClose(t *testing.T) {
-	// <head ...> present, no </head> → insert right after the open tag's '>'.
-	body := []byte(`<html><head lang="en"><body>hi`)
-	out := string(injectCosmetic(body))
-	if !strings.Contains(out, `<head lang="en"><style id="sbx-ghost-style">`) {
-		t.Fatalf("cosmetic style not placed after <head>'s '>': %s", out)
-	}
-}
-
-func TestInjectCosmeticBodyFallback(t *testing.T) {
-	body := []byte(`<html><body class="x">hi</body></html>`)
-	out := string(injectCosmetic(body))
-	if !strings.Contains(out, `<style id="sbx-ghost-style">`) {
-		t.Fatalf("cosmetic style not injected: %s", out)
-	}
-	// Inserted right before <body.
-	i := strings.Index(out, `<style id="sbx-ghost-style">`)
-	j := strings.Index(out, `<body class="x">`)
-	if i < 0 || j < 0 || i > j {
-		t.Fatalf("cosmetic style not placed before <body>: %s", out)
-	}
-}
-
-func TestInjectCosmeticNoHeadNoBody(t *testing.T) {
-	body := []byte(`<p>just a fragment</p>`)
-	out := injectCosmetic(body)
-	if string(out) != string(body) {
-		t.Fatalf("no head/body → must be unchanged.\n got: %s", out)
-	}
-}
-
-func TestInjectCosmeticCaseInsensitive(t *testing.T) {
-	body := []byte(`<HTML><HEAD></HEAD><BODY>hi</BODY></HTML>`)
-	out := string(injectCosmetic(body))
-	if !strings.Contains(out, `<style id="sbx-ghost-style">`) {
-		t.Fatalf("case-insensitive </HEAD> match failed: %s", out)
-	}
-	// Must be before </HEAD> (case-insensitive anchor).
-	i := strings.Index(out, `<style id="sbx-ghost-style">`)
-	j := strings.Index(strings.ToLower(out), `</head>`)
-	if i < 0 || j < 0 || i > j {
-		t.Fatalf("cosmetic style not placed before </HEAD>: %s", out)
-	}
-}
-
-func TestInjectLoaderAndCosmeticCompose(t *testing.T) {
-	// Both markers must be present after composing the two injects (wg client).
-	body := []byte(`<html><head></head><body>hi</body></html>`)
-	out := string(injectHTML(body, "deadbeef", true))
-	if !strings.Contains(out, bannerGuard) {
-		t.Fatalf("loader marker missing after compose: %s", out)
-	}
-	if !strings.Contains(out, cosmeticGuard) {
-		t.Fatalf("cosmetic marker missing after compose: %s", out)
-	}
-	if !strings.Contains(out, `data-mh="deadbeef"`) {
-		t.Fatalf("loader data-mh missing after compose: %s", out)
-	}
-}
-
-func TestInjectHTMLNonWGSkipsCosmetic(t *testing.T) {
-	// Non-WG (non-R3) clients get the loader but NOT the cosmetic style.
-	body := []byte(`<html><head></head><body>hi</body></html>`)
-	out := string(injectHTML(body, "x", false))
-	if !strings.Contains(out, bannerGuard) {
-		t.Fatalf("loader marker missing for non-wg: %s", out)
-	}
-	if strings.Contains(out, cosmeticGuard) {
-		t.Fatalf("cosmetic style must NOT be injected for non-wg client: %s", out)
-	}
-}
-
-func TestInjectIntoBodyGzipCarriesCosmetic(t *testing.T) {
-	// The gzip decompress→inject→recompress path must carry BOTH injects for wg.
-	body := []byte(`<html><head></head><body>hi</body></html>`)
-	gz := gzipBytes(body)
-	out, ok := injectIntoBody(gz, "gzip", "mh1", true)
-	if !ok {
-		t.Fatalf("injectIntoBody(gzip) returned ok=false")
-	}
-	plain, err := gunzipBytes(out)
-	if err != nil {
-		t.Fatalf("re-gzip output not gunzippable: %v", err)
-	}
-	if !strings.Contains(string(plain), bannerGuard) || !strings.Contains(string(plain), cosmeticGuard) {
-		t.Fatalf("gzip path lost a marker: %s", plain)
-	}
-}

packages/secubox-toolbox-ng/cmd/sbxmitm/gzip.go

@@ -75,50 +75,34 @@ func gzipBytes(in []byte) []byte {
 	return buf.Bytes()
 }
 
-// injectHTML applies BOTH HTML transforms in one pass over the DECOMPRESSED
-// body: the transparency-banner loader (always) AND, for R3 (wg) clients, the
-// ad/popup-hiding cosmetic <style> (#662 — the cutover left this unported). Both
-// are idempotent (own guard markers) and order-independent; running them in the
-// same decompressed step means the cosmetic style benefits from the gzip
-// handling exactly like the loader. The cosmetic style is gated to wg because it
-// is an R3-tunnel opt-in behaviour (mirrors the Python addon's _is_r3plus gate).
-func injectHTML(plain []byte, clientHash string, wg bool) []byte {
-	out := injectLoader(plain, clientHash, wg)
-	if wg {
-		out = injectCosmetic(out)
-	}
-	return out
-}
-
-// injectIntoBody runs the HTML injection (loader + R3 cosmetic style) over a
-// (possibly gzip-compressed) HTML body, returning the new body bytes to serve
-// and whether the body was rewritten.
+// injectIntoBody runs the transparency-banner injection over a (possibly
+// gzip-compressed) HTML body, returning the new body bytes to serve and whether
+// the body was rewritten.
 //
-//   - encoding == "" (identity): injectHTML runs directly on body; the result
+//   - encoding == "" (identity): injectLoader runs directly on body; the result
 //     is returned (ok=true). The caller MUST update Content-Length to len(out).
 //   - encoding == "gzip" (case-insensitive): the body is gunzipped, injected,
 //     then RE-gzipped so the client transfer stays compressed (the tunnel is
 //     perf-sensitive). The caller keeps Content-Encoding: gzip and sets
-//     Content-Length to len(out). BOTH the loader and the cosmetic style are
-//     injected on this decompressed body, so the cosmetic CSS lands on
-//     gzip-compressed pages too (the common case).
+//     Content-Length to len(out).
 //   - any other encoding (br/zstd/deflate — should not occur after the upstream
 //     Accept-Encoding pin, but be safe): pass through untouched, ok=false.
 //
 // Fail-open: if gunzip fails (corrupt / not-actually-gzip / bomb), the ORIGINAL
 // bytes are returned with ok=false so the page is never broken.
 //
-// idempotency / placement live entirely inside injectLoader / injectCosmetic.
+// idempotency / placement live entirely inside injectLoader (unchanged).
 func injectIntoBody(body []byte, encoding, clientHash string, wg bool) (out []byte, ok bool) {
 	switch strings.ToLower(strings.TrimSpace(encoding)) {
 	case "":
-		return injectHTML(body, clientHash, wg), true
+		return injectLoader(body, clientHash, wg), true
 	case "gzip":
 		plain, err := gunzipBytes(body)
 		if err != nil {
 			return body, false // fail open: serve the original compressed bytes
 		}
-		return gzipBytes(injectHTML(plain, clientHash, wg)), true
+		injected := injectLoader(plain, clientHash, wg)
+		return gzipBytes(injected), true
 	default:
 		return body, false // unknown encoding we cannot decode → pass through
 	}

packages/secubox-toolbox-ng/cmd/sbxmitm/main.go

@@ -204,16 +204,6 @@ type Proxy struct {
 	jarKey []byte       // anti-track HMAC fake-identity seed (nil → poison off)
 	poison bool         // master gate: poison tracker Set-Cookies (default on when jarKey present)
 	portal string       // portal base URL for /__toolbox/* reverse-proxy (banner assets)
-	ads    *adStats     // #662 — ad-block metrics aggregator (flushed to the portal)
-}
-
-// recordAdBlock forwards a 204'd ad/tracker block to the engine's metrics
-// aggregator (#662). Nil-safe so the CONNECT PoC (no aggregator) and tests can
-// run the block path without one. Non-blocking (the aggregator is O(1)).
-func (px *Proxy) recordAdBlock(adHost, site, macHash string) {
-	if px.ads != nil {
-		px.ads.recordAdBlock(adHost, site, macHash)
-	}
 }
 
 func (px *Proxy) serverTLSConfig() *tls.Config {
@@ -320,12 +310,6 @@ func (px *Proxy) mitmPipeline(tconn *tls.Conn, rawClient net.Conn, host, verdict
 	}
 
 	if verdict == "block" {
-		// #662 — tally the block BEFORE writing the 204 so the #ads dashboard
-		// (frozen since the cutover) sees it again. site = registrable(Referer)
-		// (the ad_ghost _site_of flavour); empty when there is no Referer. The
-		// per-client breakdown keys on the WG persona hash. recordAdBlock is
-		// O(1) and never blocks the block path.
-		px.recordAdBlock(host, refererSite(req.Header.Get("Referer")), clientHashFromConn(rawClient))
 		writeRaw(tconn, 204, "No Content", map[string]string{"X-SecuBox-Ng": "blocked"}, nil)
 		return
 	}
@@ -473,12 +457,7 @@ func main() {
 		jarKey: jarKey,
 		poison: *poison,
 		portal: *portal,
-		ads:    newAdStats(),
 	}
-	// #662 — start the ad-block metrics flusher: the block path tallies every
-	// 204 into px.ads, drained every 10s to the portal's /__toolbox/ad-event
-	// (best-effort, fire-and-forget) so the #ads dashboard sees blocks again.
-	go px.ads.runAdStatsFlusher(*portal)
 	if *transparent {
 		// Transparent R3 mode: raw accept loop, each conn carries its pre-DNAT
 		// destination via SO_ORIGINAL_DST (recovered in handleTransparent). The

packages/secubox-toolbox-ng/debian/changelog

@@ -1,14 +1,3 @@
-secubox-toolbox-ng (0.1.5-1~bookworm1) bookworm; urgency=medium
-
-  * ad-stats: record ad-block metrics again (frozen since the #662 cutover) —
-    aggregate blocks in-memory, flush every 10s to the portal /__toolbox/ad-event
-    which writes the SQLite store feeding the #ads dashboard. (ref #662)
-  * cosmetic: inject the ad/popup-hiding <style> (ported from ad_ghost _COSMETIC,
-    expanded with ad-specific interstitial/overlay/popunder/exit-intent tokens) on
-    R3 HTML, restoring popup-ad hiding lost at cutover. (ref #662)
-
- -- Gerald KERMA <devel@cybermind.fr>  Thu, 19 Jun 2026 08:00:00 +0000
-
 secubox-toolbox-ng (0.1.4-1~bookworm1) bookworm; urgency=medium
 
   * proxy: do NOT follow upstream redirects — relay 3xx to the client so the

packages/secubox-toolbox/secubox_toolbox/api.py

@@ -73,67 +73,6 @@ async def toolbox_bundle(mh: str = Query(default=""), wg: int = Query(default=0)
         headers={"Cache-Control": "no-store"},
     )
 
-
-# #662 — ad-block metrics ingest from the Go MITM engine (sbxmitm). The #662
-# cutover moved the BLOCK decision (204 on ad/tracker hosts) into the Go engine
-# but left the METRICS unported, so the #ads dashboard froze. The engine now
-# aggregates blocks in-memory and POSTs a snapshot here every ~10s; we persist it
-# to the SAME SQLite tables (ad_block_stats / ad_block_client_host) the dashboard
-# reads via store.ad_stats().
-#
-# UNAUTHENTICATED, by design — exactly like /__toolbox/loader.js + /__toolbox/
-# bundle above: the engine reaches the portal only over the R3 nft perimeter
-# (loopback / WG ingress), which is the trust boundary. No JWT is required for
-# these engine↔portal banner/metrics channels.
-_AD_EVENT_ROW_CAP = 2000  # bound each list so a misbehaving engine can't flood us
-
-
-@router.post("/__toolbox/ad-event")
-async def toolbox_ad_event(request: Request) -> Response:
-    """Ingest a batch of ad-block tallies from the Go engine. Best-effort: never
-    500s the engine (it is fire-and-forget) — always returns 204. See the trust
-    note above for why this is unauthenticated."""
-    try:
-        # Body-size guard: this is the only unauthenticated POST-with-body on the
-        # R3 perimeter, and the portal serves the whole board — bound the body
-        # BEFORE parsing so a misbehaving/compromised WG peer can't pressure
-        # portal memory. The legit payload (≤5000 keys × 2 maps) is well under 2 MB.
-        try:
-            clen = int(request.headers.get("content-length") or 0)
-        except (TypeError, ValueError):
-            clen = 0
-        if clen > 2 * 1024 * 1024:
-            return Response(status_code=204)
-        body = await request.json()
-        if not isinstance(body, dict):
-            return Response(status_code=204)
-        blocks = body.get("blocks") or []
-        clients = body.get("clients") or []
-        if not isinstance(blocks, list):
-            blocks = []
-        if not isinstance(clients, list):
-            clients = []
-        blocks = blocks[:_AD_EVENT_ROW_CAP]
-        clients = clients[:_AD_EVENT_ROW_CAP]
-
-        block_rows = [
-            (b["ad_host"], b.get("site", ""), "block", int(b.get("hits", 0)), int(b.get("bytes", 0)))
-            for b in blocks
-            if isinstance(b, dict) and b.get("ad_host")
-        ]
-        client_rows = [
-            (c["mac_hash"], c["ad_host"], int(c.get("hits", 0)), int(c.get("bytes", 0)))
-            for c in clients
-            if isinstance(c, dict) and c.get("mac_hash") and c.get("ad_host")
-        ]
-        if block_rows:
-            store.record_ad_blocks(block_rows)
-        if client_rows:
-            store.record_ad_client_blocks(client_rows)
-    except Exception as e:  # never raise into the engine's fire-and-forget POST
-        log.debug("ad-event ingest failed: %s", e)
-    return Response(status_code=204)
-
 # Cap geo/UA enrichment on /admin/clients/rich to the rows the UI actually shows
 # (top-5 + headroom). Beyond this, clients get bare fields — avoids ~51 cached
 # geo lookups per poll (ref #644).