diff --git a/benchmarks/aggregator-head-lag.yml b/benchmarks/aggregator-head-lag.yml index 0fd51924..6499a73e 100644 --- a/benchmarks/aggregator-head-lag.yml +++ b/benchmarks/aggregator-head-lag.yml @@ -17,9 +17,9 @@ seo_intro: | data provider. how many seconds elapse between a swap landing on chain and the same event appearing on the provider's feed. Marketing pages quote "real-time" without a number; this page quotes the - number. Mobula's WebSocket, Codex's GraphQL feed, GeckoTerminal's - REST endpoint and GMGN's Solana-only WebSocket are watched from - three regions (us-east, eu-west, sgp) against canonical-tip archive + number. Mobula's WebSocket, Codex's GraphQL feed and GeckoTerminal's + REST endpoint are watched from three regions (us-east, eu-west, sgp) + against canonical-tip archive nodes on Base, BNB Chain and Solana. Because each chain has its own base block cadence (Solana's ~400 ms slots vs Base/BNB's 2-3 s blocks), the headline winner shifts by chain. On Solana the leader @@ -42,13 +42,13 @@ abstract: | same event. methodology: - - "Aggregators measured: Mobula, Codex, GeckoTerminal, GMGN." - - "Chains: Base, BNB Chain, Solana. GMGN measured Solana-only — its WebSocket pushes per-pair routing events for Solana but only chain-level summaries for EVM chains." + - "Aggregators measured: Mobula, Codex, GeckoTerminal." + - "Chains: Base, BNB Chain, Solana." - "Regions: us-east, eu-west, sgp. Cross-region median reported in the headline." - - "Reference: archive nodes per chain, validated against block hashes. For GMGN the slot-level `route_info` stream (`gk=\"s\"`) is the canonical observation timestamp." + - "Reference: archive nodes per chain, validated against block hashes." - "Metric: gauge `head_lag_seconds`, sampled every 15 seconds. Aggregated over the 24-hour window using `quantile_over_time`." - "Success rate: presence ratio. Share of expected sampling slots where a value was actually emitted (5,760 expected per provider per day at the 15s cadence). 100% means the aggregator's feed was reachable for the full window." - - "Cardinality: 4 aggregators × up to 3 chains × 3 regions, minus inapplicable chains for chain-restricted providers." + - "Cardinality: 3 aggregators × 3 chains × 3 regions = 27 active series." findings: - "Across the unfiltered cross-chain aggregate the leader is {{best_name}} at {{best_p50}} (p50, 24 h) over {{count}} measured providers, but that headline number mixes chains with very different baseline cadences (Solana 400 ms slots vs Base/BNB 2-3 s blocks) so the per-chain breakdown below is what to read for an integration decision." @@ -171,32 +171,3 @@ providers: p50: quantile_over_time(0.50, head_lag_seconds{aggregator="geckoterminal", region="sgp"}[24h]) * 1000 series: avg_over_time(head_lag_seconds{aggregator="geckoterminal", region="sgp"}[1h]) * 1000 - # GMGN is Solana-only. We deliberately leave `chain=` out of the selectors - # so the per-chain dimension filter from lib/spec.ts handles visibility: - # - "All chains" → no chain= injected → GMGN's solana samples returned. - # - "Solana" → chain="solana" injected → returns solana samples. - # - "Base" / "BNB Chain" → chain="base"/"bnb" injected → empty result - # → tryLoadLive skips the provider entirely (see spec.ts ~L477) so - # GMGN is hidden on those tabs instead of showing zeros. - - slug: gmgn - name: GMGN - tag: WebSocket feed (Solana only) - formula: "Median seconds between a Solana slot landing on-chain and GMGN's WebSocket chain-tip stream (chain_stat, gk='s', rs) reflecting it. GMGN doesn't push per-pair events for arbitrary Solana pools, so we measure at the slot level." - queries: - p50: quantile_over_time(0.50, head_lag_seconds{aggregator="gmgn"}[24h]) * 1000 - p90: quantile_over_time(0.90, head_lag_seconds{aggregator="gmgn"}[24h]) * 1000 - p99: quantile_over_time(0.99, head_lag_seconds{aggregator="gmgn"}[24h]) * 1000 - mean: avg_over_time(head_lag_seconds{aggregator="gmgn"}[24h]) * 1000 - success: clamp_max(count_over_time(head_lag_seconds{aggregator="gmgn"}[24h]) / 5760, 1) - sample_size: sum(count_over_time(head_lag_seconds{aggregator="gmgn"}[24h])) - series: avg_over_time(head_lag_seconds{aggregator="gmgn"}[1h]) * 1000 - regions: - - region: us-east - p50: quantile_over_time(0.50, head_lag_seconds{aggregator="gmgn", region="us-east"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="gmgn", region="us-east"}[1h]) * 1000 - - region: eu-west - p50: quantile_over_time(0.50, head_lag_seconds{aggregator="gmgn", region="eu-west"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="gmgn", region="eu-west"}[1h]) * 1000 - - region: ap-southeast - p50: quantile_over_time(0.50, head_lag_seconds{aggregator="gmgn", region="sgp"}[24h]) * 1000 - series: avg_over_time(head_lag_seconds{aggregator="gmgn", region="sgp"}[1h]) * 1000 diff --git a/harnesses/aggregator-head-lag/Dockerfile b/harnesses/aggregator-head-lag/Dockerfile index 5992cf50..74ca8f0f 100644 --- a/harnesses/aggregator-head-lag/Dockerfile +++ b/harnesses/aggregator-head-lag/Dockerfile @@ -21,25 +21,11 @@ FROM debian:bookworm-slim WORKDIR /app -# Install runtime dependencies. -# -# chromium + minimal X / font libs are required for chromedp, which the -# GMGN.ai head-lag monitor uses to mint Cloudflare cf_clearance cookies -# before dialing wss://gmgn.ai/ws. When GMGN_ENABLED != true the binary -# skips that monitor and Chromium stays idle — the only cost is image -# size (~200MB extra). +# Install runtime dependencies RUN apt-get update && apt-get install -y \ ca-certificates \ - chromium \ - fonts-liberation \ - libnss3 \ - libgbm1 \ - libasound2 \ && rm -rf /var/lib/apt/lists/* -# Point chromedp at the chromium binary debian ships at /usr/bin/chromium. -ENV CHROME_PATH=/usr/bin/chromium - # Copy binary from builder COPY --from=builder /app/monitor /app/monitor diff --git a/harnesses/aggregator-head-lag/cmd/script/config.go b/harnesses/aggregator-head-lag/cmd/script/config.go index 066ecf1c..a3fd5114 100644 --- a/harnesses/aggregator-head-lag/cmd/script/config.go +++ b/harnesses/aggregator-head-lag/cmd/script/config.go @@ -13,7 +13,6 @@ type Config struct { DefinedSessionCookie string MonitorRegion string // Deployment region: us-west, us-east, singapore, etc. MobulaWSURL string // Mobula fast-trade WebSocket endpoint (allows staging to use EU-specific cluster) - GMGNEnabled bool // GMGN.ai monitor enabled (Solana per-pair head-lag). Requires Chrome installed for chromedp. } func loadEnv() (*Config, error) { @@ -25,7 +24,6 @@ func loadEnv() (*Config, error) { config.DefinedSessionCookie = strings.TrimSpace(os.Getenv("DEFINED_SESSION_COOKIE")) config.MonitorRegion = strings.TrimSpace(os.Getenv("MONITOR_REGION")) config.MobulaWSURL = strings.TrimSpace(os.Getenv("MOBULA_WS_URL")) - config.GMGNEnabled = strings.EqualFold(strings.TrimSpace(os.Getenv("GMGN_ENABLED")), "true") // Default to "unknown" if not set if config.MonitorRegion == "" { diff --git a/harnesses/aggregator-head-lag/cmd/script/gmgn_monitor.go b/harnesses/aggregator-head-lag/cmd/script/gmgn_monitor.go deleted file mode 100644 index b2f54faf..00000000 --- a/harnesses/aggregator-head-lag/cmd/script/gmgn_monitor.go +++ /dev/null @@ -1,649 +0,0 @@ -package main - -// GMGN.ai head-lag monitor — Solana only. -// -// Why Solana only: GMGN's WebSocket pushes per-pair (gk-tagged) `route_info` -// events ONLY for Solana. On Base/BSC/ETH the same `chain_stat` channel -// returns chain-level gas + coin_price but no per-pool real-time stream. -// Confirmed via devtools sniffing of gmgn.ai's own UI: their EVM chain -// pages use REST polling, not WebSocket per-pair pushes. -// -// Pipeline: -// 1. chromedp opens Chrome, navigates to https://gmgn.ai/?chain=sol, -// lets Cloudflare clear, harvests cookies (cf_clearance + __cf_bm) -// AND the live wss://gmgn.ai/ws?... URL the page would open. -// 2. A background goroutine refreshes the session every 25 min -// (cf_clearance TTL is 30-45 min). -// 3. gorilla/websocket dials the captured WS URL via getProxyDialer() -// so HTTP_PROXY/HTTPS_PROXY routes the WS through the same -// residential proxy as Codex / GeckoTerminal. -// 4. We subscribe `chain_stat` on chain="sol" and filter incoming -// `route_info` events where gk == bench Solana pool address. -// 5. Each event embeds `::` in d.p / d.a / d.b. -// We use unix_ms (GMGN's pipeline observation timestamp, ms-precision) -// as the anchor and compute head_lag = receivedAt - unix_ms. This is -// what the bench measures for the other providers too — wall-clock -// gap between the canonical observation moment and WebSocket receipt. -// -// Heartbeat: GMGN expects `{"action":"heartbeat","client_ts":}` -// ~every 30s; we send every 25s. - -import ( - "context" - "encoding/json" - "fmt" - "io" - "log" - "math/rand" - "net" - "net/http" - "net/url" - "os" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/chromedp/cdproto/fetch" - "github.com/chromedp/cdproto/network" - "github.com/chromedp/chromedp" - "github.com/gorilla/websocket" - utls "github.com/refraction-networking/utls" -) - -// GMGN monitors the same Solana pool as the rest of the head-lag bench so -// the leaderboard stays apples-to-apples. Address picked from headLagPools -// (head_lag_monitor.go) at runtime. -const gmgnChain = "sol" - -// gmgnSession is the live cookie + WS URL bundle. Refreshed every 25 min; -// readers see a coherent snapshot via the atomic pointer. -type gmgnSession struct { - WSURL string - CookieHeader string - UserAgent string - MintedAt time.Time -} - -var gmgnSessionPtr atomic.Pointer[gmgnSession] - -func runGMGNHeadLagMonitor(config *Config, stopChan <-chan struct{}, wg *sync.WaitGroup) { - defer wg.Done() - - // Print unconditionally so /logs proves the goroutine actually fired. - fmt.Println("[HEAD-LAG][GMGN] goroutine entered") - - if !config.GMGNEnabled { - fmt.Println("[HEAD-LAG][GMGN] disabled (GMGN_ENABLED != true) — set GMGN_ENABLED=true to enable") - return - } - fmt.Println("[HEAD-LAG][GMGN] starting Solana per-pair monitor…") - - var solPool HeadLagPool - for _, p := range headLagPools { - if p.ChainName == "solana" { - solPool = p - break - } - } - if solPool.Address == "" { - log.Println("[HEAD-LAG][GMGN] no Solana pool in headLagPools, aborting") - return - } - fmt.Printf("[HEAD-LAG][GMGN] target pool: %s (%s) region=%s\n", - solPool.Name, solPool.Address, config.MonitorRegion) - - // Initial cookie mint. Don't return on failure — the reconnect loop will - // keep retrying and the refresher goroutine will keep trying every 25 min. - fmt.Printf("[HEAD-LAG][GMGN] minting initial cf_clearance via chromedp (chrome at %s)…\n", - os.Getenv("CHROME_PATH")) - if err := gmgnRefreshSession(stopChan); err != nil { - fmt.Printf("[HEAD-LAG][GMGN] initial cookie mint failed: %v — will retry\n", err) - } else { - fmt.Printf("[HEAD-LAG][GMGN] initial cookie mint OK\n") - } - - // Periodic cookie refresh. - refreshWG := &sync.WaitGroup{} - refreshWG.Add(1) - go func() { - defer refreshWG.Done() - gmgnSessionRefresher(stopChan) - }() - - // Reconnect loop. - reconnect := 5 * time.Second - const reconnectMax = 60 * time.Second - failures := 0 - for { - select { - case <-stopChan: - refreshWG.Wait() - return - default: - } - - err := gmgnConnectAndConsume(config, solPool, stopChan) - if err != nil { - failures++ - fmt.Printf("[HEAD-LAG][GMGN] connection error: %v — reconnect in %v (failures=%d)\n", - err, reconnect, failures) - select { - case <-stopChan: - refreshWG.Wait() - return - case <-time.After(reconnect): - if reconnect < reconnectMax { - reconnect *= 2 - if reconnect > reconnectMax { - reconnect = reconnectMax - } - } - } - // After 5 consecutive failures, force a fresh cookie mint — - // the WS may be rejecting our stale cf_clearance / IP combo. - if failures%5 == 0 { - fmt.Printf("[HEAD-LAG][GMGN] %d consecutive failures, force cookie refresh\n", failures) - if refreshErr := gmgnRefreshSession(stopChan); refreshErr != nil { - fmt.Printf("[HEAD-LAG][GMGN] force cookie refresh failed: %v\n", refreshErr) - } - } - continue - } - reconnect = 5 * time.Second - failures = 0 - } -} - -func gmgnSessionRefresher(stopChan <-chan struct{}) { - ticker := time.NewTicker(25 * time.Minute) - defer ticker.Stop() - for { - select { - case <-stopChan: - return - case <-ticker.C: - if err := gmgnRefreshSession(stopChan); err != nil { - fmt.Printf("[HEAD-LAG][GMGN] periodic cookie refresh failed: %v\n", err) - } - } - } -} - -// gmgnRefreshSession runs chromedp once, harvests cookies + WS URL, and -// atomically swaps the global session pointer. -func gmgnRefreshSession(stopChan <-chan struct{}) error { - ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) - defer cancel() - go func() { - select { - case <-stopChan: - cancel() - case <-ctx.Done(): - } - }() - - opts := append(chromedp.DefaultExecAllocatorOptions[:], - chromedp.Flag("disable-blink-features", "AutomationControlled"), - chromedp.WindowSize(1280, 800), - chromedp.UserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"), - ) - // chromium rejects --proxy-server URLs that embed user:pass with - // ERR_NO_SUPPORTED_PROXIES, so we strip credentials and re-inject - // them via the CDP Fetch.authRequired handler below. - proxyURL, proxyUser, proxyPass := gmgnProxyParts() - if proxyURL != "" { - opts = append(opts, chromedp.ProxyServer(proxyURL)) - fmt.Printf("[HEAD-LAG][GMGN] using proxy %s (auth=%v)\n", proxyURL, proxyUser != "") - } else { - // chromium silently falls back to $HTTP_PROXY when --proxy-server is - // absent, which would re-use the rotating proxy's embedded creds and - // fail with ERR_INVALID_AUTH_CREDENTIALS. Force a direct connection. - opts = append(opts, chromedp.ProxyServer("direct://")) - fmt.Println("[HEAD-LAG][GMGN] no proxy (direct connection, GMGN_PROXY=none)") - } - allocCtx, cancelAlloc := chromedp.NewExecAllocator(ctx, opts...) - defer cancelAlloc() - - browserCtx, cancelBrowser := chromedp.NewContext(allocCtx) - defer cancelBrowser() - - var ( - wsMu sync.Mutex - wsURLs []string - ) - chromedp.ListenTarget(browserCtx, func(ev interface{}) { - if e, ok := ev.(*network.EventWebSocketCreated); ok { - wsMu.Lock() - wsURLs = append(wsURLs, e.URL) - wsMu.Unlock() - return - } - if e, ok := ev.(*fetch.EventAuthRequired); ok && proxyUser != "" { - go func() { - _ = chromedp.Run(browserCtx, - fetch.ContinueWithAuth(e.RequestID, &fetch.AuthChallengeResponse{ - Response: fetch.AuthChallengeResponseResponseProvideCredentials, - Username: proxyUser, - Password: proxyPass, - }), - ) - }() - return - } - if e, ok := ev.(*fetch.EventRequestPaused); ok { - go func() { - _ = chromedp.Run(browserCtx, fetch.ContinueRequest(e.RequestID)) - }() - } - }) - - var cookies []*network.Cookie - var ua string - runSteps := []chromedp.Action{} - if proxyUser != "" { - // Enable Fetch with auth handling so EventAuthRequired fires for - // the 407 challenge our rotating residential proxy issues. - runSteps = append(runSteps, fetch.Enable().WithHandleAuthRequests(true)) - } - runSteps = append(runSteps, - network.Enable(), - chromedp.Navigate("https://gmgn.ai/?chain=sol"), - chromedp.WaitVisible("body", chromedp.ByQuery), - chromedp.Sleep(6*time.Second), - chromedp.Evaluate(`navigator.userAgent`, &ua), - chromedp.ActionFunc(func(ctx context.Context) error { - cs, err := network.GetCookies().Do(ctx) - if err != nil { - return err - } - cookies = cs - return nil - }), - ) - err := chromedp.Run(browserCtx, runSteps...) - if err != nil { - return fmt.Errorf("chromedp: %w", err) - } - - var parts []string - for _, c := range cookies { - if !strings.HasSuffix(c.Domain, "gmgn.ai") { - continue - } - parts = append(parts, c.Name+"="+c.Value) - } - if len(parts) == 0 { - return fmt.Errorf("no gmgn.ai cookies harvested") - } - - wsMu.Lock() - var wsURL string - for _, u := range wsURLs { - if strings.HasPrefix(u, "wss://gmgn.ai/ws") { - wsURL = u - break - } - } - wsMu.Unlock() - if wsURL == "" { - return fmt.Errorf("page did not open the expected wss://gmgn.ai/ws URL") - } - - gmgnSessionPtr.Store(&gmgnSession{ - WSURL: wsURL, - CookieHeader: strings.Join(parts, "; "), - UserAgent: ua, - MintedAt: time.Now().UTC(), - }) - fmt.Printf("[HEAD-LAG][GMGN] cookie minted (%d cookies, ws=%s)\n", len(parts), wsURL) - return nil -} - -// gmgnDialer returns a websocket dialer that respects the same proxy -// choice as the chromedp mint (via GMGN_PROXY override). When the mint -// uses no proxy, the dial must also use no proxy so Cloudflare sees the -// same source IP that the cf_clearance cookie is bound to. -// -// Cloudflare on gmgn.ai 403s connections whose TLS ClientHello doesn't -// match a real browser (JA3 fingerprint), so we hand the TLS handshake -// off to utls with HelloChrome_120. EnableCompression matches Chrome's -// permessage-deflate WS extension advertisement. -func gmgnDialer() *websocket.Dialer { - dialer := &websocket.Dialer{ - NetDialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { - d := &net.Dialer{Timeout: 30 * time.Second, KeepAlive: -1} - return d.DialContext(ctx, network, addr) - }, - NetDialTLSContext: gmgnTLSDial, - HandshakeTimeout: 30 * time.Second, - EnableCompression: true, - } - cleanURL, user, pass := gmgnProxyParts() - if cleanURL == "" { - return dialer - } - u, err := url.Parse(cleanURL) - if err != nil { - return dialer - } - if user != "" { - u.User = url.UserPassword(user, pass) - } - dialer.Proxy = http.ProxyURL(u) - return dialer -} - -// gmgnTLSDial dials TCP then performs the TLS handshake using utls with -// a Chrome 120 ClientHello so the JA3 fingerprint Cloudflare sees matches -// a real browser. Without this, gmgn.ai 403s the WS upgrade despite the -// chromedp-minted cf_clearance cookie being valid. -// -// The Chrome 120 preset advertises ALPN [h2, http/1.1]. WS upgrade is -// HTTP/1.1-only — if the server picks h2 we read back a SETTINGS frame -// and gorilla errors with "malformed HTTP response". Strip h2 from the -// ALPN extension while keeping every other byte of the Chrome ClientHello -// intact (JA3 ignores ALPN payload so the fingerprint stays Chrome). -func gmgnTLSDial(ctx context.Context, network, addr string) (net.Conn, error) { - host, _, err := net.SplitHostPort(addr) - if err != nil { - return nil, err - } - d := &net.Dialer{Timeout: 30 * time.Second, KeepAlive: -1} - rawConn, err := d.DialContext(ctx, network, addr) - if err != nil { - return nil, err - } - uConn := utls.UClient(rawConn, &utls.Config{ServerName: host}, utls.HelloCustom) - spec, err := utls.UTLSIdToSpec(utls.HelloChrome_120) - if err != nil { - _ = rawConn.Close() - return nil, fmt.Errorf("utls spec: %w", err) - } - for _, ext := range spec.Extensions { - if alpn, ok := ext.(*utls.ALPNExtension); ok { - alpn.AlpnProtocols = []string{"http/1.1"} - } - } - if err := uConn.ApplyPreset(&spec); err != nil { - _ = rawConn.Close() - return nil, fmt.Errorf("utls apply preset: %w", err) - } - if err := uConn.HandshakeContext(ctx); err != nil { - _ = rawConn.Close() - return nil, fmt.Errorf("utls handshake: %w", err) - } - return uConn, nil -} - -// gmgnProxyParts returns the proxy URL stripped of any user:pass -// credentials, plus the username/password to inject later via the CDP -// Fetch.authRequired handler. chromium's --proxy-server flag rejects -// embedded credentials with ERR_NO_SUPPORTED_PROXIES. -// -// Cloudflare on gmgn.ai binds cf_clearance to the IP that minted it. -// Webshare's rotating endpoint assigns a fresh IP per TCP connection, -// so the cookie minted by chromedp is rejected on the subsequent -// gorilla WS dial (403 bad handshake). GMGN_PROXY lets us override: -// - unset → use HTTP_PROXY / HTTPS_PROXY (existing behavior) -// - "none" → skip proxy entirely (mint + dial from Railway IP) -// - "" → use this URL (e.g. a sticky-session Webshare endpoint) -func gmgnProxyParts() (cleanURL, user, pass string) { - override := strings.TrimSpace(os.Getenv("GMGN_PROXY")) - if strings.EqualFold(override, "none") { - return "", "", "" - } - raw := override - if raw == "" { - raw = strings.TrimSpace(os.Getenv("HTTP_PROXY")) - } - if raw == "" { - raw = strings.TrimSpace(os.Getenv("HTTPS_PROXY")) - } - if raw == "" { - return "", "", "" - } - u, err := url.Parse(raw) - if err != nil { - return raw, "", "" - } - if u.User != nil { - user = u.User.Username() - pass, _ = u.User.Password() - u.User = nil - } - return u.String(), user, pass -} - -func gmgnConnectAndConsume(config *Config, pool HeadLagPool, stopChan <-chan struct{}) error { - sess := gmgnSessionPtr.Load() - if sess == nil { - return fmt.Errorf("no session yet (initial mint pending)") - } - - headers := http.Header{} - headers.Set("Origin", "https://gmgn.ai") - headers.Set("User-Agent", sess.UserAgent) - headers.Set("Cache-Control", "no-cache") - headers.Set("Pragma", "no-cache") - headers.Set("Accept-Language", "en-US,en;q=0.9") - headers.Set("Accept", "*/*") - headers.Set("Sec-Fetch-Dest", "websocket") - headers.Set("Sec-Fetch-Mode", "websocket") - headers.Set("Sec-Fetch-Site", "same-origin") - headers.Set("Cookie", sess.CookieHeader) - - dialer := gmgnDialer() - conn, resp, err := dialer.Dial(sess.WSURL, headers) - if err != nil { - status := 0 - var body, cfRay, cfMitigated, server string - if resp != nil { - status = resp.StatusCode - cfRay = resp.Header.Get("Cf-Ray") - cfMitigated = resp.Header.Get("Cf-Mitigated") - server = resp.Header.Get("Server") - b, _ := io.ReadAll(io.LimitReader(resp.Body, 1024)) - _ = resp.Body.Close() - body = strings.TrimSpace(strings.ReplaceAll(string(b), "\n", " ")) - } - fmt.Printf("[HEAD-LAG][GMGN] dial 403 details: server=%q cf-ray=%q cf-mitigated=%q body=%.200q\n", - server, cfRay, cfMitigated, body) - return fmt.Errorf("dial failed (http=%d): %w", status, err) - } - defer conn.Close() - fmt.Printf("[HEAD-LAG][GMGN] WS connected, subscribing chain_stat for chain=%s\n", gmgnChain) - - subBody, _ := json.Marshal(map[string]any{ - "action": "subscribe", - "channel": "chain_stat", - "f": "w", - "id": gmgnRandHex(16), - "data": []map[string]string{{"chain": gmgnChain}}, - }) - if err := conn.WriteMessage(websocket.TextMessage, subBody); err != nil { - return fmt.Errorf("subscribe write: %w", err) - } - - hbDone := make(chan struct{}) - defer close(hbDone) - var writeMu sync.Mutex - go func() { - t := time.NewTicker(25 * time.Second) - defer t.Stop() - for { - select { - case <-hbDone: - return - case <-t.C: - hb, _ := json.Marshal(map[string]any{ - "action": "heartbeat", - "client_ts": time.Now().UnixMilli(), - }) - writeMu.Lock() - _ = conn.SetWriteDeadline(time.Now().Add(5 * time.Second)) - _ = conn.WriteMessage(websocket.TextMessage, hb) - writeMu.Unlock() - } - } - }() - - conn.SetPingHandler(func(appData string) error { - writeMu.Lock() - defer writeMu.Unlock() - return conn.WriteControl(websocket.PongMessage, []byte(appData), - time.Now().Add(time.Second)) - }) - - var msgCount int - var lastStatLog time.Time - channelCounts := map[string]int{} - for { - select { - case <-stopChan: - return nil - default: - } - _ = conn.SetReadDeadline(time.Now().Add(90 * time.Second)) - mt, data, err := conn.ReadMessage() - if err != nil { - return fmt.Errorf("read: %w", err) - } - if mt != websocket.TextMessage { - continue - } - receivedAt := time.Now().UTC() - msgCount++ - - if string(data) == "ping" { - writeMu.Lock() - _ = conn.WriteMessage(websocket.TextMessage, []byte("pong")) - writeMu.Unlock() - continue - } - - // Log the first 10 frames raw so we can see what's actually coming - // over the wire (channel, t, gk fields). Truncated to 300 chars. - if msgCount <= 10 { - preview := string(data) - if len(preview) > 300 { - preview = preview[:300] + "...(truncated)" - } - fmt.Printf("[HEAD-LAG][GMGN] sample msg #%d: %s\n", msgCount, preview) - } - - // Per-channel summary every 30s so we can see the traffic mix - // without spamming the log. - var env gmgnFrame - if json.Unmarshal(data, &env) == nil { - channelCounts[env.Channel]++ - } - if time.Since(lastStatLog) > 30*time.Second { - fmt.Printf("[HEAD-LAG][GMGN] frames=%d channels=%v target_pool=%s\n", - msgCount, channelCounts, pool.Address) - lastStatLog = time.Now() - } - - gmgnHandleFrame(data, receivedAt, pool, config.MonitorRegion) - } -} - -type gmgnFrame struct { - Channel string `json:"channel"` - Data []json.RawMessage `json:"data"` -} - -type gmgnRouteInfo struct { - T string `json:"t"` - TS int64 `json:"ts"` - C string `json:"c"` - GK string `json:"gk"` - D map[string]json.RawMessage `json:"d"` -} - -func gmgnHandleFrame(data []byte, receivedAt time.Time, pool HeadLagPool, region string) { - var env gmgnFrame - if err := json.Unmarshal(data, &env); err != nil { - return - } - if env.Channel != "chain_stat" { - return - } - for _, raw := range env.Data { - var ri gmgnRouteInfo - if err := json.Unmarshal(raw, &ri); err != nil { - continue - } - // GMGN does NOT push per-pair route_info for arbitrary Solana - // pools — verified by 90s isolated sniff: only gk="s" (slot - // updates) and one preset pool (58oQ...) ever appear. Track - // gk="s" so the metric reflects GMGN's view of the Solana tip - // (head-lag at the slot level), which is the apples-to-apples - // comparable to other aggregators' chain-tip tracking. - if ri.T != "route_info" || ri.C != gmgnChain || ri.GK != "s" { - continue - } - obsMs, slot, ok := gmgnExtractAnchor(ri.D) - if !ok { - continue - } - gmgnEmitLag(pool, obsMs, slot, receivedAt, region) - } -} - -// gmgnExtractAnchor returns (unix_ms, slot) from the first parseable -// sub-field. For gk="s" frames: `rs` = `:`, `sc` = `::`. -// `rbh` looks like `:::` so its parts[1] isn't numeric -// and falls through naturally (ParseUint fails, loop continues). -func gmgnExtractAnchor(d map[string]json.RawMessage) (int64, uint64, bool) { - for _, k := range []string{"rs", "sc", "p", "a", "b", "rbh"} { - raw, ok := d[k] - if !ok { - continue - } - var s string - if err := json.Unmarshal(raw, &s); err != nil { - continue - } - parts := strings.SplitN(s, ":", 3) - if len(parts) < 2 { - continue - } - obs, err := strconv.ParseInt(parts[0], 10, 64) - if err != nil { - continue - } - slot, err := strconv.ParseUint(parts[1], 10, 64) - if err != nil { - continue - } - return obs, slot, true - } - return 0, 0, false -} - -var gmgnEmitCount atomic.Int64 -var gmgnLastEmitLog atomic.Int64 - -func gmgnEmitLag(pool HeadLagPool, obsMs int64, slot uint64, receivedAt time.Time, region string) { - obsTime := time.UnixMilli(obsMs).UTC() - lagSeconds := receivedAt.Sub(obsTime).Seconds() - RecordHeadLag("gmgn", pool.ChainName, int64(slot), lagSeconds, region, "") - n := gmgnEmitCount.Add(1) - now := time.Now().Unix() - if now-gmgnLastEmitLog.Load() >= 10 { - gmgnLastEmitLog.Store(now) - fmt.Printf("[HEAD-LAG][GMGN] emit #%d: slot=%d lag=%.3fs chain=%s region=%s\n", - n, slot, lagSeconds, pool.ChainName, region) - } -} - -func gmgnRandHex(n int) string { - const hex = "0123456789abcdef" - b := make([]byte, n) - for i := range b { - b[i] = hex[rand.Intn(16)] - } - return string(b) -} diff --git a/harnesses/aggregator-head-lag/cmd/script/main.go b/harnesses/aggregator-head-lag/cmd/script/main.go index 1c12769e..84444379 100644 --- a/harnesses/aggregator-head-lag/cmd/script/main.go +++ b/harnesses/aggregator-head-lag/cmd/script/main.go @@ -30,11 +30,6 @@ func main() { } fmt.Println("Metrics will be exposed on :2112/metrics for Prometheus") - // GMGN integration status — printed unconditionally at startup so we can - // confirm via /logs whether the binary we deployed actually contains the - // GMGN monitor. - fmt.Printf("[GMGN-BUILD] integration v1 present | GMGN_ENABLED=%v | HTTP_PROXY=%v\n", - config.GMGNEnabled, os.Getenv("HTTP_PROXY") != "" || os.Getenv("HTTPS_PROXY") != "") fmt.Println() sigChan := make(chan os.Signal, 1) @@ -69,11 +64,6 @@ func main() { runMobulaFastTradeMonitor(config, stopChan) }() - // GMGN.ai head-lag monitor (Solana only — gated by GMGN_ENABLED env). - fmt.Println("[GMGN-BUILD] launching GMGN goroutine…") - wg.Add(1) - go runGMGNHeadLagMonitor(config, stopChan, &wg) - <-sigChan fmt.Println("\n\nShutting down monitors...") close(stopChan) diff --git a/harnesses/aggregator-head-lag/go.mod b/harnesses/aggregator-head-lag/go.mod index f7d3a0d1..ed17b543 100644 --- a/harnesses/aggregator-head-lag/go.mod +++ b/harnesses/aggregator-head-lag/go.mod @@ -7,11 +7,9 @@ require ( github.com/chromedp/chromedp v0.14.2 github.com/gorilla/websocket v1.5.3 github.com/prometheus/client_golang v1.23.2 - github.com/refraction-networking/utls v1.8.2 ) require ( - github.com/andybalholm/brotli v1.0.6 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chromedp/sysutil v1.1.0 // indirect @@ -19,14 +17,12 @@ require ( github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect github.com/gobwas/ws v1.4.0 // indirect - github.com/klauspost/compress v1.18.0 // indirect github.com/kr/text v0.2.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect - golang.org/x/crypto v0.36.0 // indirect golang.org/x/sys v0.35.0 // indirect google.golang.org/protobuf v1.36.8 // indirect ) diff --git a/harnesses/aggregator-head-lag/go.sum b/harnesses/aggregator-head-lag/go.sum index 6379815d..f594bc89 100644 --- a/harnesses/aggregator-head-lag/go.sum +++ b/harnesses/aggregator-head-lag/go.sum @@ -1,5 +1,3 @@ -github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= -github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -49,8 +47,6 @@ github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9Z github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= -github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo= -github.com/refraction-networking/utls v1.8.2/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= @@ -59,8 +55,6 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= diff --git a/public/logos/gmgn.svg b/public/logos/gmgn.svg deleted file mode 100644 index ea348c24..00000000 --- a/public/logos/gmgn.svg +++ /dev/null @@ -1,187 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/data/provider-registry.ts b/src/data/provider-registry.ts index 0f9ed892..63dbb8cd 100644 --- a/src/data/provider-registry.ts +++ b/src/data/provider-registry.ts @@ -55,12 +55,6 @@ export const PROVIDER_REGISTRY: Record = { "DEX data terminal by CoinGecko covering 200+ chains. REST API for pools, tokens, trades, and OHLCV.", twitter: "@GeckoTerminal", }, - gmgn: { - url: "https://gmgn.ai", - description: - "Solana memecoin trading terminal. WebSocket chain_stat feed for per-pair routing events on tracked Solana pools (slot, price, holder activity) used by GMGN's own trading UI.", - twitter: "@gmgnai", - }, jupiter: { url: "https://jup.ag", description: diff --git a/src/lib/logo-manifest.ts b/src/lib/logo-manifest.ts index b463f230..73fee122 100644 --- a/src/lib/logo-manifest.ts +++ b/src/lib/logo-manifest.ts @@ -46,7 +46,6 @@ const RAW: Record = { relay: "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/logos/relay.svg", lifi: "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/logos/lifi.png", geckoterminal: "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/logos/geckoterminal.png", - gmgn: "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/logos/gmgn.svg", gains: "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/logos/gains.png", blockscout: "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/logos/blockscout.svg", gmx: "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/logos/gmx.svg",