From a78acee9988a57d6a19abbfbd0bb01eba0b12121 Mon Sep 17 00:00:00 2001 From: Teodor Calin Date: Thu, 24 Sep 2026 00:11:00 +0300 Subject: [PATCH 01/19] feat(pilotctl): daemon start --transport/--proxy and proxy env passthrough `pilotctl daemon start` could not bring a node online from sandboxes whose only way out is an authenticating HTTPS proxy (Meta Muse): - config.json from `pilotctl init` holds the raw-TCP registry default 34.71.57.205:9000 and daemon start forwarded it as an explicit -registry, which stops pilot-daemon from switching to registry.pilotprotocol.network:443 (TLS) in compat mode. In compat mode the compiled-in registry/beacon defaults are now left off argv (a default-equal $PILOT_REGISTRY is dropped from the child env too). - PILOT_TRANSPORT never took effect: the daemon's -transport flag defaults to "udp", masking the env var. pilotctl now resolves --transport, then $PILOT_TRANSPORT, then config "transport" and passes an explicit -transport. New: --transport and --proxy (plus config keys "transport"/"proxy", validated by `config --set`). Both reach the daemon only when set; a pilot-daemon whose -help lacks them gets them dropped with a warning instead of a flag-parse crash. A proxy URL with credentials travels as $PILOT_PROXY (never argv, PILOT-290) and is redacted in config output. The child env explicitly keeps HTTPS_PROXY/HTTP_PROXY/ALL_PROXY/NO_PROXY (both cases), PILOT_PROXY, PILOT_TRANSPORT, SSL_CERT_FILE, SSL_CERT_DIR on both the fork and --foreground paths. --compat-beacon, --registry-trust, --registry-fingerprint, --tls-trust and the documented-but-dropped --endpoint/--motd-* are forwarded when given. Also: create ~/.pilot before the O_EXCL PID claim (a fresh HOME failed with "PID file locked" forever), and registry dial failures behind a proxy now say pilotctl's direct registry dials do not use the proxy yet (TODO(netproxy)). Co-Authored-By: Claude Opus 5.5 (1M context) --- cmd/pilotctl/daemon_transport.go | 307 ++++++++++++ cmd/pilotctl/main.go | 242 +++++++-- cmd/pilotctl/verify.go | 5 +- cmd/pilotctl/zz_daemon_transport_test.go | 595 +++++++++++++++++++++++ docs/cli-reference.md | 5 +- 5 files changed, 1120 insertions(+), 34 deletions(-) create mode 100644 cmd/pilotctl/daemon_transport.go create mode 100644 cmd/pilotctl/zz_daemon_transport_test.go diff --git a/cmd/pilotctl/daemon_transport.go b/cmd/pilotctl/daemon_transport.go new file mode 100644 index 00000000..2b2e868b --- /dev/null +++ b/cmd/pilotctl/daemon_transport.go @@ -0,0 +1,307 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "context" + "fmt" + "net/url" + "os" + "os/exec" + "strings" + "sync" + "time" +) + +// Compiled-in production endpoints. These are the same defaults cmd/daemon +// compiles in; `pilotctl init` and install.sh write them into config.json. +// Both are raw-TCP/UDP endpoints — compat mode (-transport=compat) must not +// be pinned to them, see compatSkipsDefault. +const ( + productionRegistryAddr = "34.71.57.205:9000" + productionBeaconAddr = "34.71.57.205:9001" +) + +// daemonForwardEnv is the environment `pilotctl daemon start` guarantees to +// hand to pilot-daemon, on both the fork and the --foreground exec path. The +// proxy variables drive the daemon's -proxy=auto resolution (a CONNECT proxy +// is the only way out of sandboxes such as Meta Muse), PILOT_PROXY / +// PILOT_TRANSPORT are the daemon's env-backed flag overrides, and the +// SSL_CERT_* pair lets a sandbox without a system CA bundle point Go's +// x509 at one. +var daemonForwardEnv = []string{ + "HTTPS_PROXY", "https_proxy", + "HTTP_PROXY", "http_proxy", + "ALL_PROXY", "all_proxy", + "NO_PROXY", "no_proxy", + "PILOT_PROXY", "PILOT_TRANSPORT", + "SSL_CERT_FILE", "SSL_CERT_DIR", +} + +// validateTransport accepts the two tunnel transports pilot-daemon knows. +func validateTransport(v string) error { + switch v { + case "udp", "compat": + return nil + } + return fmt.Errorf("invalid transport %q: must be 'udp' or 'compat'", v) +} + +// validateProxySetting accepts the -proxy contract: "auto" (use the +// environment's proxy in compat mode), "off", or an explicit +// http(s)://[user:pass@]host:port proxy URL. +func validateProxySetting(v string) error { + switch v { + case "auto", "off": + return nil + } + u, err := url.Parse(v) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.Hostname() == "" { + return fmt.Errorf("invalid proxy %q: must be 'auto', 'off', or an http(s)://[user:pass@]host:port URL", redactProxyURL(v)) + } + return nil +} + +// proxyHasCredentials reports whether an explicit proxy URL carries userinfo. +func proxyHasCredentials(v string) bool { + u, err := url.Parse(v) + return err == nil && u.User != nil +} + +// redactProxyURL hides proxy credentials for display: http://user:pass@h:p +// becomes http://***@h:p. Non-URL values ("auto", "off") pass through. +// +// TODO(netproxy): switch to common/netproxy's redaction helper once +// pilotctl bumps to a common release that ships it. +func redactProxyURL(v string) string { + if u, err := url.Parse(v); err == nil && u.User != nil { + return u.Scheme + "://***@" + u.Host + u.EscapedPath() + } + // Unparseable or scheme-less ("user:pass@host:port" parses as an opaque + // URL with no userinfo): never echo anything before the last '@'. + if i := strings.LastIndex(v, "@"); i >= 0 { + prefix := "" + if j := strings.Index(v, "://"); j >= 0 && j < i { + prefix = v[:j+3] + } + return prefix + "***@" + v[i+1:] + } + return v +} + +// resolveDaemonTransport picks the tunnel transport for `daemon start`: +// --transport, then $PILOT_TRANSPORT, then config.json "transport". "" means +// none was set and the daemon keeps its own default (udp). +// +// $PILOT_TRANSPORT is resolved here and handed to the daemon as an explicit +// -transport because pilot-daemon's -transport flag defaults to "udp", which +// masks the env var in every released daemon — exporting PILOT_TRANSPORT +// alone never switched a daemon started through pilotctl to compat. +func resolveDaemonTransport(flags map[string]string, cfg map[string]interface{}) (string, error) { + v := strings.TrimSpace(flagString(flags, "transport", "")) + if v == "" { + v = strings.TrimSpace(os.Getenv("PILOT_TRANSPORT")) + } + if v == "" { + if s, ok := cfg["transport"].(string); ok { + v = strings.TrimSpace(s) + } + } + if v == "" { + return "", nil + } + if err := validateTransport(v); err != nil { + return "", err + } + return v, nil +} + +// resolveDaemonProxy picks the proxy policy for `daemon start`: --proxy, then +// config.json "proxy". "" means neither was set. $PILOT_PROXY is not read +// here: it reaches the daemon through the forwarded environment and the +// daemon resolves it itself. +func resolveDaemonProxy(flags map[string]string, cfg map[string]interface{}) (string, error) { + v := strings.TrimSpace(flagString(flags, "proxy", "")) + if v == "" { + if s, ok := cfg["proxy"].(string); ok { + v = strings.TrimSpace(s) + } + } + if v == "" { + return "", nil + } + if err := validateProxySetting(v); err != nil { + return "", err + } + return v, nil +} + +// compatSkipsDefault reports whether addr must be left off the daemon command +// line because it is the compiled-in raw-TCP production default and the +// daemon runs in compat mode. pilot-daemon only switches the registry to +// registry.pilotprotocol.network:443 (TLS) in compat mode when -registry was +// NOT given explicitly; `pilotctl init` writes the raw :9000 default into +// config.json, and forwarding that verbatim pinned compat daemons to a +// non-TLS port no HTTPS proxy will CONNECT to. +func compatSkipsDefault(transport, addr, def string) bool { + return transport == "compat" && addr == def +} + +// daemonChildEnv returns the environment for the pilot-daemon child: the +// full pilotctl environment (so every daemonForwardEnv variable reaches the +// daemon untouched) plus the launch-specific overrides. +// +// - PILOT_ADMIN_TOKEN and a credential-bearing PILOT_PROXY are passed +// here rather than on argv so they never show up in /proc//cmdline +// (PILOT-290). +// - In compat mode a $PILOT_REGISTRY equal to the raw-TCP production +// default is dropped: the daemon treats an env-provided registry as +// explicit and would otherwise skip the compat TLS registry switch. +func daemonChildEnv(base []string, adminToken, proxyEnv, transport string) []string { + set := map[string]string{} + if adminToken != "" { + set["PILOT_ADMIN_TOKEN"] = adminToken + } + if proxyEnv != "" { + set["PILOT_PROXY"] = proxyEnv + } + env := make([]string, 0, len(base)+len(set)) + for _, kv := range base { + k, v, _ := strings.Cut(kv, "=") + if _, override := set[k]; override { + continue + } + if k == "PILOT_REGISTRY" && compatSkipsDefault(transport, v, productionRegistryAddr) { + continue + } + env = append(env, kv) + } + for _, k := range []string{"PILOT_ADMIN_TOKEN", "PILOT_PROXY"} { + if v, ok := set[k]; ok { + env = append(env, k+"="+v) + } + } + return env +} + +// daemonFlagProbeTimeout bounds the `pilot-daemon -help` flag probe. +const daemonFlagProbeTimeout = 5 * time.Second + +var ( + daemonFlagCacheMu sync.Mutex + daemonFlagCache = map[string]map[string]bool{} +) + +// daemonFlags returns the set of flag names the pilot-daemon binary at bin +// defines, parsed from its Go flag-package `-help` usage (" -name type"). +// Returns nil when the binary could not be probed or printed nothing +// recognisable; callers treat nil as "unknown" and keep the flag. +func daemonFlags(bin string) map[string]bool { + daemonFlagCacheMu.Lock() + defer daemonFlagCacheMu.Unlock() + if set, ok := daemonFlagCache[bin]; ok { + return set + } + ctx, cancel := context.WithTimeout(context.Background(), daemonFlagProbeTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, bin, "-help") + // Minimal environment: -help prints each flag's default, and a daemon + // with env-backed defaults would echo e.g. a credential-bearing + // $PILOT_PROXY into the captured output. + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + os.Getenv("HOME")} + out, _ := cmd.CombinedOutput() // -help exits 0 (flag.ExitOnError) or 2 on older builds + var set map[string]bool + for _, line := range strings.Split(string(out), "\n") { + if !strings.HasPrefix(line, " -") { + continue + } + name := strings.TrimPrefix(line, " -") + if i := strings.IndexAny(name, " \t"); i >= 0 { + name = name[:i] + } + if name == "" { + continue + } + if set == nil { + set = map[string]bool{} + } + set[name] = true + } + daemonFlagCache[bin] = set + return set +} + +// skewSensitiveDaemonFlags are pilot-daemon flags newer than some daemons a +// current pilotctl may still be paired with (a sibling binary left behind by +// a partial upgrade, a PILOT_DAEMON_BIN override, an npm-installed pair). +// Go's flag package aborts on an unknown flag, so forwarding one of these to +// an older daemon would turn `daemon start` into a crash loop. +var skewSensitiveDaemonFlags = []string{"transport", "proxy"} + +// dropUnsupportedDaemonFlags removes skew-sensitive flags the daemon at bin +// does not define, warning once per dropped flag. Only probes the binary +// when one of those flags is actually present in args. +func dropUnsupportedDaemonFlags(bin string, args []string) []string { + present := false + for _, a := range args { + for _, f := range skewSensitiveDaemonFlags { + if a == "--"+f { + present = true + } + } + } + if !present { + return args + } + supported := daemonFlags(bin) + if supported == nil { + return args + } + out := make([]string, 0, len(args)) + for i := 0; i < len(args); i++ { + name := strings.TrimPrefix(args[i], "--") + skew := false + for _, f := range skewSensitiveDaemonFlags { + if args[i] == "--"+f && !supported[f] { + skew = true + } + } + if !skew { + out = append(out, args[i]) + continue + } + value := "" + if i+1 < len(args) { + value = args[i+1] + i++ + } + if !jsonOutput { + fmt.Fprintf(os.Stderr, "warning: %s does not support -%s (older pilot-daemon); not passing -%s %s — upgrade pilot-daemon to use it\n", + bin, name, name, redactProxyURL(value)) + } + } + return out +} + +// envProxyURL returns the proxy an HTTPS request from this process would +// use per the environment ($HTTPS_PROXY, then $ALL_PROXY, either case), or "" +// when none is set. +func envProxyURL() string { + for _, k := range []string{"HTTPS_PROXY", "https_proxy", "ALL_PROXY", "all_proxy"} { + if v := strings.TrimSpace(os.Getenv(k)); v != "" { + return v + } + } + return "" +} + +// registryDialHint is the hint for a failed direct registry dial from +// pilotctl. Behind an egress proxy the dial fails by design — say so, +// instead of suggesting the registry is down. +func registryDialHint(addr string) string { + if p := envProxyURL(); p != "" { + return fmt.Sprintf("pilotctl dials the registry (%s) directly over TCP and does not use the proxy %s yet; daemon-backed commands (info, peers, trust, ping, send-message) reach the network through the daemon instead", + addr, redactProxyURL(p)) + } + return fmt.Sprintf("check that the registry is running at %s, or set PILOT_REGISTRY", addr) +} diff --git a/cmd/pilotctl/main.go b/cmd/pilotctl/main.go index 58fa017e..fe632469 100644 --- a/cmd/pilotctl/main.go +++ b/cmd/pilotctl/main.go @@ -287,7 +287,7 @@ func getRegistry() string { if s, ok := cfg["registry"].(string); ok && s != "" { return s } - return "34.71.57.205:9000" + return productionRegistryAddr } // getBeacon mirrors getRegistry for the beacon address: env override, @@ -304,7 +304,7 @@ func getBeacon() string { if s, ok := cfg["beacon"].(string); ok && s != "" { return s } - return "34.71.57.205:9001" + return productionBeaconAddr } func loadConfig() map[string]interface{} { @@ -602,10 +602,15 @@ func nodeIDFromDaemon() int64 { func connectRegistry() *registry.Client { addr := getRegistry() + // TODO(netproxy): dial through common/netproxy (HTTPS_PROXY CONNECT by + // hostname, TLS for the compat registry) once pilotctl bumps to a common + // release with the registry dialer option. Until then this raw-TCP dial + // bypasses any egress proxy, so registry commands fail in proxy-only + // sandboxes; registryDialHint says so. rc, err := registry.Dial(addr) if err != nil { fatalHint("connection_failed", - fmt.Sprintf("check that the registry is running at %s, or set PILOT_REGISTRY", addr), + registryDialHint(addr), "cannot reach registry at %s", addr) } return rc @@ -704,13 +709,19 @@ func maybeAutoHandshake(d *driver.Driver, addr protocol.Addr, skip bool) { } // Branch 3 — unknown peer, not trusted. Refuse if private. + // TODO(netproxy): same raw-TCP registry dial as connectRegistry — it + // bypasses HTTPS_PROXY until common/netproxy lands in pilotctl. rc, err := registry.Dial(getRegistry()) if err != nil { // Registry unreachable — be conservative and refuse rather than // silently let an untrusted tunnel attempt go through to a peer // we can't characterise. + hint := fmt.Sprintf("run: pilotctl handshake %s", addr) + if envProxyURL() != "" { + hint += " (or pass --no-auto-handshake); " + registryDialHint(getRegistry()) + } fatalHint("trust_required", - fmt.Sprintf("run: pilotctl handshake %s", addr), + hint, "cannot verify peer visibility (registry unreachable: %v); refusing tunnel to untrusted node", err) } defer rc.Close() @@ -1056,6 +1067,29 @@ Flags: --motd-feed-url message-of-the-day feed (empty to disable; env PILOT_MOTD_URL) --motd-interval message-of-the-day poll interval (default: 15m) --enterprise-control owner-only managed control attachment + --transport tunnel transport (default: $PILOT_TRANSPORT, config + "transport", else udp). compat = TLS/WSS over TCP 443 + only, for UDP-blocked hosts; the raw-TCP default + registry/beacon are then left to the daemon, which + uses registry.pilotprotocol.network:443 + --proxy outbound proxy (default: config "proxy", else the + daemon's $PILOT_PROXY or auto). auto = in compat mode + use $HTTPS_PROXY/$ALL_PROXY (honoring $NO_PROXY); + off = never; http(s)://[user:pass@]host:port = always. + A URL with credentials is passed via env, not argv + --compat-beacon beacon WSS URL for compat mode + --registry-trust registry TLS trust: pinned or system + --registry-fingerprint registry certificate SHA-256 (with --registry-trust pinned) + --tls-trust compat beacon TLS trust: system or pinned + +Environment passed through to the daemon (never scrubbed): + HTTPS_PROXY https_proxy HTTP_PROXY http_proxy ALL_PROXY all_proxy + NO_PROXY no_proxy PILOT_PROXY PILOT_TRANSPORT SSL_CERT_FILE SSL_CERT_DIR + +--transport / --proxy are forwarded only when set; if the pilot-daemon binary +is too old to know one, it is dropped with a warning instead of crashing it. +Behind an HTTPS proxy with UDP blocked (e.g. hosted agent sandboxes): + pilotctl config --set transport=compat && pilotctl daemon start `, "daemon stop": `Usage: pilotctl daemon stop @@ -1308,6 +1342,9 @@ Common keys: beacon beacon address (overrides $PILOT_BEACON) socket daemon socket path (overrides $PILOT_SOCKET) hostname default hostname passed to daemon start + transport daemon transport: udp or compat ($PILOT_TRANSPORT overrides) + proxy daemon proxy: auto, off, or http(s)://[user:pass@]host:port + (credentials are redacted when shown) `, "version": `Usage: pilotctl version @@ -1514,7 +1551,7 @@ Bootstrap: pilotctl config [--set key=value] Daemon lifecycle: - pilotctl daemon start [--config ] [--registry ] [--beacon ] [--email ] [--webhook ] [--trust-auto-approve] + pilotctl daemon start [--config ] [--registry ] [--beacon ] [--email ] [--webhook ] [--trust-auto-approve] [--transport ] [--proxy ] pilotctl daemon stop pilotctl daemon status @@ -1611,6 +1648,9 @@ Diagnostic commands: Environment: PILOT_REGISTRY Registry address (default: 34.71.57.205:9000) PILOT_SOCKET Daemon socket path (default: /tmp/pilot.sock) + PILOT_TRANSPORT daemon start transport: udp or compat (TCP 443 only) + PILOT_PROXY daemon proxy policy: auto, off, or http(s)://[user:pass@]host:port + HTTPS_PROXY proxy used by compat mode (proxy=auto) and pilotctl's HTTP calls Version: pilotctl version @@ -2087,14 +2127,32 @@ func cmdConfig(args []string) { if len(parts) != 2 { fatalCode("invalid_argument", "usage: pilotctl config --set key=value") } + // Validate the keys daemon start interprets, so a typo fails here + // rather than on the next daemon start. Empty clears the key. + if parts[1] != "" { + switch parts[0] { + case "transport": + if err := validateTransport(parts[1]); err != nil { + fatalCode("invalid_argument", "config: %v", err) + } + case "proxy": + if err := validateProxySetting(parts[1]); err != nil { + fatalCode("invalid_argument", "config: %v", err) + } + } + } cfg := loadConfig() cfg[parts[0]] = parts[1] if err := saveConfig(cfg); err != nil { fatalCode("internal", "save config: %v", err) } + shown := parts[1] + if parts[0] == "proxy" { + shown = redactProxyURL(shown) + } outputOK(map[string]interface{}{ "key": parts[0], - "value": parts[1], + "value": shown, }) return } @@ -2111,6 +2169,10 @@ func cmdConfig(args []string) { if _, ok := cfg["socket"]; !ok { cfg["socket"] = getSocket() } + // config.json is 0600 for a reason; a proxy URL may carry credentials. + if p, ok := cfg["proxy"].(string); ok { + cfg["proxy"] = redactProxyURL(p) + } if jsonOutput { output(cfg) return @@ -2201,9 +2263,9 @@ func contextCatalog() map[string]interface{} { // Daemon lifecycle "daemon start": map[string]interface{}{ - "args": []string{"[--registry ]", "[--beacon ]", "[--listen ]", "[--identity ]", "[--email ]", "[--hostname ]", "[--log-level ]", "[--public]", "[--foreground]", "[--socket ]"}, - "description": "Start the daemon as a background process. Blocks until registered, then exits", - "returns": "node_id, address, pid, socket, hostname, log_file", + "args": []string{"[--registry ]", "[--beacon ]", "[--listen ]", "[--identity ]", "[--email ]", "[--hostname ]", "[--log-level ]", "[--public]", "[--foreground]", "[--socket ]", "[--transport ]", "[--proxy ]"}, + "description": "Start the daemon as a background process. Blocks until registered, then exits. --transport compat (or config transport=compat / $PILOT_TRANSPORT) runs over TCP 443 only for UDP-blocked hosts; --proxy auto (default) then routes through $HTTPS_PROXY", + "returns": "node_id, address, pid, socket, hostname, log_file, transport (when set), proxy (when set, credentials redacted)", }, "daemon stop": map[string]interface{}{ "args": []string{}, @@ -2543,8 +2605,11 @@ func contextCatalog() map[string]interface{} { "--json": "Output structured JSON for all commands. Success: {status:ok, data:{...}}. Error: {status:error, code:string, message:string}", }, "environment": map[string]interface{}{ - "PILOT_REGISTRY": "Registry address (default: 34.71.57.205:9000)", - "PILOT_SOCKET": "Daemon socket path (default: /tmp/pilot.sock)", + "PILOT_REGISTRY": "Registry address (default: 34.71.57.205:9000)", + "PILOT_SOCKET": "Daemon socket path (default: /tmp/pilot.sock)", + "PILOT_TRANSPORT": "daemon start transport: udp or compat (TCP 443 only)", + "PILOT_PROXY": "daemon proxy policy: auto, off, or http(s)://[user:pass@]host:port", + "HTTPS_PROXY": "proxy used by compat mode (proxy=auto) and pilotctl HTTP calls; forwarded to the daemon", }, "config_file": "~/.pilot/config.json", } @@ -2662,17 +2727,52 @@ func gatewayBinaryPath() string { return path } +// daemonLaunchPlan is everything `pilotctl daemon start` hands to +// pilot-daemon: its argv (without argv[0]) plus the values that travel in +// the child environment instead of on the command line. +type daemonLaunchPlan struct { + Args []string + SocketPath string + // AdminToken is passed as $PILOT_ADMIN_TOKEN, never on argv (PILOT-290). + AdminToken string + // Transport is the resolved --transport ("" = daemon default, udp). + Transport string + // Proxy is the resolved --proxy ("" = daemon default, auto). + Proxy string + // ProxyEnv is non-empty when Proxy carries credentials: it is handed to + // the daemon as $PILOT_PROXY instead of -proxy on argv (PILOT-290). + ProxyEnv string +} + // buildDaemonArgs translates pilotctl-style flags into pilot-daemon CLI // args, applying defaults from ~/.pilot/config.json when CLI flags are // unset. This keeps existing pilotctl invocations working unchanged — // the only difference is that the daemon runs in a separate // `pilot-daemon` process rather than re-execing pilotctl. func buildDaemonArgs(args []string) (daemonArgs []string, socketPath string, adminToken string) { + plan := planDaemonLaunch(args) + return plan.Args, plan.SocketPath, plan.AdminToken +} + +// planDaemonLaunch resolves pilotctl flags, environment and config.json into +// a daemonLaunchPlan. Precedence for every setting is CLI flag, then (where +// one exists) its environment variable, then config.json, then the daemon's +// own default. +func planDaemonLaunch(args []string) daemonLaunchPlan { flags, _ := parseFlags(args) cfg := loadConfig() - socketPath = flagString(flags, "socket", "") + transport, err := resolveDaemonTransport(flags, cfg) + if err != nil { + fatalCode("invalid_argument", "daemon start: %v", err) + } + proxy, err := resolveDaemonProxy(flags, cfg) + if err != nil { + fatalCode("invalid_argument", "daemon start: %v", err) + } + + socketPath := flagString(flags, "socket", "") if socketPath == "" { socketPath = getSocket() } @@ -2725,7 +2825,7 @@ func buildDaemonArgs(args []string) (daemonArgs []string, socketPath string, adm webhookURL = w } } - adminToken = flagString(flags, "admin-token", "") + adminToken := flagString(flags, "admin-token", "") if adminToken == "" { if a, ok := cfg["admin_token"].(string); ok { adminToken = a @@ -2745,15 +2845,24 @@ func buildDaemonArgs(args []string) (daemonArgs []string, socketPath string, adm } } - daemonArgs = []string{ - "--registry", registryAddr, - "--beacon", beaconAddr, + var daemonArgs []string + // In compat mode the raw-TCP production registry/beacon defaults are + // left off argv so pilot-daemon applies its 443-only compat defaults + // (registry.pilotprotocol.network:443 over TLS). Any other address is + // an operator choice and is forwarded verbatim. + if !compatSkipsDefault(transport, registryAddr, productionRegistryAddr) { + daemonArgs = append(daemonArgs, "--registry", registryAddr) + } + if !compatSkipsDefault(transport, beaconAddr, productionBeaconAddr) { + daemonArgs = append(daemonArgs, "--beacon", beaconAddr) + } + daemonArgs = append(daemonArgs, "--listen", listenAddr, "--socket", socketPath, "--identity", identityPath, "--log-level", logLevel, "--log-format", logFormat, - } + ) // pilot-daemon's encrypt flag defaults to true; pass `=false` // only when --no-encrypt was supplied. if !encrypt { @@ -2785,7 +2894,36 @@ func buildDaemonArgs(args []string) (daemonArgs []string, socketPath string, adm if enterpriseControl != "" { daemonArgs = append(daemonArgs, "--enterprise-control", enterpriseControl) } - return daemonArgs, socketPath, adminToken + // Daemon flags that are forwarded only when given, so a plain + // `daemon start` keeps the daemon's own defaults. --endpoint and the + // motd pair were documented here for a long time but never forwarded. + for _, name := range []string{"endpoint", "compat-beacon", "registry-trust", "registry-fingerprint", "tls-trust", "motd-feed-url", "motd-interval"} { + if v, ok := flags[name]; ok { + daemonArgs = append(daemonArgs, "--"+name, v) + } + } + // --transport / --proxy are passed only when set, so a new pilotctl + // still starts an older daemon that predates them (cmdDaemonStart + // additionally drops any the daemon binary does not define). + if transport != "" { + daemonArgs = append(daemonArgs, "--transport", transport) + } + proxyEnv := "" + if proxy != "" { + if proxyHasCredentials(proxy) { + proxyEnv = proxy + } else { + daemonArgs = append(daemonArgs, "--proxy", proxy) + } + } + return daemonLaunchPlan{ + Args: daemonArgs, + SocketPath: socketPath, + AdminToken: adminToken, + Transport: transport, + Proxy: proxy, + ProxyEnv: proxyEnv, + } } // launchdAgentLabels enumerates known launchd labels for the daemon. @@ -2843,11 +2981,24 @@ func launchdAgentLoaded(label string) bool { func cmdDaemonStart(args []string) { flags, _ := parseFlags(args) + // Resolve (and validate) the launch before touching the PID file, so a + // bad --transport / --proxy fails fast with nothing to clean up. + plan := planDaemonLaunch(args) + // macOS install.sh installs a launchd plist. When present, route start // through launchctl so the agent is registered and KeepAlive supervises // the process; otherwise `pilotctl daemon stop` would have nothing to // stop (KeepAlive immediately respawns) and the user sees flapping. if plist, label := launchdAgentPlist(); plist != "" { + // launchd starts the daemon from the plist's ProgramArguments and + // its own environment: neither CLI flags nor this shell's proxy + // variables reach it. The daemon does read config.json itself. + if _, ok := flags["transport"]; ok && !jsonOutput { + fmt.Fprintf(os.Stderr, "warning: --transport is not applied to the launchd-managed daemon; persist it with: pilotctl config --set transport=%s\n", plan.Transport) + } + if _, ok := flags["proxy"]; ok && !jsonOutput { + fmt.Fprintf(os.Stderr, "warning: --proxy is not applied to the launchd-managed daemon; persist it with: pilotctl config --set proxy=\n") + } if launchdAgentLoaded(label) { fatalHint("already_exists", "stop it first with: pilotctl daemon stop", @@ -2906,6 +3057,10 @@ func cmdDaemonStart(args []string) { // Atomically claim the PID file to prevent concurrent daemon starts. // O_CREAT|O_EXCL ensures only one pilotctl daemon start can succeed; // a second concurrent invocation fails here before spawning a daemon. + // The config dir must exist first: on a fresh HOME (no `pilotctl init`, + // no ~/.pilot/bin) the open failed with ENOENT and was misreported as + // "PID file locked" on every attempt. + _ = os.MkdirAll(configDir(), 0700) if f, err := os.OpenFile(pidFilePath(), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600); err != nil { fatalHint("already_exists", "stop it first with: pilotctl daemon stop", @@ -2915,7 +3070,7 @@ func cmdDaemonStart(args []string) { f.Close() } - daemonArgs, socketPath, adminToken := buildDaemonArgs(args) + socketPath := plan.SocketPath // Clean up stale socket if _, err := os.Stat(socketPath); err == nil { @@ -2932,6 +3087,20 @@ func cmdDaemonStart(args []string) { } daemonBin := daemonBinaryPath() + // Never hand an older daemon a flag it does not define: Go's flag + // package would abort it on startup. + daemonArgs := dropUnsupportedDaemonFlags(daemonBin, plan.Args) + proxyEnv := plan.ProxyEnv + if proxyEnv != "" { + if supported := daemonFlags(daemonBin); supported != nil && !supported["proxy"] { + if !jsonOutput { + fmt.Fprintf(os.Stderr, "warning: %s does not support -proxy (older pilot-daemon); proxy %s will not be used — upgrade pilot-daemon to use it\n", + daemonBin, redactProxyURL(proxyEnv)) + } + proxyEnv = "" + } + } + daemonEnv := daemonChildEnv(os.Environ(), plan.AdminToken, proxyEnv, plan.Transport) // --foreground: replace the current process so signal/lifetime // handling matches what the user expects from systemd unit files @@ -2946,14 +3115,10 @@ func cmdDaemonStart(args []string) { // locked"), an unrecoverable restart loop under systemd. _ = os.WriteFile(pidFilePath(), []byte(strconv.Itoa(os.Getpid())+"\n"), 0600) // syscall.Exec needs argv[0] to be the binary name. Pass the - // full env. Inject PILOT_ADMIN_TOKEN so the daemon doesn't - // need the token on its argv (PILOT-290). + // full env (daemonChildEnv) — it carries PILOT_ADMIN_TOKEN so the + // daemon doesn't need the token on its argv (PILOT-290). execArgs := append([]string{daemonBin}, daemonArgs...) - env := os.Environ() - if adminToken != "" { - env = append(env, "PILOT_ADMIN_TOKEN="+adminToken) - } - if err := syscall.Exec(daemonBin, execArgs, env); err != nil { + if err := syscall.Exec(daemonBin, execArgs, daemonEnv); err != nil { fatalCode("internal", "exec %s: %v", daemonBin, err) } return @@ -2978,11 +3143,11 @@ func cmdDaemonStart(args []string) { proc.Stdout = logFile proc.Stderr = logFile proc.SysProcAttr = &syscall.SysProcAttr{Setsid: true} - // Pass admin token via env, not argv, to avoid leaking in + // Full environment plus overrides (daemonChildEnv): the proxy + // variables in daemonForwardEnv must reach the daemon, and the admin + // token travels via env, not argv, to avoid leaking in // /proc//cmdline (PILOT-290). - if adminToken != "" { - proc.Env = append(os.Environ(), "PILOT_ADMIN_TOKEN="+adminToken) - } + proc.Env = daemonEnv if err := proc.Start(); err != nil { fatalCode("internal", "start daemon: %v", err) @@ -3037,20 +3202,33 @@ func cmdDaemonStart(args []string) { address := info["address"] hn, _ := info["hostname"].(string) if jsonOutput { - outputOK(map[string]interface{}{ + fields := map[string]interface{}{ "pid": pid, "node_id": nodeID, "address": address, "hostname": hn, "socket": socketPath, "log_file": pidLogPath, - }) + } + if plan.Transport != "" { + fields["transport"] = plan.Transport + } + if plan.Proxy != "" { + fields["proxy"] = redactProxyURL(plan.Proxy) + } + outputOK(fields) } else { fmt.Printf("Daemon running (pid %d)\n", pid) fmt.Printf(" Address: %s\n", address) if hn != "" { fmt.Printf(" Hostname: %s\n", hn) } + if plan.Transport != "" { + fmt.Printf(" Transport: %s\n", plan.Transport) + } + if plan.Proxy != "" { + fmt.Printf(" Proxy: %s\n", redactProxyURL(plan.Proxy)) + } fmt.Printf(" Socket: %s\n", socketPath) fmt.Printf(" Logs: %s\n", pidLogPath) } diff --git a/cmd/pilotctl/verify.go b/cmd/pilotctl/verify.go index c674f9db..a202e2d9 100644 --- a/cmd/pilotctl/verify.go +++ b/cmd/pilotctl/verify.go @@ -298,9 +298,12 @@ func cmdRecoveryRecover(args []string) { newPub := crypto.EncodePublicKey(id.PublicKey) addr := flagString(flags, "registry", getRegistry()) + // TODO(netproxy): raw-TCP registry dial; bypasses HTTPS_PROXY until + // common/netproxy lands in pilotctl (see connectRegistry). rc, err := registry.Dial(addr) if err != nil { - fatalCode("connection_failed", "recovery recover: cannot reach registry at %s: %v", addr, err) + fatalHint("connection_failed", registryDialHint(addr), + "recovery recover: cannot reach registry at %s: %v", addr, err) } defer rc.Close() diff --git a/cmd/pilotctl/zz_daemon_transport_test.go b/cmd/pilotctl/zz_daemon_transport_test.go new file mode 100644 index 00000000..1600bb6e --- /dev/null +++ b/cmd/pilotctl/zz_daemon_transport_test.go @@ -0,0 +1,595 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// withTransportEnvCleared isolates HOME like withTempHomeFull and also blanks +// every variable planDaemonLaunch / daemonChildEnv read, so a developer shell +// exporting HTTPS_PROXY or PILOT_TRANSPORT cannot leak into assertions. +func withTransportEnvCleared(t *testing.T) string { + t.Helper() + tmp := withTempHomeFull(t) + t.Setenv("PILOT_BEACON", "") + for _, k := range daemonForwardEnv { + t.Setenv(k, "") + } + return tmp +} + +func argsHasPair(args []string, key, value string) bool { + for i := 0; i+1 < len(args); i++ { + if args[i] == key && args[i+1] == value { + return true + } + } + return false +} + +func argsHasKey(args []string, key string) bool { + for _, a := range args { + if a == key { + return true + } + } + return false +} + +func TestResolveDaemonTransportPrecedence(t *testing.T) { + withTransportEnvCleared(t) + cfg := map[string]interface{}{"transport": "udp"} + + if got, err := resolveDaemonTransport(map[string]string{}, map[string]interface{}{}); err != nil || got != "" { + t.Fatalf("unset: got %q, %v; want \"\", nil", got, err) + } + if got, _ := resolveDaemonTransport(map[string]string{}, cfg); got != "udp" { + t.Errorf("config: got %q, want udp", got) + } + t.Setenv("PILOT_TRANSPORT", "compat") + if got, _ := resolveDaemonTransport(map[string]string{}, cfg); got != "compat" { + t.Errorf("env should beat config: got %q", got) + } + if got, _ := resolveDaemonTransport(map[string]string{"transport": "udp"}, cfg); got != "udp" { + t.Errorf("flag should beat env: got %q", got) + } + if _, err := resolveDaemonTransport(map[string]string{"transport": "quic"}, cfg); err == nil { + t.Error("invalid transport must error") + } + t.Setenv("PILOT_TRANSPORT", "") + if _, err := resolveDaemonTransport(map[string]string{}, map[string]interface{}{"transport": "compact"}); err == nil { + t.Error("invalid config transport must error") + } +} + +func TestResolveDaemonProxyPrecedenceAndValidation(t *testing.T) { + withTransportEnvCleared(t) + // $PILOT_PROXY is the daemon's to resolve, never pilotctl's. + t.Setenv("PILOT_PROXY", "off") + if got, err := resolveDaemonProxy(map[string]string{}, map[string]interface{}{}); err != nil || got != "" { + t.Fatalf("unset: got %q, %v", got, err) + } + cfg := map[string]interface{}{"proxy": "auto"} + if got, _ := resolveDaemonProxy(map[string]string{}, cfg); got != "auto" { + t.Errorf("config: got %q", got) + } + if got, _ := resolveDaemonProxy(map[string]string{"proxy": "http://p.example:3128"}, cfg); got != "http://p.example:3128" { + t.Errorf("flag should beat config: got %q", got) + } + for _, ok := range []string{"auto", "off", "http://p:3128", "https://u:pw@p.example:443", "http://u@p"} { + if err := validateProxySetting(ok); err != nil { + t.Errorf("validateProxySetting(%q) = %v, want nil", ok, err) + } + } + for _, bad := range []string{"true", "socks5://p:1080", "p.example:3128", "http://", "u:pw@p:3128", "AUTO"} { + err := validateProxySetting(bad) + if err == nil { + t.Errorf("validateProxySetting(%q) = nil, want error", bad) + continue + } + if strings.Contains(err.Error(), "pw") { + t.Errorf("error for %q leaks credentials: %v", bad, err) + } + } +} + +func TestRedactProxyURL(t *testing.T) { + t.Parallel() + cases := map[string]string{ + "auto": "auto", + "off": "off", + "http://proxy:3128": "http://proxy:3128", + "http://user:s3cret@proxy:3128": "http://***@proxy:3128", + "https://user@proxy.example:8443": "https://***@proxy.example:8443", + "http://u:p%40ss@proxy:3128/": "http://***@proxy:3128/", + "user:s3cret@proxy:3128": "***@proxy:3128", + "http://us er:s3cret@proxy:3128": "http://***@proxy:3128", + } + for in, want := range cases { + got := redactProxyURL(in) + if got != want { + t.Errorf("redactProxyURL(%q) = %q, want %q", in, got, want) + } + if strings.Contains(got, "s3cret") { + t.Errorf("redactProxyURL(%q) leaked the password: %q", in, got) + } + } +} + +// TestBuildDaemonArgsUDPUnchanged pins backward compatibility: with no +// transport/proxy configured, the daemon command line is exactly what older +// pilotctl produced — the default registry/beacon are forwarded and neither +// --transport nor --proxy appears, so an older pilot-daemon still starts. +func TestBuildDaemonArgsUDPUnchanged(t *testing.T) { + withTransportEnvCleared(t) + if err := saveConfig(map[string]interface{}{ + "registry": productionRegistryAddr, + "beacon": productionBeaconAddr, + }); err != nil { + t.Fatalf("saveConfig: %v", err) + } + args, _, _ := buildDaemonArgs(nil) + if !argsHasPair(args, "--registry", productionRegistryAddr) || !argsHasPair(args, "--beacon", productionBeaconAddr) { + t.Errorf("udp mode must forward registry/beacon: %v", args) + } + for _, k := range []string{"--transport", "--proxy", "--endpoint", "--compat-beacon"} { + if argsHasKey(args, k) { + t.Errorf("unexpected %s in default args %v", k, args) + } + } +} + +// TestBuildDaemonArgsCompatFromConfig is the Meta Muse shape: config.json from +// `pilotctl init` carries the raw-TCP registry default plus transport=compat +// and proxy=auto (install.sh --transport compat). The registry/beacon defaults +// must stay off argv so the daemon's compat 443/TLS defaults apply. +func TestBuildDaemonArgsCompatFromConfig(t *testing.T) { + withTransportEnvCleared(t) + if err := saveConfig(map[string]interface{}{ + "registry": productionRegistryAddr, + "beacon": productionBeaconAddr, + "transport": "compat", + "proxy": "auto", + }); err != nil { + t.Fatalf("saveConfig: %v", err) + } + plan := planDaemonLaunch(nil) + if !argsHasPair(plan.Args, "--transport", "compat") { + t.Errorf("expected --transport compat: %v", plan.Args) + } + if !argsHasPair(plan.Args, "--proxy", "auto") { + t.Errorf("expected --proxy auto: %v", plan.Args) + } + if argsHasKey(plan.Args, "--registry") || argsHasKey(plan.Args, "--beacon") { + t.Errorf("compat mode must not pin the raw-TCP defaults: %v", plan.Args) + } + if plan.Transport != "compat" || plan.Proxy != "auto" || plan.ProxyEnv != "" { + t.Errorf("plan = %+v", plan) + } +} + +// TestBuildDaemonArgsCompatKeepsCustomRegistry: only the compiled-in default +// is dropped; an operator-chosen registry is forwarded even in compat mode. +func TestBuildDaemonArgsCompatKeepsCustomRegistry(t *testing.T) { + withTransportEnvCleared(t) + args, _, _ := buildDaemonArgs([]string{ + "--transport", "compat", + "--registry", "registry.example:443", + "--beacon", productionBeaconAddr, + }) + if !argsHasPair(args, "--registry", "registry.example:443") { + t.Errorf("custom registry must be forwarded: %v", args) + } + if argsHasKey(args, "--beacon") { + t.Errorf("default beacon must be dropped in compat mode: %v", args) + } + // An explicit --registry equal to the default is still "not explicit". + args, _, _ = buildDaemonArgs([]string{"--transport", "compat", "--registry", productionRegistryAddr}) + if argsHasKey(args, "--registry") { + t.Errorf("default registry must be dropped in compat mode: %v", args) + } +} + +// TestBuildDaemonArgsTransportFromEnv: $PILOT_TRANSPORT becomes an explicit +// -transport (the daemon's own -transport default masks the env var). +func TestBuildDaemonArgsTransportFromEnv(t *testing.T) { + withTransportEnvCleared(t) + t.Setenv("PILOT_TRANSPORT", "compat") + args, _, _ := buildDaemonArgs(nil) + if !argsHasPair(args, "--transport", "compat") { + t.Errorf("expected --transport compat from env: %v", args) + } + if argsHasKey(args, "--registry") { + t.Errorf("compat from env must drop the default registry: %v", args) + } + if argsHasKey(args, "--proxy") { + t.Errorf("--proxy must not be passed when unset: %v", args) + } +} + +// TestBuildDaemonArgsProxyCredentialsViaEnv: a proxy URL with userinfo never +// lands on argv (PILOT-290); it is carried in plan.ProxyEnv instead. +func TestBuildDaemonArgsProxyCredentialsViaEnv(t *testing.T) { + withTransportEnvCleared(t) + plan := planDaemonLaunch([]string{"--proxy", "http://user:s3cret@proxy.example:3128"}) + if strings.Contains(strings.Join(plan.Args, " "), "s3cret") || argsHasKey(plan.Args, "--proxy") { + t.Errorf("credentialed proxy leaked onto argv: %v", plan.Args) + } + if plan.ProxyEnv != "http://user:s3cret@proxy.example:3128" { + t.Errorf("ProxyEnv = %q", plan.ProxyEnv) + } + plan = planDaemonLaunch([]string{"--proxy", "http://proxy.example:3128"}) + if !argsHasPair(plan.Args, "--proxy", "http://proxy.example:3128") || plan.ProxyEnv != "" { + t.Errorf("credential-free proxy should go on argv: %+v", plan) + } +} + +func TestBuildDaemonArgsForwardsOptionalDaemonFlags(t *testing.T) { + withTransportEnvCleared(t) + args, _, _ := buildDaemonArgs([]string{ + "--endpoint", "203.0.113.7:4000", + "--compat-beacon", "wss://beacon.example/v1/compat", + "--registry-trust", "pinned", + "--registry-fingerprint", "abcd", + "--tls-trust", "system", + "--motd-feed-url", "", + "--motd-interval", "30m", + }) + for k, v := range map[string]string{ + "--endpoint": "203.0.113.7:4000", + "--compat-beacon": "wss://beacon.example/v1/compat", + "--registry-trust": "pinned", + "--registry-fingerprint": "abcd", + "--tls-trust": "system", + "--motd-feed-url": "", + "--motd-interval": "30m", + } { + if !argsHasPair(args, k, v) { + t.Errorf("expected %s %q in %v", k, v, args) + } + } +} + +func TestDaemonChildEnvForwardsProxyVars(t *testing.T) { + t.Parallel() + base := []string{"PATH=/usr/bin", "HOME=/home/agent"} + for _, k := range daemonForwardEnv { + base = append(base, k+"=value-of-"+k) + } + env := daemonChildEnv(base, "", "", "") + have := map[string]string{} + for _, kv := range env { + k, v, _ := strings.Cut(kv, "=") + have[k] = v + } + for _, k := range append([]string{"PATH", "HOME"}, daemonForwardEnv...) { + if _, ok := have[k]; !ok { + t.Errorf("%s was not forwarded to the daemon", k) + } + } + if have["HTTPS_PROXY"] != "value-of-HTTPS_PROXY" { + t.Errorf("HTTPS_PROXY = %q", have["HTTPS_PROXY"]) + } +} + +func TestDaemonChildEnvOverrides(t *testing.T) { + t.Parallel() + base := []string{ + "PILOT_ADMIN_TOKEN=stale", + "PILOT_PROXY=off", + "PILOT_REGISTRY=" + productionRegistryAddr, + "HTTPS_PROXY=http://u:p@proxy:3128", + } + count := func(env []string, key string) (n int, last string) { + for _, kv := range env { + if k, v, _ := strings.Cut(kv, "="); k == key { + n++ + last = v + } + } + return + } + + env := daemonChildEnv(base, "tok", "http://u:p@proxy2:3128", "compat") + if n, v := count(env, "PILOT_ADMIN_TOKEN"); n != 1 || v != "tok" { + t.Errorf("PILOT_ADMIN_TOKEN: n=%d v=%q", n, v) + } + if n, v := count(env, "PILOT_PROXY"); n != 1 || v != "http://u:p@proxy2:3128" { + t.Errorf("PILOT_PROXY: n=%d v=%q", n, v) + } + if n, _ := count(env, "PILOT_REGISTRY"); n != 0 { + t.Errorf("compat mode must drop the default PILOT_REGISTRY: %v", env) + } + if n, _ := count(env, "HTTPS_PROXY"); n != 1 { + t.Errorf("HTTPS_PROXY must be forwarded: %v", env) + } + + // udp mode: nothing overridden, PILOT_REGISTRY kept. + env = daemonChildEnv(base, "", "", "udp") + if n, v := count(env, "PILOT_REGISTRY"); n != 1 || v != productionRegistryAddr { + t.Errorf("udp mode must keep PILOT_REGISTRY: %v", env) + } + if n, v := count(env, "PILOT_ADMIN_TOKEN"); n != 1 || v != "stale" { + t.Errorf("no override: PILOT_ADMIN_TOKEN n=%d v=%q", n, v) + } +} + +// writeFakeDaemon writes an executable shell script standing in for +// pilot-daemon. Its -help lists exactly the given flags (Go flag-package +// format); any other invocation records argv and environment to out and, like +// Go's flag package, dies on a flag it does not define. +func writeFakeDaemon(t *testing.T, flags []string, out string) string { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("shell-script fake daemon") + } + dir := t.TempDir() + var help strings.Builder + var known strings.Builder + for _, f := range flags { + help.WriteString(" -" + f + " string\n \tdescription of " + f + "\n") + known.WriteString(" -" + f + " --" + f) + } + script := `#!/bin/sh +if [ "$1" = "-help" ]; then + echo "Usage of pilot-daemon:" >&2 + cat >&2 <<'HELP' +` + help.String() + `HELP + exit 0 +fi +: > "` + out + `.args" +for a in "$@"; do + case "$a" in + -*) + name="${a%%=*}" + case "` + known.String() + ` " in + *" $name "*) ;; + *) echo "flag provided but not defined: $name" >&2; exit 2 ;; + esac ;; + esac + printf '%s\n' "$a" >> "` + out + `.args" +done +env > "` + out + `.env" +exit 0 +` + path := filepath.Join(dir, "pilot-daemon") + if err := os.WriteFile(path, []byte(script), 0o755); err != nil { + t.Fatalf("write fake daemon: %v", err) + } + return path +} + +var baseDaemonFlags = []string{ + "registry", "beacon", "listen", "socket", "identity", "log-level", "log-format", + "encrypt", "email", "hostname", "config", "public", "webhook", "networks", + "trust-auto-approve", "enterprise-control", "endpoint", "compat-beacon", + "registry-trust", "registry-fingerprint", "tls-trust", "motd-feed-url", "motd-interval", +} + +func TestDaemonFlagsProbe(t *testing.T) { + t.Parallel() + out := filepath.Join(t.TempDir(), "probe") + bin := writeFakeDaemon(t, append([]string{"transport"}, baseDaemonFlags...), out) + set := daemonFlags(bin) + if !set["transport"] || !set["registry"] || set["proxy"] { + t.Errorf("probe parsed %v", set) + } + if got := daemonFlags(filepath.Join(t.TempDir(), "missing")); got != nil { + t.Errorf("missing binary should probe as unknown (nil), got %v", got) + } +} + +// TestDropUnsupportedDaemonFlags: an older daemon (no -proxy, or neither +// -proxy nor -transport) gets those flags stripped instead of crashing on +// them; a current daemon — or one that cannot be probed — keeps them. +func TestDropUnsupportedDaemonFlags(t *testing.T) { + t.Parallel() + args := []string{"--listen", ":0", "--transport", "compat", "--proxy", "auto", "--socket", "/tmp/x.sock"} + dir := t.TempDir() + + v1139 := writeFakeDaemon(t, append([]string{"transport"}, baseDaemonFlags...), filepath.Join(dir, "a")) + got := dropUnsupportedDaemonFlags(v1139, args) + if argsHasKey(got, "--proxy") || argsHasKey(got, "auto") { + t.Errorf("v1.13.9-style daemon must not get --proxy: %v", got) + } + if !argsHasPair(got, "--transport", "compat") || !argsHasPair(got, "--socket", "/tmp/x.sock") { + t.Errorf("supported flags must survive: %v", got) + } + + ancient := writeFakeDaemon(t, baseDaemonFlags, filepath.Join(dir, "b")) + got = dropUnsupportedDaemonFlags(ancient, args) + if argsHasKey(got, "--proxy") || argsHasKey(got, "--transport") { + t.Errorf("pre-compat daemon must get neither flag: %v", got) + } + if !argsHasPair(got, "--listen", ":0") { + t.Errorf("unrelated flags must survive: %v", got) + } + + current := writeFakeDaemon(t, append([]string{"transport", "proxy"}, baseDaemonFlags...), filepath.Join(dir, "c")) + got = dropUnsupportedDaemonFlags(current, args) + if strings.Join(got, " ") != strings.Join(args, " ") { + t.Errorf("current daemon must keep every flag: %v", got) + } + + got = dropUnsupportedDaemonFlags(filepath.Join(dir, "missing"), args) + if strings.Join(got, " ") != strings.Join(args, " ") { + t.Errorf("unprobeable daemon must keep every flag: %v", got) + } +} + +func readLines(t *testing.T, path string) []string { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return strings.Split(strings.TrimRight(string(b), "\n"), "\n") +} + +// cliEnvCleared blanks the variables that would otherwise leak from the +// developer shell into a runCLI child and skew daemon start. +func cliEnvCleared(extra map[string]string) map[string]string { + env := map[string]string{ + "PILOT_HOME": "", + "PILOT_REGISTRY": "", + "PILOT_BEACON": "", + "PILOT_ADMIN_TOKEN": "", + } + for _, k := range daemonForwardEnv { + env[k] = "" + } + for k, v := range extra { + env[k] = v + } + return env +} + +// TestCLIDaemonStartForegroundCompatProxy drives the real `daemon start +// --foreground` exec path against a fake current daemon and checks what the +// daemon actually receives: compat flags on argv, no raw-TCP registry +// default, the credentialed proxy in env (not argv), and every proxy/TLS +// variable forwarded. +func TestCLIDaemonStartForegroundCompatProxy(t *testing.T) { + t.Parallel() + dir := t.TempDir() + out := filepath.Join(dir, "daemon") + bin := writeFakeDaemon(t, append([]string{"transport", "proxy"}, baseDaemonFlags...), out) + env := cliEnvCleared(map[string]string{ + "PILOT_DAEMON_BIN": bin, + "PILOT_SOCKET": filepath.Join(dir, "pilot.sock"), + "PILOT_TRANSPORT": "compat", + "HTTPS_PROXY": "http://agent:s3cret@egress.internal:3128", + "https_proxy": "http://agent:s3cret@egress.internal:3128", + "NO_PROXY": "localhost,127.0.0.1", + "no_proxy": "localhost,127.0.0.1", + "ALL_PROXY": "http://agent:s3cret@egress.internal:3128", + "SSL_CERT_FILE": "/etc/ssl/muse/ca.pem", + "SSL_CERT_DIR": "/etc/ssl/muse", + }) + _, stderr, code := runCLI(t, []string{"daemon", "start", "--foreground", "--proxy", "http://agent:s3cret@egress.internal:3128"}, env) + if code != 0 { + t.Fatalf("daemon start --foreground exit=%d stderr=%s", code, stderr) + } + args := readLines(t, out+".args") + if !argsHasPair(args, "--transport", "compat") { + t.Errorf("daemon argv missing --transport compat: %v", args) + } + if argsHasKey(args, "--registry") || argsHasKey(args, "--beacon") { + t.Errorf("compat daemon argv must not pin the raw-TCP defaults: %v", args) + } + if strings.Contains(strings.Join(args, " "), "s3cret") { + t.Errorf("proxy credentials leaked onto daemon argv: %v", args) + } + got := map[string]string{} + for _, kv := range readLines(t, out+".env") { + if k, v, ok := strings.Cut(kv, "="); ok { + got[k] = v + } + } + if got["PILOT_PROXY"] != "http://agent:s3cret@egress.internal:3128" { + t.Errorf("PILOT_PROXY = %q, want the credentialed --proxy", got["PILOT_PROXY"]) + } + for _, k := range []string{"HTTPS_PROXY", "https_proxy", "NO_PROXY", "no_proxy", "ALL_PROXY", "SSL_CERT_FILE", "SSL_CERT_DIR", "PILOT_TRANSPORT"} { + if got[k] == "" { + t.Errorf("%s did not reach the daemon", k) + } + } +} + +// TestCLIDaemonStartForegroundOlderDaemon: config.json asks for compat + +// proxy=auto but the paired pilot-daemon predates -proxy. The start must +// still succeed (flag dropped with a warning) instead of crash-looping. +func TestCLIDaemonStartForegroundOlderDaemon(t *testing.T) { + t.Parallel() + dir := t.TempDir() + out := filepath.Join(dir, "daemon") + bin := writeFakeDaemon(t, append([]string{"transport"}, baseDaemonFlags...), out) + home := t.TempDir() + if err := os.MkdirAll(filepath.Join(home, ".pilot"), 0o700); err != nil { + t.Fatal(err) + } + cfg := `{"registry":"` + productionRegistryAddr + `","transport":"compat","proxy":"auto"}` + if err := os.WriteFile(filepath.Join(home, ".pilot", "config.json"), []byte(cfg), 0o600); err != nil { + t.Fatal(err) + } + env := cliEnvCleared(map[string]string{ + "PILOT_DAEMON_BIN": bin, + "PILOT_SOCKET": filepath.Join(dir, "pilot.sock"), + "PILOT_HOME": home, + }) + _, stderr, code := runCLI(t, []string{"daemon", "start", "--foreground"}, env) + if code != 0 { + t.Fatalf("exit=%d stderr=%s", code, stderr) + } + if !strings.Contains(stderr, "does not support -proxy") { + t.Errorf("expected a skew warning, stderr=%s", stderr) + } + args := readLines(t, out+".args") + if argsHasKey(args, "--proxy") { + t.Errorf("older daemon must not receive --proxy: %v", args) + } + if !argsHasPair(args, "--transport", "compat") || argsHasKey(args, "--registry") { + t.Errorf("compat args wrong: %v", args) + } +} + +func TestCLIDaemonStartRejectsBadTransport(t *testing.T) { + t.Parallel() + _, stderr, code := runCLI(t, []string{"daemon", "start", "--transport", "compact"}, cliEnvCleared(nil)) + if code == 0 || !strings.Contains(stderr, "invalid transport") { + t.Errorf("exit=%d stderr=%s", code, stderr) + } + _, stderr, code = runCLI(t, []string{"daemon", "start", "--proxy", "socks5://u:s3cret@p:1080"}, cliEnvCleared(nil)) + if code == 0 || !strings.Contains(stderr, "invalid proxy") { + t.Errorf("exit=%d stderr=%s", code, stderr) + } + if strings.Contains(stderr, "s3cret") { + t.Errorf("invalid-proxy error leaked credentials: %s", stderr) + } +} + +func TestCLIConfigSetProxyRedactsAndValidates(t *testing.T) { + t.Parallel() + home := t.TempDir() + env := cliEnvCleared(map[string]string{"PILOT_HOME": home}) + stdout, stderr, code := runCLI(t, []string{"--json", "config", "--set", "proxy=http://agent:s3cret@egress:3128"}, env) + if code != 0 { + t.Fatalf("exit=%d stderr=%s", code, stderr) + } + if strings.Contains(stdout, "s3cret") || !strings.Contains(stdout, "***@egress:3128") { + t.Errorf("config --set must echo the redacted proxy: %s", stdout) + } + raw, err := os.ReadFile(filepath.Join(home, ".pilot", "config.json")) + if err != nil || !strings.Contains(string(raw), "agent:s3cret@egress:3128") { + t.Errorf("config.json must keep the real value: %s %v", raw, err) + } + stdout, _, code = runCLI(t, []string{"--json", "config"}, env) + if code != 0 || strings.Contains(stdout, "s3cret") { + t.Errorf("config show leaked the proxy password (exit=%d): %s", code, stdout) + } + _, stderr, code = runCLI(t, []string{"config", "--set", "transport=compact"}, env) + if code == 0 || !strings.Contains(stderr, "invalid transport") { + t.Errorf("config --set transport=compact: exit=%d stderr=%s", code, stderr) + } + _, _, code = runCLI(t, []string{"config", "--set", "transport=compat"}, env) + if code != 0 { + t.Errorf("config --set transport=compat: exit=%d", code) + } +} + +func TestRegistryDialHintMentionsProxy(t *testing.T) { + withTransportEnvCleared(t) + if h := registryDialHint("r:9000"); !strings.Contains(h, "PILOT_REGISTRY") { + t.Errorf("no-proxy hint = %q", h) + } + t.Setenv("HTTPS_PROXY", "http://agent:s3cret@egress:3128") + h := registryDialHint("r:9000") + if !strings.Contains(h, "does not use the proxy") || strings.Contains(h, "s3cret") { + t.Errorf("proxy hint = %q", h) + } +} diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 024b0a7d..1d4dcd08 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -20,7 +20,7 @@ Bootstrap: pilotctl config [--set key=value] Daemon lifecycle: - pilotctl daemon start [--config ] [--registry ] [--beacon ] [--email ] [--webhook ] [--trust-auto-approve] + pilotctl daemon start [--config ] [--registry ] [--beacon ] [--email ] [--webhook ] [--trust-auto-approve] [--transport ] [--proxy ] pilotctl daemon stop pilotctl daemon status @@ -117,6 +117,9 @@ Diagnostic commands: Environment: PILOT_REGISTRY Registry address (default: 34.71.57.205:9000) PILOT_SOCKET Daemon socket path (default: /tmp/pilot.sock) + PILOT_TRANSPORT daemon start transport: udp or compat (TCP 443 only) + PILOT_PROXY daemon proxy policy: auto, off, or http(s)://[user:pass@]host:port + HTTPS_PROXY proxy used by compat mode (proxy=auto) and pilotctl's HTTP calls Version: pilotctl version From 9f574ce0703b7851dc36b944461f5516e3f1b33d Mon Sep 17 00:00:00 2001 From: Teodor Calin Date: Thu, 24 Sep 2026 00:11:01 +0300 Subject: [PATCH 02/19] feat(install): --transport compat for UDP-blocked / proxy-only hosts install.sh --transport compat (or PILOT_TRANSPORT=compat) merges "transport": "compat" and "proxy": "auto" into ~/.pilot/config.json via `pilotctl config --set` (atomic, 0600, other keys kept) and generates compat systemd/launchd units (-transport compat, raw-TCP registry/beacon defaults left to the daemon). A re-run without the flag keeps a compat config. Audited for a no-root, no-systemd VM whose only egress is an authenticating CONNECT-:443 proxy with poisoned local DNS for *.pilotprotocol.network: every network call is curl over HTTPS (proxy env honored, CONNECT by hostname, no wget / raw IP / non-443 / registry or beacon probes). Proxy URLs are only ever printed redacted. Download failures now name the proxy and the hosts it must allow instead of falling silently into "Go is required". With an older release (pilotctl without --transport) compat points config's registry at registry.pilotprotocol.network:443, and a pilot-daemon without -proxy gets an explicit warning when HTTPS_PROXY is set. No-systemd hosts get the `pilotctl daemon start` hint; service-manager units get a note that they do not inherit the shell's HTTPS_PROXY. Tested in docker (debian bookworm, non-root, temp HOME) on an --internal network whose only egress is an authenticating CONNECT-443 proxy, with poisoned /etc/hosts for the Pilot names: v1.13.9 download + SHA-256 verify + install succeeds; missing/wrong proxy credentials fail with a clear hint. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 33 ++++++++ README.md | 12 +++ install.sh | 229 +++++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 256 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83b6b90b..6254e481 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,8 +18,41 @@ Detailed per-release notes are on the `false` (the default) to switch app auto-updates back on. Pilot daemon/CLI binary updates are never affected. Honors the existing `PILOT_UPDATER_NO_APP_UPGRADE` as a back-compat alias. +- **`pilotctl daemon start --transport --proxy `.** + Also read from config.json (`transport`, `proxy`; set them with + `pilotctl config --set`) and, for the transport, `$PILOT_TRANSPORT`. Both are + forwarded to pilot-daemon only when set, and dropped with a warning when the + paired daemon binary predates them, so a new pilotctl still starts an older + daemon. A proxy URL with credentials travels as `$PILOT_PROXY`, never on the + daemon's argv, and is shown redacted by `pilotctl config`. +- **`daemon start` forwards the proxy/TLS environment** (`HTTPS_PROXY`, + `HTTP_PROXY`, `ALL_PROXY`, `NO_PROXY` in both cases, `PILOT_PROXY`, + `PILOT_TRANSPORT`, `SSL_CERT_FILE`, `SSL_CERT_DIR`) to the daemon on both the + fork and `--foreground` paths, and passes through `--compat-beacon`, + `--registry-trust`, `--registry-fingerprint` and `--tls-trust`. +- **`install.sh --transport compat`** (or `PILOT_TRANSPORT=compat`) for hosts + that block UDP or reach the internet only through an authenticating HTTPS + proxy: writes `transport=compat` + `proxy=auto` into config.json and + generates compat service units. Every installer download goes through + `$HTTPS_PROXY` (CONNECT to :443 by hostname only); service setup without + root/systemd/launchd degrades to a printed `pilotctl daemon start` hint, + download failures name the proxy (redacted) and the hosts it must allow. ### Fixed +- **Compat daemons started by `pilotctl` were pinned to the raw-TCP registry.** + `pilotctl init` writes `34.71.57.205:9000` into config.json and `daemon + start` forwarded it as an explicit `-registry`, which stops pilot-daemon from + switching to `registry.pilotprotocol.network:443` (TLS) in compat mode — the + handshake then failed, or the proxy refused the `:9000` CONNECT. In compat + mode the compiled-in registry/beacon defaults are now left to the daemon. +- **`PILOT_TRANSPORT=compat` had no effect through `pilotctl daemon start`.** + pilot-daemon's `-transport` flag defaults to `udp`, masking the env var; + pilotctl now resolves it and passes an explicit `-transport`. +- **`pilotctl daemon start` on a fresh home failed with "PID file locked".** + Without an existing `~/.pilot`, the PID-file claim hit ENOENT and was + reported as a concurrent start on every attempt. +- **`daemon start --endpoint` / `--motd-feed-url` / `--motd-interval`** were + documented but never forwarded to the daemon. - **The `pilotctl skills disable` opt-out now survives updates and explicit reconciles.** A forced reconcile — `pilotctl skills check`, `pilotctl update`, or an installer re-run — bypassed the disabled flag and re-injected skills a diff --git a/README.md b/README.md index 4f34e7fc..3ad5af88 100644 --- a/README.md +++ b/README.md @@ -287,6 +287,15 @@ Set a hostname and email during install: curl -fsSL https://pilotprotocol.network/install.sh | PILOT_EMAIL=user@example.com PILOT_HOSTNAME=my-agent sh ``` +UDP blocked, or the only way out is an HTTPS proxy (hosted agent sandboxes, locked-down VMs)? Install in compat mode: + +```bash +curl -fsSL https://pilotprotocol.network/install.sh | sh -s -- --transport compat +pilotctl daemon start +``` + +Compat mode keeps every connection on TCP 443 (registry over TLS, beacon over WSS). `--transport compat` writes `"transport": "compat"` and `"proxy": "auto"` to `~/.pilot/config.json`; with `proxy=auto` the daemon tunnels through `$HTTPS_PROXY` / `$ALL_PROXY` (honoring `$NO_PROXY`) when set, asking the proxy to `CONNECT` by hostname. No root, systemd or launchd needed: start the daemon with `pilotctl daemon start` from a shell that has the proxy variables. To switch an existing install: `pilotctl config --set transport=compat`, or one-off `pilotctl daemon start --transport compat --proxy `. +
What the installer does @@ -474,6 +483,9 @@ Most daemon flags have an environment variable equivalent. Useful for containeri | `PILOT_EMAIL` | `-email` | Account email | | `PILOT_HOSTNAME` | `-hostname` | Discovery hostname | | `PILOT_ADMIN_TOKEN` | `-admin-token` | Admin token for network operations | +| `PILOT_TRANSPORT` | `-transport` | `udp` (default) or `compat` (TCP 443 only). `pilotctl daemon start` turns it into an explicit `-transport` | +| `PILOT_PROXY` | `-proxy` | `auto` (default: proxy from the environment in compat mode), `off`, or `http(s)://[user:pass@]host:port` | +| `HTTPS_PROXY` / `ALL_PROXY` / `NO_PROXY` | — | Proxy used by compat mode with `proxy=auto`; `pilotctl daemon start` forwards them (and `SSL_CERT_FILE` / `SSL_CERT_DIR`) to the daemon | | `PILOT_MOTD_URL` | `-motd-feed-url` | Message-of-the-day feed URL | | `PILOT_TELEMETRY_URL` | `-telemetry-url` | Telemetry endpoint override | | `PILOT_SYN_WHITELIST` | `-syn-whitelist` | Nodes exempt from SYN rate limit | diff --git a/install.sh b/install.sh index f8b54f31..6b62ca17 100755 --- a/install.sh +++ b/install.sh @@ -9,6 +9,8 @@ set -e # Install: curl -fsSL https://pilotprotocol.network/install.sh | sh # Pin a version: curl -fsSL https://pilotprotocol.network/install.sh | sh -s -- --version v1.13.6 # Beta channel: curl -fsSL https://pilotprotocol.network/install.sh | sh -s -- --channel beta +# UDP blocked / curl -fsSL https://pilotprotocol.network/install.sh | sh -s -- --transport compat +# HTTPS proxy: (TCP 443 only; the daemon uses $HTTPS_PROXY when it is set) # Uninstall: curl -fsSL https://pilotprotocol.network/install.sh | sh -s uninstall # # Flags: @@ -19,6 +21,12 @@ set -e # never silently falls back to an unverified source build. # --yes / -y Skip the older-version confirmation prompt. # --no-warn Suppress the older-version warning entirely. +# --transport udp (default) or compat. compat writes "transport": +# "compat" and "proxy": "auto" into ~/.pilot/config.json: +# the daemon then talks TLS/WSS over TCP 443 only and, when +# $HTTPS_PROXY/$ALL_PROXY is set, goes through that proxy +# (CONNECT by hostname). For UDP-blocked hosts and agent +# sandboxes whose only way out is an HTTPS proxy. # # Legacy env vars (still honored, lower precedence than flags): # PILOT_RELEASE_TAG=vX.Y.Z Same as --version. @@ -27,6 +35,13 @@ set -e # non-interactive/headless installs (no TTY prompt). # If omitted headless, the daemon auto-synthesizes a # @nodes.pilotprotocol.network identity. +# PILOT_TRANSPORT=compat Same as --transport compat. +# +# Proxies: every download is a curl HTTPS request, so HTTPS_PROXY / https_proxy / +# ALL_PROXY / NO_PROXY are honored (curl asks the proxy to CONNECT by hostname — +# no local DNS lookup of the target). Nothing here needs UDP, a non-443 port, or +# a direct connection to the registry/beacon. Steps that need root, sudo, +# systemd or launchd are skipped with a message, never fatal. # # WHAT THIS SCRIPT DOES (read before piping to sh): # 1. Detects OS/arch (Linux/Darwin × amd64/arm64) @@ -85,8 +100,13 @@ set -e # error. REPO="pilot-protocol/pilotprotocol" -REGISTRY="${PILOT_REGISTRY:-34.71.57.205:9000}" -BEACON="${PILOT_BEACON:-34.71.57.205:9001}" +# Production defaults — the same raw-TCP/UDP endpoints compiled into +# pilot-daemon. Compat mode must not pin them explicitly (see NET_FLAGS). +DEFAULT_REGISTRY="34.71.57.205:9000" +DEFAULT_BEACON="34.71.57.205:9001" +COMPAT_REGISTRY="registry.pilotprotocol.network:443" +REGISTRY="${PILOT_REGISTRY:-$DEFAULT_REGISTRY}" +BEACON="${PILOT_BEACON:-$DEFAULT_BEACON}" PILOT_DIR="$HOME/.pilot" BIN_DIR="$PILOT_DIR/bin" @@ -132,6 +152,7 @@ PILOT_REQUESTED_CHANNEL="" PILOT_YES=0 PILOT_NO_WARN=0 PILOT_POSITIONAL="" +PILOT_REQUESTED_TRANSPORT="" while [ $# -gt 0 ]; do case "$1" in @@ -145,12 +166,17 @@ while [ $# -gt 0 ]; do PILOT_REQUESTED_CHANNEL="$2"; shift 2 ;; --channel=*) PILOT_REQUESTED_CHANNEL="${1#--channel=}"; shift ;; + --transport) + if [ $# -lt 2 ]; then echo "Error: --transport requires a value" >&2; exit 2; fi + PILOT_REQUESTED_TRANSPORT="$2"; shift 2 ;; + --transport=*) + PILOT_REQUESTED_TRANSPORT="${1#--transport=}"; shift ;; --yes|-y) PILOT_YES=1; shift ;; --no-warn) PILOT_NO_WARN=1; shift ;; -h|--help) - sed -n '4,21p' "$0" 2>/dev/null || echo "See https://pilotprotocol.network/install.sh" + sed -n '4,29p' "$0" 2>/dev/null || echo "See https://pilotprotocol.network/install.sh" exit 0 ;; --) shift @@ -179,6 +205,14 @@ if [ -n "$PILOT_REQUESTED_CHANNEL" ] \ exit 2 fi +# --transport beats the PILOT_TRANSPORT env var. Empty means "not requested on +# this run": a re-run keeps whatever transport config.json already has. +TRANSPORT="${PILOT_REQUESTED_TRANSPORT:-${PILOT_TRANSPORT:-}}" +if [ -n "$TRANSPORT" ] && [ "$TRANSPORT" != "udp" ] && [ "$TRANSPORT" != "compat" ]; then + echo "Error: --transport must be 'udp' or 'compat' (got: $TRANSPORT)" >&2 + exit 2 +fi + # Restore positional args so the existing uninstall handler still uses $1. # shellcheck disable=SC2086 # intentional word-split on PILOT_POSITIONAL set -- $PILOT_POSITIONAL @@ -192,6 +226,50 @@ if [ "${1:-}" != "uninstall" ] && [ "$(id -u)" = "0" ] && [ -z "${PILOT_ALLOW_RO exit 1 fi +# A re-run without --transport keeps the transport the existing config.json +# selects, so regenerated service units stay consistent with it. +EFFECTIVE_TRANSPORT="${TRANSPORT:-udp}" +if [ -z "$TRANSPORT" ] && [ -f "$PILOT_DIR/config.json" ] \ + && grep -q '"transport"[[:space:]]*:[[:space:]]*"compat"' "$PILOT_DIR/config.json" 2>/dev/null; then + EFFECTIVE_TRANSPORT="compat" +fi + +# --- Egress proxy --- +# +# Every download below is a curl HTTPS request, and curl honors HTTPS_PROXY / +# https_proxy / ALL_PROXY / NO_PROXY on its own, asking the proxy to CONNECT +# by hostname (no local DNS lookup of the target — which matters where local +# DNS for pilotprotocol.network is poisoned). PILOT_PROXY_URL is only used in +# messages, and only ever printed redacted: the userinfo of an +# authenticating proxy is a credential. +PILOT_PROXY_URL="${HTTPS_PROXY:-${https_proxy:-${ALL_PROXY:-${all_proxy:-}}}}" + +# redact_proxy URL — print URL with any "user:pass@" replaced by "***@". +redact_proxy() { + case "$1" in + *@*) + _rp_scheme="" + case "$1" in *://*) _rp_scheme="${1%%://*}://" ;; esac + printf '%s***@%s\n' "$_rp_scheme" "${1##*@}" ;; + *) + printf '%s\n' "$1" ;; + esac +} + +# net_hint — after a failed download, say what to check. Behind an egress +# proxy the usual cause is the proxy refusing the CONNECT (407: bad +# credentials, 403: host not allowed), not a missing release. +net_hint() { + if [ -n "$PILOT_PROXY_URL" ]; then + echo " Note: downloads go through the proxy $(redact_proxy "$PILOT_PROXY_URL")." >&2 + echo " Check that it accepts CONNECT to pilotprotocol.network:443, github.com:443" >&2 + echo " and *.githubusercontent.com:443, and that its credentials are right." >&2 + else + echo " Note: check outbound HTTPS to pilotprotocol.network and github.com. If this host" >&2 + echo " can only reach the internet through a proxy, export HTTPS_PROXY and re-run." >&2 + fi +} + # --- Manifest + version helpers --- # fetch_manifest writes the manifest JSON to $1 and returns 0 on success. @@ -386,8 +464,15 @@ echo " Pilot Protocol" echo " The network stack for AI agents." echo "" echo " Platform: ${OS}/${ARCH}" -echo " Registry: ${REGISTRY}" -echo " Beacon: ${BEACON}" +if [ "$EFFECTIVE_TRANSPORT" = "compat" ]; then + echo " Transport: compat (TLS + WSS over TCP 443 only)" +else + echo " Registry: ${REGISTRY}" + echo " Beacon: ${BEACON}" +fi +if [ -n "$PILOT_PROXY_URL" ]; then + echo " Proxy: $(redact_proxy "$PILOT_PROXY_URL") (from environment)" +fi echo "" # --- Resolve email --- @@ -548,11 +633,13 @@ if [ -z "$TAG" ]; then if [ -n "$PILOT_REQUESTED_CHANNEL" ]; then echo "Error: channel '$PILOT_REQUESTED_CHANNEL' resolved to no release (manifest reachable: $HAVE_MANIFEST)." >&2 echo " Refusing to fall back to an unverified source build for an explicit channel request." >&2 + [ "$HAVE_MANIFEST" = "1" ] || net_hint exit 1 fi if [ "${PILOT_RC:-}" = "1" ]; then echo "Error: the beta/prerelease channel resolved to no release." >&2 echo " Refusing to fall back to an unverified source build for an explicit channel request." >&2 + [ "$HAVE_MANIFEST" = "1" ] || net_hint exit 1 fi fi @@ -662,11 +749,13 @@ if [ -n "$TAG" ]; then # Archive download failed. Only the automatic default path may fall # back to a source build; an explicit request already hard-failed # above, so reaching here means no version/channel was pinned. + echo " Could not download ${URL}" >&2 TAG="" fi fi if [ -z "$TAG" ]; then + net_hint echo "No release available. Building from source..." if ! command -v go >/dev/null 2>&1; then echo "Error: Go is required to build from source." @@ -901,6 +990,94 @@ CONF echo "Config written to ${PILOT_DIR}/config.json" fi +# --- Transport: udp (default) or compat --- +# +# Merged into config.json through pilotctl (atomic write, 0600, every other +# key kept) instead of rewriting the file, so a hand-edited config survives a +# re-run. PILOT_HOME is blanked so the write lands in THIS install's +# $HOME/.pilot, which is also the file pilot-daemon auto-loads. +pilot_config_set() { + PILOT_HOME='' "$BIN_DIR/pilotctl" config --set "$1" >/dev/null 2>&1 +} + +if [ -n "$TRANSPORT" ]; then + if pilot_config_set "transport=$TRANSPORT"; then + echo "Transport set to ${TRANSPORT} in ${PILOT_DIR}/config.json" + else + echo " Note: could not save transport=${TRANSPORT} — run: pilotctl config --set transport=${TRANSPORT}" + fi +fi + +# What the installed binaries support. Both probes are local (no network). +DAEMON_HAS_TRANSPORT=false +DAEMON_HAS_PROXY=false +_daemon_help=$("$BIN_DIR/pilot-daemon" -help 2>&1 || true) +if printf '%s\n' "$_daemon_help" | grep -qE '^[[:space:]]+-transport([[:space:]]|$)'; then + DAEMON_HAS_TRANSPORT=true +fi +if printf '%s\n' "$_daemon_help" | grep -qE '^[[:space:]]+-proxy([[:space:]]|$)'; then + DAEMON_HAS_PROXY=true +fi + +if [ "$EFFECTIVE_TRANSPORT" = "compat" ]; then + # proxy=auto: in compat mode the daemon routes every outbound connection + # through $HTTPS_PROXY / $ALL_PROXY when set (honoring $NO_PROXY), and + # connects directly when not. An operator-set proxy value is kept. + if ! grep -q '"proxy"' "$PILOT_DIR/config.json" 2>/dev/null; then + pilot_config_set "proxy=auto" || true + fi + + if [ "$DAEMON_HAS_TRANSPORT" != true ]; then + echo "" + echo " WARNING: this pilot-daemon (${TAG:-source}) predates compat mode (-transport)." + echo " It will keep using UDP. Re-run without --version to get the latest release." + fi + + # pilotctl releases before --transport forward config.json's registry to + # the daemon verbatim, and a daemon given the raw-TCP default explicitly + # stays on it even in compat mode. Point such installs at the compat TLS + # registry directly — only when the file still holds the stock default. + if ! "$BIN_DIR/pilotctl" daemon start --help 2>&1 | grep -q -- '--transport' \ + && grep -q "\"registry\"[[:space:]]*:[[:space:]]*\"${DEFAULT_REGISTRY}\"" "$PILOT_DIR/config.json" 2>/dev/null; then + if pilot_config_set "registry=${COMPAT_REGISTRY}"; then + echo " Registry set to ${COMPAT_REGISTRY} for compat mode (this pilotctl" + echo " always passes config.json's registry to the daemon). Switching back to" + echo " UDP later: pilotctl config --set registry=${DEFAULT_REGISTRY}" + fi + fi + + if [ -n "$PILOT_PROXY_URL" ] && [ "$DAEMON_HAS_PROXY" != true ]; then + echo "" + echo " WARNING: HTTPS_PROXY is set, but this pilot-daemon (${TAG:-source}) cannot use a" + echo " proxy. Where the proxy is the only way out, the daemon will not come" + echo " online. Install a release whose 'pilot-daemon -help' lists -proxy." + fi +fi + +# Network flags for the service units. In compat mode the raw-TCP default +# registry/beacon are left off (an explicit -registry pins a compat daemon to +# a port no 443-only network or HTTPS proxy will carry); a custom +# PILOT_REGISTRY / PILOT_BEACON is kept. UDP units are unchanged. +if [ "$EFFECTIVE_TRANSPORT" = "compat" ] && [ "$DAEMON_HAS_TRANSPORT" = true ]; then + NET_FLAGS="-transport compat" + if [ "$REGISTRY" != "$DEFAULT_REGISTRY" ]; then NET_FLAGS="$NET_FLAGS -registry $REGISTRY"; fi + if [ "$BEACON" != "$DEFAULT_BEACON" ]; then NET_FLAGS="$NET_FLAGS -beacon $BEACON"; fi +else + NET_FLAGS="-registry $REGISTRY -beacon $BEACON" +fi + +# service_proxy_note UNIT — a service manager starts the daemon with its own +# environment, not this shell's, so an HTTPS_PROXY exported here never +# reaches it. config.json (0600, read by the daemon itself) does. +service_proxy_note() { + if [ "$EFFECTIVE_TRANSPORT" = "compat" ] && [ -n "$PILOT_PROXY_URL" ] \ + && ! grep -q '"proxy"[[:space:]]*:[[:space:]]*"http' "$PILOT_DIR/config.json" 2>/dev/null; then + echo " Note: $1 does not inherit this shell's HTTPS_PROXY. For the service to use" + echo " the proxy, save it in config.json (0600):" + echo " pilotctl config --set proxy=''" + fi +} + # Enable background auto-updates by default (opt-out). The install output and # the systemd/launchd units below promise the updater keeps binaries current; # the pilot-updater treats a MISSING control file as "disabled", so without @@ -969,8 +1146,7 @@ Wants=network-online.target Type=simple User=$(whoami) ExecStart=${BIN_DIR}/pilot-daemon \\ - -registry ${REGISTRY} \\ - -beacon ${BEACON} \\ + ${NET_FLAGS} \\ -listen :4000 \\ -socket /tmp/pilot.sock \\ -identity ${PILOT_DIR}/identity.json \\ @@ -1010,6 +1186,7 @@ USVC $PILOT_SUDO systemctl daemon-reload || true echo " Service: pilot-daemon.service" echo " Service: pilot-updater.service (auto-updates)" + service_proxy_note "pilot-daemon.service" # Auto-enable + start the updater so future releases land without # operator action. The unit file alone is not enough — without this, @@ -1049,13 +1226,18 @@ USVC esac else echo " Skipped systemd setup (run as root or with passwordless sudo to enable)" + echo " Start the daemon without a service manager: pilotctl daemon start" fi elif [ "$OS" = "linux" ]; then - # systemd is not the init system here (container / WSL / CI runner). - # There is no service to install — tell the agent the portable start path - # instead of silently leaving it with no daemon. - echo "No systemd detected (container / WSL / CI) — start the daemon manually:" + # systemd is not the init system here (container / WSL / CI runner / + # hosted agent sandbox). There is no service to install — tell the agent + # the portable start path instead of silently leaving it with no daemon. + echo "No systemd detected (container / WSL / CI / sandbox) — start the daemon manually:" echo " pilotctl daemon start" + if [ "$EFFECTIVE_TRANSPORT" = "compat" ]; then + echo " (config.json selects transport=compat; start it from a shell that has" + echo " HTTPS_PROXY set if this host reaches the internet only through a proxy)" + fi fi if [ "$OS" = "darwin" ]; then @@ -1086,6 +1268,13 @@ if [ "$OS" = "darwin" ]; then # empty value passes a blank argv element to the daemon; omitting # it lets the daemon do the documented thing instead — fall back to # ~/.pilot/account.json, then synthesise a fingerprint identity. + # One per word of NET_FLAGS (host:port / flag names only — + # validate_safe already rejected anything with spaces or markup). + PLIST_NET_ARGS="" + for _a in $NET_FLAGS; do + PLIST_NET_ARGS="${PLIST_NET_ARGS} ${_a} +" + done EXTRA_ARGS="" if [ -n "$EMAIL" ]; then EXTRA_ARGS="${EXTRA_ARGS} -email @@ -1111,11 +1300,7 @@ if [ "$OS" = "darwin" ]; then ProgramArguments ${BIN_DIR}/pilot-daemon - -registry - ${REGISTRY} - -beacon - ${BEACON} - -listen +${PLIST_NET_ARGS} -listen :4000 -socket /tmp/pilot.sock @@ -1168,6 +1353,7 @@ UPLIST echo " Service: network.pilotprotocol.pilot-daemon" echo " Service: network.pilotprotocol.pilot-updater (auto-updates)" + service_proxy_note "the launchd agent" # Auto-load the updater LaunchAgent so future releases land without # operator action. Without this, install.sh writes the plist but leaves @@ -1310,8 +1496,15 @@ echo " pilotctl ${BIN_DIR}/pilotctl" [ -f "$BIN_DIR/pilot-updater" ] && echo " pilot-updater ${BIN_DIR}/pilot-updater (auto-updates in background)" echo "" echo "Config: ${PILOT_DIR}/config.json" -echo " Registry: ${REGISTRY}" -echo " Beacon: ${BEACON}" +if [ "$EFFECTIVE_TRANSPORT" = "compat" ]; then + echo " Transport: compat (registry ${COMPAT_REGISTRY} over TLS, beacon over WSS)" + if [ -n "$PILOT_PROXY_URL" ]; then + echo " Proxy: auto -> $(redact_proxy "$PILOT_PROXY_URL") (from environment)" + fi +else + echo " Registry: ${REGISTRY}" + echo " Beacon: ${BEACON}" +fi echo " Socket: /tmp/pilot.sock" echo " Identity: ${PILOT_DIR}/identity.json" echo " Email: ${EMAIL}" From 3a89f62d54f3a345eb71e43ff82051bf64fd4671 Mon Sep 17 00:00:00 2001 From: Teodor Calin Date: Thu, 24 Sep 2026 00:23:25 +0300 Subject: [PATCH 03/19] feat(daemon): native HTTPS proxy support (-proxy / PILOT_PROXY) Hosted agent sandboxes such as Meta Muse block outbound UDP, poison local DNS for *.pilotprotocol.network, and allow only an authenticating HTTP proxy that CONNECTs to :443. Until now the daemon ignored HTTPS_PROXY entirely, so compat mode needed root, an SNI router and a mount-namespace /etc/hosts override to get online. New flag -proxy (env PILOT_PROXY, config key "proxy"), resolved once at startup by daemon.ResolveProxy into a common/netproxy policy: auto (default) -transport=compat: HTTPS_PROXY/https_proxy, falling back to ALL_PROXY/all_proxy, honoring NO_PROXY. -transport=udp: no policy, dialing exactly as before. off never proxy (HTTP clients also stop following env). http(s)://URL that proxy for every outbound TCP/HTTP connection. The policy (daemon.Config.Proxy) is applied to every outbound connection: the registry client (primary, pool and reconnects, via registry.WithDialer), the compat WSS beacon (wss.Config.Proxy), the MOTD fetch, and http.DefaultTransport for plugin HTTP clients (catalogue pins, skillinject, trustedagents, webhook, enterprise control). Targets are CONNECTed by host name and never resolved locally; TLS, SNI and pinned-fingerprint checks stay end to end. One startup line logs the transport and the redacted proxy. Compat mode now treats the compiled-in raw-TCP registry default (34.71.57.205:9000, which pilotctl passes on every daemon start) as not explicit, so it still switches to registry.pilotprotocol.network:443. -transport honors PILOT_TRANSPORT. -beacon-rtt-probe is skipped in compat mode (its UDP probes cannot leave the host). Pins github.com/pilot-protocol/common to the netproxy feature commit (v0.5.14-0.20260923210943-65ea5b1a3d2d); move to a tagged release before merge. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 2 + cmd/daemon/main.go | 31 +- cmd/daemon/proxy.go | 99 +++ cmd/daemon/proxy_test.go | 334 ++++++++++ go.mod | 2 +- go.sum | 4 +- pkg/daemon/beacon_discovery.go | 4 +- pkg/daemon/daemon.go | 33 +- pkg/daemon/proxy.go | 71 +++ pkg/daemon/transport/wss/wss.go | 13 +- pkg/daemon/transport/wss/zz_wss_proxy_test.go | 401 ++++++++++++ pkg/daemon/tunnel.go | 10 +- pkg/daemon/zz_proxy_test.go | 601 ++++++++++++++++++ 13 files changed, 1587 insertions(+), 18 deletions(-) create mode 100644 cmd/daemon/proxy.go create mode 100644 cmd/daemon/proxy_test.go create mode 100644 pkg/daemon/proxy.go create mode 100644 pkg/daemon/transport/wss/zz_wss_proxy_test.go create mode 100644 pkg/daemon/zz_proxy_test.go diff --git a/README.md b/README.md index 4f34e7fc..c9e84664 100644 --- a/README.md +++ b/README.md @@ -470,6 +470,8 @@ Most daemon flags have an environment variable equivalent. Useful for containeri |----------|----------------|---------| | `PILOT_REGISTRY` | `-registry` | Registry server address | | `PILOT_BEACON` | `-beacon` | Beacon server address | +| `PILOT_TRANSPORT` | `-transport` | Tunnel transport: `udp` (default) or `compat` (WSS on 443, for UDP-blocked hosts) | +| `PILOT_PROXY` | `-proxy` | Outbound proxy for registry, beacon and HTTP traffic: `auto` (default; with `compat`, the `HTTPS_PROXY`/`ALL_PROXY` proxy, honoring `NO_PROXY`), `off`, or `http://[user:pass@]host:port` for every connection | | `PILOT_SOCKET` | `-socket` | Unix socket path | | `PILOT_EMAIL` | `-email` | Account email | | `PILOT_HOSTNAME` | `-hostname` | Discovery hostname | diff --git a/cmd/daemon/main.go b/cmd/daemon/main.go index efff87a9..248b7c1f 100644 --- a/cmd/daemon/main.go +++ b/cmd/daemon/main.go @@ -22,6 +22,7 @@ import ( "github.com/pilot-protocol/common/config" "github.com/pilot-protocol/common/driver" "github.com/pilot-protocol/common/logging" + "github.com/pilot-protocol/common/netproxy" "github.com/pilot-protocol/pilotprotocol/internal/enterprisecontrol" "github.com/pilot-protocol/pilotprotocol/internal/managedsdk/authority" "github.com/pilot-protocol/pilotprotocol/internal/motd" @@ -52,13 +53,13 @@ var remoteLifecycleRequests = make(chan string, 1) func main() { configPath := flag.String("config", "", "path to config file (JSON)") securityProfile := flag.String("security-profile", envString("PILOT_SECURITY_PROFILE", securityProfileCompatible), "locked security profile: compatible or enterprise") - registryDefault := "34.71.57.205:9000" + registryDefault := defaultRegistryAddr registryFromEnv := false if v := os.Getenv("PILOT_REGISTRY"); v != "" { registryDefault = v registryFromEnv = true } - beaconDefault := "34.71.57.205:9001" + beaconDefault := defaultBeaconAddr beaconFromEnv := false if v := os.Getenv("PILOT_BEACON"); v != "" { beaconDefault = v @@ -108,7 +109,8 @@ func main() { beaconRTTProbe := flag.Bool("beacon-rtt-probe", false, "probe beacon RTT before selection; override hash pick when >2× slower than best (ablation test, default off)") noRxWatchdog := flag.Bool("no-rx-watchdog", false, "disable the inbound-path watchdog that soft-recovers (beacon+registry re-registration) and, on a persistent wedge, exits non-zero for supervisor respawn") noPathWatch := flag.Bool("no-path-watch", false, "disable the per-peer path watchdog that probes inbound-silent peers and resets a dead peer path in place (prefer-direct sequence) without a daemon restart") - transportMode := flag.String("transport", "udp", "tunnel transport: 'udp' (default) or 'compat' (WSS to beacon, opt-in, for UDP-blocked environments)") + transportMode := flag.String("transport", transportDefault(), "tunnel transport: 'udp' (default) or 'compat' (WSS to beacon, opt-in, for UDP-blocked environments). Env: PILOT_TRANSPORT.") + proxySpec := flag.String("proxy", envString("PILOT_PROXY", netproxy.ModeAuto), "outbound proxy for registry, beacon and HTTP connections: 'auto' (with -transport=compat, HTTPS_PROXY/ALL_PROXY from the environment, honoring NO_PROXY; nothing with -transport=udp), 'off', or a proxy URL 'http://[user:pass@]host:port' used for every connection. Env: PILOT_PROXY.") compatBeacon := flag.String("compat-beacon", "wss://beacon.pilotprotocol.network/v1/compat", "beacon WSS URL for -transport=compat") tlsTrust := flag.String("tls-trust", "system", "TLS trust store for -transport=compat: 'system' (OS trust store; current default while compat mode uses Let's Encrypt certs on beacon.pilotprotocol.network) or 'pinned' (Pilot CA root embedded in the daemon binary; will become the default in a future release once production root ships)") showVersion := flag.Bool("version", false, "print version and exit") @@ -174,13 +176,18 @@ func main() { // -registry-trust, route the registry to its TLS hostname (TCP/443 // via nginx SNI routing on the production rendezvous box) so the // daemon really does use a single port. The TCP/9000 fallback is - // still available to anyone who passes -registry explicitly. + // still available to anyone who passes a non-default -registry + // explicitly; the compiled-in default counts as not explicit (see + // compatKeepsRegistry). -beacon needs no such rule: in compat mode the + // UDP beacon address is only the relay-wrap destination on the WSS + // pipe and is never dialed. if *transportMode == "compat" { explicit := map[string]bool{} flag.Visit(func(f *flag.Flag) { explicit[f.Name] = true }) - if !explicit["registry"] && os.Getenv("PILOT_REGISTRY") == "" { - v := "registry.pilotprotocol.network:443" + if !compatKeepsRegistry(*registryAddr, explicit["registry"], os.Getenv("PILOT_REGISTRY") != "") { + v := compatRegistryAddr registryAddr = &v + registryFromEnv = false } if !explicit["registry-tls"] { v := true @@ -221,6 +228,17 @@ func main() { logging.Setup(*logLevel, *logFormat) + // Outbound proxy: resolved once, after -transport is final, and shared + // by everything that dials out — the registry client, the compat WSS + // beacon, pkg/daemon's own HTTP fetches (via daemon.Config.Proxy) and + // every plugin HTTP client (via http.DefaultTransport). + proxyPolicy, err := resolveProxyPolicy(*proxySpec, *transportMode) + if err != nil { + log.Fatalf("-proxy: %v", err) + } + installDefaultTransportProxy(proxyPolicy) + slog.Info("outbound network", "transport", *transportMode, "proxy", describeProxy(*proxySpec, *transportMode, proxyPolicy)) + // Sandbox: validate all configured file paths are under the confinement // root before the daemon touches the filesystem. Network paths are unaffected. if *sandbox { @@ -314,6 +332,7 @@ func main() { TransportMode: *transportMode, CompatBeaconURL: *compatBeacon, CompatTLSTrust: *tlsTrust, + Proxy: proxyPolicy, MOTDFeedURL: *motdFeedURL, MOTDInterval: *motdInterval, TelemetryURL: *telemetryURL, diff --git a/cmd/daemon/proxy.go b/cmd/daemon/proxy.go new file mode 100644 index 00000000..14521685 --- /dev/null +++ b/cmd/daemon/proxy.go @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "log/slog" + "net/http" + "os" + "strings" + + "github.com/pilot-protocol/common/netproxy" + "github.com/pilot-protocol/pilotprotocol/pkg/daemon" +) + +// Compiled-in production endpoints. -registry and -beacon default to these +// raw TCP / UDP addresses (pilotctl passes the same values explicitly); +// compat mode swaps the registry for its TLS host name on :443. +const ( + defaultRegistryAddr = "34.71.57.205:9000" + defaultBeaconAddr = "34.71.57.205:9001" + compatRegistryAddr = "registry.pilotprotocol.network:443" +) + +// transportDefault is the -transport default: $PILOT_TRANSPORT when it +// names a transport, otherwise "udp". An unknown value is ignored with a +// warning, as pkg/daemon has always treated it. +func transportDefault() string { + v := strings.ToLower(strings.TrimSpace(os.Getenv("PILOT_TRANSPORT"))) + switch v { + case "udp", "compat": + return v + case "": + default: + slog.Warn("ignoring unknown PILOT_TRANSPORT value", "value", v, "valid", "udp, compat") + } + return "udp" +} + +// compatKeepsRegistry reports whether a -transport=compat daemon keeps its +// configured registry instead of switching to compatRegistryAddr. Only an +// explicit choice (the -registry flag or $PILOT_REGISTRY) is kept, and the +// compiled-in raw-TCP default never counts as one: pilotctl passes it on +// every `daemon start`, and a UDP-blocked host behind a CONNECT-only egress +// proxy cannot reach it. +func compatKeepsRegistry(addr string, setByFlag, setByEnv bool) bool { + return (setByFlag || setByEnv) && strings.TrimSpace(addr) != defaultRegistryAddr +} + +// resolveProxyPolicy resolves -proxy for the transport (see +// daemon.ResolveProxy). A malformed explicit proxy URL is an error; a +// malformed proxy environment under "auto" only costs the proxy: it is +// logged and the daemon dials directly, as it did before -proxy existed. +func resolveProxyPolicy(spec, transport string) (*netproxy.Resolver, error) { + policy, err := daemon.ResolveProxy(spec, transport) + if err != nil { + if !isAutoProxy(spec) { + return nil, err + } + slog.Warn("proxy environment is malformed; dialing directly", "err", err) + return nil, nil + } + return policy, nil +} + +// describeProxy renders the resolved policy for the startup log line. +// Credentials are always redacted. +func describeProxy(spec, transport string, policy *netproxy.Resolver) string { + if policy != nil { + return policy.String() + } + if isAutoProxy(spec) && transport != "compat" { + return "none (-proxy=auto applies to -transport=compat only)" + } + return "none" +} + +// installDefaultTransportProxy makes net/http's shared DefaultTransport +// follow the policy. Every HTTP client the daemon wires in without a +// transport of its own — catalogue pins, skillinject, trustedagents, +// webhook, enterprise-control clients — uses DefaultTransport, and not all +// of them accept an injected client. nil leaves DefaultTransport alone +// (net/http's own proxy environment handling). Call before any goroutine +// issues a request. +func installDefaultTransportProxy(policy *netproxy.Resolver) { + if policy == nil { + return + } + tr, ok := http.DefaultTransport.(*http.Transport) + if !ok { + slog.Warn("http.DefaultTransport is not an *http.Transport; plugin HTTP clients do not follow -proxy") + return + } + tr.Proxy = policy.ProxyForRequest +} + +func isAutoProxy(spec string) bool { + s := strings.TrimSpace(spec) + return s == "" || strings.EqualFold(s, netproxy.ModeAuto) +} diff --git a/cmd/daemon/proxy_test.go b/cmd/daemon/proxy_test.go new file mode 100644 index 00000000..fe958b27 --- /dev/null +++ b/cmd/daemon/proxy_test.go @@ -0,0 +1,334 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "bufio" + "bytes" + "context" + "fmt" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/pilot-protocol/common/netproxy" +) + +// runMainEnv makes the test binary run the daemon's main() instead of the +// tests, so a test can start the real daemon with real flags and +// environment in a child process. +const runMainEnv = "PILOT_DAEMON_TEST_RUN_MAIN" + +func TestMain(m *testing.M) { + if os.Getenv(runMainEnv) == "1" { + main() + os.Exit(0) + } + os.Exit(m.Run()) +} + +var proxyEnvVars = []string{ + "HTTPS_PROXY", "https_proxy", "ALL_PROXY", "all_proxy", + "HTTP_PROXY", "http_proxy", "NO_PROXY", "no_proxy", "REQUEST_METHOD", +} + +func TestTransportDefault(t *testing.T) { + for _, tc := range []struct{ env, want string }{ + {"", "udp"}, + {"udp", "udp"}, + {"compat", "compat"}, + {" COMPAT ", "compat"}, + {"wss", "udp"}, + } { + t.Setenv("PILOT_TRANSPORT", tc.env) + if got := transportDefault(); got != tc.want { + t.Errorf("PILOT_TRANSPORT=%q: transportDefault() = %q, want %q", tc.env, got, tc.want) + } + } +} + +// The compiled-in raw-TCP registry never counts as an explicit choice in +// compat mode — pilotctl passes it on every `daemon start` — while any +// other address the operator set by flag or environment is kept. +func TestCompatKeepsRegistry(t *testing.T) { + for _, tc := range []struct { + addr string + byFlag, byEnv bool + want bool + }{ + {defaultRegistryAddr, false, false, false}, + {defaultRegistryAddr, true, false, false}, + {defaultRegistryAddr, false, true, false}, + {defaultRegistryAddr, true, true, false}, + {" " + defaultRegistryAddr + " ", true, false, false}, + {"10.0.0.5:9000", true, false, true}, + {"registry.corp.example:443", false, true, true}, + {"10.0.0.5:9000", false, false, false}, // config file / default only + } { + if got := compatKeepsRegistry(tc.addr, tc.byFlag, tc.byEnv); got != tc.want { + t.Errorf("compatKeepsRegistry(%q, flag=%v, env=%v) = %v, want %v", tc.addr, tc.byFlag, tc.byEnv, got, tc.want) + } + } +} + +func TestResolveProxyPolicy(t *testing.T) { + for _, k := range proxyEnvVars { + t.Setenv(k, "") + } + t.Setenv("HTTPS_PROXY", "http://muse:s3cret@egress.test:3128") + + p, err := resolveProxyPolicy("auto", "udp") + if err != nil || p != nil { + t.Fatalf("auto/udp = (%v, %v), want (nil, nil)", p, err) + } + if got := describeProxy("auto", "udp", p); got != "none (-proxy=auto applies to -transport=compat only)" { + t.Errorf("describeProxy(auto/udp) = %q", got) + } + + p, err = resolveProxyPolicy("auto", "compat") + if err != nil || p == nil || p.Mode() != netproxy.ModeAuto || !p.Enabled() { + t.Fatalf("auto/compat = (%v, %v), want an enabled auto policy", p, err) + } + if got := describeProxy("auto", "compat", p); got != "auto: http://***@egress.test:3128" { + t.Errorf("describeProxy(auto/compat) = %q", got) + } + + p, err = resolveProxyPolicy("http://ops:hunter2@flag.test:8080", "udp") + if err != nil || p == nil || p.Mode() != netproxy.ModeExplicit { + t.Fatalf("explicit/udp = (%v, %v), want an explicit policy", p, err) + } + if got := describeProxy("http://ops:hunter2@flag.test:8080", "udp", p); got != "http://***@flag.test:8080" { + t.Errorf("describeProxy(explicit) = %q", got) + } + + p, err = resolveProxyPolicy("off", "compat") + if err != nil || p == nil || p.Mode() != netproxy.ModeOff { + t.Fatalf("off/compat = (%v, %v), want an off policy", p, err) + } + + // A malformed -proxy URL is fatal (operator typo) ... + if _, err := resolveProxyPolicy("ftp://ops:hunter2@flag.test", "compat"); err == nil { + t.Fatal("malformed -proxy URL accepted") + } else if strings.Contains(err.Error(), "hunter2") { + t.Fatalf("error leaks credentials: %v", err) + } + // ... but a malformed environment under auto only costs the proxy. + t.Setenv("HTTPS_PROXY", "ftp://muse:s3cret@egress.test") + p, err = resolveProxyPolicy("auto", "compat") + if err != nil || p != nil { + t.Fatalf("auto/compat with malformed env = (%v, %v), want (nil, nil)", p, err) + } + if got := describeProxy("auto", "compat", p); got != "none" { + t.Errorf("describeProxy(auto/compat, malformed env) = %q, want none", got) + } +} + +// refusingProxy records every request it gets and refuses all of them, so +// nothing a daemon under test sends ever leaves the machine. +type refusingProxy struct { + ln net.Listener + wantAuth string + + mu sync.Mutex + targets []string + badAuths int +} + +func newRefusingProxy(t *testing.T, user, pass string) *refusingProxy { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("proxy listen: %v", err) + } + req := &http.Request{Header: http.Header{}} + req.SetBasicAuth(user, pass) + p := &refusingProxy{ln: ln, wantAuth: req.Header.Get("Authorization")} + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for { + conn, err := ln.Accept() + if err != nil { + return + } + wg.Add(1) + go func() { + defer wg.Done() + defer conn.Close() + conn.SetDeadline(time.Now().Add(10 * time.Second)) + r, err := http.ReadRequest(bufio.NewReader(conn)) + if err != nil { + return + } + p.mu.Lock() + p.targets = append(p.targets, r.Method+" "+r.RequestURI) + if r.Header.Get("Proxy-Authorization") != p.wantAuth { + p.badAuths++ + } + p.mu.Unlock() + fmt.Fprint(conn, "HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + }() + } + }() + t.Cleanup(func() { ln.Close(); wg.Wait() }) + return p +} + +func (p *refusingProxy) snapshot() ([]string, int) { + p.mu.Lock() + defer p.mu.Unlock() + return append([]string(nil), p.targets...), p.badAuths +} + +// syncBuffer is a bytes.Buffer safe for the exec copier goroutine. +type syncBuffer struct { + mu sync.Mutex + b bytes.Buffer +} + +func (s *syncBuffer) Write(p []byte) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.b.Write(p) +} + +func (s *syncBuffer) String() string { + s.mu.Lock() + defer s.mu.Unlock() + return s.b.String() +} + +// TestDaemonCompatThroughProxyEndToEnd runs the real daemon binary the way +// Meta Muse does: `pilotctl daemon start` arguments (the compiled-in raw +// TCP registry passed explicitly), transport from PILOT_TRANSPORT, and the +// egress proxy only in the environment. The compat registry must be +// CONNECTed by host name on :443, with credentials, and plugin HTTP clients +// (the catalogue pin fetch) must use the same proxy — also for ALL_PROXY +// and PILOT_PROXY, which net/http alone would not follow. The proxy refuses +// every request and the registry pin is bogus, so nothing reaches the +// network. +func TestDaemonCompatThroughProxyEndToEnd(t *testing.T) { + for _, tc := range []struct { + name, envVar, wantPrefix string + }{ + {"HTTPS_PROXY", "HTTPS_PROXY", "auto: "}, + {"ALL_PROXY", "ALL_PROXY", "auto: "}, + {"PILOT_PROXY", "PILOT_PROXY", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + proxy := newRefusingProxy(t, "muse", "s3cret") + targets, logs := runDaemonBehindProxy(t, tc.envVar+"="+fmt.Sprintf("http://muse:s3cret@%s", proxy.ln.Addr()), proxy) + t.Logf("proxy requests: %q", targets) + + if _, badAuths := proxy.snapshot(); badAuths != 0 { + t.Errorf("%d proxy request(s) lacked the proxy credentials", badAuths) + } + for _, target := range targets { + if !strings.HasPrefix(target, "CONNECT ") { + t.Errorf("non-CONNECT proxy request %q", target) + } + if strings.Contains(target, "34.71.57.205") { + t.Errorf("proxy was asked for the raw-TCP registry/beacon: %q", target) + } + } + if !contains(targets, "CONNECT raw.githubusercontent.com:443") { + t.Errorf("catalogue fetch did not use the proxy; targets %q", targets) + } + if strings.Contains(logs, "s3cret") { + t.Errorf("daemon output leaks the proxy password:\n%s", logs) + } + value := fmt.Sprintf("%shttp://***@%s", tc.wantPrefix, proxy.ln.Addr()) + if strings.Contains(value, " ") { + value = strconv.Quote(value) // slog's text handler quotes only when needed + } + wantLog := `msg="outbound network" transport=compat proxy=` + value + if !strings.Contains(logs, wantLog) { + t.Errorf("daemon output lacks %s\n%s", wantLog, logs) + } + }) + } +} + +// runDaemonBehindProxy starts main() in a child process with proxyEnv and +// returns the proxy's request targets once the registry CONNECT arrived, +// plus the daemon's output. +func runDaemonBehindProxy(t *testing.T, proxyEnv string, proxy *refusingProxy) ([]string, string) { + t.Helper() + home := t.TempDir() + sockDir, err := os.MkdirTemp("", "pdm") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(sockDir) }) + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, os.Args[0], + "--registry", defaultRegistryAddr, + "--beacon", defaultBeaconAddr, + "--listen", ":0", + "--socket", filepath.Join(sockDir, "s"), + "--identity", filepath.Join(home, "identity.json"), + "--log-level", "info", + "--log-format", "text", + "-registry-trust=pinned", + "-registry-fingerprint="+strings.Repeat("00", 32), + "-no-skillinject", + "-motd-feed-url=", + ) + cmd.Env = []string{ + runMainEnv + "=1", + "HOME=" + home, + "PATH=" + os.Getenv("PATH"), + "TMPDIR=" + os.TempDir(), + "PILOT_TRANSPORT=compat", + "PILOT_NO_SKILLINJECT=1", + "PILOT_APPSTORE_ROOT=" + filepath.Join(home, "apps"), + proxyEnv, + } + var out syncBuffer + cmd.Stdout = &out + cmd.Stderr = &out + if err := cmd.Start(); err != nil { + t.Fatalf("start daemon: %v", err) + } + defer func() { + cancel() + _ = cmd.Wait() + }() + + const registryTarget = "CONNECT registry.pilotprotocol.network:443" + deadline := time.Now().Add(45 * time.Second) + for { + targets, _ := proxy.snapshot() + if contains(targets, registryTarget) { + break + } + if time.Now().After(deadline) { + t.Fatalf("proxy never saw %q; saw %q\ndaemon output:\n%s", registryTarget, targets, out.String()) + } + time.Sleep(50 * time.Millisecond) + } + cancel() + _ = cmd.Wait() + targets, _ := proxy.snapshot() + return targets, out.String() +} + +func contains(list []string, s string) bool { + for _, v := range list { + if v == s { + return true + } + } + return false +} diff --git a/go.mod b/go.mod index 008a5517..3eeae20b 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/coder/websocket v1.8.15 github.com/pilot-protocol/app-store v1.0.2 github.com/pilot-protocol/beacon v0.2.9 - github.com/pilot-protocol/common v0.5.13 + github.com/pilot-protocol/common v0.5.14-0.20260923210943-65ea5b1a3d2d github.com/pilot-protocol/dataexchange v0.2.2 github.com/pilot-protocol/eventstream v0.2.4 github.com/pilot-protocol/handshake v0.2.8 diff --git a/go.sum b/go.sum index 59b33fce..8a23fa4f 100644 --- a/go.sum +++ b/go.sum @@ -237,8 +237,8 @@ github.com/pilot-protocol/app-store v1.0.2 h1:oK7cNl3e/gfxVhhkUFKNLRN256+7sDSBw8 github.com/pilot-protocol/app-store v1.0.2/go.mod h1:deltPnaQkiTgMcxWU+honz3+Bl2R1cthhuZra4pQ4PI= github.com/pilot-protocol/beacon v0.2.9 h1:VqXAtRKl4YhZVkDmJzuNhS1bfpK0n9qbGKOGLfVYSyo= github.com/pilot-protocol/beacon v0.2.9/go.mod h1:DE8masXGku/IwfHL8lN/4CUTe5YsxMXFe2fRGJcH7w8= -github.com/pilot-protocol/common v0.5.13 h1:h9NmPh37ZZujktpSAwgq2H2ai7jG240nQTetGNWtZ3U= -github.com/pilot-protocol/common v0.5.13/go.mod h1:Ybc6f1A37s3ShoEh1nBMVL9DPyYlxvkqPTvtbxaNWg4= +github.com/pilot-protocol/common v0.5.14-0.20260923210943-65ea5b1a3d2d h1:mwYvwpPTFmE1KHZf3ynK5LqD+Boo8irP1hx0+4noeBI= +github.com/pilot-protocol/common v0.5.14-0.20260923210943-65ea5b1a3d2d/go.mod h1:Ybc6f1A37s3ShoEh1nBMVL9DPyYlxvkqPTvtbxaNWg4= github.com/pilot-protocol/dataexchange v0.2.2 h1:h1VJFqFCdMDtX1E2E8zxTQPw+9rWnMohXN8b6HpB2R0= github.com/pilot-protocol/dataexchange v0.2.2/go.mod h1:TUj2QtNMZ4oMnOag1j5k5qDxz2457S8uaB2FJif1Ulk= github.com/pilot-protocol/eventstream v0.2.4 h1:SyB64wqo+Qpz0hIAuzHGkI9Npwk5SFSg1aqNFLueUIQ= diff --git a/pkg/daemon/beacon_discovery.go b/pkg/daemon/beacon_discovery.go index 6c331246..2c63aa4f 100644 --- a/pkg/daemon/beacon_discovery.go +++ b/pkg/daemon/beacon_discovery.go @@ -212,7 +212,9 @@ func (d *Daemon) beaconRefreshTick(firstTick bool) { // per refresh tick is bounded by one probe round-trip (~200ms typical, // 2s max). The override is logged at Debug so ablation tests can grep // for "beacon RTT probe". - if d.config.BeaconRTTProbe && len(decision.NewList) > 1 { + // Compat mode has no UDP path (the probes would only time out or trip + // an egress guard), so Start logs once and the probe never runs. + if d.config.BeaconRTTProbe && d.config.TransportMode != "compat" && len(decision.NewList) > 1 { rttMap := d.probeBeaconsParallel(decision.NewList, 2*time.Second) if len(rttMap) > 0 { rttPick := routing.PickBeaconWithRTT(decision.NewList, identityPubKey, rttMap) diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index bbc1b87b..4b1634be 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -8,6 +8,7 @@ import ( "crypto/sha256" "crypto/subtle" "crypto/tls" + "crypto/x509" "encoding/base64" "encoding/hex" "encoding/json" @@ -29,6 +30,7 @@ import ( "github.com/pilot-protocol/common/crypto" "github.com/pilot-protocol/common/daemonapi" "github.com/pilot-protocol/common/fsutil" + "github.com/pilot-protocol/common/netproxy" "github.com/pilot-protocol/common/protocol" registry "github.com/pilot-protocol/common/registry/client" registrywire "github.com/pilot-protocol/common/registry/wire" @@ -212,6 +214,21 @@ type Config struct { // behind TLS-intercepting corp proxies). CompatTLSTrust string + // Proxy is the outbound proxy policy, normally built by ResolveProxy + // from -proxy and the transport mode. When non-nil it governs every + // TCP/HTTP connection the daemon opens itself: the registry (primary, + // pool and every reconnect), the compat-mode WSS beacon and the MOTD + // fetch. Targets stay host names end to end — the proxy is asked to + // CONNECT by name and TLS runs through the tunnel to the real server. + // nil keeps the historical behaviour: registry and beacon are dialed + // directly and HTTP fetches follow net/http's proxy environment. + Proxy *netproxy.Resolver + + // systemRoots replaces the OS trust store behind the "system" trust + // settings (RegistryTrust, CompatTLSTrust). Test seam only: it is + // always nil outside this package's tests. + systemRoots *x509.CertPool + // Tuning (zero = use defaults) KeepaliveInterval time.Duration // default 60s IdleTimeout time.Duration // default 120s @@ -885,6 +902,9 @@ func (d *Daemon) Start() error { slog.Info("compat mode enabled — skipping STUN; will dial WSS beacon after register", "compat_beacon", d.config.CompatBeaconURL, "tls_trust", d.config.CompatTLSTrust) + if d.config.BeaconRTTProbe { + slog.Info("compat mode: -beacon-rtt-probe disabled (its raw UDP probes cannot leave a UDP-blocked host)") + } } else if d.config.Endpoint != "" { registrationAddr = d.config.Endpoint slog.Info("using fixed endpoint", "endpoint", registrationAddr) @@ -1052,11 +1072,15 @@ func (d *Daemon) Start() error { if terr != nil { return fmt.Errorf("compat tls config: %w", terr) } + if tlsCfg.RootCAs == nil { + tlsCfg.RootCAs = d.config.systemRoots // "system" trust; nil = OS store + } ccCtx, ccCancel := context.WithTimeout(context.Background(), 30*time.Second) defer ccCancel() if cerr := d.tunnels.ConnectCompat(ccCtx, ConnectCompatConfig{ BeaconURL: d.config.CompatBeaconURL, TLSConfig: tlsCfg, + Proxy: d.httpProxyFunc(), Identity: d.identity, NodeID: d.nodeID, }); cerr != nil { @@ -2224,6 +2248,7 @@ func (d *Daemon) dialRegistryClient() (*registry.Client, error) { const maxRegistryDialAttempts = 10 const regConnPoolSize = 4 registryDialBackoff := 500 * time.Millisecond + dialOpts := d.registryDialOptions() for attempt := 1; attempt <= maxRegistryDialAttempts; attempt++ { if d.config.RegistryTLS { trust := d.config.RegistryTrust @@ -2235,14 +2260,14 @@ func (d *Daemon) dialRegistryClient() (*registry.Client, error) { if d.config.RegistryFingerprint == "" { return nil, fmt.Errorf("registry TLS with -registry-trust=pinned requires RegistryFingerprint") } - rc, err = registry.DialTLSPinned(d.config.RegistryAddr, d.config.RegistryFingerprint) + rc, err = registry.DialTLSPinned(d.config.RegistryAddr, d.config.RegistryFingerprint, dialOpts...) case "system": - rc, err = registry.DialTLSPool(d.config.RegistryAddr, &tls.Config{MinVersion: tls.VersionTLS12}, regConnPoolSize) + rc, err = registry.DialTLSPool(d.config.RegistryAddr, &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: d.config.systemRoots}, regConnPoolSize, dialOpts...) default: return nil, fmt.Errorf("invalid -registry-trust %q: must be 'pinned' or 'system'", trust) } } else { - rc, err = registry.DialPool(d.config.RegistryAddr, regConnPoolSize) + rc, err = registry.DialPool(d.config.RegistryAddr, regConnPoolSize, dialOpts...) } if err == nil { break @@ -4852,7 +4877,7 @@ func (d *Daemon) motdPollLoop() { if interval <= 0 { interval = motd.DefaultInterval } - client := &http.Client{Timeout: 10 * time.Second} + client := d.newHTTPClient(10 * time.Second) // Fire once on startup so the banner is warm shortly after boot, // then settle into the interval. diff --git a/pkg/daemon/proxy.go b/pkg/daemon/proxy.go new file mode 100644 index 00000000..818170a6 --- /dev/null +++ b/pkg/daemon/proxy.go @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package daemon + +import ( + "net/http" + "net/url" + "strings" + "time" + + "github.com/pilot-protocol/common/netproxy" + registry "github.com/pilot-protocol/common/registry/client" +) + +// ResolveProxy turns a -proxy setting into the daemon's outbound proxy +// policy (Config.Proxy) for the given transport mode: +// +// - "auto" or "": with transportMode "compat", the proxy from the +// environment — HTTPS_PROXY / https_proxy, falling back to ALL_PROXY / +// all_proxy, with NO_PROXY / no_proxy honoured. With any other transport +// it returns nil: no policy, the daemon dials exactly as it always has. +// - "off": a policy that never proxies. Unlike nil, it also stops the +// daemon's HTTP fetches from following proxy environment variables. +// - "http://[user:pass@]host:port" or "https://...": that proxy for every +// outbound TCP/HTTP connection, whatever the transport. +// +// Errors never echo proxy credentials. +func ResolveProxy(spec, transportMode string) (*netproxy.Resolver, error) { + s := strings.TrimSpace(spec) + if (s == "" || strings.EqualFold(s, netproxy.ModeAuto)) && transportMode != "compat" { + return nil, nil + } + return netproxy.Parse(s) +} + +// registryDialOptions routes every registry connection — the primary, each +// pool member and every reconnect — through the proxy policy. With no policy, +// or one that proxies nothing, the client keeps its direct dial. +func (d *Daemon) registryDialOptions() []registry.DialOption { + if !d.config.Proxy.Enabled() { + return nil + } + return []registry.DialOption{registry.WithDialer(netproxy.NewDialer(d.config.Proxy).DialContext)} +} + +// httpProxyFunc returns the http.Transport.Proxy function for connections +// the daemon makes over HTTP(S) or WSS, or nil when there is no policy. +func (d *Daemon) httpProxyFunc() func(*http.Request) (*url.URL, error) { + if d.config.Proxy == nil { + return nil + } + return d.config.Proxy.ProxyForRequest +} + +// newHTTPClient returns a client for daemon-owned HTTP fetches. Without a +// proxy policy it is a plain client on http.DefaultTransport, as before; +// with one, its transport routes through the policy. +func (d *Daemon) newHTTPClient(timeout time.Duration) *http.Client { + client := &http.Client{Timeout: timeout} + if proxy := d.httpProxyFunc(); proxy != nil { + var tr *http.Transport + if base, ok := http.DefaultTransport.(*http.Transport); ok { + tr = base.Clone() + } else { + tr = &http.Transport{} + } + tr.Proxy = proxy + client.Transport = tr + } + return client +} diff --git a/pkg/daemon/transport/wss/wss.go b/pkg/daemon/transport/wss/wss.go index 13619f01..f2fc3962 100644 --- a/pkg/daemon/transport/wss/wss.go +++ b/pkg/daemon/transport/wss/wss.go @@ -39,6 +39,7 @@ import ( "log/slog" "net" "net/http" + "net/url" "sync" "sync/atomic" "time" @@ -66,8 +67,8 @@ const DefaultIdlePingInterval = 30 * time.Second const DefaultIdlePingTimeout = 10 * time.Second // DefaultDialTimeout caps the time we spend on a single dial attempt -// (DNS + TCP + TLS + WS upgrade + auth challenge). Beyond this we -// fail fast and let the reconnect loop try again. +// (DNS + TCP + proxy CONNECT + TLS + WS upgrade + auth challenge). +// Beyond this we fail fast and let the reconnect loop try again. const DefaultDialTimeout = 20 * time.Second // DefaultRecvBuffer is the buffered channel size for inbound frames. @@ -109,6 +110,13 @@ type Config struct { // store. Always non-nil — the caller picks the policy. TLSConfig *tls.Config + // Proxy picks the HTTP proxy for the WSS dial, with the signature of + // http.Transport.Proxy (e.g. netproxy.Resolver.ProxyForRequest). A + // wss:// URL is tunnelled with CONNECT by host name, so the beacon + // name is never resolved locally and TLS (TLSConfig, SNI) stays + // end-to-end with the beacon. nil dials directly. + Proxy func(*http.Request) (*url.URL, error) + // Identity provides the Ed25519 keypair used for the auth challenge. Identity *crypto.Identity @@ -256,6 +264,7 @@ func Dial(ctx context.Context, cfg Config) (*Transport, error) { func (t *Transport) dialAndAuth(ctx context.Context) (*websocket.Conn, error) { httpClient := &http.Client{ Transport: &http.Transport{ + Proxy: t.cfg.Proxy, TLSClientConfig: t.cfg.TLSConfig.Clone(), }, } diff --git a/pkg/daemon/transport/wss/zz_wss_proxy_test.go b/pkg/daemon/transport/wss/zz_wss_proxy_test.go new file mode 100644 index 00000000..d4b33123 --- /dev/null +++ b/pkg/daemon/transport/wss/zz_wss_proxy_test.go @@ -0,0 +1,401 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package wss_test + +import ( + "bufio" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "fmt" + "io" + "math/big" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/pilot-protocol/common/netproxy" + "github.com/pilot-protocol/pilotprotocol/pkg/daemon/transport/wss" +) + +// The compat beacon behind an authenticating HTTP CONNECT proxy — the only +// egress in a Meta Muse-style sandbox. The beacon is served only as +// beacon.pilot.invalid, a reserved name that never resolves locally, and +// only the proxy knows where it lives. A working transport therefore proves +// the proxy was asked to CONNECT by host name and that TLS ran end-to-end +// with the beacon (the certificate covers only that name). + +const proxiedBeaconHost = "beacon.pilot.invalid" + +// connectProxy is a minimal authenticating CONNECT proxy. It routes +// *.pilot.invalid to loopback, answers 407 without the expected Basic +// credentials, refuses every other method and host, and records each +// request target. +type connectProxy struct { + ln net.Listener + wantAuth string + + mu sync.Mutex + targets []string + denied []int + live []net.Conn + wg sync.WaitGroup +} + +func newConnectProxy(t *testing.T, user, pass string) *connectProxy { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("proxy listen: %v", err) + } + req := &http.Request{Header: http.Header{}} + req.SetBasicAuth(user, pass) + p := &connectProxy{ln: ln, wantAuth: req.Header.Get("Authorization")} + p.wg.Add(1) + go p.accept() + t.Cleanup(func() { + ln.Close() + p.mu.Lock() + for _, c := range p.live { + c.Close() + } + p.mu.Unlock() + p.wg.Wait() + }) + return p +} + +func (p *connectProxy) url(user, pass string) string { + return fmt.Sprintf("http://%s:%s@%s", user, pass, p.ln.Addr()) +} + +func (p *connectProxy) connects() []string { + p.mu.Lock() + defer p.mu.Unlock() + return append([]string(nil), p.targets...) +} + +func (p *connectProxy) deniedStatuses() []int { + p.mu.Lock() + defer p.mu.Unlock() + return append([]int(nil), p.denied...) +} + +func (p *connectProxy) track(c net.Conn) { + p.mu.Lock() + p.live = append(p.live, c) + p.mu.Unlock() +} + +func (p *connectProxy) accept() { + defer p.wg.Done() + for { + conn, err := p.ln.Accept() + if err != nil { + return + } + p.track(conn) + p.wg.Add(1) + go func() { + defer p.wg.Done() + p.serve(conn) + }() + } +} + +func (p *connectProxy) deny(conn net.Conn, code int) { + p.mu.Lock() + p.denied = append(p.denied, code) + p.mu.Unlock() + fmt.Fprintf(conn, "HTTP/1.1 %d %s\r\nContent-Length: 0\r\n\r\n", code, http.StatusText(code)) +} + +func (p *connectProxy) serve(conn net.Conn) { + defer conn.Close() + br := bufio.NewReader(conn) + req, err := http.ReadRequest(br) + if err != nil { + return + } + p.mu.Lock() + p.targets = append(p.targets, req.RequestURI) + p.mu.Unlock() + if req.Method != http.MethodConnect { + p.deny(conn, http.StatusMethodNotAllowed) + return + } + if req.Header.Get("Proxy-Authorization") != p.wantAuth { + p.deny(conn, http.StatusProxyAuthRequired) + return + } + host, port, err := net.SplitHostPort(req.RequestURI) + if err != nil || !strings.HasSuffix(host, ".pilot.invalid") { + p.deny(conn, http.StatusForbidden) + return + } + up, err := net.DialTimeout("tcp", net.JoinHostPort("127.0.0.1", port), 5*time.Second) + if err != nil { + p.deny(conn, http.StatusBadGateway) + return + } + p.track(up) + defer up.Close() + if _, err := io.WriteString(conn, "HTTP/1.1 200 Connection established\r\n\r\n"); err != nil { + return + } + done := make(chan struct{}, 2) + go func() { io.Copy(up, br); up.Close(); done <- struct{}{} }() + go func() { io.Copy(conn, up); conn.Close(); done <- struct{}{} }() + <-done + <-done +} + +// newProxiedFakeBeacon serves the fake beacon protocol over TLS with a +// certificate for proxiedBeaconHost only, and records the SNI of every +// handshake. +func newProxiedFakeBeacon(t *testing.T, nodeID uint32) (*fakeBeacon, *x509.CertPool, func() []string) { + t.Helper() + cert, pool := proxiedCert(t, proxiedBeaconHost) + fb := &fakeBeacon{expectedID: nodeID, t: t} + mux := http.NewServeMux() + mux.HandleFunc("/", fb.handle) + fb.srv = httptest.NewUnstartedServer(mux) + var mu sync.Mutex + var snis []string + fb.srv.TLS = &tls.Config{ + Certificates: []tls.Certificate{cert}, + GetConfigForClient: func(hello *tls.ClientHelloInfo) (*tls.Config, error) { + mu.Lock() + snis = append(snis, hello.ServerName) + mu.Unlock() + return nil, nil + }, + } + fb.srv.StartTLS() + t.Cleanup(fb.srv.Close) + return fb, pool, func() []string { + mu.Lock() + defer mu.Unlock() + return append([]string(nil), snis...) + } +} + +// proxiedURL is the beacon URL a daemon is given: the unresolvable name +// plus the real listener's port. +func proxiedURL(fb *fakeBeacon) string { + _, port, _ := net.SplitHostPort(fb.srv.Listener.Addr().String()) + return "wss://" + net.JoinHostPort(proxiedBeaconHost, port) + "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/v1/compat" +} + +func proxiedTarget(fb *fakeBeacon) string { + _, port, _ := net.SplitHostPort(fb.srv.Listener.Addr().String()) + return net.JoinHostPort(proxiedBeaconHost, port) +} + +func proxiedCert(t *testing.T, host string) (tls.Certificate, *x509.CertPool) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("genkey: %v", err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(time.Now().UnixNano()), + Subject: pkix.Name{CommonName: host}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IsCA: true, + DNSNames: []string{host}, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatalf("create cert: %v", err) + } + leaf, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parse cert: %v", err) + } + pool := x509.NewCertPool() + pool.AddCert(leaf) + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key, Leaf: leaf}, pool +} + +func assertOnlyConnects(t *testing.T, p *connectProxy, target string, min int) { + t.Helper() + got := p.connects() + if len(got) < min { + t.Fatalf("proxy saw %d CONNECTs %q, want at least %d", len(got), got, min) + } + for _, g := range got { + if g != target { + t.Fatalf("proxy saw request target %q, want only %q (a host name, never an IP)", g, target) + } + } +} + +func TestDial_ThroughAuthenticatingConnectProxy(t *testing.T) { + t.Parallel() + const nodeID uint32 = 7001 + fb, pool, snis := newProxiedFakeBeacon(t, nodeID) + proxy := newConnectProxy(t, "muse", "s3cret") + res, err := netproxy.Explicit(proxy.url("muse", "s3cret")) + if err != nil { + t.Fatalf("netproxy.Explicit: %v", err) + } + + tr, err := wss.Dial(context.Background(), wss.Config{ + URL: proxiedURL(fb), + TLSConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, + Proxy: res.ProxyForRequest, + Identity: mustID(t), + NodeID: nodeID, + DialTimeout: 5 * time.Second, + }) + if err != nil { + t.Fatalf("Dial through proxy: %v", err) + } + defer tr.Close() + + if _, err := tr.Send([]byte("via-proxy"), nil); err != nil { + t.Fatalf("Send: %v", err) + } + frame, _, err := tr.Recv() + if err != nil { + t.Fatalf("Recv: %v", err) + } + if string(frame) != "echo:via-proxy" { + t.Fatalf("Recv = %q, want %q", frame, "echo:via-proxy") + } + assertOnlyConnects(t, proxy, proxiedTarget(fb), 1) + if got := snis(); len(got) != 1 || got[0] != proxiedBeaconHost { + t.Fatalf("beacon saw SNI %q, want [%q]", got, proxiedBeaconHost) + } +} + +// The reconnect path uses the same proxy: after the beacon drops the +// connection, the supervisor's redial is CONNECTed by name again. +func TestReconnect_ThroughConnectProxy(t *testing.T) { + t.Parallel() + const nodeID uint32 = 7002 + fb, pool, _ := newProxiedFakeBeacon(t, nodeID) + fb.killAfterFrame = 1 + proxy := newConnectProxy(t, "muse", "s3cret") + res, err := netproxy.Explicit(proxy.url("muse", "s3cret")) + if err != nil { + t.Fatalf("netproxy.Explicit: %v", err) + } + + tr, err := wss.Dial(context.Background(), wss.Config{ + URL: proxiedURL(fb), + TLSConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, + Proxy: res.ProxyForRequest, + Identity: mustID(t), + NodeID: nodeID, + DialTimeout: 5 * time.Second, + }) + if err != nil { + t.Fatalf("Dial through proxy: %v", err) + } + defer tr.Close() + + if _, err := tr.Send([]byte("kill"), nil); err != nil { + t.Fatalf("Send (kill): %v", err) + } + deadline := time.Now().Add(10 * time.Second) + for fb.authCount.Load() < 2 { + if time.Now().After(deadline) { + t.Fatalf("transport never re-authenticated through the proxy (auths=%d, connects=%q)", fb.authCount.Load(), proxy.connects()) + } + time.Sleep(50 * time.Millisecond) + } + assertOnlyConnects(t, proxy, proxiedTarget(fb), 2) +} + +// With the proxy taken from the environment, NO_PROXY is honoured: an +// exempted beacon is dialed directly — which, for a name that only the +// proxy can reach, fails without the proxy ever being asked. +func TestDial_ProxyFromEnvironmentHonoursNoProxy(t *testing.T) { + const nodeID uint32 = 7003 + fb, pool, _ := newProxiedFakeBeacon(t, nodeID) + proxy := newConnectProxy(t, "muse", "s3cret") + for _, k := range []string{"HTTPS_PROXY", "https_proxy", "ALL_PROXY", "all_proxy", "HTTP_PROXY", "http_proxy", "NO_PROXY", "no_proxy", "REQUEST_METHOD"} { + t.Setenv(k, "") + } + t.Setenv("HTTPS_PROXY", proxy.url("muse", "s3cret")) + + dial := func() error { + res, err := netproxy.FromEnvironment() + if err != nil { + t.Fatalf("netproxy.FromEnvironment: %v", err) + } + tr, err := wss.Dial(context.Background(), wss.Config{ + URL: proxiedURL(fb), + TLSConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, + Proxy: res.ProxyForRequest, + Identity: mustID(t), + NodeID: nodeID, + DialTimeout: 5 * time.Second, + }) + if err == nil { + tr.Close() + } + return err + } + + if err := dial(); err != nil { + t.Fatalf("Dial with HTTPS_PROXY: %v", err) + } + assertOnlyConnects(t, proxy, proxiedTarget(fb), 1) + + t.Setenv("NO_PROXY", ".pilot.invalid") + if err := dial(); err == nil { + t.Fatal("Dial of a NO_PROXY-exempt, locally unresolvable beacon succeeded; want a direct-dial failure") + } + if got := proxy.connects(); len(got) != 1 { + t.Fatalf("proxy saw %d requests %q after NO_PROXY exemption, want still 1", len(got), got) + } +} + +// Wrong proxy credentials fail the dial with the proxy's 407, and the +// error never carries the credentials. +func TestDial_ProxyAuthFailureDoesNotLeakCredentials(t *testing.T) { + t.Parallel() + const nodeID uint32 = 7004 + fb, pool, _ := newProxiedFakeBeacon(t, nodeID) + proxy := newConnectProxy(t, "muse", "s3cret") + res, err := netproxy.Explicit(proxy.url("muse", "wr0ng-pa55")) + if err != nil { + t.Fatalf("netproxy.Explicit: %v", err) + } + + _, err = wss.Dial(context.Background(), wss.Config{ + URL: proxiedURL(fb), + TLSConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, + Proxy: res.ProxyForRequest, + Identity: mustID(t), + NodeID: nodeID, + DialTimeout: 5 * time.Second, + }) + if err == nil { + t.Fatal("Dial with wrong proxy credentials succeeded") + } + if strings.Contains(err.Error(), "wr0ng-pa55") || strings.Contains(err.Error(), "muse") { + t.Fatalf("dial error leaks proxy credentials: %v", err) + } + if got := proxy.deniedStatuses(); len(got) != 1 || got[0] != http.StatusProxyAuthRequired { + t.Fatalf("proxy denials = %v, want [407]", got) + } + if fb.authCount.Load() != 0 { + t.Fatal("beacon was reached despite the proxy refusing the CONNECT") + } +} diff --git a/pkg/daemon/tunnel.go b/pkg/daemon/tunnel.go index aa62b8a8..5b217cca 100644 --- a/pkg/daemon/tunnel.go +++ b/pkg/daemon/tunnel.go @@ -13,6 +13,8 @@ import ( "fmt" "log/slog" "net" + "net/http" + "net/url" "sync" "sync/atomic" "syscall" @@ -1151,8 +1153,11 @@ func (tm *TunnelManager) Listen(addr string) error { type ConnectCompatConfig struct { BeaconURL string TLSConfig *tls.Config - Identity *crypto.Identity - NodeID uint32 + // Proxy is the http.Transport.Proxy function for the WSS dial (see + // wss.Config.Proxy). nil dials the beacon directly. + Proxy func(*http.Request) (*url.URL, error) + Identity *crypto.Identity + NodeID uint32 } // ConnectCompat opens a compat-mode (WSS) tunnel to the beacon @@ -1169,6 +1174,7 @@ func (tm *TunnelManager) ConnectCompat(ctx context.Context, cfg ConnectCompatCon wssTr, err := wssTransport.Dial(ctx, wssTransport.Config{ URL: cfg.BeaconURL, TLSConfig: cfg.TLSConfig, + Proxy: cfg.Proxy, Identity: cfg.Identity, NodeID: cfg.NodeID, }) diff --git a/pkg/daemon/zz_proxy_test.go b/pkg/daemon/zz_proxy_test.go new file mode 100644 index 00000000..732e0038 --- /dev/null +++ b/pkg/daemon/zz_proxy_test.go @@ -0,0 +1,601 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package daemon + +import ( + "bufio" + "context" + "crypto/ecdsa" + "crypto/ed25519" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" + "encoding/hex" + "encoding/json" + "encoding/pem" + "fmt" + "io" + "math/big" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/coder/websocket" + + "github.com/pilot-protocol/common/crypto" + "github.com/pilot-protocol/common/netproxy" + rendezvous "github.com/pilot-protocol/rendezvous" +) + +// proxyEnvVars are every variable netproxy.FromEnvironment reads; tests +// clear them so the developer's or CI runner's own proxy never leaks in. +var proxyEnvVars = []string{ + "HTTPS_PROXY", "https_proxy", "ALL_PROXY", "all_proxy", + "HTTP_PROXY", "http_proxy", "NO_PROXY", "no_proxy", "REQUEST_METHOD", +} + +func clearProxyEnv(t *testing.T) { + t.Helper() + for _, k := range proxyEnvVars { + t.Setenv(k, "") + } +} + +// --- ResolveProxy: -proxy × transport × environment ------------------------ + +func TestResolveProxyMatrix(t *testing.T) { + const envProxy = "http://muse:s3cret@egress.test:3128" + const flagProxy = "http://ops:hunter2@flag-proxy.test:8080" + type want struct { + policy bool // non-nil policy + mode string // policy Mode() + enabled bool + // proxy chosen for a pilot host and for a NO_PROXY'd host; "" = direct + pilot, exempt string + } + cases := []struct { + name, spec, transport string + env map[string]string + want want + }{ + {name: "auto udp ignores env", spec: "auto", transport: "udp", + env: map[string]string{"HTTPS_PROXY": envProxy}, + want: want{}}, + {name: "empty spec udp", spec: "", transport: "udp", + env: map[string]string{"HTTPS_PROXY": envProxy}, + want: want{}}, + {name: "auto default transport", spec: "auto", transport: "", + env: map[string]string{"HTTPS_PROXY": envProxy}, + want: want{}}, + {name: "auto compat HTTPS_PROXY", spec: "auto", transport: "compat", + env: map[string]string{"HTTPS_PROXY": envProxy, "NO_PROXY": ".internal.test"}, + want: want{policy: true, mode: netproxy.ModeAuto, enabled: true, pilot: "egress.test:3128"}}, + {name: "AUTO compat lower-case env", spec: " AUTO ", transport: "compat", + env: map[string]string{"https_proxy": envProxy, "no_proxy": "svc.internal.test"}, + want: want{policy: true, mode: netproxy.ModeAuto, enabled: true, pilot: "egress.test:3128"}}, + {name: "auto compat ALL_PROXY fallback", spec: "auto", transport: "compat", + env: map[string]string{"ALL_PROXY": "http://all.test:1080"}, + want: want{policy: true, mode: netproxy.ModeAuto, enabled: true, pilot: "all.test:1080", exempt: "all.test:1080"}}, + {name: "auto compat no env", spec: "auto", transport: "compat", + want: want{policy: true, mode: netproxy.ModeAuto}}, + {name: "off udp", spec: "off", transport: "udp", + env: map[string]string{"HTTPS_PROXY": envProxy}, + want: want{policy: true, mode: netproxy.ModeOff}}, + {name: "off compat", spec: "OFF", transport: "compat", + env: map[string]string{"HTTPS_PROXY": envProxy}, + want: want{policy: true, mode: netproxy.ModeOff}}, + {name: "url udp", spec: flagProxy, transport: "udp", + env: map[string]string{"HTTPS_PROXY": envProxy, "NO_PROXY": ".internal.test"}, + want: want{policy: true, mode: netproxy.ModeExplicit, enabled: true, pilot: "flag-proxy.test:8080", exempt: "flag-proxy.test:8080"}}, + {name: "url compat", spec: flagProxy, transport: "compat", + env: map[string]string{"NO_PROXY": ".internal.test"}, + want: want{policy: true, mode: netproxy.ModeExplicit, enabled: true, pilot: "flag-proxy.test:8080", exempt: "flag-proxy.test:8080"}}, + {name: "https url default port", spec: "https://tls-proxy.test", transport: "udp", + want: want{policy: true, mode: netproxy.ModeExplicit, enabled: true, pilot: "tls-proxy.test", exempt: "tls-proxy.test"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + clearProxyEnv(t) + for k, v := range tc.env { + t.Setenv(k, v) + } + policy, err := ResolveProxy(tc.spec, tc.transport) + if err != nil { + t.Fatalf("ResolveProxy(%q, %q): %v", tc.spec, tc.transport, err) + } + if (policy != nil) != tc.want.policy { + t.Fatalf("policy = %v, want non-nil=%v", policy, tc.want.policy) + } + if policy == nil { + return + } + if got := policy.Mode(); got != tc.want.mode { + t.Errorf("Mode = %q, want %q", got, tc.want.mode) + } + if got := policy.Enabled(); got != tc.want.enabled { + t.Errorf("Enabled = %v, want %v", got, tc.want.enabled) + } + check := func(target, want string) { + t.Helper() + u, err := policy.ProxyForAddr(target) + if err != nil { + t.Fatalf("ProxyForAddr(%s): %v", target, err) + } + got := "" + if u != nil { + got = u.Host + } + if got != want { + t.Errorf("proxy for %s = %q, want %q", target, got, want) + } + req := &http.Request{URL: &url.URL{Scheme: "https", Host: target}} + ru, err := policy.ProxyForRequest(req) + if err != nil { + t.Fatalf("ProxyForRequest(%s): %v", target, err) + } + if (ru == nil) != (u == nil) || (ru != nil && ru.Host != u.Host) { + t.Errorf("ProxyForRequest(%s) = %v, ProxyForAddr = %v; the registry and HTTP paths must agree", target, ru, u) + } + } + check("registry.pilotprotocol.network:443", tc.want.pilot) + check("svc.internal.test:443", tc.want.exempt) + if s := policy.String(); strings.Contains(s, "s3cret") || strings.Contains(s, "hunter2") || strings.Contains(s, "muse") || strings.Contains(s, "ops:") { + t.Errorf("String() leaks credentials: %q", s) + } + }) + } +} + +func TestResolveProxyRejectsMalformedSettings(t *testing.T) { + clearProxyEnv(t) + if _, err := ResolveProxy("socks5://user:pw@proxy.test:1080", "udp"); err == nil { + t.Fatal("unsupported proxy scheme accepted") + } + t.Setenv("HTTPS_PROXY", "ftp://muse:s3cret@proxy.test:21") + _, err := ResolveProxy("auto", "compat") + if err == nil { + t.Fatal("malformed HTTPS_PROXY accepted in compat mode") + } + if strings.Contains(err.Error(), "s3cret") { + t.Fatalf("error leaks credentials: %v", err) + } + // udp + auto never reads the environment, so a bad value is harmless. + if p, err := ResolveProxy("auto", "udp"); err != nil || p != nil { + t.Fatalf("ResolveProxy(auto, udp) with bad env = (%v, %v), want (nil, nil)", p, err) + } +} + +// --- a Muse-style egress: authenticating CONNECT-only proxy ----------------- + +// proxyTestConnect is an authenticating CONNECT proxy that routes +// *.pilot.invalid (names that never resolve locally) to loopback and +// records every request target. +type proxyTestConnect struct { + ln net.Listener + wantAuth string + + mu sync.Mutex + targets []string + live []net.Conn + wg sync.WaitGroup +} + +func newProxyTestConnect(t *testing.T, user, pass string) *proxyTestConnect { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("proxy listen: %v", err) + } + req := &http.Request{Header: http.Header{}} + req.SetBasicAuth(user, pass) + p := &proxyTestConnect{ln: ln, wantAuth: req.Header.Get("Authorization")} + p.wg.Add(1) + go func() { + defer p.wg.Done() + for { + conn, err := ln.Accept() + if err != nil { + return + } + p.track(conn) + p.wg.Add(1) + go func() { + defer p.wg.Done() + p.serve(conn) + }() + } + }() + t.Cleanup(func() { + ln.Close() + p.mu.Lock() + for _, c := range p.live { + c.Close() + } + p.mu.Unlock() + p.wg.Wait() + }) + return p +} + +func (p *proxyTestConnect) url(user, pass string) string { + return fmt.Sprintf("http://%s:%s@%s", user, pass, p.ln.Addr()) +} + +func (p *proxyTestConnect) track(c net.Conn) { + p.mu.Lock() + p.live = append(p.live, c) + p.mu.Unlock() +} + +// counts returns how many requests named each target. +func (p *proxyTestConnect) counts() map[string]int { + p.mu.Lock() + defer p.mu.Unlock() + m := map[string]int{} + for _, target := range p.targets { + m[target]++ + } + return m +} + +func (p *proxyTestConnect) serve(conn net.Conn) { + defer conn.Close() + br := bufio.NewReader(conn) + req, err := http.ReadRequest(br) + if err != nil { + return + } + p.mu.Lock() + p.targets = append(p.targets, req.RequestURI) + p.mu.Unlock() + deny := func(code int) { + fmt.Fprintf(conn, "HTTP/1.1 %d %s\r\nContent-Length: 0\r\n\r\n", code, http.StatusText(code)) + } + if req.Method != http.MethodConnect { + deny(http.StatusMethodNotAllowed) + return + } + if req.Header.Get("Proxy-Authorization") != p.wantAuth { + deny(http.StatusProxyAuthRequired) + return + } + host, port, err := net.SplitHostPort(req.RequestURI) + if err != nil || !strings.HasSuffix(host, ".pilot.invalid") { + deny(http.StatusForbidden) + return + } + up, err := net.DialTimeout("tcp", net.JoinHostPort("127.0.0.1", port), 5*time.Second) + if err != nil { + deny(http.StatusBadGateway) + return + } + p.track(up) + defer up.Close() + if _, err := io.WriteString(conn, "HTTP/1.1 200 Connection established\r\n\r\n"); err != nil { + return + } + done := make(chan struct{}, 2) + go func() { io.Copy(up, br); up.Close(); done <- struct{}{} }() + go func() { io.Copy(conn, up); conn.Close(); done <- struct{}{} }() + <-done + <-done +} + +// proxyTestCert issues a self-signed certificate for one host name only +// (no IP SANs) and adds it to pool. +func proxyTestCert(t *testing.T, host string, pool *x509.CertPool) (tls.Certificate, []byte, *ecdsa.PrivateKey) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("genkey: %v", err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(time.Now().UnixNano()), + Subject: pkix.Name{CommonName: host}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IsCA: true, + DNSNames: []string{host}, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatalf("create cert: %v", err) + } + leaf, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parse cert: %v", err) + } + if pool != nil { + pool.AddCert(leaf) + } + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key, Leaf: leaf}, der, key +} + +// startProxiedRegistry runs a real rendezvous registry over TLS with a +// certificate for registry.pilot.invalid, and returns the address a daemon +// must use (the unresolvable name plus the real port) and the leaf's pin. +func startProxiedRegistry(t *testing.T, pool *x509.CertPool) (*rendezvous.Server, string, string) { + t.Helper() + _, der, key := proxyTestCert(t, "registry.pilot.invalid", pool) + dir := t.TempDir() + certFile, keyFile := filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem") + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + t.Fatalf("marshal key: %v", err) + } + if err := os.WriteFile(certFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(keyFile, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600); err != nil { + t.Fatal(err) + } + reg := rendezvous.New("") + if err := reg.SetTLS(certFile, keyFile); err != nil { + t.Fatalf("registry SetTLS: %v", err) + } + go func() { _ = reg.ListenAndServe("127.0.0.1:0") }() + select { + case <-reg.Ready(): + case <-time.After(5 * time.Second): + t.Fatal("registry failed to start") + } + t.Cleanup(func() { reg.Close() }) + _, port, _ := net.SplitHostPort(reg.Addr().String()) + sum := sha256.Sum256(der) + return reg, net.JoinHostPort("registry.pilot.invalid", port), hex.EncodeToString(sum[:]) +} + +// proxiedBeacon is a compat WSS beacon stand-in served over TLS as +// beacon.pilot.invalid. It completes the Ed25519 auth challenge only for a +// node whose key matches the registry's record, then drains frames. +type proxiedBeacon struct { + srv *httptest.Server + authed atomic.Uint32 // node ID of the last authenticated daemon + snis chan string +} + +func startProxiedBeacon(t *testing.T, pool *x509.CertPool, lookup func(uint32) ([]byte, bool)) (*proxiedBeacon, string) { + t.Helper() + cert, _, _ := proxyTestCert(t, "beacon.pilot.invalid", pool) + b := &proxiedBeacon{snis: make(chan string, 16)} + b.srv = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b.handle(w, r, lookup) + })) + b.srv.TLS = &tls.Config{ + Certificates: []tls.Certificate{cert}, + GetConfigForClient: func(hello *tls.ClientHelloInfo) (*tls.Config, error) { + select { + case b.snis <- hello.ServerName: + default: + } + return nil, nil + }, + } + b.srv.StartTLS() + t.Cleanup(b.srv.Close) + _, port, _ := net.SplitHostPort(b.srv.Listener.Addr().String()) + return b, net.JoinHostPort("beacon.pilot.invalid", port) +} + +func (b *proxiedBeacon) handle(w http.ResponseWriter, r *http.Request, lookup func(uint32) ([]byte, bool)) { + conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{"pilot.v1"}}) + if err != nil { + return + } + defer conn.CloseNow() + ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) + defer cancel() + const nonce, ts = "0123456789abcdef0123456789abcdef", int64(1700000000) + challenge, _ := json.Marshal(map[string]interface{}{"type": "auth_challenge", "nonce": nonce, "ts": ts}) + if err := conn.Write(ctx, websocket.MessageText, challenge); err != nil { + return + } + _, body, err := conn.Read(ctx) + if err != nil { + return + } + var reply struct { + NodeID uint32 `json:"node_id"` + PublicKey string `json:"public_key"` + Sig string `json:"sig"` + } + if json.Unmarshal(body, &reply) != nil { + return + } + registered, ok := lookup(reply.NodeID) + sig, _ := base64.StdEncoding.DecodeString(reply.Sig) + msg := fmt.Sprintf("compat_auth:%d:%d:%s", reply.NodeID, ts, nonce) + if !ok || !ed25519.Verify(ed25519.PublicKey(registered), []byte(msg), sig) { + conn.Close(websocket.StatusPolicyViolation, "auth_fail") + return + } + ok2, _ := json.Marshal(map[string]string{"type": "auth_ok"}) + if err := conn.Write(ctx, websocket.MessageText, ok2); err != nil { + return + } + b.authed.Store(reply.NodeID) + for { + if _, _, err := conn.Read(r.Context()); err != nil { + return + } + } +} + +// TestStartCompatModeThroughConnectProxy is the Meta Muse sandbox in +// miniature: no route to the registry or the beacon except an +// authenticating CONNECT proxy taken from HTTPS_PROXY, and both services +// reachable only by host names that do not resolve locally. A compat daemon +// with -proxy=auto must register over TLS (system trust, full pool), bring +// the WSS tunnel up, and reconnect the registry — all through the proxy. +func TestStartCompatModeThroughConnectProxy(t *testing.T) { + clearProxyEnv(t) + proxy := newProxyTestConnect(t, "muse", "s3cret") + + // The policy snapshots the environment. Clear it again before the + // fixtures start: the in-process registry's own HTTP client (its + // GitHub release poller) would otherwise follow HTTPS_PROXY too, and + // net/http caches that environment for the whole test binary. + t.Setenv("HTTPS_PROXY", proxy.url("muse", "s3cret")) + policy, err := ResolveProxy("auto", "compat") + if err != nil { + t.Fatalf("ResolveProxy: %v", err) + } + if !policy.Enabled() { + t.Fatalf("policy %s is not enabled", policy) + } + t.Setenv("HTTPS_PROXY", "") + + roots := x509.NewCertPool() + reg, regAddr, _ := startProxiedRegistry(t, roots) + beacon, beaconHost := startProxiedBeacon(t, roots, reg.LookupPublicKey) + + sockDir, err := os.MkdirTemp("", "pdx") + if err != nil { + t.Fatalf("mkdtemp: %v", err) + } + t.Cleanup(func() { os.RemoveAll(sockDir) }) + d := New(Config{ + RegistryAddr: regAddr, + RegistryTLS: true, + RegistryTrust: "system", + TransportMode: "compat", + CompatBeaconURL: "wss://" + beaconHost + "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/v1/compat", + CompatTLSTrust: "system", + Proxy: policy, + SocketPath: sockDir + "/s", + IdentityPath: t.TempDir() + "/id.json", + Email: "proxy-compat@example.test", + Encrypt: true, + DisablePolicyRunner: true, + systemRoots: roots, + }) + if err := d.Start(); err != nil { + t.Fatalf("Start behind the proxy: %v", err) + } + t.Cleanup(func() { _ = d.Stop() }) + + if d.NodeID() == 0 { + t.Fatal("daemon has no node ID after Start") + } + if got := beacon.authed.Load(); got != d.NodeID() { + t.Fatalf("beacon authenticated node %d, want %d", got, d.NodeID()) + } + select { + case sni := <-beacon.snis: + if sni != "beacon.pilot.invalid" { + t.Fatalf("beacon SNI = %q, want beacon.pilot.invalid", sni) + } + default: + t.Fatal("beacon saw no TLS handshake") + } + + counts := proxy.counts() + if counts[regAddr] < 4 { + t.Errorf("registry CONNECTs = %d, want >= 4 (primary + pool)", counts[regAddr]) + } + if counts[beaconHost] != 1 { + t.Errorf("beacon CONNECTs = %d, want 1", counts[beaconHost]) + } + for target := range counts { + if target != regAddr && target != beaconHost { + t.Errorf("unexpected proxy request target %q (all: %v)", target, counts) + } + } + + // The reconnect path (rx-watchdog soft recovery, half-open registry) + // builds a fresh client through the same proxy. + before := counts[regAddr] + if err := d.forceReconnectRegistry(); err != nil { + t.Fatalf("forceReconnectRegistry: %v", err) + } + if _, err := d.reg().Lookup(d.NodeID()); err != nil { + t.Fatalf("Lookup on the reconnected registry client: %v", err) + } + if after := proxy.counts()[regAddr]; after < before+4 { + t.Fatalf("registry CONNECTs after reconnect = %d, want >= %d", after, before+4) + } +} + +// Pinned registry trust (the fallback for sandboxes without a CA bundle) +// works through an explicit -proxy URL, in UDP mode too. +func TestDialRegistryClientPinnedThroughExplicitProxy(t *testing.T) { + t.Parallel() + proxy := newProxyTestConnect(t, "muse", "s3cret") + _, regAddr, pin := startProxiedRegistry(t, nil) + policy, err := ResolveProxy(proxy.url("muse", "s3cret"), "udp") + if err != nil { + t.Fatalf("ResolveProxy: %v", err) + } + d := New(Config{ + RegistryAddr: regAddr, + RegistryTLS: true, + RegistryTrust: "pinned", + RegistryFingerprint: pin, + Proxy: policy, + }) + rc, err := d.dialRegistryClient() + if err != nil { + t.Fatalf("dialRegistryClient: %v", err) + } + defer rc.Close() + id, err := crypto.GenerateIdentity() + if err != nil { + t.Fatalf("GenerateIdentity: %v", err) + } + resp, err := rc.RegisterWithKey("127.0.0.1:4000", crypto.EncodePublicKey(id.PublicKey), "proxy-pinned@example.test", nil) + if err != nil { + t.Fatalf("register through the proxy: %v", err) + } + if id, _ := resp["node_id"].(float64); id == 0 { + t.Fatalf("register response has no node_id: %v", resp) + } + counts := proxy.counts() + if len(counts) != 1 || counts[regAddr] == 0 { + t.Fatalf("proxy request targets = %v, want only %q", counts, regAddr) + } +} + +// Without a policy nothing changes: no dialer option, no HTTP proxy +// override, and the MOTD client stays on http.DefaultTransport. +func TestNoProxyPolicyKeepsHistoricalDialing(t *testing.T) { + t.Parallel() + d := New(Config{}) + if opts := d.registryDialOptions(); opts != nil { + t.Fatalf("registryDialOptions without a policy = %d options, want none", len(opts)) + } + if d.httpProxyFunc() != nil { + t.Fatal("httpProxyFunc without a policy is non-nil") + } + if c := d.newHTTPClient(time.Second); c.Transport != nil { + t.Fatalf("HTTP client transport = %T, want nil (http.DefaultTransport)", c.Transport) + } + + off := New(Config{Proxy: netproxy.Off()}) + if opts := off.registryDialOptions(); opts != nil { + t.Fatal("-proxy=off still installs a registry dialer") + } + c := off.newHTTPClient(time.Second) + tr, ok := c.Transport.(*http.Transport) + if !ok || tr.Proxy == nil { + t.Fatalf("-proxy=off HTTP client transport = %T, want an *http.Transport with the policy's Proxy", c.Transport) + } + u, err := tr.Proxy(&http.Request{URL: &url.URL{Scheme: "https", Host: "raw.githubusercontent.com"}}) + if err != nil || u != nil { + t.Fatalf("-proxy=off HTTP proxy = (%v, %v), want direct", u, err) + } +} From 1c2be8f4845319dcb9fb874896160dd466ce3ea5 Mon Sep 17 00:00:00 2001 From: Teodor Calin Date: Thu, 24 Sep 2026 01:32:19 +0300 Subject: [PATCH 04/19] build(deps): pin github.com/pilot-protocol/common v0.5.14 Replace the feature-branch pseudo-version with the tagged release that ships netproxy (HTTP CONNECT dialer, independent proxy env parsing, credential-safe URL handling) and registry/client WithDialer. Co-Authored-By: Claude Opus 5.5 (1M context) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b7ca35b7..d47c91bd 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/coder/websocket v1.8.15 github.com/pilot-protocol/app-store v1.0.3 github.com/pilot-protocol/beacon v0.2.9 - github.com/pilot-protocol/common v0.5.14-0.20260923210943-65ea5b1a3d2d + github.com/pilot-protocol/common v0.5.14 github.com/pilot-protocol/dataexchange v0.2.2 github.com/pilot-protocol/eventstream v0.2.4 github.com/pilot-protocol/handshake v0.2.8 diff --git a/go.sum b/go.sum index 56497d0d..9c976ac4 100644 --- a/go.sum +++ b/go.sum @@ -237,8 +237,8 @@ github.com/pilot-protocol/app-store v1.0.3 h1:LfY6iZW6awTPV9nnQn+u/sTG+2UfJPxzS+ github.com/pilot-protocol/app-store v1.0.3/go.mod h1:Sz1vIZ92zNMWEnTJTXNxw7QmlbPRXs73qY/duNC0tgs= github.com/pilot-protocol/beacon v0.2.9 h1:VqXAtRKl4YhZVkDmJzuNhS1bfpK0n9qbGKOGLfVYSyo= github.com/pilot-protocol/beacon v0.2.9/go.mod h1:DE8masXGku/IwfHL8lN/4CUTe5YsxMXFe2fRGJcH7w8= -github.com/pilot-protocol/common v0.5.14-0.20260923210943-65ea5b1a3d2d h1:mwYvwpPTFmE1KHZf3ynK5LqD+Boo8irP1hx0+4noeBI= -github.com/pilot-protocol/common v0.5.14-0.20260923210943-65ea5b1a3d2d/go.mod h1:Ybc6f1A37s3ShoEh1nBMVL9DPyYlxvkqPTvtbxaNWg4= +github.com/pilot-protocol/common v0.5.14 h1:CKLjgRimNlksUe0OePHkOE32aPlemjfdRMBq8X/CDiw= +github.com/pilot-protocol/common v0.5.14/go.mod h1:OTXD84ScrBbKTAhawgFlLw9HtisgNvr83Zs1yDAuG9k= github.com/pilot-protocol/dataexchange v0.2.2 h1:h1VJFqFCdMDtX1E2E8zxTQPw+9rWnMohXN8b6HpB2R0= github.com/pilot-protocol/dataexchange v0.2.2/go.mod h1:TUj2QtNMZ4oMnOag1j5k5qDxz2457S8uaB2FJif1Ulk= github.com/pilot-protocol/eventstream v0.2.4 h1:SyB64wqo+Qpz0hIAuzHGkI9Npwk5SFSg1aqNFLueUIQ= From a3ed59f4a928cab2082bb2e7eea1261aa20f1376 Mon Sep 17 00:00:00 2001 From: Teodor Calin Date: Thu, 24 Sep 2026 01:32:46 +0300 Subject: [PATCH 05/19] feat: -transport=auto, proxy precedence, proxy-aware pilotctl, Muse onboarding Unifies the daemon (-proxy) and pilotctl/install.sh (--transport/--proxy) branches and fixes every confirmed review finding on both. pilot-daemon - -transport=auto (also $PILOT_TRANSPORT, config "transport"): udp when the beacon answers a UDP discover (one round trip; two attempts within 1.5s), else compat when the compat beacon accepts TCP through the proxy compat would use, else udp as before. Logged once. Never picks compat for a private registry/beacon. The binary default stays udp. pkg/daemon exports SelectTransport/NormalizeTransport; the embedder path (TransportMode "" or "auto") uses the same check. - Precedence for -transport, -proxy, -registry-trust, -registry-fingerprint: flag, then $PILOT_TRANSPORT / $PILOT_PROXY / $PILOT_REGISTRY_TRUST / $PILOT_REGISTRY_FINGERPRINT, then config.json, then default. Literal flag defaults: -help never prints $PILOT_PROXY. - registry_trust/registry_fingerprint from config/env survive compat mode; a fingerprint alone selects pinned trust. compat keeps the raw registry with an explicit -registry-tls=false (TCP/9000 fallback) and a custom registry from config.json. registry.pilotprotocol.network:443 always gets TLS, also in udp mode. - internal/proxyconf (on common/netproxy): off also accepts none/no/false/direct; bare words and scheme-less values are errors, not proxy host names. Loopback targets are never proxied (DefaultTransport, daemon HTTP clients, registry and WSS dials), even with an explicit URL. - WSS beacon dials through a netproxy dialer (wss.Config.DialContext replaces Config.Proxy): an https:// proxy is verified with the system roots; the beacon TLS config applies to the beacon only. - Unusable HTTP_PROXY/ALL_PROXY no longer drop a valid HTTPS_PROXY (common v0.5.14); skipped variables are logged. Certificate errors against the system store name SSL_CERT_FILE and the fingerprint. pilotctl - daemon start asks a daemon that supports it for -transport=auto when no transport is configured; the daemon is probed once (-help) and flags/values it predates are dropped (auto -> udp) with a warning. The ready summary reports the transport the daemon chose. - --proxy, then $PILOT_PROXY, then config "proxy"; any '@' means credentials, which travel as $PILOT_PROXY only, and nothing on argv contradicts them. Redaction/validation via internal/proxyconf. - lookup/register/rotate-key, the auto-handshake visibility check and recovery dial the registry through the egress proxy (CONNECT by name), using registry.pilotprotocol.network:443 over TLS when proxied or in compat mode (pinned with registry_fingerprint when configured), with a one-shot TLS fallback when the raw registry is unreachable directly. - config --set transport accepts auto and normalizes; leaving compat restores the raw-TCP registry an older compat install saved. - withTempHomeFull clears PILOT_TRANSPORT/PILOT_PROXY/*_PROXY. install.sh - --transport auto|udp|compat; fresh installs save auto when the daemon supports it; no "proxy" key is written; units take the transport from config.json; --transport udp restores the raw registry. - Root is allowed in a Linux container/VM without systemd (hosted agent sandboxes); regular hosts still refuse without PILOT_ALLOW_ROOT. Docs: README compat/proxy section (precedence, SSL_CERT_FILE, pinned registry), env table, CHANGELOG, regenerated docs/cli-reference.md. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 86 +++- README.md | 20 +- cmd/daemon/main.go | 123 ++++-- cmd/daemon/netflags.go | 211 ++++++++++ cmd/daemon/netflags_test.go | 342 ++++++++++++++++ cmd/daemon/proxy.go | 63 +-- cmd/daemon/proxy_test.go | 124 +++--- cmd/pilotctl/daemon_transport.go | 382 ++++++++++------- cmd/pilotctl/main.go | 187 +++++---- cmd/pilotctl/registry_dial.go | 196 +++++++++ cmd/pilotctl/verify.go | 9 +- cmd/pilotctl/zz_daemon_transport_test.go | 119 +++--- cmd/pilotctl/zz_lifecycle_test.go | 6 + cmd/pilotctl/zz_proxy_route_test.go | 385 ++++++++++++++++++ docs/cli-reference.md | 8 +- install.sh | 201 +++++---- internal/proxyconf/proxyconf.go | 181 ++++++++ internal/proxyconf/proxyconf_test.go | 252 ++++++++++++ pkg/daemon/daemon.go | 85 ++-- pkg/daemon/proxy.go | 65 ++- pkg/daemon/transport/wss/wss.go | 17 +- pkg/daemon/transport/wss/zz_wss_proxy_test.go | 83 +++- pkg/daemon/transport_auto.go | 177 ++++++++ pkg/daemon/tunnel.go | 23 +- pkg/daemon/zz_transport_auto_test.go | 258 ++++++++++++ 25 files changed, 3015 insertions(+), 588 deletions(-) create mode 100644 cmd/daemon/netflags.go create mode 100644 cmd/daemon/netflags_test.go create mode 100644 cmd/pilotctl/registry_dial.go create mode 100644 cmd/pilotctl/zz_proxy_route_test.go create mode 100644 internal/proxyconf/proxyconf.go create mode 100644 internal/proxyconf/proxyconf_test.go create mode 100644 pkg/daemon/transport_auto.go create mode 100644 pkg/daemon/zz_transport_auto_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 6254e481..489b93c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,25 +18,66 @@ Detailed per-release notes are on the `false` (the default) to switch app auto-updates back on. Pilot daemon/CLI binary updates are never affected. Honors the existing `PILOT_UPDATER_NO_APP_UPGRADE` as a back-compat alias. -- **`pilotctl daemon start --transport --proxy `.** - Also read from config.json (`transport`, `proxy`; set them with - `pilotctl config --set`) and, for the transport, `$PILOT_TRANSPORT`. Both are - forwarded to pilot-daemon only when set, and dropped with a warning when the - paired daemon binary predates them, so a new pilotctl still starts an older - daemon. A proxy URL with credentials travels as `$PILOT_PROXY`, never on the - daemon's argv, and is shown redacted by `pilotctl config`. +- **Works behind an HTTPS proxy with UDP blocked (Meta Muse and other hosted + agent sandboxes) — no flags needed.** pilot-daemon gets `-proxy + ` (`$PILOT_PROXY`, config.json `proxy`): with compat mode the + registry, the WSS beacon and every plugin HTTP client go through + `$HTTPS_PROXY` / `$ALL_PROXY` (honoring `$NO_PROXY`), CONNECTing by host + name so poisoned local DNS does not matter, with TLS end to end; an + explicit `http(s)://[user:pass@]host:port` proxies every connection except + loopback. `off` also accepts `none`, `no`, `false`, `direct`; any other bare + word is an error instead of a proxy host name. Proxy credentials never + appear in logs, errors or `-help`. +- **`-transport=auto`.** UDP when the beacon answers a UDP discover (one round + trip), otherwise compat when the compat beacon is reachable over TCP 443 + (through the proxy, if any), otherwise udp as before; the decision is logged + once (`transport auto-selected`). It never moves a node with a private + registry or beacon onto the public compat beacon. pilot-daemon's own default + stays `udp`; `pilotctl daemon start` asks for `auto` whenever no transport is + configured and the daemon supports it, and fresh installs save + `transport=auto`. +- **`$PILOT_TRANSPORT` and config.json `transport` are honored by pilot-daemon + itself** (both used to be masked by the `-transport` default). Precedence for + `-transport`, `-proxy`, `-registry-trust` and `-registry-fingerprint`: flag, + then `$PILOT_TRANSPORT` / `$PILOT_PROXY` / `$PILOT_REGISTRY_TRUST` / + `$PILOT_REGISTRY_FINGERPRINT`, then config.json, then the default — so a + credential-bearing proxy handed over in the environment beats a saved + `"proxy": "auto"`. +- **Pinned registry trust from config/env for sandboxes without a CA bundle.** + `registry_trust` / `registry_fingerprint` in config.json (or the env vars + above) now survive compat mode, and a fingerprint alone selects pinned + trust there. Certificate errors name the fix: `SSL_CERT_FILE` / + `SSL_CERT_DIR` (forwarded by pilotctl) or the registry fingerprint. +- **`pilotctl daemon start --transport --proxy `**, + also from `$PILOT_TRANSPORT` / `$PILOT_PROXY` and config.json. The daemon + binary is probed once (`-help`): flags or values it predates are dropped + (auto becomes udp) with a warning, so a new pilotctl still starts an older + daemon, and an old pilotctl (v1.13.9) starts the new daemon unchanged. A + proxy URL with credentials (any `@`) travels as `$PILOT_PROXY`, never on the + daemon's argv, and is shown redacted. The ready summary reports the + transport the daemon actually chose. +- **pilotctl's own registry connections follow the proxy.** `lookup`, + `register`, `rotate-key`, the auto-handshake visibility check and `recovery + recover` dial through `$PILOT_PROXY` / config `proxy` / `$HTTPS_PROXY` + (via common/netproxy), using `registry.pilotprotocol.network:443` over TLS + when proxied or in compat mode, and fall back to it when the raw-TCP + registry is unreachable directly. `proxy=off` restores direct dials. - **`daemon start` forwards the proxy/TLS environment** (`HTTPS_PROXY`, `HTTP_PROXY`, `ALL_PROXY`, `NO_PROXY` in both cases, `PILOT_PROXY`, - `PILOT_TRANSPORT`, `SSL_CERT_FILE`, `SSL_CERT_DIR`) to the daemon on both the - fork and `--foreground` paths, and passes through `--compat-beacon`, - `--registry-trust`, `--registry-fingerprint` and `--tls-trust`. -- **`install.sh --transport compat`** (or `PILOT_TRANSPORT=compat`) for hosts - that block UDP or reach the internet only through an authenticating HTTPS - proxy: writes `transport=compat` + `proxy=auto` into config.json and - generates compat service units. Every installer download goes through - `$HTTPS_PROXY` (CONNECT to :443 by hostname only); service setup without - root/systemd/launchd degrades to a printed `pilotctl daemon start` hint, - download failures name the proxy (redacted) and the hosts it must allow. + `PILOT_TRANSPORT`, `PILOT_REGISTRY_TRUST`, `PILOT_REGISTRY_FINGERPRINT`, + `SSL_CERT_FILE`, `SSL_CERT_DIR`) on both the fork and `--foreground` paths, + and passes through `--compat-beacon`, `--registry-trust`, + `--registry-fingerprint` and `--tls-trust`. +- **`install.sh --transport `** (or `PILOT_TRANSPORT`). Fresh + installs save `auto`; `compat` skips the UDP probe. No `proxy` key is + written (the daemon default already uses the environment's proxy). Service + units take the transport from config.json, so `pilotctl config --set + transport=` applies to them too. Every installer download goes through + `$HTTPS_PROXY`; service setup without root/systemd/launchd degrades to a + printed `pilotctl daemon start` hint; download failures name the proxy + (redacted) and the hosts it must allow. In a Linux container/VM without + systemd the installer runs as root without `PILOT_ALLOW_ROOT` (hosted + sandboxes run the agent as root); regular hosts still refuse root. ### Fixed - **Compat daemons started by `pilotctl` were pinned to the raw-TCP registry.** @@ -53,6 +94,17 @@ Detailed per-release notes are on the reported as a concurrent start on every attempt. - **`daemon start --endpoint` / `--motd-feed-url` / `--motd-interval`** were documented but never forwarded to the daemon. +- **`-transport=compat -registry=34.71.57.205:9000 -registry-tls=false`** (the + raw TCP/9000 registry fallback) keeps the raw registry again instead of + sending plaintext to the TLS registry on :443; a custom registry in + config.json is also kept in compat mode. +- **Switching back from compat.** A udp daemon given + `registry.pilotprotocol.network:443` now uses TLS for it, and `install.sh + --transport udp` / `pilotctl config --set transport=udp|auto` restore the + raw-TCP registry an older compat install saved. +- **An https:// proxy was verified with the beacon's pinned roots** under + `-tls-trust=pinned`; the proxy's certificate is now checked against the + system roots and the beacon's trust store applies to the beacon only. - **The `pilotctl skills disable` opt-out now survives updates and explicit reconciles.** A forced reconcile — `pilotctl skills check`, `pilotctl update`, or an installer re-run — bypassed the disabled flag and re-injected skills a diff --git a/README.md b/README.md index 72d6f87e..e6397fcb 100644 --- a/README.md +++ b/README.md @@ -287,14 +287,19 @@ Set a hostname and email during install: curl -fsSL https://pilotprotocol.network/install.sh | PILOT_EMAIL=user@example.com PILOT_HOSTNAME=my-agent sh ``` -UDP blocked, or the only way out is an HTTPS proxy (hosted agent sandboxes, locked-down VMs)? Install in compat mode: +**UDP blocked, or the only way out is an HTTPS proxy** (hosted agent sandboxes such as Meta Muse, locked-down VMs)? The plain install works: ```bash -curl -fsSL https://pilotprotocol.network/install.sh | sh -s -- --transport compat +curl -fsSL https://pilotprotocol.network/install.sh | sh pilotctl daemon start ``` -Compat mode keeps every connection on TCP 443 (registry over TLS, beacon over WSS). `--transport compat` writes `"transport": "compat"` and `"proxy": "auto"` to `~/.pilot/config.json`; with `proxy=auto` the daemon tunnels through `$HTTPS_PROXY` / `$ALL_PROXY` (honoring `$NO_PROXY`) when set, asking the proxy to `CONNECT` by hostname. No root, systemd or launchd needed: start the daemon with `pilotctl daemon start` from a shell that has the proxy variables. To switch an existing install: `pilotctl config --set transport=compat`, or one-off `pilotctl daemon start --transport compat --proxy `. +New installs use transport `auto`: the daemon probes the beacon over UDP once at startup (one round trip when UDP works, at most ~1.5s when it does not) and, when there is no answer but TCP 443 is reachable, runs in **compat** mode — registry over TLS on `registry.pilotprotocol.network:443`, beacon over WSS on `beacon.pilotprotocol.network:443`. In compat mode the daemon tunnels through `$HTTPS_PROXY` / `$ALL_PROXY` (honoring `$NO_PROXY`), asking the proxy to `CONNECT` by host name, so poisoned local DNS for the Pilot hosts does not matter. `pilotctl`'s own registry commands (`lookup`, the auto-handshake check, `recovery`) go through the same proxy. To skip the UDP probe, pick compat explicitly: `sh -s -- --transport compat`, `pilotctl config --set transport=compat`, or `pilotctl daemon start --transport compat`. + +- **No root, systemd or launchd needed.** Start the daemon with `pilotctl daemon start` from a shell that has the proxy variables. In a container or VM without systemd the installer also runs as root (the agent user in hosted sandboxes); on a regular host it still refuses root unless `PILOT_ALLOW_ROOT=1`. +- **Proxy with credentials:** keep them out of `ps` — export `HTTPS_PROXY` / `PILOT_PROXY`, or `pilotctl daemon start --proxy http://user:pass@host:port` (pilotctl hands a URL with credentials to the daemon in its environment, never on its command line). For a systemd/launchd service, which does not see your shell's variables, save it: `pilotctl config --set proxy=http://user:pass@host:port` (config.json is 0600). +- **Precedence** for transport and proxy: command-line flag, then `$PILOT_TRANSPORT` / `$PILOT_PROXY`, then `config.json` (`transport`, `proxy`), then the default (`auto` from pilotctl, `udp` for a bare `pilot-daemon`; proxy `auto`). `-proxy` accepts `auto`, `off` (also `none`, `direct`) or an `http://` / `https://` URL; anything else is an error. Loopback targets (local webhooks, sidecars) are never sent to a proxy. +- **No CA bundle in the sandbox?** Point Go at one with `SSL_CERT_FILE=/path/to/ca-certificates.crt` (or `SSL_CERT_DIR`) — `pilotctl daemon start` forwards both. The registry can instead be pinned: `PILOT_REGISTRY_FINGERPRINT=` (or `pilotctl config --set registry_fingerprint=...`), which selects `registry_trust=pinned`. The WSS beacon has no fingerprint option, so it needs the CA bundle.
What the installer does @@ -479,15 +484,14 @@ Most daemon flags have an environment variable equivalent. Useful for containeri |----------|----------------|---------| | `PILOT_REGISTRY` | `-registry` | Registry server address | | `PILOT_BEACON` | `-beacon` | Beacon server address | -| `PILOT_TRANSPORT` | `-transport` | Tunnel transport: `udp` (default) or `compat` (WSS on 443, for UDP-blocked hosts) | -| `PILOT_PROXY` | `-proxy` | Outbound proxy for registry, beacon and HTTP traffic: `auto` (default; with `compat`, the `HTTPS_PROXY`/`ALL_PROXY` proxy, honoring `NO_PROXY`), `off`, or `http://[user:pass@]host:port` for every connection | +| `PILOT_TRANSPORT` | `-transport` | Tunnel transport: `udp` (daemon default), `compat` (TLS registry + WSS beacon on TCP 443 only) or `auto` (udp when the beacon answers over UDP, else compat). Beats `config.json` | +| `PILOT_PROXY` | `-proxy` | Outbound proxy: `auto` (default; with compat, the `HTTPS_PROXY`/`ALL_PROXY` proxy honoring `NO_PROXY`), `off` (also `none`, `direct`), or `http(s)://[user:pass@]host:port` for every connection except loopback. Beats `config.json` | +| `PILOT_REGISTRY_TRUST` / `PILOT_REGISTRY_FINGERPRINT` | `-registry-trust` / `-registry-fingerprint` | Pin the TLS registry (hosts without a CA bundle); a fingerprint alone selects pinned trust in compat mode. Beat `config.json` | +| `HTTPS_PROXY` / `ALL_PROXY` / `NO_PROXY` | — | Proxy used with `-proxy=auto` in compat mode and by `pilotctl`'s own registry dials; `pilotctl daemon start` forwards them (and `SSL_CERT_FILE` / `SSL_CERT_DIR`) to the daemon | | `PILOT_SOCKET` | `-socket` | Unix socket path | | `PILOT_EMAIL` | `-email` | Account email | | `PILOT_HOSTNAME` | `-hostname` | Discovery hostname | | `PILOT_ADMIN_TOKEN` | `-admin-token` | Admin token for network operations | -| `PILOT_TRANSPORT` | `-transport` | `udp` (default) or `compat` (TCP 443 only). `pilotctl daemon start` turns it into an explicit `-transport` | -| `PILOT_PROXY` | `-proxy` | `auto` (default: proxy from the environment in compat mode), `off`, or `http(s)://[user:pass@]host:port` | -| `HTTPS_PROXY` / `ALL_PROXY` / `NO_PROXY` | — | Proxy used by compat mode with `proxy=auto`; `pilotctl daemon start` forwards them (and `SSL_CERT_FILE` / `SSL_CERT_DIR`) to the daemon | | `PILOT_MOTD_URL` | `-motd-feed-url` | Message-of-the-day feed URL | | `PILOT_TELEMETRY_URL` | `-telemetry-url` | Telemetry endpoint override | | `PILOT_SYN_WHITELIST` | `-syn-whitelist` | Nodes exempt from SYN rate limit | diff --git a/cmd/daemon/main.go b/cmd/daemon/main.go index 248b7c1f..752e146e 100644 --- a/cmd/daemon/main.go +++ b/cmd/daemon/main.go @@ -22,10 +22,10 @@ import ( "github.com/pilot-protocol/common/config" "github.com/pilot-protocol/common/driver" "github.com/pilot-protocol/common/logging" - "github.com/pilot-protocol/common/netproxy" "github.com/pilot-protocol/pilotprotocol/internal/enterprisecontrol" "github.com/pilot-protocol/pilotprotocol/internal/managedsdk/authority" "github.com/pilot-protocol/pilotprotocol/internal/motd" + "github.com/pilot-protocol/pilotprotocol/internal/proxyconf" "github.com/pilot-protocol/pilotprotocol/pkg/daemon" // L11 plugin imports — cmd/daemon (L12) is the only place these @@ -73,8 +73,8 @@ func main() { advertiseEndpoint := flag.String("advertise-endpoint", "", "override STUN-discovered endpoint for registry advertisement (host:port) — for k8s pods where STUN returns unreachable IPs. When set, STUN still runs but the advertised address uses this value") encrypt := flag.Bool("encrypt", true, "enable tunnel-layer encryption (X25519 + AES-256-GCM)") registryTLS := flag.Bool("registry-tls", false, "use TLS for registry connection") - registryFingerprint := flag.String("registry-fingerprint", "", "hex SHA-256 fingerprint of registry TLS certificate (required when -registry-trust=pinned)") - registryTrust := flag.String("registry-trust", "pinned", "trust store for -registry-tls: 'pinned' (verify cert against -registry-fingerprint) or 'system' (OS x509 root store — used for compat-mode registry on registry.pilotprotocol.network:443 with Let's Encrypt)") + registryFingerprint := flag.String("registry-fingerprint", "", "hex SHA-256 fingerprint of registry TLS certificate (required when -registry-trust=pinned). With compat mode and no -registry-trust, a fingerprint selects pinned trust — the fallback for hosts without a CA bundle. Precedence: this flag, $PILOT_REGISTRY_FINGERPRINT, config.json \"registry_fingerprint\".") + registryTrust := flag.String("registry-trust", "pinned", "trust store for -registry-tls: 'pinned' (verify cert against -registry-fingerprint) or 'system' (OS x509 root store; set SSL_CERT_FILE/SSL_CERT_DIR where the host has no CA bundle — used for compat-mode registry on registry.pilotprotocol.network:443 with Let's Encrypt). Precedence: this flag, $PILOT_REGISTRY_TRUST, config.json \"registry_trust\".") identityPath := flag.String("identity", "", "path to persist Ed25519 identity (enables stable identity across restarts)") email := flag.String("email", "", "email address for account identification and key recovery") owner := flag.String("owner", "", "(deprecated: use -email) owner identifier for key rotation recovery") @@ -109,10 +109,14 @@ func main() { beaconRTTProbe := flag.Bool("beacon-rtt-probe", false, "probe beacon RTT before selection; override hash pick when >2× slower than best (ablation test, default off)") noRxWatchdog := flag.Bool("no-rx-watchdog", false, "disable the inbound-path watchdog that soft-recovers (beacon+registry re-registration) and, on a persistent wedge, exits non-zero for supervisor respawn") noPathWatch := flag.Bool("no-path-watch", false, "disable the per-peer path watchdog that probes inbound-silent peers and resets a dead peer path in place (prefer-direct sequence) without a daemon restart") - transportMode := flag.String("transport", transportDefault(), "tunnel transport: 'udp' (default) or 'compat' (WSS to beacon, opt-in, for UDP-blocked environments). Env: PILOT_TRANSPORT.") - proxySpec := flag.String("proxy", envString("PILOT_PROXY", netproxy.ModeAuto), "outbound proxy for registry, beacon and HTTP connections: 'auto' (with -transport=compat, HTTPS_PROXY/ALL_PROXY from the environment, honoring NO_PROXY; nothing with -transport=udp), 'off', or a proxy URL 'http://[user:pass@]host:port' used for every connection. Env: PILOT_PROXY.") - compatBeacon := flag.String("compat-beacon", "wss://beacon.pilotprotocol.network/v1/compat", "beacon WSS URL for -transport=compat") - tlsTrust := flag.String("tls-trust", "system", "TLS trust store for -transport=compat: 'system' (OS trust store; current default while compat mode uses Let's Encrypt certs on beacon.pilotprotocol.network) or 'pinned' (Pilot CA root embedded in the daemon binary; will become the default in a future release once production root ships)") + // -transport and -proxy have literal defaults: their environment + // variables beat config.json (see flagSources.envOverConfig), and -help + // must never print an environment value — PILOT_PROXY can hold proxy + // credentials. + transportMode := flag.String("transport", "", "tunnel transport: 'udp' (the default), 'compat' (registry over TLS and beacon over WSS, TCP 443 only, for UDP-blocked or proxy-only hosts) or 'auto' (udp when the beacon answers over UDP, otherwise compat when TCP 443 is reachable, through the proxy if there is one). Precedence: this flag, $PILOT_TRANSPORT, config.json \"transport\", udp.") + proxySpec := flag.String("proxy", "", "outbound proxy for registry, beacon and HTTP connections: 'auto' (the default: with compat, HTTPS_PROXY/ALL_PROXY from the environment, honoring NO_PROXY; nothing with udp), 'off' (also none, no, false, direct), or an http:// or https:// proxy URL, http://[user:pass@]host:port, used for every connection except loopback. Precedence: this flag, $PILOT_PROXY, config.json \"proxy\", auto.") + compatBeacon := flag.String("compat-beacon", defaultCompatBeacon, "beacon WSS URL for -transport=compat") + tlsTrust := flag.String("tls-trust", "system", "TLS trust store for -transport=compat: 'system' (OS trust store; current default while compat mode uses Let's Encrypt certs on beacon.pilotprotocol.network — on a host without a CA bundle set SSL_CERT_FILE or SSL_CERT_DIR) or 'pinned' (Pilot CA root embedded in the daemon binary; will become the default in a future release once production root ships)") showVersion := flag.Bool("version", false, "print version and exit") logLevel := flag.String("log-level", "info", "log level (debug, info, warn, error)") logFormat := flag.String("log-format", "text", "log format (text, json)") @@ -158,12 +162,17 @@ func main() { } } } + var fileCfg map[string]interface{} if *configPath != "" { cfg, err := config.Load(*configPath) if err != nil { log.Fatalf("load config: %v", err) } - config.ApplyToFlags(cfg) + fileCfg = cfg + } + sources := newFlagSources(flag.CommandLine, fileCfg) + if fileCfg != nil { + config.ApplyToFlags(fileCfg) } if *enterpriseControlPath == "" { if discovered, ok := discoverManagedEnterpriseControl(); ok { @@ -171,34 +180,75 @@ func main() { } } - // Compat-mode 443-only defaults. When -transport=compat is selected - // and the operator hasn't explicitly overridden -registry/-registry-tls/ - // -registry-trust, route the registry to its TLS hostname (TCP/443 - // via nginx SNI routing on the production rendezvous box) so the - // daemon really does use a single port. The TCP/9000 fallback is - // still available to anyone who passes a non-default -registry - // explicitly; the compiled-in default counts as not explicit (see - // compatKeepsRegistry). -beacon needs no such rule: in compat mode the - // UDP beacon address is only the relay-wrap destination on the WSS - // pipe and is never dialed. - if *transportMode == "compat" { - explicit := map[string]bool{} - flag.Visit(func(f *flag.Flag) { explicit[f.Name] = true }) - if !compatKeepsRegistry(*registryAddr, explicit["registry"], os.Getenv("PILOT_REGISTRY") != "") { - v := compatRegistryAddr - registryAddr = &v - registryFromEnv = false - } - if !explicit["registry-tls"] { - v := true - registryTLS = &v - } - if !explicit["registry-trust"] { - v := "system" - registryTrust = &v - slog.Warn("compat-mode registry-trust defaulted to 'system' (Let's Encrypt validation). Override with -registry-trust=pinned if using pinned certificates (supply -registry-fingerprint).") + logging.Setup(*logLevel, *logFormat) + + // Launch-time settings: flag, then environment, then config.json. + configTransport := *transportMode + var transportSrc string + *transportMode, transportSrc = sources.envOverConfig("transport", *transportMode, "PILOT_TRANSPORT") + *proxySpec, _ = sources.envOverConfig("proxy", *proxySpec, "PILOT_PROXY") + var trustSrc string + *registryTrust, trustSrc = sources.envOverConfig("registry-trust", *registryTrust, "PILOT_REGISTRY_TRUST") + *registryFingerprint, _ = sources.envOverConfig("registry-fingerprint", *registryFingerprint, "PILOT_REGISTRY_FINGERPRINT") + + transport, err := daemon.NormalizeTransport(*transportMode) + if err != nil && transportSrc == srcEnv { + // A stray environment variable never stops the daemon. + slog.Warn("ignoring unknown PILOT_TRANSPORT value", "value", *transportMode, "valid", "udp, compat, auto") + transportSrc = srcDefault + if sources.config["transport"] { + transportSrc = srcConfig + } + transport, err = daemon.NormalizeTransport(configTransport) + } + if err != nil { + log.Fatalf("-transport: %v", err) + } + if transport == "" { + transport = daemon.TransportUDP + } + if _, err := proxyconf.Normalize(*proxySpec); err != nil { + log.Fatalf("-proxy: %v", err) + } + + reg := registrySettings{ + Addr: *registryAddr, + AddrExplicit: sources.explicit("registry") || registryFromEnv, + TLS: *registryTLS, + TLSExplicit: sources.explicit("registry-tls"), + Trust: *registryTrust, + TrustExplicit: sources.explicit("registry-trust") || trustSrc == srcEnv, + Fingerprint: *registryFingerprint, + } + + // -transport=auto: probe once, before anything depends on the mode. + if transport == daemon.TransportAuto { + mode, reason, err := resolveAutoTransport(reg, *beaconAddr, sources.explicit("beacon") || beaconFromEnv, + *compatBeacon, sources.explicit("compat-beacon"), *proxySpec) + if err != nil { + log.Fatalf("-proxy: %v", err) } + slog.Info("transport auto-selected", "transport", mode, "reason", reason) + transport = mode } + *transportMode = transport + + // Registry defaults for the transport (see applyRegistryDefaults): + // compat moves the compiled-in raw-TCP registry to + // registry.pilotprotocol.network:443 over TLS, so the daemon really + // uses a single port; an explicit non-default -registry, or an + // explicit -registry-tls=false (the TCP/9000 fallback), is kept. + // -beacon needs no such rule: in compat mode the UDP beacon address is + // only the relay-wrap destination on the WSS pipe and is never dialed. + final := applyRegistryDefaults(transport, reg) + if final.Addr != reg.Addr { + registryFromEnv = false + } + if final.Trust != reg.Trust { + slog.Info("registry trust defaulted for the TLS registry", "registry_trust", final.Trust, + "hint", "override with -registry-trust (config registry_trust, env PILOT_REGISTRY_TRUST); pinned needs -registry-fingerprint") + } + *registryAddr, *registryTLS, *registryTrust = final.Addr, final.TLS, final.Trust profileOptions := daemonSecurityOptions{ RegistryAddr: *registryAddr, @@ -226,8 +276,6 @@ func main() { *noSkillinject = profileOptions.DisableSkillinject *motdFeedURL = profileOptions.MOTDFeedURL - logging.Setup(*logLevel, *logFormat) - // Outbound proxy: resolved once, after -transport is final, and shared // by everything that dials out — the registry client, the compat WSS // beacon, pkg/daemon's own HTTP fetches (via daemon.Config.Proxy) and @@ -237,7 +285,8 @@ func main() { log.Fatalf("-proxy: %v", err) } installDefaultTransportProxy(proxyPolicy) - slog.Info("outbound network", "transport", *transportMode, "proxy", describeProxy(*proxySpec, *transportMode, proxyPolicy)) + slog.Info("outbound network", "transport", *transportMode, "proxy", describeProxy(*proxySpec, *transportMode, proxyPolicy), + "transport_from", transportSrc, "registry", *registryAddr, "registry_tls", *registryTLS) // Sandbox: validate all configured file paths are under the confinement // root before the daemon touches the filesystem. Network paths are unaffected. diff --git a/cmd/daemon/netflags.go b/cmd/daemon/netflags.go new file mode 100644 index 00000000..d72f8d77 --- /dev/null +++ b/cmd/daemon/netflags.go @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "context" + "flag" + "log/slog" + "net" + "os" + "strings" + + "github.com/pilot-protocol/pilotprotocol/internal/proxyconf" + "github.com/pilot-protocol/pilotprotocol/pkg/daemon" +) + +// Compiled-in production endpoints. -registry and -beacon default to these +// raw TCP / UDP addresses (pilotctl passes the same values explicitly); +// compat mode swaps the registry for its TLS host name on :443. +const ( + defaultRegistryAddr = "34.71.57.205:9000" + defaultBeaconAddr = "34.71.57.205:9001" + compatRegistryAddr = "registry.pilotprotocol.network:443" + defaultCompatBeacon = "wss://beacon.pilotprotocol.network/v1/compat" +) + +// Where a setting's final value came from, for logs and precedence. +const ( + srcFlag = "flag" + srcEnv = "env" + srcConfig = "config" + srcDefault = "default" +) + +// flagSources records which flags the command line set and which +// config.json will set (common/config.ApplyToFlags sets a flag's value +// without marking it as set, so flag.Visit alone cannot tell a config +// value from a default). +type flagSources struct { + cmdline map[string]bool + config map[string]bool +} + +// newFlagSources must run after flag.Parse and before config.ApplyToFlags. +// cfg may be nil (no config file). +func newFlagSources(fs *flag.FlagSet, cfg map[string]interface{}) flagSources { + s := flagSources{cmdline: map[string]bool{}, config: map[string]bool{}} + fs.Visit(func(f *flag.Flag) { s.cmdline[f.Name] = true }) + fs.VisitAll(func(f *flag.Flag) { + if s.cmdline[f.Name] { + return + } + v, ok := cfg[f.Name] + if !ok { + v, ok = cfg[strings.ReplaceAll(f.Name, "-", "_")] + } + if !ok { + return + } + switch v.(type) { // the types ApplyToFlags applies + case string, float64, bool: + s.config[f.Name] = true + } + }) + return s +} + +// explicit reports whether the operator chose the flag's value, on the +// command line or in config.json. +func (s flagSources) explicit(name string) bool { + return s.cmdline[name] || s.config[name] +} + +// envOverConfig resolves a setting whose environment variable beats +// config.json: the command line, then the environment, then config.json +// (already applied to cur by ApplyToFlags), then the flag's default. It +// exists for settings that a launcher hands over per start — pilotctl passes +// a credential-bearing proxy URL as $PILOT_PROXY so it never appears on the +// daemon's argv, and a persistent config.json default must not override it. +func (s flagSources) envOverConfig(name, cur, envVar string) (string, string) { + if s.cmdline[name] { + return cur, srcFlag + } + if v := strings.TrimSpace(getenv(envVar)); v != "" { + return v, srcEnv + } + if s.config[name] { + return cur, srcConfig + } + return cur, srcDefault +} + +// getenv is os.Getenv (a test seam). +var getenv = os.Getenv + +// registrySettings is the registry part of the daemon's configuration +// after flags, config.json and the environment are merged. +type registrySettings struct { + Addr string + // AddrExplicit: set by -registry, $PILOT_REGISTRY or config.json. + AddrExplicit bool + TLS bool + TLSExplicit bool + Trust string + TrustExplicit bool + Fingerprint string +} + +// compatKeepsRegistry reports whether a -transport=compat daemon keeps its +// configured registry instead of switching to compatRegistryAddr: +// +// - an explicit -registry-tls=false keeps the address: the operator asked +// for the raw TCP registry (the TCP/9000 fallback for hosts where UDP is +// blocked but TCP 9000 is open); +// - otherwise only an explicit, non-default address is kept. The +// compiled-in raw-TCP default never counts as a choice: pilotctl and +// `pilotctl init`'s config.json pass it on every start, and a +// UDP-blocked host behind a CONNECT-only egress proxy cannot reach it. +func compatKeepsRegistry(r registrySettings) bool { + if r.TLSExplicit && !r.TLS { + return true + } + return r.AddrExplicit && strings.TrimSpace(r.Addr) != defaultRegistryAddr +} + +// applyRegistryDefaults adapts the registry to the final transport: +// +// - compat: the registry moves to its TLS host name on :443 unless +// compatKeepsRegistry, and TLS is on unless -registry-tls was chosen; +// - either transport: the compat registry address (registry. +// pilotprotocol.network:443) is TLS-only, so TLS is on for it unless +// -registry-tls was chosen — an install switched back from compat to +// udp keeps working; +// - whenever TLS was turned on here and -registry-trust was not chosen, +// trust is "pinned" when a -registry-fingerprint is configured (the +// fallback for sandboxes without a CA bundle) and "system" otherwise +// (the production registry has a Let's Encrypt certificate). +// +// Choices made in config.json or the environment count as explicit, so a +// pinned registry configured there is never overridden. +func applyRegistryDefaults(transport string, r registrySettings) registrySettings { + tlsDefaulted := false + if transport == daemon.TransportCompat { + if !compatKeepsRegistry(r) { + r.Addr = compatRegistryAddr + } + if !r.TLSExplicit { + r.TLS = true + tlsDefaulted = true + } + } + if strings.EqualFold(strings.TrimSpace(r.Addr), compatRegistryAddr) && !r.TLSExplicit && !r.TLS { + r.TLS = true + tlsDefaulted = true + } + if tlsDefaulted && !r.TrustExplicit { + if strings.TrimSpace(r.Fingerprint) != "" { + r.Trust = "pinned" + } else { + r.Trust = "system" + } + } + return r +} + +// autoCompatBlocker returns why -transport=auto must not pick compat for +// this configuration ("" when it may): a private registry that compat would +// keep but that is not known to speak TLS, or a private UDP beacon with the +// public compat beacon — compat would silently move such a node onto the +// public network. +func autoCompatBlocker(r registrySettings, beacon string, beaconExplicit, compatBeaconExplicit bool) string { + if compatKeepsRegistry(r) && !strings.EqualFold(strings.TrimSpace(r.Addr), compatRegistryAddr) && !(r.TLSExplicit && r.TLS) { + return "custom -registry " + r.Addr + " (raw TCP)" + } + if beaconExplicit && strings.TrimSpace(beacon) != defaultBeaconAddr && !compatBeaconExplicit { + return "custom -beacon " + beacon + " without -compat-beacon" + } + return "" +} + +// autoProbe is daemon.SelectTransport (a test seam). +var autoProbe = daemon.SelectTransport + +// resolveAutoTransport decides -transport=auto (see daemon.SelectTransport) +// for this configuration: compat is only considered when it would reach +// the same network (autoCompatBlocker), and its TCP check runs through the +// proxy compat mode would use (-proxy, auto = the environment's). The error +// is a malformed -proxy. +func resolveAutoTransport(reg registrySettings, beacon string, beaconExplicit bool, compatBeacon string, compatBeaconExplicit bool, proxySpec string) (mode, reason string, err error) { + if why := autoCompatBlocker(reg, beacon, beaconExplicit, compatBeaconExplicit); why != "" { + return daemon.TransportUDP, why + "; auto stays on udp (pass -transport=compat to force compat)", nil + } + policy, err := daemon.ResolveProxy(proxySpec, daemon.TransportCompat) + if err != nil { + if !isAutoProxy(proxySpec) { + return "", "", err + } + slog.Warn("proxy environment is malformed; the compat check dials directly", "err", err) + policy = nil + } + var dial func(ctx context.Context, network, addr string) (net.Conn, error) + if policy.Enabled() { + dial = proxyconf.DialContext(policy, nil) + } + mode, reason = autoProbe(context.Background(), daemon.AutoTransportProbe{ + BeaconAddr: beacon, + CompatBeaconURL: compatBeacon, + Dial: dial, + }) + return mode, reason, nil +} diff --git a/cmd/daemon/netflags_test.go b/cmd/daemon/netflags_test.go new file mode 100644 index 00000000..fa33b5a2 --- /dev/null +++ b/cmd/daemon/netflags_test.go @@ -0,0 +1,342 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "context" + "flag" + "fmt" + "net" + "os" + "os/exec" + "strings" + "testing" + "time" + + "github.com/pilot-protocol/pilotprotocol/pkg/daemon" +) + +func TestFlagSourcesEnvOverConfig(t *testing.T) { + fs := flag.NewFlagSet("t", flag.ContinueOnError) + fs.String("proxy", "", "") + fs.String("transport", "", "") + fs.String("registry-trust", "pinned", "") + fs.String("registry-fingerprint", "", "") + if err := fs.Parse([]string{"-transport", "udp"}); err != nil { + t.Fatal(err) + } + cfg := map[string]interface{}{"proxy": "auto", "transport": "compat", "registry_trust": "system", "registry-fingerprint": 7.0} + src := newFlagSources(fs, cfg) + if !src.cmdline["transport"] || src.config["transport"] { + t.Fatalf("transport: cmdline=%v config=%v, want cmdline only", src.cmdline["transport"], src.config["transport"]) + } + for _, name := range []string{"proxy", "registry-trust", "registry-fingerprint"} { + if !src.config[name] || !src.explicit(name) { + t.Errorf("%s not recognised as set by config.json", name) + } + } + + env := map[string]string{} + getenv = func(k string) string { return env[k] } + t.Cleanup(func() { getenv = os.Getenv }) + + // flag > env > config > default + env["PILOT_TRANSPORT"] = "auto" + if v, from := src.envOverConfig("transport", "udp", "PILOT_TRANSPORT"); v != "udp" || from != srcFlag { + t.Errorf("transport = (%q, %s), want the flag", v, from) + } + env["PILOT_PROXY"] = "http://muse:s3cret@egress.test:3128" + if v, from := src.envOverConfig("proxy", "auto", "PILOT_PROXY"); v != env["PILOT_PROXY"] || from != srcEnv { + t.Errorf("proxy = (%q, %s), want $PILOT_PROXY over config.json's auto", v, from) + } + delete(env, "PILOT_PROXY") + if v, from := src.envOverConfig("proxy", "auto", "PILOT_PROXY"); v != "auto" || from != srcConfig { + t.Errorf("proxy = (%q, %s), want config.json", v, from) + } + src2 := newFlagSources(fs, nil) + if v, from := src2.envOverConfig("registry-trust", "pinned", "PILOT_REGISTRY_TRUST"); v != "pinned" || from != srcDefault { + t.Errorf("registry-trust = (%q, %s), want the default", v, from) + } +} + +func TestCompatKeepsRegistry(t *testing.T) { + for _, tc := range []struct { + name string + r registrySettings + want bool + }{ + {"default, not chosen", registrySettings{Addr: defaultRegistryAddr}, false}, + {"default passed by pilotctl", registrySettings{Addr: defaultRegistryAddr, AddrExplicit: true}, false}, + {"default with spaces", registrySettings{Addr: " " + defaultRegistryAddr + " ", AddrExplicit: true}, false}, + {"custom", registrySettings{Addr: "10.0.0.5:9000", AddrExplicit: true}, true}, + {"custom TLS host", registrySettings{Addr: "registry.corp.example:443", AddrExplicit: true}, true}, + {"custom from default only", registrySettings{Addr: "10.0.0.5:9000"}, false}, + // The TCP/9000 fallback: -registry-tls=false asks for the raw registry. + {"default + explicit -registry-tls=false", registrySettings{Addr: defaultRegistryAddr, AddrExplicit: true, TLSExplicit: true}, true}, + {"default + explicit -registry-tls=true", registrySettings{Addr: defaultRegistryAddr, AddrExplicit: true, TLS: true, TLSExplicit: true}, false}, + } { + if got := compatKeepsRegistry(tc.r); got != tc.want { + t.Errorf("%s: compatKeepsRegistry = %v, want %v", tc.name, got, tc.want) + } + } +} + +func TestApplyRegistryDefaults(t *testing.T) { + const fp = "c1f958f6bcff667cf6a08d5066cc031a9086115a7667835877ca62a3019b3da9" + pilotctl := registrySettings{Addr: defaultRegistryAddr, AddrExplicit: true, Trust: "pinned"} + for _, tc := range []struct { + name string + transport string + in, want registrySettings + }{ + {"udp unchanged", daemon.TransportUDP, pilotctl, pilotctl}, + {"compat moves the default registry to TLS/443, system trust", daemon.TransportCompat, pilotctl, + registrySettings{Addr: compatRegistryAddr, AddrExplicit: true, TLS: true, Trust: "system"}}, + {"compat with a fingerprint (config/env) pins", daemon.TransportCompat, + registrySettings{Addr: defaultRegistryAddr, AddrExplicit: true, Trust: "pinned", Fingerprint: fp}, + registrySettings{Addr: compatRegistryAddr, AddrExplicit: true, TLS: true, Trust: "pinned", Fingerprint: fp}}, + {"compat keeps an explicit system trust even with a fingerprint", daemon.TransportCompat, + registrySettings{Addr: defaultRegistryAddr, Trust: "system", TrustExplicit: true, Fingerprint: fp}, + registrySettings{Addr: compatRegistryAddr, TLS: true, Trust: "system", TrustExplicit: true, Fingerprint: fp}}, + {"compat keeps explicit pinned trust (config registry_trust)", daemon.TransportCompat, + registrySettings{Addr: defaultRegistryAddr, Trust: "pinned", TrustExplicit: true, Fingerprint: fp}, + registrySettings{Addr: compatRegistryAddr, TLS: true, Trust: "pinned", TrustExplicit: true, Fingerprint: fp}}, + {"compat + explicit -registry-tls=false keeps the raw registry (TCP/9000 fallback)", daemon.TransportCompat, + registrySettings{Addr: defaultRegistryAddr, AddrExplicit: true, TLSExplicit: true, Trust: "pinned"}, + registrySettings{Addr: defaultRegistryAddr, AddrExplicit: true, TLSExplicit: true, Trust: "pinned"}}, + {"compat keeps a custom registry, TLS on", daemon.TransportCompat, + registrySettings{Addr: "10.0.0.5:9443", AddrExplicit: true, Trust: "pinned"}, + registrySettings{Addr: "10.0.0.5:9443", AddrExplicit: true, TLS: true, Trust: "system"}}, + {"udp with the compat registry address turns TLS on (switch back from compat)", daemon.TransportUDP, + registrySettings{Addr: compatRegistryAddr, AddrExplicit: true, Trust: "pinned"}, + registrySettings{Addr: compatRegistryAddr, AddrExplicit: true, TLS: true, Trust: "system"}}, + {"udp with the compat registry and explicit -registry-tls=false is left alone", daemon.TransportUDP, + registrySettings{Addr: compatRegistryAddr, AddrExplicit: true, TLSExplicit: true, Trust: "pinned"}, + registrySettings{Addr: compatRegistryAddr, AddrExplicit: true, TLSExplicit: true, Trust: "pinned"}}, + } { + if got := applyRegistryDefaults(tc.transport, tc.in); got != tc.want { + t.Errorf("%s:\n got %+v\nwant %+v", tc.name, got, tc.want) + } + } +} + +func TestAutoCompatBlocker(t *testing.T) { + std := registrySettings{Addr: defaultRegistryAddr, AddrExplicit: true} + if why := autoCompatBlocker(std, defaultBeaconAddr, true, false); why != "" { + t.Errorf("production defaults blocked: %s", why) + } + if why := autoCompatBlocker(registrySettings{Addr: compatRegistryAddr, AddrExplicit: true}, defaultBeaconAddr, true, false); why != "" { + t.Errorf("compat registry blocked: %s", why) + } + if why := autoCompatBlocker(registrySettings{Addr: "10.0.0.5:9000", AddrExplicit: true}, defaultBeaconAddr, true, false); why == "" { + t.Error("private raw registry not blocked") + } + if why := autoCompatBlocker(registrySettings{Addr: "reg.corp:443", AddrExplicit: true, TLS: true, TLSExplicit: true}, defaultBeaconAddr, true, false); why != "" { + t.Errorf("private TLS registry blocked: %s", why) + } + if why := autoCompatBlocker(std, "10.0.0.6:9001", true, false); why == "" { + t.Error("private beacon with the public compat beacon not blocked") + } + if why := autoCompatBlocker(std, "10.0.0.6:9001", true, true); why != "" { + t.Errorf("private beacon with its own compat beacon blocked: %s", why) + } +} + +// resolveAutoTransport runs the compat check through the proxy compat mode +// would use, and never probes a configuration auto must not move. +func TestResolveAutoTransport(t *testing.T) { + for _, k := range proxyEnvVars { + t.Setenv(k, "") + } + var got []daemon.AutoTransportProbe + autoProbe = func(ctx context.Context, p daemon.AutoTransportProbe) (string, string) { + got = append(got, p) + return daemon.TransportCompat, "stub" + } + t.Cleanup(func() { autoProbe = daemon.SelectTransport }) + std := registrySettings{Addr: defaultRegistryAddr, AddrExplicit: true} + + mode, _, err := resolveAutoTransport(std, defaultBeaconAddr, true, defaultCompatBeacon, false, "auto") + if err != nil || mode != daemon.TransportCompat || len(got) != 1 { + t.Fatalf("auto = (%q, %v), probes %d", mode, err, len(got)) + } + if got[0].Dial != nil || got[0].BeaconAddr != defaultBeaconAddr || got[0].CompatBeaconURL != defaultCompatBeacon { + t.Errorf("probe without a proxy = %+v, want a direct check of the production beacons", got[0]) + } + + t.Setenv("HTTPS_PROXY", "http://muse:s3cret@egress.test:3128") + if _, _, err := resolveAutoTransport(std, defaultBeaconAddr, true, defaultCompatBeacon, false, "auto"); err != nil || got[1].Dial == nil { + t.Errorf("with HTTPS_PROXY the compat check does not use the proxy (err %v)", err) + } + if _, _, err := resolveAutoTransport(std, defaultBeaconAddr, true, defaultCompatBeacon, false, "off"); err != nil || got[2].Dial != nil { + t.Errorf("-proxy=off still proxies the compat check (err %v)", err) + } + if _, _, err := resolveAutoTransport(std, defaultBeaconAddr, true, defaultCompatBeacon, false, "ftp://x"); err == nil { + t.Error("malformed -proxy accepted") + } + + mode, reason, err := resolveAutoTransport(registrySettings{Addr: "10.0.0.5:9000", AddrExplicit: true}, defaultBeaconAddr, true, defaultCompatBeacon, false, "auto") + if err != nil || mode != daemon.TransportUDP || len(got) != 3 { + t.Errorf("private registry: (%q, %q, %v), probes %d — want udp without probing", mode, reason, err, len(got)) + } +} + +// -help and flag errors print every flag's default. $PILOT_PROXY (which +// pilotctl uses for credential-bearing proxy URLs) must never show up. +func TestHelpNeverPrintsProxyCredentials(t *testing.T) { + for _, args := range [][]string{{"-h"}, {"--help"}, {"-no-such-flag"}} { + cmd := exec.Command(os.Args[0], args...) + cmd.Env = []string{ + runMainEnv + "=1", + "HOME=" + t.TempDir(), + "PATH=" + os.Getenv("PATH"), + "PILOT_PROXY=http://muse:s3cret@egress.test:3128", + "PILOT_REGISTRY_FINGERPRINT=" + strings.Repeat("ab", 32), + } + out, _ := cmd.CombinedOutput() + if strings.Contains(string(out), "s3cret") || strings.Contains(string(out), "muse") { + t.Errorf("pilot-daemon %v prints the proxy credentials:\n%s", args, out) + } + if !strings.Contains(string(out), "-proxy") || !strings.Contains(string(out), "'auto'") { + t.Errorf("pilot-daemon %v usage lacks -proxy / auto:\n%s", args, out) + } + } +} + +// silentUDP returns a UDP address that receives but never answers: a +// beacon behind a UDP-blocking firewall. +func silentUDP(t *testing.T) string { + t.Helper() + conn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close() }) + return conn.LocalAddr().String() +} + +// An old pilotctl (v1.13.9) passes the raw-TCP registry and beacon on +// every start and knows nothing of -transport or -proxy; config.json +// carries transport=compat. The new daemon must still come up in compat +// through the environment's proxy. +func TestOldPilotctlArgsWithCompatConfig(t *testing.T) { + proxy := newRefusingProxy(t, "muse", "s3cret") + targets, logs := runDaemon(t, proxy, daemonRun{ + args: []string{ // v1.13.9 buildDaemonArgs, minus the per-test paths + "--registry", defaultRegistryAddr, + "--beacon", defaultBeaconAddr, + }, + config: `{"registry":"` + defaultRegistryAddr + `","beacon":"` + defaultBeaconAddr + `","transport":"compat",` + + `"registry_trust":"pinned","registry_fingerprint":"` + strings.Repeat("00", 32) + `"}`, + env: []string{"HTTPS_PROXY=" + fmt.Sprintf("http://muse:s3cret@%s", proxy.ln.Addr())}, + await: "CONNECT registry.pilotprotocol.network:443", + }) + for _, target := range targets { + if strings.Contains(target, "34.71.57.205") { + t.Errorf("proxy was asked for the raw-TCP registry: %q", target) + } + } + if !strings.Contains(logs, "transport_from=config") { + t.Errorf("transport not taken from config.json:\n%s", logs) + } + // config.json's pinned trust survives compat mode. + if strings.Contains(logs, "registry_trust=system") { + t.Errorf("config.json registry_trust=pinned was overridden:\n%s", logs) + } +} + +// A credential-bearing proxy handed over as $PILOT_PROXY (pilotctl's +// --proxy with credentials) beats config.json's "proxy":"auto", and an +// explicit URL proxies the registry in udp mode too. +func TestPilotProxyEnvBeatsConfigAuto(t *testing.T) { + proxy := newRefusingProxy(t, "muse", "s3cret") + _, logs := runDaemon(t, proxy, daemonRun{ + args: []string{ + "--registry", "registry.pilot.invalid:9000", + "--beacon", silentUDP(t), + }, + config: `{"proxy":"auto","transport":"udp"}`, + env: []string{"PILOT_PROXY=" + fmt.Sprintf("http://muse:s3cret@%s", proxy.ln.Addr())}, + await: "CONNECT registry.pilot.invalid:9000", + }) + if _, bad := proxy.snapshot(); bad != 0 { + t.Errorf("%d proxy request(s) without the credentials", bad) + } + if strings.Contains(logs, "s3cret") { + t.Errorf("daemon output leaks the proxy password:\n%s", logs) + } +} + +// -transport=compat with an explicit -registry-tls=false keeps the raw +// TCP registry (the TCP/9000 fallback) instead of sending plaintext to the +// TLS registry on :443. +func TestCompatRegistryTLSFalseKeepsRawRegistry(t *testing.T) { + proxy := newRefusingProxy(t, "muse", "s3cret") + targets, _ := runDaemon(t, proxy, daemonRun{ + args: []string{ + "--registry", defaultRegistryAddr, + "--beacon", defaultBeaconAddr, + "-transport=compat", + "-registry-tls=false", + }, + env: []string{"HTTPS_PROXY=" + fmt.Sprintf("http://muse:s3cret@%s", proxy.ln.Addr())}, + await: "CONNECT " + defaultRegistryAddr, + }) + for _, target := range targets { + if strings.Contains(target, "registry.pilotprotocol.network") { + t.Errorf("compat + -registry-tls=false switched to the TLS registry: %q", target) + } + } +} + +// -transport=auto on a UDP-blocked host whose only way out is the proxy: +// the UDP probe gets no answer, the compat beacon is reachable through the +// proxy, so the daemon runs compat and registers through the proxy. +func TestAutoTransportFallsBackToCompatThroughProxy(t *testing.T) { + proxy := newRefusingProxy(t, "muse", "s3cret") + proxy.allow("beacon.pilotprotocol.network:443") + start := time.Now() + targets, logs := runDaemon(t, proxy, daemonRun{ + args: []string{ + "--registry", defaultRegistryAddr, + "--beacon", silentUDP(t), + "-compat-beacon", defaultCompatBeacon, + "-transport=auto", + "-registry-fingerprint=" + strings.Repeat("00", 32), + }, + env: []string{"HTTPS_PROXY=" + fmt.Sprintf("http://muse:s3cret@%s", proxy.ln.Addr())}, + await: "CONNECT registry.pilotprotocol.network:443", + }) + if !strings.Contains(logs, `msg="transport auto-selected" transport=compat`) { + t.Errorf("no auto-selection log line:\n%s", logs) + } + if n := strings.Count(logs, "transport auto-selected"); n != 1 { + t.Errorf("auto decision logged %d times, want once", n) + } + if !contains(targets, "CONNECT beacon.pilotprotocol.network:443") { + t.Errorf("compat check did not go through the proxy: %q", targets) + } + for _, target := range targets { + if strings.Contains(target, "34.71.57.205") { + t.Errorf("proxy was asked for the raw-TCP registry: %q", target) + } + } + t.Logf("auto → compat → registry CONNECT in %s", time.Since(start)) +} + +// auto never moves a private deployment onto the public compat beacon. +func TestAutoTransportKeepsPrivateRegistryOnUDP(t *testing.T) { + proxy := newRefusingProxy(t, "muse", "s3cret") + _, logs := runDaemon(t, proxy, daemonRun{ + args: []string{ + "--registry", "registry.pilot.invalid:9000", + "--beacon", silentUDP(t), + "-transport=auto", + "-proxy", fmt.Sprintf("http://muse:s3cret@%s", proxy.ln.Addr()), + }, + await: "CONNECT registry.pilot.invalid:9000", + }) + if !strings.Contains(logs, `msg="transport auto-selected" transport=udp`) { + t.Errorf("private registry did not stay on udp:\n%s", logs) + } +} diff --git a/cmd/daemon/proxy.go b/cmd/daemon/proxy.go index 14521685..bb716010 100644 --- a/cmd/daemon/proxy.go +++ b/cmd/daemon/proxy.go @@ -5,51 +5,18 @@ package main import ( "log/slog" "net/http" - "os" - "strings" "github.com/pilot-protocol/common/netproxy" + "github.com/pilot-protocol/pilotprotocol/internal/proxyconf" "github.com/pilot-protocol/pilotprotocol/pkg/daemon" ) -// Compiled-in production endpoints. -registry and -beacon default to these -// raw TCP / UDP addresses (pilotctl passes the same values explicitly); -// compat mode swaps the registry for its TLS host name on :443. -const ( - defaultRegistryAddr = "34.71.57.205:9000" - defaultBeaconAddr = "34.71.57.205:9001" - compatRegistryAddr = "registry.pilotprotocol.network:443" -) - -// transportDefault is the -transport default: $PILOT_TRANSPORT when it -// names a transport, otherwise "udp". An unknown value is ignored with a -// warning, as pkg/daemon has always treated it. -func transportDefault() string { - v := strings.ToLower(strings.TrimSpace(os.Getenv("PILOT_TRANSPORT"))) - switch v { - case "udp", "compat": - return v - case "": - default: - slog.Warn("ignoring unknown PILOT_TRANSPORT value", "value", v, "valid", "udp, compat") - } - return "udp" -} - -// compatKeepsRegistry reports whether a -transport=compat daemon keeps its -// configured registry instead of switching to compatRegistryAddr. Only an -// explicit choice (the -registry flag or $PILOT_REGISTRY) is kept, and the -// compiled-in raw-TCP default never counts as one: pilotctl passes it on -// every `daemon start`, and a UDP-blocked host behind a CONNECT-only egress -// proxy cannot reach it. -func compatKeepsRegistry(addr string, setByFlag, setByEnv bool) bool { - return (setByFlag || setByEnv) && strings.TrimSpace(addr) != defaultRegistryAddr -} - // resolveProxyPolicy resolves -proxy for the transport (see -// daemon.ResolveProxy). A malformed explicit proxy URL is an error; a -// malformed proxy environment under "auto" only costs the proxy: it is -// logged and the daemon dials directly, as it did before -proxy existed. +// daemon.ResolveProxy). A malformed -proxy value is an error. Under "auto" +// only an unusable HTTPS_PROXY / https_proxy (the variable that names the +// TLS proxy) can fail; that costs the proxy, not the daemon: it is logged +// and the daemon dials directly, as it did before -proxy existed. Unusable +// HTTP_PROXY / ALL_PROXY values are skipped by netproxy and only logged. func resolveProxyPolicy(spec, transport string) (*netproxy.Resolver, error) { policy, err := daemon.ResolveProxy(spec, transport) if err != nil { @@ -59,6 +26,9 @@ func resolveProxyPolicy(spec, transport string) (*netproxy.Resolver, error) { slog.Warn("proxy environment is malformed; dialing directly", "err", err) return nil, nil } + for _, w := range policy.Warnings() { + slog.Warn("ignoring unusable proxy environment variable", "err", w) + } return policy, nil } @@ -68,7 +38,7 @@ func describeProxy(spec, transport string, policy *netproxy.Resolver) string { if policy != nil { return policy.String() } - if isAutoProxy(spec) && transport != "compat" { + if isAutoProxy(spec) && transport != daemon.TransportCompat { return "none (-proxy=auto applies to -transport=compat only)" } return "none" @@ -78,9 +48,10 @@ func describeProxy(spec, transport string, policy *netproxy.Resolver) string { // follow the policy. Every HTTP client the daemon wires in without a // transport of its own — catalogue pins, skillinject, trustedagents, // webhook, enterprise-control clients — uses DefaultTransport, and not all -// of them accept an injected client. nil leaves DefaultTransport alone -// (net/http's own proxy environment handling). Call before any goroutine -// issues a request. +// of them accept an injected client. Loopback targets (a local webhook or +// sidecar) always go direct. nil leaves DefaultTransport alone (net/http's +// own proxy environment handling). Call before any goroutine issues a +// request. func installDefaultTransportProxy(policy *netproxy.Resolver) { if policy == nil { return @@ -90,10 +61,10 @@ func installDefaultTransportProxy(policy *netproxy.Resolver) { slog.Warn("http.DefaultTransport is not an *http.Transport; plugin HTTP clients do not follow -proxy") return } - tr.Proxy = policy.ProxyForRequest + tr.Proxy = proxyconf.RequestProxy(policy) } func isAutoProxy(spec string) bool { - s := strings.TrimSpace(spec) - return s == "" || strings.EqualFold(s, netproxy.ModeAuto) + s, err := proxyconf.Normalize(spec) + return err == nil && s == proxyconf.Auto } diff --git a/cmd/daemon/proxy_test.go b/cmd/daemon/proxy_test.go index fe958b27..02615faf 100644 --- a/cmd/daemon/proxy_test.go +++ b/cmd/daemon/proxy_test.go @@ -39,45 +39,6 @@ var proxyEnvVars = []string{ "HTTP_PROXY", "http_proxy", "NO_PROXY", "no_proxy", "REQUEST_METHOD", } -func TestTransportDefault(t *testing.T) { - for _, tc := range []struct{ env, want string }{ - {"", "udp"}, - {"udp", "udp"}, - {"compat", "compat"}, - {" COMPAT ", "compat"}, - {"wss", "udp"}, - } { - t.Setenv("PILOT_TRANSPORT", tc.env) - if got := transportDefault(); got != tc.want { - t.Errorf("PILOT_TRANSPORT=%q: transportDefault() = %q, want %q", tc.env, got, tc.want) - } - } -} - -// The compiled-in raw-TCP registry never counts as an explicit choice in -// compat mode — pilotctl passes it on every `daemon start` — while any -// other address the operator set by flag or environment is kept. -func TestCompatKeepsRegistry(t *testing.T) { - for _, tc := range []struct { - addr string - byFlag, byEnv bool - want bool - }{ - {defaultRegistryAddr, false, false, false}, - {defaultRegistryAddr, true, false, false}, - {defaultRegistryAddr, false, true, false}, - {defaultRegistryAddr, true, true, false}, - {" " + defaultRegistryAddr + " ", true, false, false}, - {"10.0.0.5:9000", true, false, true}, - {"registry.corp.example:443", false, true, true}, - {"10.0.0.5:9000", false, false, false}, // config file / default only - } { - if got := compatKeepsRegistry(tc.addr, tc.byFlag, tc.byEnv); got != tc.want { - t.Errorf("compatKeepsRegistry(%q, flag=%v, env=%v) = %v, want %v", tc.addr, tc.byFlag, tc.byEnv, got, tc.want) - } - } -} - func TestResolveProxyPolicy(t *testing.T) { for _, k := range proxyEnvVars { t.Setenv(k, "") @@ -139,6 +100,18 @@ type refusingProxy struct { mu sync.Mutex targets []string badAuths int + allowed map[string]bool // CONNECT targets answered 200 (then closed) +} + +// allow makes the proxy accept CONNECT target (answer 200, then close the +// tunnel): enough for a reachability check, useless for anything else. +func (p *refusingProxy) allow(target string) { + p.mu.Lock() + defer p.mu.Unlock() + if p.allowed == nil { + p.allowed = map[string]bool{} + } + p.allowed[target] = true } func newRefusingProxy(t *testing.T, user, pass string) *refusingProxy { @@ -170,10 +143,16 @@ func newRefusingProxy(t *testing.T, user, pass string) *refusingProxy { } p.mu.Lock() p.targets = append(p.targets, r.Method+" "+r.RequestURI) - if r.Header.Get("Proxy-Authorization") != p.wantAuth { + authOK := r.Header.Get("Proxy-Authorization") == p.wantAuth + if !authOK { p.badAuths++ } + ok := authOK && r.Method == http.MethodConnect && p.allowed[r.RequestURI] p.mu.Unlock() + if ok { + fmt.Fprint(conn, "HTTP/1.1 200 Connection established\r\n\r\n") + return + } fmt.Fprint(conn, "HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") }() } @@ -262,6 +241,30 @@ func TestDaemonCompatThroughProxyEndToEnd(t *testing.T) { // returns the proxy's request targets once the registry CONNECT arrived, // plus the daemon's output. func runDaemonBehindProxy(t *testing.T, proxyEnv string, proxy *refusingProxy) ([]string, string) { + t.Helper() + return runDaemon(t, proxy, daemonRun{ + env: []string{"PILOT_TRANSPORT=compat", proxyEnv}, + await: "CONNECT registry.pilotprotocol.network:443", + }) +} + +// daemonRun describes one child daemon: args replaces the default +// pilotctl-style network flags (registry, beacon, pinned bogus registry +// trust), env is added to a minimal environment, config (when set) is +// written to $HOME/.pilot/config.json, and await is the proxy request the +// run waits for. +type daemonRun struct { + args []string + env []string + config string + await string +} + +// runDaemon starts main() in a child process and returns the proxy's +// request targets once run.await arrived, plus the daemon's output. The +// proxy refuses everything it is not told to allow, so nothing reaches the +// network. +func runDaemon(t *testing.T, proxy *refusingProxy, run daemonRun) ([]string, string) { t.Helper() home := t.TempDir() sockDir, err := os.MkdirTemp("", "pdm") @@ -269,32 +272,44 @@ func runDaemonBehindProxy(t *testing.T, proxyEnv string, proxy *refusingProxy) ( t.Fatal(err) } t.Cleanup(func() { os.RemoveAll(sockDir) }) - - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() - cmd := exec.CommandContext(ctx, os.Args[0], - "--registry", defaultRegistryAddr, - "--beacon", defaultBeaconAddr, + if run.config != "" { + if err := os.MkdirAll(filepath.Join(home, ".pilot"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(home, ".pilot", "config.json"), []byte(run.config), 0o600); err != nil { + t.Fatal(err) + } + } + args := run.args + if args == nil { + args = []string{ + "--registry", defaultRegistryAddr, + "--beacon", defaultBeaconAddr, + "-registry-trust=pinned", + "-registry-fingerprint=" + strings.Repeat("00", 32), + } + } + args = append(append([]string(nil), args...), "--listen", ":0", "--socket", filepath.Join(sockDir, "s"), "--identity", filepath.Join(home, "identity.json"), "--log-level", "info", "--log-format", "text", - "-registry-trust=pinned", - "-registry-fingerprint="+strings.Repeat("00", 32), "-no-skillinject", "-motd-feed-url=", ) - cmd.Env = []string{ + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, os.Args[0], args...) + cmd.Env = append([]string{ runMainEnv + "=1", "HOME=" + home, "PATH=" + os.Getenv("PATH"), "TMPDIR=" + os.TempDir(), - "PILOT_TRANSPORT=compat", "PILOT_NO_SKILLINJECT=1", "PILOT_APPSTORE_ROOT=" + filepath.Join(home, "apps"), - proxyEnv, - } + }, run.env...) var out syncBuffer cmd.Stdout = &out cmd.Stderr = &out @@ -306,15 +321,14 @@ func runDaemonBehindProxy(t *testing.T, proxyEnv string, proxy *refusingProxy) ( _ = cmd.Wait() }() - const registryTarget = "CONNECT registry.pilotprotocol.network:443" deadline := time.Now().Add(45 * time.Second) for { targets, _ := proxy.snapshot() - if contains(targets, registryTarget) { + if contains(targets, run.await) { break } if time.Now().After(deadline) { - t.Fatalf("proxy never saw %q; saw %q\ndaemon output:\n%s", registryTarget, targets, out.String()) + t.Fatalf("proxy never saw %q; saw %q\ndaemon output:\n%s", run.await, targets, out.String()) } time.Sleep(50 * time.Millisecond) } diff --git a/cmd/pilotctl/daemon_transport.go b/cmd/pilotctl/daemon_transport.go index 2b2e868b..26625415 100644 --- a/cmd/pilotctl/daemon_transport.go +++ b/cmd/pilotctl/daemon_transport.go @@ -3,30 +3,35 @@ package main import ( + "bufio" "context" "fmt" - "net/url" "os" "os/exec" + "regexp" "strings" "sync" "time" + + "github.com/pilot-protocol/pilotprotocol/internal/proxyconf" ) // Compiled-in production endpoints. These are the same defaults cmd/daemon // compiles in; `pilotctl init` and install.sh write them into config.json. // Both are raw-TCP/UDP endpoints — compat mode (-transport=compat) must not -// be pinned to them, see compatSkipsDefault. +// be pinned to them, see compatSkipsDefault. compatRegistryAddr is the TLS +// registry compat mode (and any proxied connection) uses instead. const ( productionRegistryAddr = "34.71.57.205:9000" productionBeaconAddr = "34.71.57.205:9001" + compatRegistryAddr = "registry.pilotprotocol.network:443" ) // daemonForwardEnv is the environment `pilotctl daemon start` guarantees to // hand to pilot-daemon, on both the fork and the --foreground exec path. The // proxy variables drive the daemon's -proxy=auto resolution (a CONNECT proxy -// is the only way out of sandboxes such as Meta Muse), PILOT_PROXY / -// PILOT_TRANSPORT are the daemon's env-backed flag overrides, and the +// is the only way out of sandboxes such as Meta Muse), the PILOT_* ones are +// the daemon's environment overrides (they beat config.json), and the // SSL_CERT_* pair lets a sandbox without a system CA bundle point Go's // x509 at one. var daemonForwardEnv = []string{ @@ -35,68 +40,62 @@ var daemonForwardEnv = []string{ "ALL_PROXY", "all_proxy", "NO_PROXY", "no_proxy", "PILOT_PROXY", "PILOT_TRANSPORT", + "PILOT_REGISTRY_TRUST", "PILOT_REGISTRY_FINGERPRINT", "SSL_CERT_FILE", "SSL_CERT_DIR", } -// validateTransport accepts the two tunnel transports pilot-daemon knows. +// normalizeTransport lower-cases and validates a transport: udp, compat or +// auto ("" stays ""). +func normalizeTransport(v string) (string, error) { + s := strings.ToLower(strings.TrimSpace(v)) + switch s { + case "", "udp", "compat", "auto": + return s, nil + } + return "", fmt.Errorf("invalid transport %q: must be 'udp', 'compat' or 'auto'", v) +} + +// validateTransport accepts the tunnel transports pilot-daemon knows. func validateTransport(v string) error { - switch v { - case "udp", "compat": - return nil + s, err := normalizeTransport(v) + if err == nil && s == "" { + err = fmt.Errorf("invalid transport %q: must be 'udp', 'compat' or 'auto'", v) } - return fmt.Errorf("invalid transport %q: must be 'udp' or 'compat'", v) + return err } // validateProxySetting accepts the -proxy contract: "auto" (use the -// environment's proxy in compat mode), "off", or an explicit -// http(s)://[user:pass@]host:port proxy URL. +// environment's proxy in compat mode), "off" (or none/no/false/direct), or +// an explicit http(s)://[user:pass@]host:port proxy URL. See +// internal/proxyconf, which pilot-daemon uses too. func validateProxySetting(v string) error { - switch v { - case "auto", "off": - return nil - } - u, err := url.Parse(v) - if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.Hostname() == "" { - return fmt.Errorf("invalid proxy %q: must be 'auto', 'off', or an http(s)://[user:pass@]host:port URL", redactProxyURL(v)) - } - return nil + _, err := proxyconf.Normalize(v) + return err } -// proxyHasCredentials reports whether an explicit proxy URL carries userinfo. +// proxyHasCredentials reports whether an explicit proxy URL carries +// userinfo. Any '@' counts, whatever url.Parse would make of the value, so +// a password with an unescaped '#', '/' or '?' still travels in the +// environment, never on argv. func proxyHasCredentials(v string) bool { - u, err := url.Parse(v) - return err == nil && u.User != nil + return proxyconf.HasCredentials(v) } // redactProxyURL hides proxy credentials for display: http://user:pass@h:p -// becomes http://***@h:p. Non-URL values ("auto", "off") pass through. -// -// TODO(netproxy): switch to common/netproxy's redaction helper once -// pilotctl bumps to a common release that ships it. +// becomes http://***@h:p. "auto" and "off" pass through. func redactProxyURL(v string) string { - if u, err := url.Parse(v); err == nil && u.User != nil { - return u.Scheme + "://***@" + u.Host + u.EscapedPath() - } - // Unparseable or scheme-less ("user:pass@host:port" parses as an opaque - // URL with no userinfo): never echo anything before the last '@'. - if i := strings.LastIndex(v, "@"); i >= 0 { - prefix := "" - if j := strings.Index(v, "://"); j >= 0 && j < i { - prefix = v[:j+3] - } - return prefix + "***@" + v[i+1:] - } - return v + return proxyconf.Redact(v) } // resolveDaemonTransport picks the tunnel transport for `daemon start`: // --transport, then $PILOT_TRANSPORT, then config.json "transport". "" means -// none was set and the daemon keeps its own default (udp). +// none was set: cmdDaemonStart then asks a daemon that supports it for +// auto, and otherwise leaves the daemon on its default (udp). // -// $PILOT_TRANSPORT is resolved here and handed to the daemon as an explicit -// -transport because pilot-daemon's -transport flag defaults to "udp", which -// masks the env var in every released daemon — exporting PILOT_TRANSPORT -// alone never switched a daemon started through pilotctl to compat. +// The chosen value is handed to the daemon as an explicit -transport: a +// released pilot-daemon's -transport flag defaults to "udp", which masks +// the env var, so exporting PILOT_TRANSPORT alone never switched a daemon +// started through pilotctl. func resolveDaemonTransport(flags map[string]string, cfg map[string]interface{}) (string, error) { v := strings.TrimSpace(flagString(flags, "transport", "")) if v == "" { @@ -107,21 +106,19 @@ func resolveDaemonTransport(flags map[string]string, cfg map[string]interface{}) v = strings.TrimSpace(s) } } - if v == "" { - return "", nil - } - if err := validateTransport(v); err != nil { - return "", err - } - return v, nil + return normalizeTransport(v) } -// resolveDaemonProxy picks the proxy policy for `daemon start`: --proxy, then -// config.json "proxy". "" means neither was set. $PILOT_PROXY is not read -// here: it reaches the daemon through the forwarded environment and the -// daemon resolves it itself. +// resolveDaemonProxy picks the proxy for `daemon start`: --proxy, then +// $PILOT_PROXY, then config.json "proxy" — the same precedence pilot-daemon +// applies, so an explicitly passed flag or environment value always beats +// a persistent config default. "" means none was set (the daemon default, +// auto). The value is normalized ("none" becomes "off"). func resolveDaemonProxy(flags map[string]string, cfg map[string]interface{}) (string, error) { v := strings.TrimSpace(flagString(flags, "proxy", "")) + if v == "" { + v = strings.TrimSpace(os.Getenv("PILOT_PROXY")) + } if v == "" { if s, ok := cfg["proxy"].(string); ok { v = strings.TrimSpace(s) @@ -130,19 +127,16 @@ func resolveDaemonProxy(flags map[string]string, cfg map[string]interface{}) (st if v == "" { return "", nil } - if err := validateProxySetting(v); err != nil { - return "", err - } - return v, nil + return proxyconf.Normalize(v) } // compatSkipsDefault reports whether addr must be left off the daemon command // line because it is the compiled-in raw-TCP production default and the -// daemon runs in compat mode. pilot-daemon only switches the registry to -// registry.pilotprotocol.network:443 (TLS) in compat mode when -registry was -// NOT given explicitly; `pilotctl init` writes the raw :9000 default into -// config.json, and forwarding that verbatim pinned compat daemons to a -// non-TLS port no HTTPS proxy will CONNECT to. +// daemon runs in compat mode. An older pilot-daemon only switches the +// registry to registry.pilotprotocol.network:443 (TLS) in compat mode when +// -registry was NOT given explicitly; `pilotctl init` writes the raw :9000 +// default into config.json, and forwarding that verbatim pinned compat +// daemons to a non-TLS port no HTTPS proxy will CONNECT to. func compatSkipsDefault(transport, addr, def string) bool { return transport == "compat" && addr == def } @@ -155,7 +149,7 @@ func compatSkipsDefault(transport, addr, def string) bool { // here rather than on argv so they never show up in /proc//cmdline // (PILOT-290). // - In compat mode a $PILOT_REGISTRY equal to the raw-TCP production -// default is dropped: the daemon treats an env-provided registry as +// default is dropped: an older daemon treats an env-provided registry as // explicit and would otherwise skip the compat TLS registry switch. func daemonChildEnv(base []string, adminToken, proxyEnv, transport string) []string { set := map[string]string{} @@ -189,14 +183,16 @@ const daemonFlagProbeTimeout = 5 * time.Second var ( daemonFlagCacheMu sync.Mutex - daemonFlagCache = map[string]map[string]bool{} + daemonFlagCache = map[string]map[string]string{} ) -// daemonFlags returns the set of flag names the pilot-daemon binary at bin -// defines, parsed from its Go flag-package `-help` usage (" -name type"). -// Returns nil when the binary could not be probed or printed nothing -// recognisable; callers treat nil as "unknown" and keep the flag. -func daemonFlags(bin string) map[string]bool { +// daemonFlagUsage returns the flags the pilot-daemon binary at bin defines, +// mapped to their usage text, parsed from its Go flag-package `-help` +// output (" -name type" followed by indented usage lines). The binary is +// probed once per process. Returns nil when it could not be probed or +// printed nothing recognisable; callers treat nil as "unknown" and keep +// the flag. +func daemonFlagUsage(bin string) map[string]string { daemonFlagCacheMu.Lock() defer daemonFlagCacheMu.Unlock() if set, ok := daemonFlagCache[bin]; ok { @@ -205,103 +201,207 @@ func daemonFlags(bin string) map[string]bool { ctx, cancel := context.WithTimeout(context.Background(), daemonFlagProbeTimeout) defer cancel() cmd := exec.CommandContext(ctx, bin, "-help") - // Minimal environment: -help prints each flag's default, and a daemon - // with env-backed defaults would echo e.g. a credential-bearing + // Minimal environment: -help prints each flag's default, and an older + // daemon with env-backed defaults would echo e.g. a credential-bearing // $PILOT_PROXY into the captured output. cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + os.Getenv("HOME")} out, _ := cmd.CombinedOutput() // -help exits 0 (flag.ExitOnError) or 2 on older builds - var set map[string]bool - for _, line := range strings.Split(string(out), "\n") { - if !strings.HasPrefix(line, " -") { - continue - } - name := strings.TrimPrefix(line, " -") - if i := strings.IndexAny(name, " \t"); i >= 0 { - name = name[:i] - } - if name == "" { + var set map[string]string + current := "" + sc := bufio.NewScanner(strings.NewReader(string(out))) + sc.Buffer(make([]byte, 0, 64*1024), 1024*1024) + for sc.Scan() { + line := sc.Text() + if strings.HasPrefix(line, " -") { + name := strings.TrimPrefix(line, " -") + if i := strings.IndexAny(name, " \t"); i >= 0 { + name = name[:i] + } + current = name + if name == "" { + continue + } + if set == nil { + set = map[string]string{} + } + set[name] = "" continue } - if set == nil { - set = map[string]bool{} + if current != "" && (strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t")) { + set[current] += strings.TrimSpace(line) + " " } - set[name] = true } daemonFlagCache[bin] = set return set } -// skewSensitiveDaemonFlags are pilot-daemon flags newer than some daemons a -// current pilotctl may still be paired with (a sibling binary left behind by -// a partial upgrade, a PILOT_DAEMON_BIN override, an npm-installed pair). -// Go's flag package aborts on an unknown flag, so forwarding one of these to -// an older daemon would turn `daemon start` into a crash loop. -var skewSensitiveDaemonFlags = []string{"transport", "proxy"} +// daemonFlags returns the set of flag names the pilot-daemon binary at bin +// defines (see daemonFlagUsage), nil when unknown. +func daemonFlags(bin string) map[string]bool { + usage := daemonFlagUsage(bin) + if usage == nil { + return nil + } + set := make(map[string]bool, len(usage)) + for name := range usage { + set[name] = true + } + return set +} -// dropUnsupportedDaemonFlags removes skew-sensitive flags the daemon at bin -// does not define, warning once per dropped flag. Only probes the binary -// when one of those flags is actually present in args. -func dropUnsupportedDaemonFlags(bin string, args []string) []string { - present := false - for _, a := range args { - for _, f := range skewSensitiveDaemonFlags { - if a == "--"+f { - present = true +// autoTransportRe matches the -transport usage of a daemon that accepts +// -transport=auto. +var autoTransportRe = regexp.MustCompile(`'auto'`) + +// daemonSupportsAutoTransport reports whether the daemon at bin accepts +// -transport=auto, and whether that is known at all. +func daemonSupportsAutoTransport(bin string) (supported, known bool) { + usage := daemonFlagUsage(bin) + if usage == nil { + return false, false + } + u, ok := usage["transport"] + return ok && autoTransportRe.MatchString(u), true +} + +// adaptDaemonArgs fits a launch plan to the pilot-daemon binary at bin +// (probed once): a flag or value the daemon does not know would abort it +// on startup, and a new pilotctl is still paired with older daemons (a +// sibling binary left behind by a partial upgrade, a PILOT_DAEMON_BIN +// override, an npm-installed pair). +// +// - No transport configured: a daemon that supports -transport=auto gets +// it (udp when UDP works, compat when only TCP 443 does); an older one +// keeps its default, udp, silently. +// - -transport=auto on a daemon without it becomes -transport=udp (an +// older daemon would also read "auto" from config.json and fail), with +// a warning. +// - -transport / -proxy on a daemon without the flag are dropped with a +// warning, and so is a proxy handed over as $PILOT_PROXY. +// +// It returns the argv and the $PILOT_PROXY value to use, and the transport +// actually requested ("" = the daemon's default). +func adaptDaemonArgs(bin string, plan daemonLaunchPlan) (args []string, proxyEnv, transport string) { + args = append([]string(nil), plan.Args...) + proxyEnv = plan.ProxyEnv + transport = plan.Transport + flags := daemonFlags(bin) + warn := func(format string, a ...interface{}) { + if !jsonOutput { + fmt.Fprintf(os.Stderr, "warning: "+format+"\n", a...) + } + } + + switch { + case transport == "": + if ok, _ := daemonSupportsAutoTransport(bin); ok { + args = append(args, "--transport", "auto") + transport = "auto" + } + case transport == "auto": + if ok, known := daemonSupportsAutoTransport(bin); known && !ok { + if flags["transport"] { + warn("%s does not support -transport=auto (older pilot-daemon); starting it with -transport=udp — upgrade pilot-daemon to use auto", bin) + args = replaceFlagValue(args, "--transport", "udp") + transport = "udp" + } else { + warn("%s does not support -transport (older pilot-daemon); not passing -transport auto — upgrade pilot-daemon to use it", bin) + args = removeFlag(args, "--transport") + transport = "" } } + default: + if flags != nil && !flags["transport"] { + warn("%s does not support -transport (older pilot-daemon); not passing -transport %s — upgrade pilot-daemon to use it", bin, transport) + args = removeFlag(args, "--transport") + transport = "" + } + } + + if flags != nil && !flags["proxy"] { + if hasFlag(args, "--proxy") { + warn("%s does not support -proxy (older pilot-daemon); not passing -proxy %s — upgrade pilot-daemon to use it", bin, redactProxyURL(flagValue(args, "--proxy"))) + args = removeFlag(args, "--proxy") + } + if proxyEnv != "" { + warn("%s does not support -proxy (older pilot-daemon); proxy %s will not be used — upgrade pilot-daemon to use it", bin, redactProxyURL(proxyEnv)) + proxyEnv = "" + } } - if !present { - return args + return args, proxyEnv, transport +} + +func hasFlag(args []string, name string) bool { + for _, a := range args { + if a == name { + return true + } } - supported := daemonFlags(bin) - if supported == nil { - return args + return false +} + +func flagValue(args []string, name string) string { + for i := 0; i+1 < len(args); i++ { + if args[i] == name { + return args[i+1] + } } + return "" +} + +// removeFlag drops every "name value" pair from args. +func removeFlag(args []string, name string) []string { out := make([]string, 0, len(args)) for i := 0; i < len(args); i++ { - name := strings.TrimPrefix(args[i], "--") - skew := false - for _, f := range skewSensitiveDaemonFlags { - if args[i] == "--"+f && !supported[f] { - skew = true - } - } - if !skew { - out = append(out, args[i]) + if args[i] == name { + i++ // skip the value continue } - value := "" - if i+1 < len(args) { - value = args[i+1] - i++ - } - if !jsonOutput { - fmt.Fprintf(os.Stderr, "warning: %s does not support -%s (older pilot-daemon); not passing -%s %s — upgrade pilot-daemon to use it\n", - bin, name, name, redactProxyURL(value)) - } + out = append(out, args[i]) } return out } -// envProxyURL returns the proxy an HTTPS request from this process would -// use per the environment ($HTTPS_PROXY, then $ALL_PROXY, either case), or "" -// when none is set. -func envProxyURL() string { - for _, k := range []string{"HTTPS_PROXY", "https_proxy", "ALL_PROXY", "all_proxy"} { - if v := strings.TrimSpace(os.Getenv(k)); v != "" { - return v +// replaceFlagValue sets the value of every "name value" pair in args. +func replaceFlagValue(args []string, name, value string) []string { + out := append([]string(nil), args...) + for i := 0; i+1 < len(out); i++ { + if out[i] == name { + out[i+1] = value } } - return "" + return out } -// registryDialHint is the hint for a failed direct registry dial from -// pilotctl. Behind an egress proxy the dial fails by design — say so, -// instead of suggesting the registry is down. -func registryDialHint(addr string) string { - if p := envProxyURL(); p != "" { - return fmt.Sprintf("pilotctl dials the registry (%s) directly over TCP and does not use the proxy %s yet; daemon-backed commands (info, peers, trust, ping, send-message) reach the network through the daemon instead", - addr, redactProxyURL(p)) +// daemonLogTransportRe finds the transport pilot-daemon reports on its +// "outbound network" startup line, in text or JSON log format. +var daemonLogTransportRe = regexp.MustCompile(`outbound network"?[ ,]+"?transport"?[=:]"?(udp|compat)`) + +// transportFromDaemonLog returns the transport the daemon logged at +// startup ("" if unknown) — the only place the outcome of -transport=auto +// is visible to pilotctl. +func transportFromDaemonLog(path string) string { + b, err := os.ReadFile(path) + if err != nil { + return "" + } + m := daemonLogTransportRe.FindAllSubmatch(b, -1) + if len(m) == 0 { + return "" + } + return string(m[len(m)-1][1]) +} + +// effectiveTransport is what `daemon start` reports: the transport the +// daemon logged at startup, else the one requested ("" when neither is +// known), and whether -transport=auto chose it. +func effectiveTransport(requested, logPath string) (transport string, auto bool) { + auto = requested == "auto" + if logged := transportFromDaemonLog(logPath); logged != "" { + return logged, auto + } + if auto { + return "", true } - return fmt.Sprintf("check that the registry is running at %s, or set PILOT_REGISTRY", addr) + return requested, false } diff --git a/cmd/pilotctl/main.go b/cmd/pilotctl/main.go index fe632469..fc95d008 100644 --- a/cmd/pilotctl/main.go +++ b/cmd/pilotctl/main.go @@ -31,6 +31,7 @@ import ( registry "github.com/pilot-protocol/common/registry/client" "github.com/pilot-protocol/dataexchange" "github.com/pilot-protocol/eventstream" + "github.com/pilot-protocol/pilotprotocol/internal/proxyconf" "github.com/pilot-protocol/policy/policylang" "github.com/pilot-protocol/trustedagents" ) @@ -600,20 +601,11 @@ func nodeIDFromDaemon() int64 { return int64(nid) } +// connectRegistry dials the configured registry along the same network +// the daemon uses (see registryRoute: egress proxy, compat TLS registry), +// or exits with a hint. func connectRegistry() *registry.Client { - addr := getRegistry() - // TODO(netproxy): dial through common/netproxy (HTTPS_PROXY CONNECT by - // hostname, TLS for the compat registry) once pilotctl bumps to a common - // release with the registry dialer option. Until then this raw-TCP dial - // bypasses any egress proxy, so registry commands fail in proxy-only - // sandboxes; registryDialHint says so. - rc, err := registry.Dial(addr) - if err != nil { - fatalHint("connection_failed", - registryDialHint(addr), - "cannot reach registry at %s", addr) - } - return rc + return connectRegistryAt(getRegistry(), "registry") } func resolveHostnameToAddr(d *driver.Driver, hostname string) (protocol.Addr, uint32, error) { @@ -708,17 +700,17 @@ func maybeAutoHandshake(d *driver.Driver, addr protocol.Addr, skip bool) { return } - // Branch 3 — unknown peer, not trusted. Refuse if private. - // TODO(netproxy): same raw-TCP registry dial as connectRegistry — it - // bypasses HTTPS_PROXY until common/netproxy lands in pilotctl. - rc, err := registry.Dial(getRegistry()) + // Branch 3 — unknown peer, not trusted. Refuse if private. The + // visibility check reaches the registry the way connectRegistry does + // (through the egress proxy when there is one). + rc, route, err := dialRegistry(getRegistry()) if err != nil { // Registry unreachable — be conservative and refuse rather than // silently let an untrusted tunnel attempt go through to a peer // we can't characterise. hint := fmt.Sprintf("run: pilotctl handshake %s", addr) - if envProxyURL() != "" { - hint += " (or pass --no-auto-handshake); " + registryDialHint(getRegistry()) + if route.Addr != "" { + hint += "; " + registryDialHint(route) } fatalHint("trust_required", hint, @@ -1063,32 +1055,41 @@ Flags: --log-format log format: text, json (default: text) --no-encrypt disable tunnel encryption --foreground run in foreground (no fork; for systemd / shell wrappers) - --wait how long to wait for daemon to become ready (default: 15s) + --wait how long to wait for daemon to become ready (default: + 15s; 30s with --transport compat or auto) --motd-feed-url message-of-the-day feed (empty to disable; env PILOT_MOTD_URL) --motd-interval message-of-the-day poll interval (default: 15m) --enterprise-control owner-only managed control attachment - --transport tunnel transport (default: $PILOT_TRANSPORT, config - "transport", else udp). compat = TLS/WSS over TCP 443 - only, for UDP-blocked hosts; the raw-TCP default - registry/beacon are then left to the daemon, which - uses registry.pilotprotocol.network:443 - --proxy outbound proxy (default: config "proxy", else the - daemon's $PILOT_PROXY or auto). auto = in compat mode - use $HTTPS_PROXY/$ALL_PROXY (honoring $NO_PROXY); - off = never; http(s)://[user:pass@]host:port = always. - A URL with credentials is passed via env, not argv + --transport + tunnel transport. Precedence: this flag, + $PILOT_TRANSPORT, config "transport", else auto when + the daemon supports it (older daemons: udp). + udp = UDP tunnels; compat = registry over TLS and + beacon over WSS, TCP 443 only (UDP-blocked or + proxy-only hosts); auto = udp when the beacon + answers over UDP, else compat when TCP 443 is + reachable (through the proxy, if any) + --proxy outbound proxy. Precedence: this flag, $PILOT_PROXY, + config "proxy", else auto. auto = with compat, use + $HTTPS_PROXY/$ALL_PROXY (honoring $NO_PROXY); + off (or none) = never; http(s)://[user:pass@]host:port + = every connection except loopback. A URL with + credentials is passed via env, never on argv --compat-beacon beacon WSS URL for compat mode --registry-trust registry TLS trust: pinned or system - --registry-fingerprint registry certificate SHA-256 (with --registry-trust pinned) + --registry-fingerprint registry certificate SHA-256 (pins the compat registry; + for hosts without a CA bundle) --tls-trust compat beacon TLS trust: system or pinned Environment passed through to the daemon (never scrubbed): HTTPS_PROXY https_proxy HTTP_PROXY http_proxy ALL_PROXY all_proxy - NO_PROXY no_proxy PILOT_PROXY PILOT_TRANSPORT SSL_CERT_FILE SSL_CERT_DIR + NO_PROXY no_proxy PILOT_PROXY PILOT_TRANSPORT PILOT_REGISTRY_TRUST + PILOT_REGISTRY_FINGERPRINT SSL_CERT_FILE SSL_CERT_DIR ---transport / --proxy are forwarded only when set; if the pilot-daemon binary -is too old to know one, it is dropped with a warning instead of crashing it. -Behind an HTTPS proxy with UDP blocked (e.g. hosted agent sandboxes): +A pilot-daemon too old for --transport, --transport auto or --proxy gets the +flag dropped (auto becomes udp) with a warning instead of crashing it. +Behind an HTTPS proxy with UDP blocked (e.g. hosted agent sandboxes), plain +"pilotctl daemon start" picks compat by itself; to skip the UDP probe: pilotctl config --set transport=compat && pilotctl daemon start `, "daemon stop": `Usage: pilotctl daemon stop @@ -1342,9 +1343,11 @@ Common keys: beacon beacon address (overrides $PILOT_BEACON) socket daemon socket path (overrides $PILOT_SOCKET) hostname default hostname passed to daemon start - transport daemon transport: udp or compat ($PILOT_TRANSPORT overrides) + transport daemon transport: udp, compat or auto ($PILOT_TRANSPORT overrides) proxy daemon proxy: auto, off, or http(s)://[user:pass@]host:port - (credentials are redacted when shown) + ($PILOT_PROXY overrides; credentials are redacted when shown) + registry_fingerprint / registry_trust + pin the TLS registry (hosts without a CA bundle) `, "version": `Usage: pilotctl version @@ -1551,7 +1554,7 @@ Bootstrap: pilotctl config [--set key=value] Daemon lifecycle: - pilotctl daemon start [--config ] [--registry ] [--beacon ] [--email ] [--webhook ] [--trust-auto-approve] [--transport ] [--proxy ] + pilotctl daemon start [--config ] [--registry ] [--beacon ] [--email ] [--webhook ] [--trust-auto-approve] [--transport ] [--proxy ] pilotctl daemon stop pilotctl daemon status @@ -1648,9 +1651,9 @@ Diagnostic commands: Environment: PILOT_REGISTRY Registry address (default: 34.71.57.205:9000) PILOT_SOCKET Daemon socket path (default: /tmp/pilot.sock) - PILOT_TRANSPORT daemon start transport: udp or compat (TCP 443 only) - PILOT_PROXY daemon proxy policy: auto, off, or http(s)://[user:pass@]host:port - HTTPS_PROXY proxy used by compat mode (proxy=auto) and pilotctl's HTTP calls + PILOT_TRANSPORT daemon start transport: udp, compat (TCP 443 only) or auto + PILOT_PROXY proxy policy: auto, off, or http(s)://[user:pass@]host:port + HTTPS_PROXY proxy for compat mode (proxy=auto) and pilotctl's own connections Version: pilotctl version @@ -2127,33 +2130,50 @@ func cmdConfig(args []string) { if len(parts) != 2 { fatalCode("invalid_argument", "usage: pilotctl config --set key=value") } - // Validate the keys daemon start interprets, so a typo fails here - // rather than on the next daemon start. Empty clears the key. - if parts[1] != "" { + // Validate the keys daemon start (and pilot-daemon, which reads + // config.json itself) interprets, so a typo fails here rather than + // on the next daemon start. Empty clears the key. + value := parts[1] + if value != "" { switch parts[0] { case "transport": - if err := validateTransport(parts[1]); err != nil { + t, err := normalizeTransport(value) + if err == nil && t == "" { + err = validateTransport(value) + } + if err != nil { fatalCode("invalid_argument", "config: %v", err) } + value = t case "proxy": - if err := validateProxySetting(parts[1]); err != nil { + p, err := proxyconf.Normalize(value) + if err != nil { fatalCode("invalid_argument", "config: %v", err) } + value = p } } cfg := loadConfig() - cfg[parts[0]] = parts[1] + cfg[parts[0]] = value + result := map[string]interface{}{"key": parts[0], "value": value} + // Leaving compat: an install made with a pilotctl that predated + // --transport pointed the registry at the compat TLS host + // (install.sh --transport compat). A udp daemon needs the raw-TCP + // registry back (current daemons cope either way; older ones + // would dial :443 without TLS). + if parts[0] == "transport" && value != "compat" { + if r, _ := cfg["registry"].(string); strings.EqualFold(strings.TrimSpace(r), compatRegistryAddr) { + cfg["registry"] = productionRegistryAddr + result["registry"] = productionRegistryAddr + } + } if err := saveConfig(cfg); err != nil { fatalCode("internal", "save config: %v", err) } - shown := parts[1] if parts[0] == "proxy" { - shown = redactProxyURL(shown) + result["value"] = redactProxyURL(value) } - outputOK(map[string]interface{}{ - "key": parts[0], - "value": shown, - }) + outputOK(result) return } @@ -2263,9 +2283,9 @@ func contextCatalog() map[string]interface{} { // Daemon lifecycle "daemon start": map[string]interface{}{ - "args": []string{"[--registry ]", "[--beacon ]", "[--listen ]", "[--identity ]", "[--email ]", "[--hostname ]", "[--log-level ]", "[--public]", "[--foreground]", "[--socket ]", "[--transport ]", "[--proxy ]"}, - "description": "Start the daemon as a background process. Blocks until registered, then exits. --transport compat (or config transport=compat / $PILOT_TRANSPORT) runs over TCP 443 only for UDP-blocked hosts; --proxy auto (default) then routes through $HTTPS_PROXY", - "returns": "node_id, address, pid, socket, hostname, log_file, transport (when set), proxy (when set, credentials redacted)", + "args": []string{"[--registry ]", "[--beacon ]", "[--listen ]", "[--identity ]", "[--email ]", "[--hostname ]", "[--log-level ]", "[--public]", "[--foreground]", "[--socket ]", "[--transport ]", "[--proxy ]"}, + "description": "Start the daemon as a background process. Blocks until registered, then exits. The default transport is auto: UDP when it works, else compat (TLS/WSS over TCP 443 only, through $HTTPS_PROXY when set) — so UDP-blocked and proxy-only hosts work without flags. --transport udp|compat (or config transport / $PILOT_TRANSPORT) forces one", + "returns": "node_id, address, pid, socket, hostname, log_file, transport (as the daemon reported it), transport_auto (when auto chose it), proxy (when set, credentials redacted)", }, "daemon stop": map[string]interface{}{ "args": []string{}, @@ -2607,9 +2627,9 @@ func contextCatalog() map[string]interface{} { "environment": map[string]interface{}{ "PILOT_REGISTRY": "Registry address (default: 34.71.57.205:9000)", "PILOT_SOCKET": "Daemon socket path (default: /tmp/pilot.sock)", - "PILOT_TRANSPORT": "daemon start transport: udp or compat (TCP 443 only)", - "PILOT_PROXY": "daemon proxy policy: auto, off, or http(s)://[user:pass@]host:port", - "HTTPS_PROXY": "proxy used by compat mode (proxy=auto) and pilotctl HTTP calls; forwarded to the daemon", + "PILOT_TRANSPORT": "daemon start transport: udp, compat (TCP 443 only) or auto", + "PILOT_PROXY": "proxy policy: auto, off, or http(s)://[user:pass@]host:port (beats config.json)", + "HTTPS_PROXY": "proxy for compat mode (proxy=auto) and pilotctl's own connections; forwarded to the daemon", }, "config_file": "~/.pilot/config.json", } @@ -2735,7 +2755,8 @@ type daemonLaunchPlan struct { SocketPath string // AdminToken is passed as $PILOT_ADMIN_TOKEN, never on argv (PILOT-290). AdminToken string - // Transport is the resolved --transport ("" = daemon default, udp). + // Transport is the resolved --transport ("" = not configured: + // cmdDaemonStart asks a daemon that supports it for auto). Transport string // Proxy is the resolved --proxy ("" = daemon default, auto). Proxy string @@ -3087,20 +3108,12 @@ func cmdDaemonStart(args []string) { } daemonBin := daemonBinaryPath() - // Never hand an older daemon a flag it does not define: Go's flag - // package would abort it on startup. - daemonArgs := dropUnsupportedDaemonFlags(daemonBin, plan.Args) - proxyEnv := plan.ProxyEnv - if proxyEnv != "" { - if supported := daemonFlags(daemonBin); supported != nil && !supported["proxy"] { - if !jsonOutput { - fmt.Fprintf(os.Stderr, "warning: %s does not support -proxy (older pilot-daemon); proxy %s will not be used — upgrade pilot-daemon to use it\n", - daemonBin, redactProxyURL(proxyEnv)) - } - proxyEnv = "" - } - } - daemonEnv := daemonChildEnv(os.Environ(), plan.AdminToken, proxyEnv, plan.Transport) + // Fit the launch to the daemon binary (probed once): never hand an + // older daemon a flag or value it does not define — Go's flag package + // would abort it on startup — and ask a current one for + // -transport=auto when no transport is configured. + daemonArgs, proxyEnv, requestedTransport := adaptDaemonArgs(daemonBin, plan) + daemonEnv := daemonChildEnv(os.Environ(), plan.AdminToken, proxyEnv, requestedTransport) // --foreground: replace the current process so signal/lifetime // handling matches what the user expects from systemd unit files @@ -3175,8 +3188,14 @@ func cmdDaemonStart(args []string) { fmt.Fprintf(os.Stderr, "starting daemon (pid %d, socket %s)...", pid, socketPath) } - // Wait for daemon to become ready (socket appears and responds) - waitDur := flagDuration(flags, "wait", 15*time.Second) + // Wait for daemon to become ready (socket appears and responds). + // compat (and auto, which may pick it) registers over TLS and brings + // up the WSS tunnel, possibly through a proxy: allow it more time. + defaultWait := 15 * time.Second + if requestedTransport == "compat" || requestedTransport == "auto" { + defaultWait = 30 * time.Second + } + waitDur := flagDuration(flags, "wait", defaultWait) deadline := time.Now().Add(waitDur) dots := 0 for time.Now().Before(deadline) { @@ -3210,8 +3229,13 @@ func cmdDaemonStart(args []string) { "socket": socketPath, "log_file": pidLogPath, } - if plan.Transport != "" { - fields["transport"] = plan.Transport + if t, auto := effectiveTransport(requestedTransport, pidLogPath); t != "" || auto { + if t != "" { + fields["transport"] = t + } + if auto { + fields["transport_auto"] = true + } } if plan.Proxy != "" { fields["proxy"] = redactProxyURL(plan.Proxy) @@ -3223,8 +3247,11 @@ func cmdDaemonStart(args []string) { if hn != "" { fmt.Printf(" Hostname: %s\n", hn) } - if plan.Transport != "" { - fmt.Printf(" Transport: %s\n", plan.Transport) + if t, auto := effectiveTransport(requestedTransport, pidLogPath); t != "" { + if auto { + t += " (auto)" + } + fmt.Printf(" Transport: %s\n", t) } if plan.Proxy != "" { fmt.Printf(" Proxy: %s\n", redactProxyURL(plan.Proxy)) diff --git a/cmd/pilotctl/registry_dial.go b/cmd/pilotctl/registry_dial.go new file mode 100644 index 00000000..8bce1421 --- /dev/null +++ b/cmd/pilotctl/registry_dial.go @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "crypto/tls" + "fmt" + "net" + "os" + "strings" + + "github.com/pilot-protocol/common/netproxy" + registry "github.com/pilot-protocol/common/registry/client" + "github.com/pilot-protocol/pilotprotocol/internal/proxyconf" +) + +// registryRoute is how pilotctl reaches the registry for its own commands +// (lookup, register, rotate-key, the auto-handshake visibility check, +// recovery, ...). It follows the same network the daemon uses, so nothing +// pilotctl does bypasses an egress proxy: +// +// - the proxy comes from $PILOT_PROXY, else config.json "proxy", else +// "auto" — the environment's HTTPS_PROXY / ALL_PROXY (honouring +// NO_PROXY). pilotctl is a short-lived client whose HTTP calls already +// follow HTTPS_PROXY, so auto applies whatever the daemon's transport; +// set proxy=off to dial directly. Loopback registries are never +// proxied. +// - the compiled-in raw-TCP registry (34.71.57.205:9000) is replaced by +// registry.pilotprotocol.network:443 over TLS when the registry would +// be proxied (proxies allow CONNECT to :443 only) or the transport is +// compat (UDP-blocked hosts are usually TCP-443-only too). That +// address always uses TLS, verified against the system roots, or +// pinned to registry_fingerprint / $PILOT_REGISTRY_FINGERPRINT when +// one is configured (sandboxes without a CA bundle). +type registryRoute struct { + Addr string + TLS bool + Fingerprint string // non-empty: pinned trust + Proxy *netproxy.Resolver + // Switched: Addr replaced the raw-TCP default. + Switched bool + // FallbackTLS: Addr is the raw-TCP default dialed directly; on failure + // dialRegistry retries the compat TLS registry (UDP- and TCP/9000- + // blocked hosts with direct TCP 443). + FallbackTLS bool +} + +// proxied reports whether dialing r.Addr goes through a proxy. +func (r registryRoute) proxied() bool { + return r.proxyFor(r.Addr) != "" +} + +// proxyFor returns the redacted proxy for target, "" for a direct dial. +func (r registryRoute) proxyFor(target string) string { + if !r.Proxy.Enabled() { + return "" + } + if host, _, err := net.SplitHostPort(target); err == nil && proxyconf.IsLoopbackHost(host) { + return "" + } + u, err := r.Proxy.ProxyForAddr(target) + if err != nil || u == nil { + return "" + } + return netproxy.Redact(u) +} + +// pilotctlProxySpec is the proxy setting for pilotctl's own connections: +// $PILOT_PROXY, then config.json "proxy", then auto. +func pilotctlProxySpec(cfg map[string]interface{}) string { + if v := strings.TrimSpace(os.Getenv("PILOT_PROXY")); v != "" { + return v + } + if s, ok := cfg["proxy"].(string); ok && strings.TrimSpace(s) != "" { + return s + } + return proxyconf.Auto +} + +// configuredTransport is $PILOT_TRANSPORT, else config.json "transport", +// normalized; "" when neither is set or the value is unknown. +func configuredTransport(cfg map[string]interface{}) string { + v := strings.TrimSpace(os.Getenv("PILOT_TRANSPORT")) + if v == "" { + v, _ = cfg["transport"].(string) + } + t, err := normalizeTransport(v) + if err != nil { + return "" + } + return t +} + +// registryFingerprintSetting is $PILOT_REGISTRY_FINGERPRINT, else +// config.json "registry_fingerprint" — used only when trust is pinned +// ($PILOT_REGISTRY_TRUST / config.json "registry_trust", defaulting to +// pinned when a fingerprint is configured). +func registryFingerprintSetting(cfg map[string]interface{}) string { + fp := strings.TrimSpace(os.Getenv("PILOT_REGISTRY_FINGERPRINT")) + if fp == "" { + fp, _ = cfg["registry_fingerprint"].(string) + } + trust := strings.TrimSpace(os.Getenv("PILOT_REGISTRY_TRUST")) + if trust == "" { + trust, _ = cfg["registry_trust"].(string) + } + if strings.EqualFold(strings.TrimSpace(trust), "system") { + return "" + } + return strings.TrimSpace(fp) +} + +// planRegistryRoute resolves how to reach addr. The error is an invalid +// proxy setting. +func planRegistryRoute(addr string) (registryRoute, error) { + cfg := loadConfig() + policy, err := proxyconf.Resolve(pilotctlProxySpec(cfg)) + if err != nil { + return registryRoute{}, err + } + r := registryRoute{Addr: strings.TrimSpace(addr), Proxy: policy} + if r.Addr == productionRegistryAddr { + if configuredTransport(cfg) == "compat" || r.proxyFor(compatRegistryAddr) != "" { + r.Addr = compatRegistryAddr + r.Switched = true + } else { + r.FallbackTLS = true + } + } + if strings.EqualFold(r.Addr, compatRegistryAddr) { + r.TLS = true + r.Fingerprint = registryFingerprintSetting(cfg) + } + return r, nil +} + +// dial opens the registry connection the route describes. +func (r registryRoute) dial() (*registry.Client, error) { + var opts []registry.DialOption + if r.Proxy.Enabled() { + opts = append(opts, registry.WithDialer(proxyconf.DialContext(r.Proxy, nil))) + } + if !r.TLS { + return registry.Dial(r.Addr, opts...) + } + if r.Fingerprint != "" { + return registry.DialTLSPinned(r.Addr, r.Fingerprint, opts...) + } + return registry.DialTLS(r.Addr, &tls.Config{MinVersion: tls.VersionTLS12}, opts...) +} + +// dialRegistry connects to the registry at addr along its registryRoute, +// retrying the compat TLS registry once when a direct dial of the raw-TCP +// default fails. The returned route is the one that was tried last. +func dialRegistry(addr string) (*registry.Client, registryRoute, error) { + route, err := planRegistryRoute(addr) + if err != nil { + return nil, route, err + } + rc, err := route.dial() + if err == nil || !route.FallbackTLS { + return rc, route, err + } + fallback := route + fallback.Addr, fallback.TLS, fallback.Switched, fallback.FallbackTLS = compatRegistryAddr, true, true, false + fallback.Fingerprint = registryFingerprintSetting(loadConfig()) + if rc, ferr := fallback.dial(); ferr == nil { + return rc, fallback, nil + } + return nil, route, err +} + +// registryDialHint says what to check after a failed registry dial. +func registryDialHint(route registryRoute) string { + if p := route.proxyFor(route.Addr); p != "" { + return fmt.Sprintf("the registry %s is reached through the proxy %s: check that it allows CONNECT to %s and that its credentials are right; if this host can reach the registry directly, set PILOT_PROXY=off (or pilotctl config --set proxy=off)", + route.Addr, p, route.Addr) + } + if route.TLS { + return fmt.Sprintf("check outbound TCP 443 to %s; if TLS verification fails, point SSL_CERT_FILE at a CA bundle or set PILOT_REGISTRY_FINGERPRINT", route.Addr) + } + return fmt.Sprintf("check that the registry is running at %s, or set PILOT_REGISTRY", route.Addr) +} + +// connectRegistryAt dials addr or exits with a hint. +func connectRegistryAt(addr, what string) *registry.Client { + rc, route, err := dialRegistry(addr) + if err != nil { + if route.Addr == "" { + fatalCode("invalid_argument", "%s: %v", what, err) + } + fatalHint("connection_failed", registryDialHint(route), + "%s: cannot reach registry at %s: %v", what, route.Addr, err) + } + return rc +} diff --git a/cmd/pilotctl/verify.go b/cmd/pilotctl/verify.go index a202e2d9..f7525d36 100644 --- a/cmd/pilotctl/verify.go +++ b/cmd/pilotctl/verify.go @@ -11,7 +11,6 @@ import ( "github.com/pilot-protocol/common/badgeverify" "github.com/pilot-protocol/common/crypto" "github.com/pilot-protocol/common/protocol" - registry "github.com/pilot-protocol/common/registry/client" ) // loadJSONFile reads a small JSON credential file into v, exiting on error. @@ -298,13 +297,7 @@ func cmdRecoveryRecover(args []string) { newPub := crypto.EncodePublicKey(id.PublicKey) addr := flagString(flags, "registry", getRegistry()) - // TODO(netproxy): raw-TCP registry dial; bypasses HTTPS_PROXY until - // common/netproxy lands in pilotctl (see connectRegistry). - rc, err := registry.Dial(addr) - if err != nil { - fatalHint("connection_failed", registryDialHint(addr), - "recovery recover: cannot reach registry at %s: %v", addr, err) - } + rc := connectRegistryAt(addr, "recovery recover") defer rc.Close() resp, err := rc.RecoverIdentity(nodeID, recovery, recoverySig, newPub) diff --git a/cmd/pilotctl/zz_daemon_transport_test.go b/cmd/pilotctl/zz_daemon_transport_test.go index 1600bb6e..4db644f0 100644 --- a/cmd/pilotctl/zz_daemon_transport_test.go +++ b/cmd/pilotctl/zz_daemon_transport_test.go @@ -69,8 +69,6 @@ func TestResolveDaemonTransportPrecedence(t *testing.T) { func TestResolveDaemonProxyPrecedenceAndValidation(t *testing.T) { withTransportEnvCleared(t) - // $PILOT_PROXY is the daemon's to resolve, never pilotctl's. - t.Setenv("PILOT_PROXY", "off") if got, err := resolveDaemonProxy(map[string]string{}, map[string]interface{}{}); err != nil || got != "" { t.Fatalf("unset: got %q, %v", got, err) } @@ -78,15 +76,25 @@ func TestResolveDaemonProxyPrecedenceAndValidation(t *testing.T) { if got, _ := resolveDaemonProxy(map[string]string{}, cfg); got != "auto" { t.Errorf("config: got %q", got) } + // $PILOT_PROXY beats config.json, as it does in pilot-daemon: an + // explicitly passed value always beats a persistent default. + t.Setenv("PILOT_PROXY", "http://u:s3cret@env.example:3128") + if got, _ := resolveDaemonProxy(map[string]string{}, cfg); got != "http://u:s3cret@env.example:3128" { + t.Errorf("env should beat config: got %q", got) + } if got, _ := resolveDaemonProxy(map[string]string{"proxy": "http://p.example:3128"}, cfg); got != "http://p.example:3128" { - t.Errorf("flag should beat config: got %q", got) + t.Errorf("flag should beat env and config: got %q", got) } - for _, ok := range []string{"auto", "off", "http://p:3128", "https://u:pw@p.example:443", "http://u@p"} { + t.Setenv("PILOT_PROXY", "") + if got, _ := resolveDaemonProxy(map[string]string{"proxy": "None"}, cfg); got != "off" { + t.Errorf("none should normalize to off: got %q", got) + } + for _, ok := range []string{"auto", "AUTO", "off", "none", "direct", "http://p:3128", "https://u:pw@p.example:443", "http://u@p"} { if err := validateProxySetting(ok); err != nil { t.Errorf("validateProxySetting(%q) = %v, want nil", ok, err) } } - for _, bad := range []string{"true", "socks5://p:1080", "p.example:3128", "http://", "u:pw@p:3128", "AUTO"} { + for _, bad := range []string{"true", "proxy", "socks5://p:1080", "p.example:3128", "http://", "u:pw@p:3128"} { err := validateProxySetting(bad) if err == nil { t.Errorf("validateProxySetting(%q) = nil, want error", bad) @@ -101,14 +109,15 @@ func TestResolveDaemonProxyPrecedenceAndValidation(t *testing.T) { func TestRedactProxyURL(t *testing.T) { t.Parallel() cases := map[string]string{ - "auto": "auto", - "off": "off", - "http://proxy:3128": "http://proxy:3128", - "http://user:s3cret@proxy:3128": "http://***@proxy:3128", - "https://user@proxy.example:8443": "https://***@proxy.example:8443", - "http://u:p%40ss@proxy:3128/": "http://***@proxy:3128/", - "user:s3cret@proxy:3128": "***@proxy:3128", - "http://us er:s3cret@proxy:3128": "http://***@proxy:3128", + "auto": "auto", + "off": "off", + "http://proxy:3128": "http://proxy:3128", + "http://user:s3cret@proxy:3128": "http://***@proxy:3128", + "https://user@proxy.example:8443": "https://***@proxy.example:8443", + "http://u:p%40ss@proxy:3128/": "http://***@proxy:3128", + "user:s3cret@proxy:3128": "***@proxy:3128", + "http://us er:s3cret@proxy:3128": "http://***@proxy:3128", + "http://agent:1234#s3cret@egress:3128": "http://***@egress:3128", } for in, want := range cases { got := redactProxyURL(in) @@ -121,6 +130,25 @@ func TestRedactProxyURL(t *testing.T) { } } +// A password with an unescaped '#', '/' or '?' makes url.Parse see no +// userinfo; it must still count as credentials and stay off argv. +func TestProxyCredentialsWithReservedCharactersStayOffArgv(t *testing.T) { + withTransportEnvCleared(t) + for _, proxy := range []string{ + "http://agent:1234#Xyz9@egress:3128", + "http://agent:12/34@egress:3128", + "http://agent:12?34@egress:3128", + } { + plan := planDaemonLaunch([]string{"--proxy", proxy}) + if argsHasKey(plan.Args, "--proxy") || strings.Contains(strings.Join(plan.Args, " "), "agent") { + t.Errorf("%s: credentials on argv: %v", proxy, plan.Args) + } + if plan.ProxyEnv != proxy { + t.Errorf("%s: ProxyEnv = %q", proxy, plan.ProxyEnv) + } + } +} + // TestBuildDaemonArgsUDPUnchanged pins backward compatibility: with no // transport/proxy configured, the daemon command line is exactly what older // pilotctl produced — the default registry/beacon are forwarded and neither @@ -146,8 +174,9 @@ func TestBuildDaemonArgsUDPUnchanged(t *testing.T) { // TestBuildDaemonArgsCompatFromConfig is the Meta Muse shape: config.json from // `pilotctl init` carries the raw-TCP registry default plus transport=compat -// and proxy=auto (install.sh --transport compat). The registry/beacon defaults -// must stay off argv so the daemon's compat 443/TLS defaults apply. +// (install.sh --transport compat) and, hand-set, proxy=auto. The +// registry/beacon defaults must stay off argv so even an older daemon's +// compat 443/TLS defaults apply. func TestBuildDaemonArgsCompatFromConfig(t *testing.T) { withTransportEnvCleared(t) if err := saveConfig(map[string]interface{}{ @@ -332,8 +361,12 @@ func writeFakeDaemon(t *testing.T, flags []string, out string) string { var help strings.Builder var known strings.Builder for _, f := range flags { - help.WriteString(" -" + f + " string\n \tdescription of " + f + "\n") - known.WriteString(" -" + f + " --" + f) + name, usage, ok := strings.Cut(f, "=") + if !ok { + usage = "description of " + name + } + help.WriteString(" -" + name + " string\n \t" + usage + "\n") + known.WriteString(" -" + name + " --" + name) } script := `#!/bin/sh if [ "$1" = "-help" ]; then @@ -384,44 +417,6 @@ func TestDaemonFlagsProbe(t *testing.T) { } } -// TestDropUnsupportedDaemonFlags: an older daemon (no -proxy, or neither -// -proxy nor -transport) gets those flags stripped instead of crashing on -// them; a current daemon — or one that cannot be probed — keeps them. -func TestDropUnsupportedDaemonFlags(t *testing.T) { - t.Parallel() - args := []string{"--listen", ":0", "--transport", "compat", "--proxy", "auto", "--socket", "/tmp/x.sock"} - dir := t.TempDir() - - v1139 := writeFakeDaemon(t, append([]string{"transport"}, baseDaemonFlags...), filepath.Join(dir, "a")) - got := dropUnsupportedDaemonFlags(v1139, args) - if argsHasKey(got, "--proxy") || argsHasKey(got, "auto") { - t.Errorf("v1.13.9-style daemon must not get --proxy: %v", got) - } - if !argsHasPair(got, "--transport", "compat") || !argsHasPair(got, "--socket", "/tmp/x.sock") { - t.Errorf("supported flags must survive: %v", got) - } - - ancient := writeFakeDaemon(t, baseDaemonFlags, filepath.Join(dir, "b")) - got = dropUnsupportedDaemonFlags(ancient, args) - if argsHasKey(got, "--proxy") || argsHasKey(got, "--transport") { - t.Errorf("pre-compat daemon must get neither flag: %v", got) - } - if !argsHasPair(got, "--listen", ":0") { - t.Errorf("unrelated flags must survive: %v", got) - } - - current := writeFakeDaemon(t, append([]string{"transport", "proxy"}, baseDaemonFlags...), filepath.Join(dir, "c")) - got = dropUnsupportedDaemonFlags(current, args) - if strings.Join(got, " ") != strings.Join(args, " ") { - t.Errorf("current daemon must keep every flag: %v", got) - } - - got = dropUnsupportedDaemonFlags(filepath.Join(dir, "missing"), args) - if strings.Join(got, " ") != strings.Join(args, " ") { - t.Errorf("unprobeable daemon must keep every flag: %v", got) - } -} - func readLines(t *testing.T, path string) []string { t.Helper() b, err := os.ReadFile(path) @@ -584,12 +579,20 @@ func TestCLIConfigSetProxyRedactsAndValidates(t *testing.T) { func TestRegistryDialHintMentionsProxy(t *testing.T) { withTransportEnvCleared(t) - if h := registryDialHint("r:9000"); !strings.Contains(h, "PILOT_REGISTRY") { + route, err := planRegistryRoute("r.example:9000") + if err != nil { + t.Fatal(err) + } + if h := registryDialHint(route); !strings.Contains(h, "PILOT_REGISTRY") { t.Errorf("no-proxy hint = %q", h) } t.Setenv("HTTPS_PROXY", "http://agent:s3cret@egress:3128") - h := registryDialHint("r:9000") - if !strings.Contains(h, "does not use the proxy") || strings.Contains(h, "s3cret") { + route, err = planRegistryRoute("r.example:9000") + if err != nil { + t.Fatal(err) + } + h := registryDialHint(route) + if !strings.Contains(h, "through the proxy http://***@egress:3128") || !strings.Contains(h, "proxy=off") || strings.Contains(h, "s3cret") { t.Errorf("proxy hint = %q", h) } } diff --git a/cmd/pilotctl/zz_lifecycle_test.go b/cmd/pilotctl/zz_lifecycle_test.go index 6f04b181..f106f354 100644 --- a/cmd/pilotctl/zz_lifecycle_test.go +++ b/cmd/pilotctl/zz_lifecycle_test.go @@ -22,7 +22,13 @@ func withTempHomeFull(t *testing.T) string { t.Setenv("PILOT_HOME", "") t.Setenv("PILOT_SOCKET", "") t.Setenv("PILOT_REGISTRY", "") + t.Setenv("PILOT_BEACON", "") t.Setenv("PILOT_ADMIN_TOKEN", "") + // A developer or CI shell exporting PILOT_TRANSPORT=compat or a proxy + // must not change what daemon start / registry dials plan. + for _, k := range daemonForwardEnv { + t.Setenv(k, "") + } return tmp } diff --git a/cmd/pilotctl/zz_proxy_route_test.go b/cmd/pilotctl/zz_proxy_route_test.go new file mode 100644 index 00000000..52218fc8 --- /dev/null +++ b/cmd/pilotctl/zz_proxy_route_test.go @@ -0,0 +1,385 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "bufio" + "fmt" + "io" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +const autoUsage = "transport=tunnel transport: 'udp' (the default), 'compat' (...) or 'auto' (...)" + +// v1.13.9's -transport usage: no 'auto'. +const v1139TransportUsage = "transport=tunnel transport: 'udp' (default) or 'compat' (WSS to beacon, opt-in, for UDP-blocked environments)" + +func TestDaemonFlagUsageParsesUsage(t *testing.T) { + t.Parallel() + dir := t.TempDir() + cur := writeFakeDaemon(t, append([]string{autoUsage, "proxy"}, baseDaemonFlags...), filepath.Join(dir, "a")) + if ok, known := daemonSupportsAutoTransport(cur); !ok || !known { + t.Errorf("current daemon: auto=(%v, %v), want supported", ok, known) + } + old := writeFakeDaemon(t, append([]string{v1139TransportUsage}, baseDaemonFlags...), filepath.Join(dir, "b")) + if ok, known := daemonSupportsAutoTransport(old); ok || !known { + t.Errorf("v1.13.9 daemon: auto=(%v, %v), want known unsupported", ok, known) + } + if ok, known := daemonSupportsAutoTransport(filepath.Join(dir, "missing")); ok || known { + t.Errorf("missing daemon: auto=(%v, %v), want unknown", ok, known) + } +} + +// adaptDaemonArgs: new pilotctl × old/new daemon. Nothing a daemon does not +// know ever reaches its argv, and an unconfigured transport becomes auto +// only where auto exists. +func TestAdaptDaemonArgs(t *testing.T) { + withTransportEnvCleared(t) + dir := t.TempDir() + current := writeFakeDaemon(t, append([]string{autoUsage, "proxy"}, baseDaemonFlags...), filepath.Join(dir, "cur")) + v1139 := writeFakeDaemon(t, append([]string{v1139TransportUsage}, baseDaemonFlags...), filepath.Join(dir, "v1139")) + ancient := writeFakeDaemon(t, baseDaemonFlags, filepath.Join(dir, "ancient")) + base := []string{"--registry", productionRegistryAddr, "--listen", ":0"} + + for _, tc := range []struct { + name string + bin string + plan daemonLaunchPlan + wantTransport string // value of --transport on argv, "" = absent + wantProxyArg string + wantProxyEnv string + }{ + {"unset + current → auto", current, daemonLaunchPlan{Args: base}, "auto", "", ""}, + {"unset + v1.13.9 → daemon default", v1139, daemonLaunchPlan{Args: base}, "", "", ""}, + {"unset + ancient → daemon default", ancient, daemonLaunchPlan{Args: base}, "", "", ""}, + {"unset + unprobeable → daemon default", filepath.Join(dir, "missing"), daemonLaunchPlan{Args: base}, "", "", ""}, + {"auto + v1.13.9 → udp", v1139, daemonLaunchPlan{Args: append(base, "--transport", "auto"), Transport: "auto"}, "udp", "", ""}, + {"auto + ancient → dropped", ancient, daemonLaunchPlan{Args: append(base, "--transport", "auto"), Transport: "auto"}, "", "", ""}, + {"compat + v1.13.9 kept", v1139, daemonLaunchPlan{Args: append(base, "--transport", "compat"), Transport: "compat"}, "compat", "", ""}, + {"compat + ancient dropped", ancient, daemonLaunchPlan{Args: append(base, "--transport", "compat"), Transport: "compat"}, "", "", ""}, + {"proxy + v1.13.9 dropped", v1139, + daemonLaunchPlan{Args: append(base, "--transport", "compat", "--proxy", "off"), Transport: "compat", Proxy: "off"}, "compat", "", ""}, + {"cred proxy env + v1.13.9 dropped", v1139, + daemonLaunchPlan{Args: base, Transport: "", Proxy: "http://u:p@x:1", ProxyEnv: "http://u:p@x:1"}, "", "", ""}, + {"current keeps everything", current, + daemonLaunchPlan{Args: append(base, "--transport", "compat", "--proxy", "off"), Transport: "compat", Proxy: "off"}, "compat", "off", ""}, + {"current keeps the env proxy", current, + daemonLaunchPlan{Args: base, Proxy: "http://u:p@x:1", ProxyEnv: "http://u:p@x:1"}, "auto", "", "http://u:p@x:1"}, + } { + t.Run(tc.name, func(t *testing.T) { + args, proxyEnv, _ := adaptDaemonArgs(tc.bin, tc.plan) + if got := flagValue(args, "--transport"); got != tc.wantTransport || (tc.wantTransport == "" && hasFlag(args, "--transport")) { + t.Errorf("--transport = %q (args %v), want %q", got, args, tc.wantTransport) + } + if got := flagValue(args, "--proxy"); got != tc.wantProxyArg || (tc.wantProxyArg == "" && hasFlag(args, "--proxy")) { + t.Errorf("--proxy = %q (args %v), want %q", got, args, tc.wantProxyArg) + } + if proxyEnv != tc.wantProxyEnv { + t.Errorf("proxy env = %q, want %q", proxyEnv, tc.wantProxyEnv) + } + if !argsHasPair(args, "--registry", productionRegistryAddr) || !argsHasPair(args, "--listen", ":0") { + t.Errorf("unrelated args lost: %v", args) + } + }) + } +} + +// The real `daemon start --foreground` path against a current daemon with +// no transport configured anywhere: the daemon is asked for auto. +func TestCLIDaemonStartImplicitAuto(t *testing.T) { + t.Parallel() + dir := t.TempDir() + out := filepath.Join(dir, "daemon") + bin := writeFakeDaemon(t, append([]string{autoUsage, "proxy"}, baseDaemonFlags...), out) + env := cliEnvCleared(map[string]string{ + "PILOT_DAEMON_BIN": bin, + "PILOT_SOCKET": filepath.Join(dir, "pilot.sock"), + }) + _, stderr, code := runCLI(t, []string{"daemon", "start", "--foreground"}, env) + if code != 0 { + t.Fatalf("exit=%d stderr=%s", code, stderr) + } + args := readLines(t, out+".args") + if !argsHasPair(args, "--transport", "auto") { + t.Errorf("daemon argv lacks --transport auto: %v", args) + } + if !argsHasPair(args, "--registry", productionRegistryAddr) { + t.Errorf("auto must keep the registry on argv (an old daemon given udp needs it): %v", args) + } +} + +// config.json "proxy":"auto" must not override a credential-bearing +// --proxy: the URL travels as $PILOT_PROXY (which pilot-daemon ranks above +// config.json) and nothing on argv contradicts it. +func TestCLIDaemonStartCredProxyBeatsConfigAuto(t *testing.T) { + t.Parallel() + dir := t.TempDir() + out := filepath.Join(dir, "daemon") + bin := writeFakeDaemon(t, append([]string{autoUsage, "proxy"}, baseDaemonFlags...), out) + home := t.TempDir() + if err := os.MkdirAll(filepath.Join(home, ".pilot"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(home, ".pilot", "config.json"), []byte(`{"transport":"compat","proxy":"auto"}`), 0o600); err != nil { + t.Fatal(err) + } + env := cliEnvCleared(map[string]string{ + "PILOT_DAEMON_BIN": bin, + "PILOT_SOCKET": filepath.Join(dir, "pilot.sock"), + "PILOT_HOME": home, + }) + _, stderr, code := runCLI(t, []string{"daemon", "start", "--foreground", "--proxy", "http://u:s3cret@corp:3128"}, env) + if code != 0 { + t.Fatalf("exit=%d stderr=%s", code, stderr) + } + args := readLines(t, out+".args") + if hasFlag(args, "--proxy") { + t.Errorf("argv carries a --proxy that would outrank the credentialed one: %v", args) + } + got := map[string]string{} + for _, kv := range readLines(t, out+".env") { + if k, v, ok := strings.Cut(kv, "="); ok { + got[k] = v + } + } + if got["PILOT_PROXY"] != "http://u:s3cret@corp:3128" { + t.Errorf("PILOT_PROXY = %q", got["PILOT_PROXY"]) + } + + // Same with the URL exported as $PILOT_PROXY instead of --proxy. + env["PILOT_PROXY"] = "http://u:s3cret@corp:3128" + if _, stderr, code = runCLI(t, []string{"daemon", "start", "--foreground"}, env); code != 0 { + t.Fatalf("exit=%d stderr=%s", code, stderr) + } + if args := readLines(t, out+".args"); hasFlag(args, "--proxy") { + t.Errorf("$PILOT_PROXY lost to config.json's auto on argv: %v", args) + } +} + +func TestPlanRegistryRoute(t *testing.T) { + const fp = "c1f958f6bcff667cf6a08d5066cc031a9086115a7667835877ca62a3019b3da9" + for _, tc := range []struct { + name string + addr string + env map[string]string + cfg map[string]interface{} + wantAddr string + wantTLS bool + proxied bool + wantFP string + }{ + {name: "plain host: raw default, direct", addr: productionRegistryAddr, + wantAddr: productionRegistryAddr}, + {name: "HTTPS_PROXY: compat TLS registry through the proxy", addr: productionRegistryAddr, + env: map[string]string{"HTTPS_PROXY": "http://u:p@egress.test:3128"}, + wantAddr: compatRegistryAddr, wantTLS: true, proxied: true}, + {name: "config transport=compat, no proxy: compat TLS registry direct", addr: productionRegistryAddr, + cfg: map[string]interface{}{"transport": "compat"}, + wantAddr: compatRegistryAddr, wantTLS: true}, + {name: "PILOT_PROXY=off beats HTTPS_PROXY", addr: productionRegistryAddr, + env: map[string]string{"HTTPS_PROXY": "http://egress.test:3128", "PILOT_PROXY": "off"}, + wantAddr: productionRegistryAddr}, + {name: "config proxy=none", addr: productionRegistryAddr, + env: map[string]string{"HTTPS_PROXY": "http://egress.test:3128"}, + cfg: map[string]interface{}{"proxy": "none"}, + wantAddr: productionRegistryAddr}, + // NO_PROXY exempts the TLS registry's name, so the raw default is + // kept; the raw IP itself is not exempt and still goes via the proxy. + {name: "NO_PROXY exempts the TLS registry name", addr: productionRegistryAddr, + env: map[string]string{"HTTPS_PROXY": "http://egress.test:3128", "NO_PROXY": ".pilotprotocol.network"}, + wantAddr: productionRegistryAddr, proxied: true}, + {name: "NO_PROXY exempting both", addr: productionRegistryAddr, + env: map[string]string{"HTTPS_PROXY": "http://egress.test:3128", "NO_PROXY": ".pilotprotocol.network,34.71.57.205"}, + wantAddr: productionRegistryAddr}, + {name: "loopback registry never proxied", addr: "127.0.0.1:9000", + env: map[string]string{"HTTPS_PROXY": "http://egress.test:3128"}, + wantAddr: "127.0.0.1:9000"}, + {name: "custom registry proxied as is", addr: "registry.corp.test:9000", + env: map[string]string{"HTTPS_PROXY": "http://egress.test:3128"}, + wantAddr: "registry.corp.test:9000", proxied: true}, + {name: "fingerprint from env pins", addr: productionRegistryAddr, + env: map[string]string{"HTTPS_PROXY": "http://egress.test:3128", "PILOT_REGISTRY_FINGERPRINT": fp}, + wantAddr: compatRegistryAddr, wantTLS: true, proxied: true, wantFP: fp}, + {name: "fingerprint from config, trust=system wins", addr: compatRegistryAddr, + cfg: map[string]interface{}{"registry_fingerprint": fp, "registry_trust": "system"}, + wantAddr: compatRegistryAddr, wantTLS: true}, + {name: "fingerprint from config pins", addr: compatRegistryAddr, + cfg: map[string]interface{}{"registry_fingerprint": fp}, + wantAddr: compatRegistryAddr, wantTLS: true, wantFP: fp}, + } { + t.Run(tc.name, func(t *testing.T) { + withTransportEnvCleared(t) + for _, k := range []string{"HTTPS_PROXY", "NO_PROXY", "PILOT_PROXY", "PILOT_REGISTRY_FINGERPRINT", "PILOT_REGISTRY_TRUST", "PILOT_TRANSPORT"} { + t.Setenv(k, tc.env[k]) + } + if tc.cfg != nil { + if err := saveConfig(tc.cfg); err != nil { + t.Fatal(err) + } + } + r, err := planRegistryRoute(tc.addr) + if err != nil { + t.Fatalf("planRegistryRoute: %v", err) + } + if r.Addr != tc.wantAddr || r.TLS != tc.wantTLS || r.proxied() != tc.proxied || r.Fingerprint != tc.wantFP { + t.Errorf("route = {Addr:%s TLS:%v proxied:%v FP:%q}, want {%s %v %v %q}", + r.Addr, r.TLS, r.proxied(), r.Fingerprint, tc.wantAddr, tc.wantTLS, tc.proxied, tc.wantFP) + } + }) + } + t.Run("invalid proxy setting", func(t *testing.T) { + withTransportEnvCleared(t) + t.Setenv("PILOT_PROXY", "proxy.test:3128") + if _, err := planRegistryRoute(productionRegistryAddr); err == nil { + t.Error("bare host:port proxy accepted") + } + }) +} + +// connectProxyToLoopback is an authenticating CONNECT proxy that routes +// *.pilot.invalid to loopback and records the CONNECT targets. +func connectProxyToLoopback(t *testing.T, user, pass string) (string, func() []string) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + req := &http.Request{Header: http.Header{}} + req.SetBasicAuth(user, pass) + want := req.Header.Get("Authorization") + var mu sync.Mutex + var targets []string + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go func() { + defer c.Close() + br := bufio.NewReader(c) + r, err := http.ReadRequest(br) + if err != nil { + return + } + mu.Lock() + targets = append(targets, r.Method+" "+r.RequestURI) + mu.Unlock() + host, port, _ := net.SplitHostPort(r.RequestURI) + if r.Method != http.MethodConnect || r.Header.Get("Proxy-Authorization") != want || !strings.HasSuffix(host, ".pilot.invalid") { + fmt.Fprint(c, "HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\n\r\n") + return + } + up, err := net.DialTimeout("tcp", net.JoinHostPort("127.0.0.1", port), 5*time.Second) + if err != nil { + fmt.Fprint(c, "HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n") + return + } + defer up.Close() + fmt.Fprint(c, "HTTP/1.1 200 Connection established\r\n\r\n") + go func() { io.Copy(up, br); up.Close() }() + io.Copy(c, up) + }() + } + }() + t.Cleanup(func() { ln.Close() }) + return fmt.Sprintf("http://%s:%s@%s", user, pass, ln.Addr()), func() []string { + mu.Lock() + defer mu.Unlock() + return append([]string(nil), targets...) + } +} + +// pilotctl's own registry commands go through the egress proxy (CONNECT by +// host name, with credentials) — the auto-handshake visibility check and +// `lookup` no longer trip a sandbox's direct-egress guard. +func TestRegistryCommandsUseTheProxy(t *testing.T) { + r := newFakeRegistry(t) + r.onOK("lookup", map[string]interface{}{"node_id": float64(99), "address": "0:0000.0000.0063", "public": true}) + proxyURL, targets := connectProxyToLoopback(t, "muse", "s3cret") + _, port, _ := net.SplitHostPort(r.addr()) + regAddr := net.JoinHostPort("registry.pilot.invalid", port) + + withTransportEnvCleared(t) + t.Setenv("HTTPS_PROXY", proxyURL) + rc, route, err := dialRegistry(regAddr) + if err != nil { + t.Fatalf("dialRegistry through the proxy: %v", err) + } + defer rc.Close() + if !route.proxied() { + t.Error("route not proxied") + } + if _, err := rc.Lookup(99); err != nil { + t.Fatalf("Lookup through the proxy: %v", err) + } + if got := targets(); len(got) != 1 || got[0] != "CONNECT "+regAddr { + t.Fatalf("proxy requests = %q, want one CONNECT %s", got, regAddr) + } + + // The CLI path too: `pilotctl lookup` in a child process. + stdout, stderr, code := runCLI(t, []string{"--json", "lookup", "99"}, cliEnvCleared(map[string]string{ + "PILOT_REGISTRY": regAddr, + "HTTPS_PROXY": proxyURL, + })) + if code != 0 || !strings.Contains(stdout, "0:0000.0000.0063") { + t.Fatalf("pilotctl lookup via proxy: exit=%d stdout=%s stderr=%s", code, stdout, stderr) + } + if got := targets(); len(got) != 2 { + t.Fatalf("CLI lookup did not go through the proxy: %q", got) + } +} + +func TestConfigSetTransportSwitchBackRestoresRegistry(t *testing.T) { + t.Parallel() + home := t.TempDir() + env := cliEnvCleared(map[string]string{"PILOT_HOME": home}) + if _, stderr, code := runCLI(t, []string{"config", "--set", "registry=" + compatRegistryAddr}, env); code != 0 { + t.Fatalf("exit=%d stderr=%s", code, stderr) + } + stdout, stderr, code := runCLI(t, []string{"--json", "config", "--set", "transport=udp"}, env) + if code != 0 { + t.Fatalf("exit=%d stderr=%s", code, stderr) + } + if !strings.Contains(stdout, productionRegistryAddr) { + t.Errorf("switch back to udp did not report the restored registry: %s", stdout) + } + raw, _ := os.ReadFile(filepath.Join(home, ".pilot", "config.json")) + if !strings.Contains(string(raw), productionRegistryAddr) || strings.Contains(string(raw), compatRegistryAddr) { + t.Errorf("config.json registry not restored: %s", raw) + } + // auto and none normalize. + if stdout, _, code := runCLI(t, []string{"--json", "config", "--set", "transport=AUTO"}, env); code != 0 || !strings.Contains(stdout, `"auto"`) { + t.Errorf("transport=AUTO: exit=%d %s", code, stdout) + } + if stdout, _, code := runCLI(t, []string{"--json", "config", "--set", "proxy=none"}, env); code != 0 || !strings.Contains(stdout, `"off"`) { + t.Errorf("proxy=none: exit=%d %s", code, stdout) + } +} + +func TestTransportFromDaemonLog(t *testing.T) { + t.Parallel() + dir := t.TempDir() + text := filepath.Join(dir, "text.log") + os.WriteFile(text, []byte(`time=x level=INFO msg="transport auto-selected" transport=compat reason="..." +time=x level=INFO msg="outbound network" transport=compat proxy="auto: http://***@p:3128" transport_from=default +`), 0o600) + if got, auto := effectiveTransport("auto", text); got != "compat" || !auto { + t.Errorf("text log: (%q, %v)", got, auto) + } + js := filepath.Join(dir, "json.log") + os.WriteFile(js, []byte(`{"time":"x","level":"INFO","msg":"outbound network","transport":"udp","proxy":"none"}`+"\n"), 0o600) + if got, _ := effectiveTransport("", js); got != "udp" { + t.Errorf("json log: %q", got) + } + if got, auto := effectiveTransport("compat", filepath.Join(dir, "missing")); got != "compat" || auto { + t.Errorf("no log: (%q, %v)", got, auto) + } +} diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 1d4dcd08..cd6e67f0 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -20,7 +20,7 @@ Bootstrap: pilotctl config [--set key=value] Daemon lifecycle: - pilotctl daemon start [--config ] [--registry ] [--beacon ] [--email ] [--webhook ] [--trust-auto-approve] [--transport ] [--proxy ] + pilotctl daemon start [--config ] [--registry ] [--beacon ] [--email ] [--webhook ] [--trust-auto-approve] [--transport ] [--proxy ] pilotctl daemon stop pilotctl daemon status @@ -117,9 +117,9 @@ Diagnostic commands: Environment: PILOT_REGISTRY Registry address (default: 34.71.57.205:9000) PILOT_SOCKET Daemon socket path (default: /tmp/pilot.sock) - PILOT_TRANSPORT daemon start transport: udp or compat (TCP 443 only) - PILOT_PROXY daemon proxy policy: auto, off, or http(s)://[user:pass@]host:port - HTTPS_PROXY proxy used by compat mode (proxy=auto) and pilotctl's HTTP calls + PILOT_TRANSPORT daemon start transport: udp, compat (TCP 443 only) or auto + PILOT_PROXY proxy policy: auto, off, or http(s)://[user:pass@]host:port + HTTPS_PROXY proxy for compat mode (proxy=auto) and pilotctl's own connections Version: pilotctl version diff --git a/install.sh b/install.sh index 6b62ca17..563d8070 100755 --- a/install.sh +++ b/install.sh @@ -9,8 +9,10 @@ set -e # Install: curl -fsSL https://pilotprotocol.network/install.sh | sh # Pin a version: curl -fsSL https://pilotprotocol.network/install.sh | sh -s -- --version v1.13.6 # Beta channel: curl -fsSL https://pilotprotocol.network/install.sh | sh -s -- --channel beta -# UDP blocked / curl -fsSL https://pilotprotocol.network/install.sh | sh -s -- --transport compat -# HTTPS proxy: (TCP 443 only; the daemon uses $HTTPS_PROXY when it is set) +# UDP blocked / curl -fsSL https://pilotprotocol.network/install.sh | sh +# HTTPS proxy: (nothing extra: transport "auto" picks TLS/WSS over TCP 443 +# through $HTTPS_PROXY when UDP does not work; add +# `-s -- --transport compat` to skip the UDP probe) # Uninstall: curl -fsSL https://pilotprotocol.network/install.sh | sh -s uninstall # # Flags: @@ -21,11 +23,11 @@ set -e # never silently falls back to an unverified source build. # --yes / -y Skip the older-version confirmation prompt. # --no-warn Suppress the older-version warning entirely. -# --transport udp (default) or compat. compat writes "transport": -# "compat" and "proxy": "auto" into ~/.pilot/config.json: -# the daemon then talks TLS/WSS over TCP 443 only and, when -# $HTTPS_PROXY/$ALL_PROXY is set, goes through that proxy -# (CONNECT by hostname). For UDP-blocked hosts and agent +# --transport auto (default for new installs), udp or compat, saved +# as "transport" in ~/.pilot/config.json. auto: UDP when +# the beacon answers over UDP, else compat. compat: TLS/WSS +# over TCP 443 only, through $HTTPS_PROXY/$ALL_PROXY when set +# (CONNECT by hostname) — for UDP-blocked hosts and agent # sandboxes whose only way out is an HTTPS proxy. # # Legacy env vars (still honored, lower precedence than flags): @@ -36,6 +38,8 @@ set -e # If omitted headless, the daemon auto-synthesizes a # @nodes.pilotprotocol.network identity. # PILOT_TRANSPORT=compat Same as --transport compat. +# PILOT_ALLOW_ROOT=1 Install as root on a host with systemd/launchd +# (not needed in containers/VMs without systemd). # # Proxies: every download is a curl HTTPS request, so HTTPS_PROXY / https_proxy / # ALL_PROXY / NO_PROXY are honored (curl asks the proxy to CONNECT by hostname — @@ -176,7 +180,7 @@ while [ $# -gt 0 ]; do --no-warn) PILOT_NO_WARN=1; shift ;; -h|--help) - sed -n '4,29p' "$0" 2>/dev/null || echo "See https://pilotprotocol.network/install.sh" + sed -n '4,33p' "$0" 2>/dev/null || echo "See https://pilotprotocol.network/install.sh" exit 0 ;; --) shift @@ -207,32 +211,45 @@ fi # --transport beats the PILOT_TRANSPORT env var. Empty means "not requested on # this run": a re-run keeps whatever transport config.json already has. -TRANSPORT="${PILOT_REQUESTED_TRANSPORT:-${PILOT_TRANSPORT:-}}" -if [ -n "$TRANSPORT" ] && [ "$TRANSPORT" != "udp" ] && [ "$TRANSPORT" != "compat" ]; then - echo "Error: --transport must be 'udp' or 'compat' (got: $TRANSPORT)" >&2 - exit 2 -fi +TRANSPORT="$(printf '%s' "${PILOT_REQUESTED_TRANSPORT:-${PILOT_TRANSPORT:-}}" | tr '[:upper:]' '[:lower:]')" +case "$TRANSPORT" in + ""|udp|compat|auto) ;; + *) + echo "Error: --transport must be 'udp', 'compat' or 'auto' (got: $TRANSPORT)" >&2 + exit 2 ;; +esac # Restore positional args so the existing uninstall handler still uses $1. # shellcheck disable=SC2086 # intentional word-split on PILOT_POSITIONAL set -- $PILOT_POSITIONAL -# Refuse to run as root — daemon must run as the invoking user so identity.json -# and received files land under that user's home, not /root. +# Refuse to run as root on a regular host — the daemon must run as the +# invoking user so identity.json and received files land under that user's +# home, not /root. A Linux container or VM without systemd (CI runners, +# hosted agent sandboxes such as Meta Muse, where the agent IS root) has no +# other user to install for and no system service to protect, so root is +# allowed there. if [ "${1:-}" != "uninstall" ] && [ "$(id -u)" = "0" ] && [ -z "${PILOT_ALLOW_ROOT:-}" ]; then - echo "Error: refusing to install as root." - echo " Run as a regular user; the installer uses sudo only when needed." - echo " Set PILOT_ALLOW_ROOT=1 to override (not recommended)." - exit 1 + if [ "$(uname -s)" = "Linux" ] && [ ! -d /run/systemd/system ]; then + echo "Note: installing as root (no systemd: container/VM sandbox) into ${HOME}/.pilot" + else + echo "Error: refusing to install as root." + echo " Run as a regular user; the installer uses sudo only when needed." + echo " Set PILOT_ALLOW_ROOT=1 to override (not recommended)." + exit 1 + fi fi -# A re-run without --transport keeps the transport the existing config.json -# selects, so regenerated service units stay consistent with it. -EFFECTIVE_TRANSPORT="${TRANSPORT:-udp}" -if [ -z "$TRANSPORT" ] && [ -f "$PILOT_DIR/config.json" ] \ - && grep -q '"transport"[[:space:]]*:[[:space:]]*"compat"' "$PILOT_DIR/config.json" 2>/dev/null; then - EFFECTIVE_TRANSPORT="compat" +# The transport already saved in config.json, if any ("udp", "compat", +# "auto"). A re-run without --transport keeps it, so regenerated service +# units stay consistent with it. +CONFIG_TRANSPORT="" +if [ -f "$PILOT_DIR/config.json" ]; then + CONFIG_TRANSPORT=$(sed -n 's/.*"transport"[[:space:]]*:[[:space:]]*"\([A-Za-z]*\)".*/\1/p' "$PILOT_DIR/config.json" 2>/dev/null | head -n 1 | tr '[:upper:]' '[:lower:]') fi +# Without any choice, new installs get auto (settled below, once the +# installed daemon is known to support it). +EFFECTIVE_TRANSPORT="${TRANSPORT:-${CONFIG_TRANSPORT:-auto}}" # --- Egress proxy --- # @@ -464,12 +481,17 @@ echo " Pilot Protocol" echo " The network stack for AI agents." echo "" echo " Platform: ${OS}/${ARCH}" -if [ "$EFFECTIVE_TRANSPORT" = "compat" ]; then - echo " Transport: compat (TLS + WSS over TCP 443 only)" -else - echo " Registry: ${REGISTRY}" - echo " Beacon: ${BEACON}" -fi +case "$EFFECTIVE_TRANSPORT" in + compat) + echo " Transport: compat (TLS + WSS over TCP 443 only)" ;; + auto) + echo " Transport: auto (UDP when it works, else TLS + WSS over TCP 443)" + echo " Registry: ${REGISTRY}" + echo " Beacon: ${BEACON}" ;; + *) + echo " Registry: ${REGISTRY}" + echo " Beacon: ${BEACON}" ;; +esac if [ -n "$PILOT_PROXY_URL" ]; then echo " Proxy: $(redact_proxy "$PILOT_PROXY_URL") (from environment)" fi @@ -976,6 +998,7 @@ fi # further below. Guarding on the file itself makes the documented opt-outs # reachable at install time instead of only after the fact. Defaults are # unchanged — a host with no config still gets the standard one. +CONFIG_WRITTEN=false if [ "$UPDATING" != true ] && [ ! -f "$PILOT_DIR/config.json" ]; then cat > "$PILOT_DIR/config.json" </dev/null 2>&1 } -if [ -n "$TRANSPORT" ]; then - if pilot_config_set "transport=$TRANSPORT"; then - echo "Transport set to ${TRANSPORT} in ${PILOT_DIR}/config.json" - else - echo " Note: could not save transport=${TRANSPORT} — run: pilotctl config --set transport=${TRANSPORT}" - fi -fi - -# What the installed binaries support. Both probes are local (no network). +# What the installed binaries support. The probes are local (no network). DAEMON_HAS_TRANSPORT=false DAEMON_HAS_PROXY=false +DAEMON_HAS_AUTO=false _daemon_help=$("$BIN_DIR/pilot-daemon" -help 2>&1 || true) if printf '%s\n' "$_daemon_help" | grep -qE '^[[:space:]]+-transport([[:space:]]|$)'; then DAEMON_HAS_TRANSPORT=true + # -transport=auto: its usage line names 'auto'. + if printf '%s\n' "$_daemon_help" | sed -n '/^[[:space:]]*-transport/,/^[[:space:]]*-[a-z]/p' | grep -q "'auto'"; then + DAEMON_HAS_AUTO=true + fi fi if printf '%s\n' "$_daemon_help" | grep -qE '^[[:space:]]+-proxy([[:space:]]|$)'; then DAEMON_HAS_PROXY=true fi -if [ "$EFFECTIVE_TRANSPORT" = "compat" ]; then - # proxy=auto: in compat mode the daemon routes every outbound connection - # through $HTTPS_PROXY / $ALL_PROXY when set (honoring $NO_PROXY), and - # connects directly when not. An operator-set proxy value is kept. - if ! grep -q '"proxy"' "$PILOT_DIR/config.json" 2>/dev/null; then - pilot_config_set "proxy=auto" || true +# auto needs a daemon that knows it: an older one would fail on +# "transport":"auto" in config.json. Fall back to its default (udp). +if [ "$EFFECTIVE_TRANSPORT" = "auto" ] && [ "$DAEMON_HAS_AUTO" != true ]; then + if [ "$TRANSPORT" = "auto" ]; then + echo " Note: this pilot-daemon (${TAG:-source}) predates -transport=auto; keeping its default (udp)." fi + TRANSPORT_TO_SAVE="" + EFFECTIVE_TRANSPORT="udp" +elif [ -n "$TRANSPORT" ]; then + TRANSPORT_TO_SAVE="$TRANSPORT" +elif [ -z "$CONFIG_TRANSPORT" ] && [ "$CONFIG_WRITTEN" = true ]; then + # Fresh install with nothing chosen: save auto, so the service units + # and a directly started pilot-daemon auto-detect too (pilotctl daemon + # start asks for auto whenever nothing is configured). Existing installs + # keep whatever they run today. + TRANSPORT_TO_SAVE="auto" +else + TRANSPORT_TO_SAVE="" +fi +if [ -n "$TRANSPORT_TO_SAVE" ]; then + if pilot_config_set "transport=$TRANSPORT_TO_SAVE"; then + echo "Transport set to ${TRANSPORT_TO_SAVE} in ${PILOT_DIR}/config.json" + else + echo " Note: could not save transport=${TRANSPORT_TO_SAVE} — run: pilotctl config --set transport=${TRANSPORT_TO_SAVE}" + fi +fi + +if [ "$EFFECTIVE_TRANSPORT" = "compat" ]; then + # No "proxy" key is written: the daemon's default, auto, already uses + # $HTTPS_PROXY / $ALL_PROXY in compat mode, and a saved "auto" would only + # get in the way of a proxy passed later with --proxy or $PILOT_PROXY. if [ "$DAEMON_HAS_TRANSPORT" != true ]; then echo "" echo " WARNING: this pilot-daemon (${TAG:-source}) predates compat mode (-transport)." @@ -1042,26 +1087,35 @@ if [ "$EFFECTIVE_TRANSPORT" = "compat" ]; then if pilot_config_set "registry=${COMPAT_REGISTRY}"; then echo " Registry set to ${COMPAT_REGISTRY} for compat mode (this pilotctl" echo " always passes config.json's registry to the daemon). Switching back to" - echo " UDP later: pilotctl config --set registry=${DEFAULT_REGISTRY}" + echo " UDP later: re-run this installer with --transport udp" fi fi - - if [ -n "$PILOT_PROXY_URL" ] && [ "$DAEMON_HAS_PROXY" != true ]; then - echo "" - echo " WARNING: HTTPS_PROXY is set, but this pilot-daemon (${TAG:-source}) cannot use a" - echo " proxy. Where the proxy is the only way out, the daemon will not come" - echo " online. Install a release whose 'pilot-daemon -help' lists -proxy." +elif grep -q "\"registry\"[[:space:]]*:[[:space:]]*\"${COMPAT_REGISTRY}\"" "$PILOT_DIR/config.json" 2>/dev/null; then + # Leaving compat after an install that pointed the registry at the + # compat TLS host: a udp daemon needs the raw-TCP registry back. + if pilot_config_set "registry=${DEFAULT_REGISTRY}"; then + echo " Registry restored to ${DEFAULT_REGISTRY} for transport ${EFFECTIVE_TRANSPORT}" fi fi -# Network flags for the service units. In compat mode the raw-TCP default -# registry/beacon are left off (an explicit -registry pins a compat daemon to -# a port no 443-only network or HTTPS proxy will carry); a custom -# PILOT_REGISTRY / PILOT_BEACON is kept. UDP units are unchanged. +if [ "$EFFECTIVE_TRANSPORT" != "udp" ] && [ -n "$PILOT_PROXY_URL" ] && [ "$DAEMON_HAS_PROXY" != true ]; then + echo "" + echo " WARNING: HTTPS_PROXY is set, but this pilot-daemon (${TAG:-source}) cannot use a" + echo " proxy. Where the proxy is the only way out, the daemon will not come" + echo " online. Install a release whose 'pilot-daemon -help' lists -proxy." +fi + +# Network flags for the service units. The transport itself comes from +# config.json, which the daemon reads, so `pilotctl config --set transport=` +# applies to the service too. In compat mode the raw-TCP default +# registry/beacon are left off (an older daemon given -registry explicitly +# stays pinned to a port no 443-only network or HTTPS proxy will carry); a +# custom PILOT_REGISTRY / PILOT_BEACON is kept. if [ "$EFFECTIVE_TRANSPORT" = "compat" ] && [ "$DAEMON_HAS_TRANSPORT" = true ]; then - NET_FLAGS="-transport compat" + NET_FLAGS="" if [ "$REGISTRY" != "$DEFAULT_REGISTRY" ]; then NET_FLAGS="$NET_FLAGS -registry $REGISTRY"; fi if [ "$BEACON" != "$DEFAULT_BEACON" ]; then NET_FLAGS="$NET_FLAGS -beacon $BEACON"; fi + NET_FLAGS="${NET_FLAGS# }" else NET_FLAGS="-registry $REGISTRY -beacon $BEACON" fi @@ -1070,7 +1124,7 @@ fi # environment, not this shell's, so an HTTPS_PROXY exported here never # reaches it. config.json (0600, read by the daemon itself) does. service_proxy_note() { - if [ "$EFFECTIVE_TRANSPORT" = "compat" ] && [ -n "$PILOT_PROXY_URL" ] \ + if [ "$EFFECTIVE_TRANSPORT" != "udp" ] && [ -n "$PILOT_PROXY_URL" ] \ && ! grep -q '"proxy"[[:space:]]*:[[:space:]]*"http' "$PILOT_DIR/config.json" 2>/dev/null; then echo " Note: $1 does not inherit this shell's HTTPS_PROXY. For the service to use" echo " the proxy, save it in config.json (0600):" @@ -1234,9 +1288,9 @@ elif [ "$OS" = "linux" ]; then # the portable start path instead of silently leaving it with no daemon. echo "No systemd detected (container / WSL / CI / sandbox) — start the daemon manually:" echo " pilotctl daemon start" - if [ "$EFFECTIVE_TRANSPORT" = "compat" ]; then - echo " (config.json selects transport=compat; start it from a shell that has" - echo " HTTPS_PROXY set if this host reaches the internet only through a proxy)" + if [ "$EFFECTIVE_TRANSPORT" != "udp" ]; then + echo " (transport=${EFFECTIVE_TRANSPORT}; start it from a shell that has HTTPS_PROXY" + echo " set if this host reaches the internet only through a proxy)" fi fi @@ -1496,14 +1550,19 @@ echo " pilotctl ${BIN_DIR}/pilotctl" [ -f "$BIN_DIR/pilot-updater" ] && echo " pilot-updater ${BIN_DIR}/pilot-updater (auto-updates in background)" echo "" echo "Config: ${PILOT_DIR}/config.json" -if [ "$EFFECTIVE_TRANSPORT" = "compat" ]; then - echo " Transport: compat (registry ${COMPAT_REGISTRY} over TLS, beacon over WSS)" - if [ -n "$PILOT_PROXY_URL" ]; then - echo " Proxy: auto -> $(redact_proxy "$PILOT_PROXY_URL") (from environment)" - fi -else - echo " Registry: ${REGISTRY}" - echo " Beacon: ${BEACON}" +case "$EFFECTIVE_TRANSPORT" in + compat) + echo " Transport: compat (registry ${COMPAT_REGISTRY} over TLS, beacon over WSS)" ;; + auto) + echo " Transport: auto (UDP when it works, else compat over TCP 443)" + echo " Registry: ${REGISTRY}" + echo " Beacon: ${BEACON}" ;; + *) + echo " Registry: ${REGISTRY}" + echo " Beacon: ${BEACON}" ;; +esac +if [ "$EFFECTIVE_TRANSPORT" != "udp" ] && [ -n "$PILOT_PROXY_URL" ]; then + echo " Proxy: auto -> $(redact_proxy "$PILOT_PROXY_URL") (from environment)" fi echo " Socket: /tmp/pilot.sock" echo " Identity: ${PILOT_DIR}/identity.json" diff --git a/internal/proxyconf/proxyconf.go b/internal/proxyconf/proxyconf.go new file mode 100644 index 00000000..1dd45965 --- /dev/null +++ b/internal/proxyconf/proxyconf.go @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package proxyconf holds the -proxy rules that pilot-daemon and pilotctl +// share on top of github.com/pilot-protocol/common/netproxy: which +// spellings a proxy setting accepts, how it is shown without credentials, +// and the loopback exemption every outbound path applies. +// +// A proxy setting is one of: +// +// - "auto" (or empty): the proxy from the environment (HTTPS_PROXY, +// https_proxy, ALL_PROXY, all_proxy, honouring NO_PROXY). Whether auto +// applies at all for a given transport is the caller's policy. +// - "off", or one of its aliases "none", "no", "false", "direct": never +// use a proxy. +// - an explicit "http://[user:pass@]host[:port]" or "https://..." URL. +// +// Anything else — a bare word such as "proxy", a host:port without a +// scheme, or another scheme — is rejected, so a typo can never turn into a +// proxy host name. Errors and display strings never contain credentials. +package proxyconf + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net" + "net/http" + "net/url" + "strings" + + "github.com/pilot-protocol/common/netproxy" +) + +// The normalized setting values. +const ( + Auto = netproxy.ModeAuto + Off = netproxy.ModeOff +) + +// offAliases are accepted, case-insensitively, as "off". "none" is what the +// daemon's startup log prints when nothing is proxied, so operators copy it. +var offAliases = map[string]bool{ + "off": true, "none": true, "no": true, "false": true, "direct": true, +} + +// Normalize maps a proxy setting to Auto, Off, or the trimmed explicit +// proxy URL, which it validates. The error never includes credentials. +func Normalize(spec string) (string, error) { + s := strings.TrimSpace(spec) + lower := strings.ToLower(s) + switch { + case lower == "" || lower == Auto: + return Auto, nil + case offAliases[lower]: + return Off, nil + } + scheme, _, ok := strings.Cut(s, "://") + if !ok { + return "", fmt.Errorf("invalid proxy %q: use auto, off, or an http:// or https:// proxy URL", Redact(s)) + } + switch strings.ToLower(scheme) { + case "http", "https": + default: + return "", fmt.Errorf("invalid proxy %q: the scheme must be http or https", Redact(s)) + } + if _, err := netproxy.Explicit(s); err != nil { + return "", fmt.Errorf("invalid proxy %q: %v", Redact(s), unwrapNetproxy(err)) + } + return s, nil +} + +// Resolve builds the resolver for a proxy setting (see Normalize): Auto +// reads the environment now, Off never proxies, a URL proxies everything. +// It applies no transport policy. +func Resolve(spec string) (*netproxy.Resolver, error) { + s, err := Normalize(spec) + if err != nil { + return nil, err + } + switch s { + case Auto: + return netproxy.FromEnvironment() + case Off: + return netproxy.Off(), nil + } + return netproxy.Explicit(s) +} + +// HasCredentials reports whether an explicit proxy setting carries userinfo. +// netproxy takes everything before the last '@' as credentials, so any '@' +// counts — including in a value url.Parse would misread (an unescaped '/', +// '?' or '#' in the password). +func HasCredentials(spec string) bool { + return strings.Contains(spec, "@") +} + +// Redact renders a proxy setting for display: "auto", "off" and URLs +// without credentials unchanged (URLs reduced to scheme://host:port), +// userinfo replaced by "***". Values that do not parse keep only what +// follows the last '@'. +func Redact(spec string) string { + s := strings.TrimSpace(spec) + lower := strings.ToLower(s) + if lower == "" || lower == Auto || offAliases[lower] { + return s + } + if strings.Contains(s, "://") { + if r, err := netproxy.Explicit(s); err == nil { + return r.String() + } + } + if i := strings.LastIndex(s, "@"); i >= 0 { + prefix := "" + if j := strings.Index(s, "://"); j >= 0 && j < i { + prefix = s[:j+3] + } + return prefix + "***@" + s[i+1:] + } + return s +} + +// IsLoopbackHost reports whether host (a name or IP literal, optionally in +// brackets) is this machine: localhost, *.localhost or a loopback address. +// Such targets are never sent to a proxy, even an explicit one — the proxy +// would reach its own loopback, and the request would leave the host. +func IsLoopbackHost(host string) bool { + h := strings.TrimSuffix(strings.ToLower(strings.Trim(host, "[]")), ".") + if h == "localhost" || strings.HasSuffix(h, ".localhost") { + return true + } + if i := strings.IndexByte(h, '%'); i >= 0 { + h = h[:i] + } + ip := net.ParseIP(h) + return ip != nil && ip.IsLoopback() +} + +// RequestProxy returns an http.Transport.Proxy function that follows r but +// never proxies loopback targets. nil for a nil resolver (net/http's own +// environment handling then applies wherever the caller leaves Proxy +// unset). +func RequestProxy(r *netproxy.Resolver) func(*http.Request) (*url.URL, error) { + if r == nil { + return nil + } + return func(req *http.Request) (*url.URL, error) { + if req != nil && req.URL != nil && IsLoopbackHost(req.URL.Hostname()) { + return nil, nil + } + return r.ProxyForRequest(req) + } +} + +// DialContext returns a dial function that tunnels through the proxy r +// picks for each target (CONNECT by host name, never resolved locally) and +// dials loopback targets, and targets r does not proxy, directly. +// proxyTLS configures the TLS session with an https:// proxy — never the +// target's TLS, which the caller runs end to end over the returned conn; +// nil verifies the proxy against the system roots. +func DialContext(r *netproxy.Resolver, proxyTLS *tls.Config) func(ctx context.Context, network, addr string) (net.Conn, error) { + d := &netproxy.Dialer{Resolver: r, TLSConfig: proxyTLS} + var direct net.Dialer + return func(ctx context.Context, network, addr string) (net.Conn, error) { + if host, _, err := net.SplitHostPort(addr); err == nil && IsLoopbackHost(host) { + return direct.DialContext(ctx, network, addr) + } + return d.DialContext(ctx, network, addr) + } +} + +// unwrapNetproxy drops netproxy's "netproxy: " prefix for messages that +// already say which setting failed. +func unwrapNetproxy(err error) string { + msg := err.Error() + var ee *netproxy.EnvError + if errors.As(err, &ee) { + msg = ee.Err.Error() + } + return strings.TrimPrefix(msg, "netproxy: ") +} diff --git a/internal/proxyconf/proxyconf_test.go b/internal/proxyconf/proxyconf_test.go new file mode 100644 index 00000000..f1139bd4 --- /dev/null +++ b/internal/proxyconf/proxyconf_test.go @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package proxyconf + +import ( + "bufio" + "context" + "fmt" + "io" + "net" + "net/http" + "strings" + "sync" + "testing" + "time" + + "github.com/pilot-protocol/common/netproxy" +) + +func TestNormalize(t *testing.T) { + for _, tc := range []struct{ in, want string }{ + {"", Auto}, + {" auto ", Auto}, + {"AUTO", Auto}, + {"off", Off}, + {"OFF", Off}, + {"none", Off}, + {" None ", Off}, + {"no", Off}, + {"false", Off}, + {"direct", Off}, + {"http://proxy.test:3128", "http://proxy.test:3128"}, + {"https://proxy.test", "https://proxy.test"}, + {" http://u:p@proxy.test:3128 ", "http://u:p@proxy.test:3128"}, + // netproxy splits the userinfo at the last '@', so an unescaped + // '/', '#' or '?' in the password is fine. + {"http://agent:12#34/x@egress.test:3128", "http://agent:12#34/x@egress.test:3128"}, + } { + got, err := Normalize(tc.in) + if err != nil || got != tc.want { + t.Errorf("Normalize(%q) = (%q, %v), want %q", tc.in, got, err, tc.want) + } + } +} + +func TestNormalizeRejectsWordsAndOtherSchemes(t *testing.T) { + for _, in := range []string{ + "proxy", + "yes", + "true", + "proxy.test:3128", // no scheme: never guess + "muse:s3cret@proxy.test:3128", // no scheme, with credentials + "socks5://muse:s3cret@proxy:1080", // unsupported scheme + "ftp://proxy.test", + "http://", + "http://muse:s3cret@", + } { + _, err := Normalize(in) + if err == nil { + t.Errorf("Normalize(%q) accepted", in) + continue + } + if strings.Contains(err.Error(), "s3cret") { + t.Errorf("Normalize(%q) error leaks the password: %v", in, err) + } + } +} + +func TestResolve(t *testing.T) { + for _, k := range []string{"HTTPS_PROXY", "https_proxy", "ALL_PROXY", "all_proxy", "HTTP_PROXY", "http_proxy", "NO_PROXY", "no_proxy"} { + t.Setenv(k, "") + } + t.Setenv("HTTPS_PROXY", "http://env.test:3128") + r, err := Resolve("auto") + if err != nil || r.Mode() != netproxy.ModeAuto || !r.Enabled() { + t.Fatalf("Resolve(auto) = (%v, %v)", r, err) + } + for _, off := range []string{"off", "none", "direct"} { + r, err = Resolve(off) + if err != nil || r.Mode() != netproxy.ModeOff || r.Enabled() { + t.Fatalf("Resolve(%s) = (%v, %v), want off", off, r, err) + } + } + r, err = Resolve("http://flag.test:8080") + if err != nil || r.Mode() != netproxy.ModeExplicit { + t.Fatalf("Resolve(url) = (%v, %v)", r, err) + } + if _, err := Resolve("none.example"); err == nil { + t.Fatal("Resolve accepted a bare host name") + } +} + +func TestHasCredentialsAndRedact(t *testing.T) { + for _, tc := range []struct { + in string + creds bool + shown string + }{ + {"auto", false, "auto"}, + {"off", false, "off"}, + {"http://proxy.test:3128", false, "http://proxy.test:3128"}, + {"http://muse:s3cret@proxy.test:3128", true, "http://***@proxy.test:3128"}, + {"http://agent:1234#Xyz9@egress.test:3128", true, "http://***@egress.test:3128"}, + {"http://agent:12/34?x@egress.test:3128", true, "http://***@egress.test:3128"}, + {"muse:s3cret@proxy.test:3128", true, "***@proxy.test:3128"}, + {"socks5://muse:s3cret@proxy:1080", true, "socks5://***@proxy:1080"}, + } { + if got := HasCredentials(tc.in); got != tc.creds { + t.Errorf("HasCredentials(%q) = %v, want %v", tc.in, got, tc.creds) + } + if got := Redact(tc.in); got != tc.shown { + t.Errorf("Redact(%q) = %q, want %q", tc.in, got, tc.shown) + } + } +} + +func TestIsLoopbackHost(t *testing.T) { + for host, want := range map[string]bool{ + "localhost": true, + "LOCALHOST.": true, + "api.localhost": true, + "127.0.0.1": true, + "127.3.2.1": true, + "::1": true, + "[::1]": true, + "example.com": false, + "10.0.0.1": false, + "registry.pilot.invalid": false, + "": false, + "localhost.example.com": false, + } { + if got := IsLoopbackHost(host); got != want { + t.Errorf("IsLoopbackHost(%q) = %v, want %v", host, got, want) + } + } +} + +// An explicit proxy normally takes every target; loopback is the exception +// on both the HTTP and the raw-dial path. +func TestLoopbackNeverProxied(t *testing.T) { + r, err := netproxy.Explicit("http://proxy.test:3128") + if err != nil { + t.Fatal(err) + } + pf := RequestProxy(r) + for _, target := range []string{"http://127.0.0.1:8080/hook", "http://localhost:5002/analyze", "https://[::1]:9443/"} { + req, _ := http.NewRequest(http.MethodGet, target, nil) + if u, err := pf(req); err != nil || u != nil { + t.Errorf("proxy for %s = (%v, %v), want direct", target, u, err) + } + } + req, _ := http.NewRequest(http.MethodGet, "https://raw.githubusercontent.com/x", nil) + if u, err := pf(req); err != nil || u == nil || u.Host != "proxy.test:3128" { + t.Errorf("proxy for a remote target = (%v, %v), want proxy.test:3128", u, err) + } + if RequestProxy(nil) != nil { + t.Error("RequestProxy(nil) is not nil") + } +} + +// DialContext sends a loopback target straight to it, and a remote target +// through the proxy with CONNECT by name. +func TestDialContextLoopbackDirectRemoteViaProxy(t *testing.T) { + echo, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer echo.Close() + go func() { + for { + c, err := echo.Accept() + if err != nil { + return + } + go func() { defer c.Close(); io.Copy(c, c) }() + } + }() + + var mu sync.Mutex + var connects []string + proxyLn, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer proxyLn.Close() + go func() { + for { + c, err := proxyLn.Accept() + if err != nil { + return + } + go func() { + defer c.Close() + br := bufio.NewReader(c) + req, err := http.ReadRequest(br) + if err != nil { + return + } + mu.Lock() + connects = append(connects, req.Method+" "+req.RequestURI) + mu.Unlock() + up, err := net.Dial("tcp", echo.Addr().String()) + if err != nil { + return + } + defer up.Close() + fmt.Fprint(c, "HTTP/1.1 200 OK\r\n\r\n") + go io.Copy(up, br) + io.Copy(c, up) + }() + } + }() + + r, err := netproxy.Explicit("http://" + proxyLn.Addr().String()) + if err != nil { + t.Fatal(err) + } + dial := DialContext(r, nil) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + roundTrip := func(addr string) { + t.Helper() + c, err := dial(ctx, "tcp", addr) + if err != nil { + t.Fatalf("dial %s: %v", addr, err) + } + defer c.Close() + c.SetDeadline(time.Now().Add(5 * time.Second)) + if _, err := c.Write([]byte("ping")); err != nil { + t.Fatal(err) + } + buf := make([]byte, 4) + if _, err := io.ReadFull(c, buf); err != nil || string(buf) != "ping" { + t.Fatalf("echo via %s = (%q, %v)", addr, buf, err) + } + } + + roundTrip(echo.Addr().String()) + mu.Lock() + if len(connects) != 0 { + t.Fatalf("loopback dial went to the proxy: %q", connects) + } + mu.Unlock() + + roundTrip("registry.pilot.invalid:443") + mu.Lock() + defer mu.Unlock() + if len(connects) != 1 || connects[0] != "CONNECT registry.pilot.invalid:443" { + t.Fatalf("proxy requests = %q, want one CONNECT registry.pilot.invalid:443", connects) + } +} diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index 4b1634be..e6b65a6f 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -843,39 +843,70 @@ func (d *Daemon) Start() error { _ = synthesised // reserved for future log/metric tagging // 0b. Auto-detect transport mode. PILOT_TRANSPORT env var lets the - // operator force a mode at install time. When nothing is set, probe - // UDP reachability to the beacon; on UDP-blocked hosts the daemon - // auto-falls back to compat (WSS/443) so it can reach peers without - // a manual restart. + // operator force a mode at install time. When nothing is set (or + // TransportMode is "auto"), probe UDP reachability to the beacon; on + // UDP-blocked hosts that can reach the compat beacon over TCP the + // daemon auto-falls back to compat (WSS/443) so it can reach peers + // without a manual restart. cmd/daemon resolves -transport=auto itself + // (it also switches the registry for compat); this path serves + // embedders that leave TransportMode empty. if envTransport := os.Getenv("PILOT_TRANSPORT"); envTransport != "" && d.config.TransportMode == "" { switch envTransport { - case "udp", "compat": + case TransportUDP, TransportCompat: d.config.TransportMode = envTransport slog.Info("transport set from PILOT_TRANSPORT env", "mode", envTransport) + case TransportAuto: default: - slog.Warn("ignoring unknown PILOT_TRANSPORT value", "value", envTransport, "valid", "udp, compat") + slog.Warn("ignoring unknown PILOT_TRANSPORT value", "value", envTransport, "valid", "udp, compat, auto") } } + if d.config.TransportMode == TransportAuto { + d.config.TransportMode = "" + } if d.config.TransportMode == "" { stunBeacon := firstBeacon(d.config.BeaconAddr) switch { case stunBeacon == "": // No beacon to probe — leave transport on the UDP default. - case probeUDPReachable(stunBeacon): - // Positive evidence UDP works end-to-end; stay on UDP. case d.config.CompatBeaconURL == "": - // UDP looks blocked but we have no compat beacon to fall back - // to. Switching to compat would strand the daemon, so stay on - // UDP and warn the operator to configure compat explicitly. - slog.Warn("UDP probe to beacon failed but no compat beacon configured — staying on UDP", - "beacon", stunBeacon, - "hint", "set -compat-beacon and PILOT_TRANSPORT=compat to use WSS/443") + if !probeUDPReachable(stunBeacon) { + // UDP looks blocked but we have no compat beacon to fall + // back to. Switching to compat would strand the daemon, so + // stay on UDP and warn the operator to configure compat. + slog.Warn("UDP probe to beacon failed but no compat beacon configured — staying on UDP", + "beacon", stunBeacon, + "hint", "set -compat-beacon and PILOT_TRANSPORT=compat to use WSS/443") + } default: - d.config.TransportMode = "compat" - slog.Warn("UDP probe to beacon failed — auto-falling back to compat mode (WSS/443)", - "beacon", stunBeacon, - "compat_beacon", d.config.CompatBeaconURL, - "hint", "set PILOT_TRANSPORT=udp to force UDP") + // Compat would use the environment's proxy (-proxy=auto) + // unless the embedder set a policy; check reachability the + // same way. + policy := d.config.Proxy + if policy == nil { + if p, err := ResolveProxy(proxyAutoSpec, TransportCompat); err == nil { + policy = p + } else { + slog.Warn("proxy environment unusable; compat check dials directly", "error", err) + } + } + mode, reason := SelectTransport(context.Background(), AutoTransportProbe{ + BeaconAddr: stunBeacon, + CompatBeaconURL: d.config.CompatBeaconURL, + Dial: d.dialerFor(policy), + }) + if mode == TransportCompat { + d.config.TransportMode = TransportCompat + if d.config.Proxy == nil { + d.config.Proxy = policy + } + slog.Warn("UDP probe to beacon failed — auto-falling back to compat mode (WSS/443)", + "beacon", stunBeacon, + "compat_beacon", d.config.CompatBeaconURL, + "reason", reason, + "hint", "set PILOT_TRANSPORT=udp to force UDP") + } else { + slog.Info("transport auto-selected", "transport", TransportUDP, "reason", reason) + } } } @@ -1078,12 +1109,15 @@ func (d *Daemon) Start() error { ccCtx, ccCancel := context.WithTimeout(context.Background(), 30*time.Second) defer ccCancel() if cerr := d.tunnels.ConnectCompat(ccCtx, ConnectCompatConfig{ - BeaconURL: d.config.CompatBeaconURL, - TLSConfig: tlsCfg, - Proxy: d.httpProxyFunc(), - Identity: d.identity, - NodeID: d.nodeID, + BeaconURL: d.config.CompatBeaconURL, + TLSConfig: tlsCfg, + DialContext: d.proxyDialer(), + Identity: d.identity, + NodeID: d.nodeID, }); cerr != nil { + if hint := tlsTrustHint(cerr, "beacon"); hint != "" && d.config.CompatTLSTrust == "system" { + return fmt.Errorf("compat connect: %w; %s", cerr, hint) + } return fmt.Errorf("compat connect: %w", cerr) } slog.Info("compat mode tunnel up", @@ -2273,6 +2307,9 @@ func (d *Daemon) dialRegistryClient() (*registry.Client, error) { break } if attempt == maxRegistryDialAttempts { + if hint := tlsTrustHint(err, "registry"); hint != "" { + return nil, fmt.Errorf("registry dial (after %d attempts): %w; %s", attempt, err, hint) + } return nil, fmt.Errorf("registry dial (after %d attempts): %w", attempt, err) } slog.Warn("registry dial failed, retrying", diff --git a/pkg/daemon/proxy.go b/pkg/daemon/proxy.go index 818170a6..939d7699 100644 --- a/pkg/daemon/proxy.go +++ b/pkg/daemon/proxy.go @@ -3,13 +3,16 @@ package daemon import ( + "context" + "crypto/tls" + "net" "net/http" "net/url" - "strings" "time" "github.com/pilot-protocol/common/netproxy" registry "github.com/pilot-protocol/common/registry/client" + "github.com/pilot-protocol/pilotprotocol/internal/proxyconf" ) // ResolveProxy turns a -proxy setting into the daemon's outbound proxy @@ -19,37 +22,69 @@ import ( // environment — HTTPS_PROXY / https_proxy, falling back to ALL_PROXY / // all_proxy, with NO_PROXY / no_proxy honoured. With any other transport // it returns nil: no policy, the daemon dials exactly as it always has. -// - "off": a policy that never proxies. Unlike nil, it also stops the -// daemon's HTTP fetches from following proxy environment variables. +// - "off" (or "none", "no", "false", "direct"): a policy that never +// proxies. Unlike nil, it also stops the daemon's HTTP fetches from +// following proxy environment variables. // - "http://[user:pass@]host:port" or "https://...": that proxy for every // outbound TCP/HTTP connection, whatever the transport. // -// Errors never echo proxy credentials. +// Any other value (a bare word, a host:port without a scheme, another +// scheme) is an error. Loopback targets are never proxied, whatever the +// policy. Errors never echo proxy credentials. func ResolveProxy(spec, transportMode string) (*netproxy.Resolver, error) { - s := strings.TrimSpace(spec) - if (s == "" || strings.EqualFold(s, netproxy.ModeAuto)) && transportMode != "compat" { + s, err := proxyconf.Normalize(spec) + if err != nil { + return nil, err + } + if s == proxyconf.Auto && transportMode != "compat" { return nil, nil } - return netproxy.Parse(s) + return proxyconf.Resolve(s) +} + +// proxyAutoSpec is the -proxy default: the environment's proxy, in compat +// mode only. +const proxyAutoSpec = proxyconf.Auto + +// proxyTLSConfig is the TLS configuration for the session with an https:// +// proxy itself. The proxy is verified against the system roots (or the +// test seam), never against the pinned roots the beacon or registry trust +// settings select: those pin Pilot's servers, not the operator's proxy. +func (d *Daemon) proxyTLSConfig() *tls.Config { + return &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: d.config.systemRoots} +} + +// proxyDialer returns the dial function for raw TCP connections the daemon +// opens itself (the registry, the compat WSS beacon), or nil when there is +// no policy or it proxies nothing. Loopback targets are dialed directly. +func (d *Daemon) proxyDialer() func(ctx context.Context, network, addr string) (net.Conn, error) { + return d.dialerFor(d.config.Proxy) +} + +// dialerFor is proxyDialer for an arbitrary policy. +func (d *Daemon) dialerFor(policy *netproxy.Resolver) func(ctx context.Context, network, addr string) (net.Conn, error) { + if !policy.Enabled() { + return nil + } + return proxyconf.DialContext(policy, d.proxyTLSConfig()) } // registryDialOptions routes every registry connection — the primary, each // pool member and every reconnect — through the proxy policy. With no policy, // or one that proxies nothing, the client keeps its direct dial. func (d *Daemon) registryDialOptions() []registry.DialOption { - if !d.config.Proxy.Enabled() { + dial := d.proxyDialer() + if dial == nil { return nil } - return []registry.DialOption{registry.WithDialer(netproxy.NewDialer(d.config.Proxy).DialContext)} + return []registry.DialOption{registry.WithDialer(dial)} } -// httpProxyFunc returns the http.Transport.Proxy function for connections -// the daemon makes over HTTP(S) or WSS, or nil when there is no policy. +// httpProxyFunc returns the http.Transport.Proxy function for HTTP fetches +// the daemon makes itself, or nil when there is no policy. Loopback targets +// always go direct. func (d *Daemon) httpProxyFunc() func(*http.Request) (*url.URL, error) { - if d.config.Proxy == nil { - return nil - } - return d.config.Proxy.ProxyForRequest + return proxyconf.RequestProxy(d.config.Proxy) } // newHTTPClient returns a client for daemon-owned HTTP fetches. Without a diff --git a/pkg/daemon/transport/wss/wss.go b/pkg/daemon/transport/wss/wss.go index f2fc3962..6be1e4d9 100644 --- a/pkg/daemon/transport/wss/wss.go +++ b/pkg/daemon/transport/wss/wss.go @@ -39,7 +39,6 @@ import ( "log/slog" "net" "net/http" - "net/url" "sync" "sync/atomic" "time" @@ -110,12 +109,14 @@ type Config struct { // store. Always non-nil — the caller picks the policy. TLSConfig *tls.Config - // Proxy picks the HTTP proxy for the WSS dial, with the signature of - // http.Transport.Proxy (e.g. netproxy.Resolver.ProxyForRequest). A - // wss:// URL is tunnelled with CONNECT by host name, so the beacon - // name is never resolved locally and TLS (TLSConfig, SNI) stays - // end-to-end with the beacon. nil dials directly. - Proxy func(*http.Request) (*url.URL, error) + // DialContext opens the TCP connection to the beacon, e.g. a + // netproxy.Dialer that tunnels through an HTTP CONNECT proxy by host + // name, so the beacon name is never resolved locally. TLS (TLSConfig, + // SNI) then runs end to end with the beacon over that connection; + // TLSConfig never applies to the proxy itself — an https:// proxy's + // TLS is the dialer's business, so a pinned beacon trust store cannot + // reject the operator's proxy certificate. nil dials directly. + DialContext func(ctx context.Context, network, addr string) (net.Conn, error) // Identity provides the Ed25519 keypair used for the auth challenge. Identity *crypto.Identity @@ -264,7 +265,7 @@ func Dial(ctx context.Context, cfg Config) (*Transport, error) { func (t *Transport) dialAndAuth(ctx context.Context) (*websocket.Conn, error) { httpClient := &http.Client{ Transport: &http.Transport{ - Proxy: t.cfg.Proxy, + DialContext: t.cfg.DialContext, TLSClientConfig: t.cfg.TLSConfig.Clone(), }, } diff --git a/pkg/daemon/transport/wss/zz_wss_proxy_test.go b/pkg/daemon/transport/wss/zz_wss_proxy_test.go index d4b33123..6ed78173 100644 --- a/pkg/daemon/transport/wss/zz_wss_proxy_test.go +++ b/pkg/daemon/transport/wss/zz_wss_proxy_test.go @@ -256,7 +256,7 @@ func TestDial_ThroughAuthenticatingConnectProxy(t *testing.T) { tr, err := wss.Dial(context.Background(), wss.Config{ URL: proxiedURL(fb), TLSConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, - Proxy: res.ProxyForRequest, + DialContext: netproxy.NewDialer(res).DialContext, Identity: mustID(t), NodeID: nodeID, DialTimeout: 5 * time.Second, @@ -298,7 +298,7 @@ func TestReconnect_ThroughConnectProxy(t *testing.T) { tr, err := wss.Dial(context.Background(), wss.Config{ URL: proxiedURL(fb), TLSConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, - Proxy: res.ProxyForRequest, + DialContext: netproxy.NewDialer(res).DialContext, Identity: mustID(t), NodeID: nodeID, DialTimeout: 5 * time.Second, @@ -341,7 +341,7 @@ func TestDial_ProxyFromEnvironmentHonoursNoProxy(t *testing.T) { tr, err := wss.Dial(context.Background(), wss.Config{ URL: proxiedURL(fb), TLSConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, - Proxy: res.ProxyForRequest, + DialContext: netproxy.NewDialer(res).DialContext, Identity: mustID(t), NodeID: nodeID, DialTimeout: 5 * time.Second, @@ -381,7 +381,7 @@ func TestDial_ProxyAuthFailureDoesNotLeakCredentials(t *testing.T) { _, err = wss.Dial(context.Background(), wss.Config{ URL: proxiedURL(fb), TLSConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, - Proxy: res.ProxyForRequest, + DialContext: netproxy.NewDialer(res).DialContext, Identity: mustID(t), NodeID: nodeID, DialTimeout: 5 * time.Second, @@ -399,3 +399,78 @@ func TestDial_ProxyAuthFailureDoesNotLeakCredentials(t *testing.T) { t.Fatal("beacon was reached despite the proxy refusing the CONNECT") } } + +// newTLSConnectProxy is connectProxy served over TLS (an https:// proxy) +// with a certificate for "localhost" from its own CA. +func newTLSConnectProxy(t *testing.T, user, pass string) (*connectProxy, *x509.CertPool) { + t.Helper() + cert, pool := proxiedCert(t, "localhost") + raw, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("proxy listen: %v", err) + } + ln := tls.NewListener(raw, &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12}) + req := &http.Request{Header: http.Header{}} + req.SetBasicAuth(user, pass) + p := &connectProxy{ln: ln, wantAuth: req.Header.Get("Authorization")} + p.wg.Add(1) + go p.accept() + t.Cleanup(func() { + ln.Close() + p.mu.Lock() + for _, c := range p.live { + c.Close() + } + p.mu.Unlock() + p.wg.Wait() + }) + return p, pool +} + +// An https:// proxy is verified with its own trust store (the dialer's), +// never with the beacon's TLSConfig: a beacon pinned to Pilot's roots must +// not reject the operator's proxy certificate, and the proxy's roots must +// not be able to vouch for the beacon. +func TestDial_ThroughHTTPSProxyKeepsBeaconTrustSeparate(t *testing.T) { + t.Parallel() + const nodeID uint32 = 7005 + fb, beaconPool, snis := newProxiedFakeBeacon(t, nodeID) + proxy, proxyPool := newTLSConnectProxy(t, "muse", "s3cret") + _, port, _ := net.SplitHostPort(proxy.ln.Addr().String()) + res, err := netproxy.Explicit("https://muse:s3cret@" + net.JoinHostPort("localhost", port)) + if err != nil { + t.Fatalf("netproxy.Explicit: %v", err) + } + dialer := &netproxy.Dialer{Resolver: res, TLSConfig: &tls.Config{RootCAs: proxyPool, MinVersion: tls.VersionTLS12}} + + tr, err := wss.Dial(context.Background(), wss.Config{ + URL: proxiedURL(fb), + TLSConfig: &tls.Config{RootCAs: beaconPool, MinVersion: tls.VersionTLS12}, // "pinned": beacon CA only + DialContext: dialer.DialContext, + Identity: mustID(t), + NodeID: nodeID, + DialTimeout: 5 * time.Second, + }) + if err != nil { + t.Fatalf("Dial through an https:// proxy with a pinned beacon trust store: %v", err) + } + tr.Close() + assertOnlyConnects(t, proxy, proxiedTarget(fb), 1) + if got := snis(); len(got) != 1 || got[0] != proxiedBeaconHost { + t.Fatalf("beacon saw SNI %q, want [%q]", got, proxiedBeaconHost) + } + + // The beacon is still verified with TLSConfig alone: trusting only the + // proxy's CA for the beacon fails. + _, err = wss.Dial(context.Background(), wss.Config{ + URL: proxiedURL(fb), + TLSConfig: &tls.Config{RootCAs: proxyPool, MinVersion: tls.VersionTLS12}, + DialContext: dialer.DialContext, + Identity: mustID(t), + NodeID: nodeID, + DialTimeout: 5 * time.Second, + }) + if err == nil { + t.Fatal("beacon accepted with only the proxy's CA trusted") + } +} diff --git a/pkg/daemon/transport_auto.go b/pkg/daemon/transport_auto.go new file mode 100644 index 00000000..4d07021c --- /dev/null +++ b/pkg/daemon/transport_auto.go @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package daemon + +import ( + "context" + "crypto/x509" + "errors" + "fmt" + "net" + "net/url" + "strings" + "time" + + "github.com/pilot-protocol/pilotprotocol/pkg/daemon/routing" +) + +// Transport modes accepted by Config.TransportMode and -transport. +const ( + TransportUDP = "udp" + TransportCompat = "compat" + // TransportAuto picks udp or compat at startup (SelectTransport). + // Config.TransportMode "" behaves the same way for embedders. + TransportAuto = "auto" +) + +// NormalizeTransport lower-cases and validates a transport name. "" stays +// "" (the caller's default). +func NormalizeTransport(v string) (string, error) { + s := strings.ToLower(strings.TrimSpace(v)) + switch s { + case "", TransportUDP, TransportCompat, TransportAuto: + return s, nil + } + return "", fmt.Errorf("invalid transport %q: must be udp, compat or auto", v) +} + +// defaultCompatCheckTimeout bounds the TCP reachability check of the +// compat beacon in SelectTransport. +const defaultCompatCheckTimeout = 5 * time.Second + +// AutoTransportProbe configures SelectTransport. +type AutoTransportProbe struct { + // BeaconAddr is the UDP beacon ("host:port"; the first entry of a + // comma-separated list is probed). + BeaconAddr string + // CompatBeaconURL is the WSS beacon compat mode would dial. Empty + // means compat is not available and auto always picks udp. + CompatBeaconURL string + // Dial opens the compat reachability check, normally through the + // proxy policy compat mode would use. nil dials directly. + Dial func(ctx context.Context, network, addr string) (net.Conn, error) + // UDPTimeout bounds the UDP probe (0 = udpProbeTimeout). A reply + // returns as soon as it arrives, so on a working UDP path the probe + // costs one round trip. + UDPTimeout time.Duration + // TCPTimeout bounds the compat check (0 = 5s). It only runs when the + // UDP probe got no answer. + TCPTimeout time.Duration +} + +// SelectTransport decides -transport=auto: +// +// - udp when the beacon answers a UDP discover — the transport a daemon +// would have used anyway, found in one round trip; +// - otherwise compat, when the compat beacon's host:port accepts a TCP +// connection through p.Dial (i.e. through the egress proxy, if there +// is one): the host blocks UDP but can reach TCP 443; +// - otherwise udp, exactly as before -transport=auto existed: nothing +// is reachable yet (no network at boot, beacon outage), and a compat +// daemon could not start either. +// +// reason is a short, log-ready explanation of the choice. +func SelectTransport(ctx context.Context, p AutoTransportProbe) (mode, reason string) { + beacon := firstBeacon(p.BeaconAddr) + if beacon == "" { + return TransportUDP, "no UDP beacon configured" + } + udpTimeout := p.UDPTimeout + if udpTimeout <= 0 { + udpTimeout = udpProbeTimeout + } + if probeUDPReachableWithin(beacon, udpTimeout) { + return TransportUDP, "beacon " + beacon + " answered over UDP" + } + if strings.TrimSpace(p.CompatBeaconURL) == "" { + return TransportUDP, "no UDP answer from beacon " + beacon + ", and no compat beacon configured" + } + target, err := compatBeaconHostPort(p.CompatBeaconURL) + if err != nil { + return TransportUDP, "no UDP answer from beacon " + beacon + "; compat beacon unusable: " + err.Error() + } + tcpTimeout := p.TCPTimeout + if tcpTimeout <= 0 { + tcpTimeout = defaultCompatCheckTimeout + } + cctx, cancel := context.WithTimeout(ctx, tcpTimeout) + defer cancel() + dial := p.Dial + if dial == nil { + var d net.Dialer + dial = d.DialContext + } + conn, err := dial(cctx, "tcp", target) + if err != nil { + return TransportUDP, fmt.Sprintf("no UDP answer from beacon %s, and compat beacon %s unreachable (%v)", beacon, target, err) + } + conn.Close() + return TransportCompat, fmt.Sprintf("no UDP answer from beacon %s within %s; compat beacon %s reachable over TCP", beacon, udpTimeout, target) +} + +// probeUDPReachableWithin is probeUDPReachable with a caller-chosen bound, +// split over two discover attempts so one lost datagram does not count as +// a blocked path. +func probeUDPReachableWithin(beaconAddr string, timeout time.Duration) bool { + if _, _, err := net.SplitHostPort(beaconAddr); err != nil { + return false + } + per := timeout / 2 + if per <= 0 { + per = timeout + } + for i := 0; i < 2; i++ { + if _, err := routing.ProbeBeaconRTT(beaconAddr, 0, per); err == nil { + return true + } + } + return false +} + +// compatBeaconHostPort extracts the TCP target of a ws:// or wss:// URL. +func compatBeaconHostPort(raw string) (string, error) { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil { + return "", fmt.Errorf("parse %q: %w", raw, err) + } + port := u.Port() + switch strings.ToLower(u.Scheme) { + case "wss", "https": + if port == "" { + port = "443" + } + case "ws", "http": + if port == "" { + port = "80" + } + default: + return "", fmt.Errorf("unsupported scheme in %q", raw) + } + if u.Hostname() == "" { + return "", fmt.Errorf("no host in %q", raw) + } + return net.JoinHostPort(u.Hostname(), port), nil +} + +// tlsTrustHint explains a certificate verification failure against the +// system trust store, which in a minimal sandbox usually means there is no +// CA bundle. "" for any other error. +func tlsTrustHint(err error, what string) string { + if err == nil { + return "" + } + var ua x509.UnknownAuthorityError + var sr x509.SystemRootsError + msg := err.Error() + if !errors.As(err, &ua) && !errors.As(err, &sr) && + !strings.Contains(msg, "certificate signed by unknown authority") && + !strings.Contains(msg, "failed to load system roots") { + return "" + } + switch what { + case "registry": + return "cannot verify the registry certificate: point SSL_CERT_FILE (or SSL_CERT_DIR) at a CA bundle, or pin it with -registry-trust=pinned -registry-fingerprint= (config registry_trust/registry_fingerprint, or env PILOT_REGISTRY_TRUST/PILOT_REGISTRY_FINGERPRINT)" + default: + return "cannot verify the beacon certificate: point SSL_CERT_FILE (or SSL_CERT_DIR) at a CA bundle" + } +} diff --git a/pkg/daemon/tunnel.go b/pkg/daemon/tunnel.go index 5b217cca..30b8087a 100644 --- a/pkg/daemon/tunnel.go +++ b/pkg/daemon/tunnel.go @@ -13,8 +13,6 @@ import ( "fmt" "log/slog" "net" - "net/http" - "net/url" "sync" "sync/atomic" "syscall" @@ -1153,11 +1151,12 @@ func (tm *TunnelManager) Listen(addr string) error { type ConnectCompatConfig struct { BeaconURL string TLSConfig *tls.Config - // Proxy is the http.Transport.Proxy function for the WSS dial (see - // wss.Config.Proxy). nil dials the beacon directly. - Proxy func(*http.Request) (*url.URL, error) - Identity *crypto.Identity - NodeID uint32 + // DialContext opens the TCP connection for the WSS dial (see + // wss.Config.DialContext), e.g. through the proxy policy. nil dials + // the beacon directly. + DialContext func(ctx context.Context, network, addr string) (net.Conn, error) + Identity *crypto.Identity + NodeID uint32 } // ConnectCompat opens a compat-mode (WSS) tunnel to the beacon @@ -1172,11 +1171,11 @@ type ConnectCompatConfig struct { // today for symmetric-NAT peers. func (tm *TunnelManager) ConnectCompat(ctx context.Context, cfg ConnectCompatConfig) error { wssTr, err := wssTransport.Dial(ctx, wssTransport.Config{ - URL: cfg.BeaconURL, - TLSConfig: cfg.TLSConfig, - Proxy: cfg.Proxy, - Identity: cfg.Identity, - NodeID: cfg.NodeID, + URL: cfg.BeaconURL, + TLSConfig: cfg.TLSConfig, + DialContext: cfg.DialContext, + Identity: cfg.Identity, + NodeID: cfg.NodeID, }) if err != nil { return fmt.Errorf("compat dial: %w", err) diff --git a/pkg/daemon/zz_transport_auto_test.go b/pkg/daemon/zz_transport_auto_test.go new file mode 100644 index 00000000..2d162154 --- /dev/null +++ b/pkg/daemon/zz_transport_auto_test.go @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package daemon + +import ( + "context" + "encoding/binary" + "errors" + "net" + "net/http" + "net/url" + "strings" + "testing" + "time" + + "github.com/pilot-protocol/common/netproxy" + "github.com/pilot-protocol/common/protocol" +) + +// udpBeacon answers BeaconMsgDiscover like a real beacon when answer is +// true, and silently drops everything otherwise (a UDP-blocked path). +func udpBeacon(t *testing.T, answer bool) string { + t.Helper() + conn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)}) + if err != nil { + t.Fatalf("udp listen: %v", err) + } + t.Cleanup(func() { conn.Close() }) + go func() { + buf := make([]byte, 64) + for { + n, from, err := conn.ReadFromUDP(buf) + if err != nil { + return + } + if !answer || n < 5 || buf[0] != protocol.BeaconMsgDiscover { + continue + } + reply := []byte{protocol.BeaconMsgDiscoverReply, 4} + reply = append(reply, from.IP.To4()...) + reply = binary.BigEndian.AppendUint16(reply, uint16(from.Port)) + _, _ = conn.WriteToUDP(reply, from) + } + }() + return conn.LocalAddr().String() +} + +func tcpListener(t *testing.T) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("tcp listen: %v", err) + } + t.Cleanup(func() { ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + c.Close() + } + }() + return ln.Addr().String() +} + +// When UDP works, auto is udp — the transport the daemon always used — +// found in one round trip, and the compat side is never touched. +func TestSelectTransportUDPWorks(t *testing.T) { + t.Parallel() + beacon := udpBeacon(t, true) + dialed := false + start := time.Now() + mode, reason := SelectTransport(context.Background(), AutoTransportProbe{ + BeaconAddr: beacon, + CompatBeaconURL: "wss://beacon.pilot.invalid/v1/compat", + Dial: func(ctx context.Context, network, addr string) (net.Conn, error) { + dialed = true + return nil, errors.New("must not be called") + }, + }) + if mode != TransportUDP { + t.Fatalf("mode = %q (%s), want udp", mode, reason) + } + if elapsed := time.Since(start); elapsed > 500*time.Millisecond { + t.Errorf("UDP-reachable probe took %s; want about one round trip", elapsed) + } + if dialed { + t.Error("compat check ran although UDP answered") + } +} + +// UDP blocked, compat beacon reachable over TCP: compat, within the probe +// bound plus the TCP connect. +func TestSelectTransportUDPBlockedFallsBackToCompat(t *testing.T) { + t.Parallel() + beacon := udpBeacon(t, false) + compat := tcpListener(t) + start := time.Now() + mode, reason := SelectTransport(context.Background(), AutoTransportProbe{ + BeaconAddr: beacon, + CompatBeaconURL: "wss://" + compat + "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/v1/compat", + UDPTimeout: 300 * time.Millisecond, + }) + if mode != TransportCompat { + t.Fatalf("mode = %q (%s), want compat", mode, reason) + } + if elapsed := time.Since(start); elapsed > 2*time.Second { + t.Errorf("fallback took %s, want the bounded probe (~300ms) plus a local connect", elapsed) + } + if !strings.Contains(reason, "no UDP answer") { + t.Errorf("reason %q does not say why", reason) + } +} + +// Nothing reachable (no network yet, beacon outage): stay on udp, as a +// daemon always did — a compat daemon could not start either. +func TestSelectTransportNothingReachableStaysUDP(t *testing.T) { + t.Parallel() + beacon := udpBeacon(t, false) + // Take a port and close it so connects are refused. + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + refused := ln.Addr().String() + ln.Close() + mode, reason := SelectTransport(context.Background(), AutoTransportProbe{ + BeaconAddr: beacon, + CompatBeaconURL: "wss://" + refused + "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/v1/compat", + UDPTimeout: 200 * time.Millisecond, + TCPTimeout: time.Second, + }) + if mode != TransportUDP { + t.Fatalf("mode = %q (%s), want udp", mode, reason) + } + for _, tc := range []struct{ beacon, compat string }{ + {"", "wss://beacon.pilot.invalid/v1/compat"}, + {beacon, ""}, + {beacon, "ftp://beacon.pilot.invalid"}, + } { + if mode, reason := SelectTransport(context.Background(), AutoTransportProbe{ + BeaconAddr: tc.beacon, CompatBeaconURL: tc.compat, UDPTimeout: 100 * time.Millisecond, + }); mode != TransportUDP { + t.Errorf("SelectTransport(%q, %q) = %q (%s), want udp", tc.beacon, tc.compat, mode, reason) + } + } +} + +// Behind an egress proxy the compat check goes through the proxy, by host +// name — the Muse case: UDP silently dropped, direct TCP killed. +func TestSelectTransportCompatCheckUsesProxy(t *testing.T) { + clearProxyEnv(t) + beacon := udpBeacon(t, false) + proxy := newProxyTestConnect(t, "muse", "s3cret") + target := tcpListener(t) + _, port, _ := net.SplitHostPort(target) + policy, err := ResolveProxy(proxy.url("muse", "s3cret"), TransportCompat) + if err != nil { + t.Fatal(err) + } + d := New(Config{Proxy: policy}) + mode, reason := SelectTransport(context.Background(), AutoTransportProbe{ + BeaconAddr: beacon, + CompatBeaconURL: "wss://beacon.pilot.invalid:" + port + "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/v1/compat", + Dial: d.proxyDialer(), + UDPTimeout: 200 * time.Millisecond, + }) + if mode != TransportCompat { + t.Fatalf("mode = %q (%s), want compat", mode, reason) + } + if got := proxy.counts()["beacon.pilot.invalid:"+port]; got != 1 { + t.Fatalf("proxy CONNECTs = %v, want one for beacon.pilot.invalid:%s", proxy.counts(), port) + } +} + +func TestNormalizeTransport(t *testing.T) { + for in, want := range map[string]string{"": "", "udp": "udp", " COMPAT ": "compat", "Auto": "auto"} { + if got, err := NormalizeTransport(in); err != nil || got != want { + t.Errorf("NormalizeTransport(%q) = (%q, %v), want %q", in, got, err, want) + } + } + if _, err := NormalizeTransport("wss"); err == nil { + t.Error("NormalizeTransport accepted wss") + } +} + +// -proxy=none and the other "off" spellings disable the proxy; any other +// bare word is an error instead of a proxy host name. +func TestResolveProxyWords(t *testing.T) { + clearProxyEnv(t) + t.Setenv("HTTPS_PROXY", "http://env.test:3128") + for _, off := range []string{"none", "NONE", "no", "false", "direct", "off"} { + p, err := ResolveProxy(off, TransportCompat) + if err != nil || p == nil || p.Mode() != netproxy.ModeOff { + t.Errorf("ResolveProxy(%q) = (%v, %v), want off", off, p, err) + } + } + for _, bad := range []string{"proxy", "true", "yes", "env.test:3128", "u:s3cret@env.test:3128"} { + _, err := ResolveProxy(bad, TransportCompat) + if err == nil { + t.Errorf("ResolveProxy(%q) accepted", bad) + } else if strings.Contains(err.Error(), "s3cret") { + t.Errorf("ResolveProxy(%q) error leaks the password: %v", bad, err) + } + } + // A bad HTTP_PROXY no longer throws away a good HTTPS_PROXY (common + // v0.5.14): compat still proxies TLS targets. + t.Setenv("HTTP_PROXY", "socks5://127.0.0.1:1080") + p, err := ResolveProxy("auto", TransportCompat) + if err != nil || !p.Enabled() { + t.Fatalf("auto/compat with a bad HTTP_PROXY = (%v, %v), want the HTTPS_PROXY policy", p, err) + } + if u, _ := p.ProxyForAddr("registry.pilotprotocol.network:443"); u == nil || u.Host != "env.test:3128" { + t.Fatalf("registry proxy = %v, want env.test:3128", u) + } +} + +// An explicit -proxy URL takes every outbound target except this machine: +// loopback webhooks, sidecars and a local registry go direct. +func TestExplicitProxyNeverTakesLoopback(t *testing.T) { + t.Parallel() + proxy := newProxyTestConnect(t, "muse", "s3cret") + policy, err := ResolveProxy(proxy.url("muse", "s3cret"), TransportUDP) + if err != nil { + t.Fatal(err) + } + d := New(Config{Proxy: policy}) + pf := d.httpProxyFunc() + for _, target := range []string{"http://127.0.0.1:8080/hook", "http://localhost:5002/analyze"} { + u, err := pf(&http.Request{URL: mustURL(t, target)}) + if err != nil || u != nil { + t.Errorf("HTTP proxy for %s = (%v, %v), want direct", target, u, err) + } + } + if u, _ := pf(&http.Request{URL: mustURL(t, "https://raw.githubusercontent.com/x")}); u == nil { + t.Error("remote HTTP target not proxied") + } + + local := tcpListener(t) + conn, err := d.proxyDialer()(context.Background(), "tcp", local) + if err != nil { + t.Fatalf("dial loopback %s: %v", local, err) + } + conn.Close() + if n := len(proxy.counts()); n != 0 { + t.Fatalf("loopback dial reached the proxy: %v", proxy.counts()) + } +} + +func mustURL(t *testing.T, raw string) *url.URL { + t.Helper() + u, err := url.Parse(raw) + if err != nil { + t.Fatal(err) + } + return u +} From ce86a650dc8987386ca4fe454bb7d06af477a2e9 Mon Sep 17 00:00:00 2001 From: Teodor Calin Date: Thu, 24 Sep 2026 02:38:22 +0300 Subject: [PATCH 06/19] fix: rotating proxy credentials, auto/compat and registry routing re-review Fixes the six re-review findings on feat/native-https-proxy. muse-proxy-cred-rotation-unhandled (high) - New -proxy-cmd / $PILOT_PROXY_CMD / config.json proxy_cmd: a command whose stdout is the current proxy URL. internal/proxyconf.Policy wraps it: re-run every 60s (Config.ProxyRefreshInterval) and whenever a CONNECT gets 407 (or the malformed status line sandbox proxies send), then the dial is retried once with the new URL. The registry client (primary, pool, every redial), the WSS beacon (every reconnect), daemon HTTP clients (retrying RoundTripper) and http.DefaultTransport (refresh on 407 via OnProxyConnectResponse) all follow it. Output and stderr are never logged; PILOT_ADMIN_TOKEN/PILOT_WEBHOOK_SECRET are kept from the command. A failing first run falls back to the launch environment's proxy. - pkg/daemon: Config.ProxyPolicy (*ProxyPolicy), StaticProxyPolicy, NewCommandProxyPolicy; Start runs the refresher until Stop. - install.sh saves proxy_cmd=bash -c 'printf %s "${https_proxy:-$HTTPS_PROXY}"' in a Linux container/VM without systemd whose HTTPS_PROXY carries credentials (or PILOT_PROXY_CMD when set); README documents it. version-skew-config-transport-auto-bricks-older-daemon (medium) - install.sh never saves transport=auto (--transport auto removes a saved transport); service units get Environment PILOT_TRANSPORT_DEFAULT=auto, which a pre-auto daemon ignores. pilot-daemon honours $PILOT_TRANSPORT_DEFAULT only when flag, $PILOT_TRANSPORT and config.json choose nothing. - install.sh --version and pilotctl update --pin rewrite a saved transport=auto to udp. pilotctl config --set transport= (proxy=, proxy_cmd=) removes the key. compat-explicit-registry-tls-now-pinned-fatal (low) - applyRegistryDefaults defaults trust (pinned with a fingerprint, else system) whenever TLS is on in compat or for the compat registry address, also when -registry-tls was explicit. auto-compat-check-tcp-only-fatal-on-beacon-outage (low) - SelectTransport's compat check now does TLS + GET of the beacon path and requires 426 Upgrade Required (live WebSocket endpoint); a TCP front with the beacon down (502/503/close) keeps auto on udp. daemon-proxied-udp-registry-stays-raw-9000 (low) - When the registry dial goes through the proxy (any transport), the compiled-in raw registry moves to registry.pilotprotocol.network:443 over TLS, the rule pilotctl already applied. pilotctl-auto-proxy-regardless-of-transport (low) - pilotctl's registry routes follow the daemon: an explicit proxy URL always; the environment's proxy only when the transport is compat (the running daemon's, from the new info "transport" field, else $PILOT_TRANSPORT / config.json). udp hosts dial directly; unknown transport tries direct first and the proxied TLS registry as fallback for the production registry only; private raw registries are never sent to the environment's proxy outside compat. Tests: proxyconf policy unit tests (407 retry for dials and HTTP, NO_PROXY, Run loop, command runner); pkg/daemon compat start through a rotating proxy (registry + WSS reconnect); cmd/daemon end-to-end runs for PILOT_PROXY_CMD rotation, command failure fallback, proxied udp registry, explicit -registry-tls trust, PILOT_TRANSPORT_DEFAULT; SelectTransport front-up/beacon-down; pilotctl route matrix, private registry direct, fitTransportToDaemon, config key clearing. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 76 +++- README.md | 7 +- cmd/daemon/main.go | 61 ++- cmd/daemon/netflags.go | 92 ++-- cmd/daemon/netflags_test.go | 87 +++- cmd/daemon/proxy.go | 78 +++- cmd/daemon/proxy_refresh_test.go | 165 +++++++ cmd/daemon/proxy_test.go | 177 +++++--- cmd/pilotctl/daemon_transport.go | 27 +- cmd/pilotctl/main.go | 14 +- cmd/pilotctl/registry_dial.go | 210 ++++++--- cmd/pilotctl/updates.go | 19 +- cmd/pilotctl/zz_daemon_transport_test.go | 20 + cmd/pilotctl/zz_proxy_route_test.go | 239 ++++++++--- go.mod | 2 +- install.sh | 143 ++++-- internal/proxyconf/policy.go | 525 +++++++++++++++++++++++ internal/proxyconf/policy_test.go | 501 +++++++++++++++++++++ internal/proxyconf/proxyconf.go | 24 +- pkg/daemon/daemon.go | 35 +- pkg/daemon/ipc.go | 1 + pkg/daemon/proxy.go | 56 ++- pkg/daemon/transport_auto.go | 92 +++- pkg/daemon/zz_proxy_refresh_test.go | 158 +++++++ pkg/daemon/zz_proxy_test.go | 54 ++- pkg/daemon/zz_transport_auto_test.go | 49 ++- 26 files changed, 2571 insertions(+), 341 deletions(-) create mode 100644 cmd/daemon/proxy_refresh_test.go create mode 100644 internal/proxyconf/policy.go create mode 100644 internal/proxyconf/policy_test.go create mode 100644 pkg/daemon/zz_proxy_refresh_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 489b93c4..0ab879a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,14 +28,28 @@ Detailed per-release notes are on the loopback. `off` also accepts `none`, `no`, `false`, `direct`; any other bare word is an error instead of a proxy host name. Proxy credentials never appear in logs, errors or `-help`. +- **Rotating proxy credentials: `-proxy-cmd` / `$PILOT_PROXY_CMD` / + config.json `proxy_cmd`.** A command whose output is the current proxy URL + (e.g. `bash -c 'printf %s "$https_proxy"'`); the daemon re-runs it every + 60s and whenever the proxy answers a CONNECT with 407, and retries that + connection once with the new credentials. Registry redials, WSS beacon + reconnects and every HTTP client (plugins included) follow it, so a sandbox + that rotates its proxy credentials every few minutes (Meta Muse) no longer + leaves a node "online with all apps broken" until a restart. The installer + saves such a command in a Linux container/VM without systemd whose + `HTTPS_PROXY` carries credentials. The command's output is never logged. - **`-transport=auto`.** UDP when the beacon answers a UDP discover (one round - trip), otherwise compat when the compat beacon is reachable over TCP 443 - (through the proxy, if any), otherwise udp as before; the decision is logged - once (`transport auto-selected`). It never moves a node with a private - registry or beacon onto the public compat beacon. pilot-daemon's own default - stays `udp`; `pilotctl daemon start` asks for `auto` whenever no transport is - configured and the daemon supports it, and fresh installs save - `transport=auto`. + trip), otherwise compat when the compat beacon itself answers over TCP 443 + (through the proxy, if any: a TLS GET of the beacon path must return `426 + Upgrade Required` — a front that accepts TCP while the beacon is down does + not count), otherwise udp as before; the decision is logged once + (`transport auto-selected`). It never moves a node with a private registry + or beacon onto the public compat beacon. pilot-daemon's own default stays + `udp` (or `$PILOT_TRANSPORT_DEFAULT`, which applies only when nothing else + chooses); `pilotctl daemon start` asks for `auto` whenever no transport is + configured and the daemon supports it, and the systemd/launchd units the + installer writes set `PILOT_TRANSPORT_DEFAULT=auto`. `auto` is never saved + in config.json, where a daemon that predates it would refuse to start. - **`$PILOT_TRANSPORT` and config.json `transport` are honored by pilot-daemon itself** (both used to be masked by the `-transport` default). Precedence for `-transport`, `-proxy`, `-registry-trust` and `-registry-fingerprint`: flag, @@ -56,23 +70,29 @@ Detailed per-release notes are on the proxy URL with credentials (any `@`) travels as `$PILOT_PROXY`, never on the daemon's argv, and is shown redacted. The ready summary reports the transport the daemon actually chose. -- **pilotctl's own registry connections follow the proxy.** `lookup`, - `register`, `rotate-key`, the auto-handshake visibility check and `recovery - recover` dial through `$PILOT_PROXY` / config `proxy` / `$HTTPS_PROXY` - (via common/netproxy), using `registry.pilotprotocol.network:443` over TLS - when proxied or in compat mode, and fall back to it when the raw-TCP - registry is unreachable directly. `proxy=off` restores direct dials. +- **pilotctl's own registry connections follow the daemon's network.** + `lookup`, `register`, `rotate-key`, the auto-handshake visibility check and + `recovery recover` dial through an explicit `$PILOT_PROXY` / config `proxy` + URL, and through `$HTTPS_PROXY` / `$ALL_PROXY` only when the daemon runs + compat (asked over IPC; the info reply now carries `transport`), else + `$PILOT_TRANSPORT` / config.json — on a udp host that merely exports a proxy + they dial directly, as the daemon does, so a private raw-TCP registry keeps + working. Proxied or compat dials use `registry.pilotprotocol.network:443` + over TLS; a direct raw-TCP attempt falls back to it (through the + environment's proxy when the transport is unknown). `proxy=off` restores + direct dials. - **`daemon start` forwards the proxy/TLS environment** (`HTTPS_PROXY`, `HTTP_PROXY`, `ALL_PROXY`, `NO_PROXY` in both cases, `PILOT_PROXY`, `PILOT_TRANSPORT`, `PILOT_REGISTRY_TRUST`, `PILOT_REGISTRY_FINGERPRINT`, `SSL_CERT_FILE`, `SSL_CERT_DIR`) on both the fork and `--foreground` paths, and passes through `--compat-beacon`, `--registry-trust`, `--registry-fingerprint` and `--tls-trust`. -- **`install.sh --transport `** (or `PILOT_TRANSPORT`). Fresh - installs save `auto`; `compat` skips the UDP probe. No `proxy` key is - written (the daemon default already uses the environment's proxy). Service - units take the transport from config.json, so `pilotctl config --set - transport=` applies to them too. Every installer download goes through +- **`install.sh --transport `** (or `PILOT_TRANSPORT`). + `udp` and `compat` are saved; `auto` (the default) is not — `--transport + auto` removes a saved transport. `compat` skips the UDP probe. No `proxy` + key is written (the daemon default already uses the environment's proxy). + Service units take the transport from config.json, so `pilotctl config + --set transport=` applies to them too. Every installer download goes through `$HTTPS_PROXY`; service setup without root/systemd/launchd degrades to a printed `pilotctl daemon start` hint; download failures name the proxy (redacted) and the hosts it must allow. In a Linux container/VM without @@ -80,6 +100,26 @@ Detailed per-release notes are on the sandboxes run the agent as root); regular hosts still refuse root. ### Fixed +- **Downgrading after `transport=auto` was saved no longer bricks the daemon.** + A pilot-daemon that predates `auto` exits on `"transport":"auto"` in + config.json. Reinstalling an older release with `install.sh --version` and + `pilotctl update --pin ` now rewrite it to `udp` for such a daemon, and + nothing writes `auto` implicitly any more. `pilotctl config --set + transport=` (also `proxy=`, `proxy_cmd=`) removes the key instead of saving + an empty value. +- **`-transport=compat -registry-tls` without `-registry-trust`** (also + `registry_tls` in config.json) exited with "registry TLS with + -registry-trust=pinned requires RegistryFingerprint"; trust defaults to + `system` (or `pinned` with a configured fingerprint) again, and so does TLS + to `registry.pilotprotocol.network:443` in udp mode. +- **An explicit proxy in udp mode asked the proxy for the raw-TCP registry** + (`CONNECT 34.71.57.205:9000`, which CONNECT-443-only proxies refuse). When + the registry dial goes through a proxy, the compiled-in registry moves to + `registry.pilotprotocol.network:443` over TLS in any transport, as pilotctl + already did. +- **`-transport=auto` exited during a beacon outage** because a TCP connect + to the compat front counted as a working compat path; see the 426 check + above — auto now stays on udp and the daemon starts degraded. - **Compat daemons started by `pilotctl` were pinned to the raw-TCP registry.** `pilotctl init` writes `34.71.57.205:9000` into config.json and `daemon start` forwarded it as an explicit `-registry`, which stops pilot-daemon from diff --git a/README.md b/README.md index e6397fcb..3d5afdfa 100644 --- a/README.md +++ b/README.md @@ -294,11 +294,12 @@ curl -fsSL https://pilotprotocol.network/install.sh | sh pilotctl daemon start ``` -New installs use transport `auto`: the daemon probes the beacon over UDP once at startup (one round trip when UDP works, at most ~1.5s when it does not) and, when there is no answer but TCP 443 is reachable, runs in **compat** mode — registry over TLS on `registry.pilotprotocol.network:443`, beacon over WSS on `beacon.pilotprotocol.network:443`. In compat mode the daemon tunnels through `$HTTPS_PROXY` / `$ALL_PROXY` (honoring `$NO_PROXY`), asking the proxy to `CONNECT` by host name, so poisoned local DNS for the Pilot hosts does not matter. `pilotctl`'s own registry commands (`lookup`, the auto-handshake check, `recovery`) go through the same proxy. To skip the UDP probe, pick compat explicitly: `sh -s -- --transport compat`, `pilotctl config --set transport=compat`, or `pilotctl daemon start --transport compat`. +New installs use transport `auto`: the daemon probes the beacon over UDP once at startup (one round trip when UDP works, at most ~1.5s when it does not) and, when there is no answer but the compat beacon answers over TCP 443 (a TLS request that a live beacon answers with `426 Upgrade Required`), runs in **compat** mode — registry over TLS on `registry.pilotprotocol.network:443`, beacon over WSS on `beacon.pilotprotocol.network:443`. When neither answers (no network yet, beacon outage) it stays on UDP, as before. In compat mode the daemon tunnels through `$HTTPS_PROXY` / `$ALL_PROXY` (honoring `$NO_PROXY`), asking the proxy to `CONNECT` by host name, so poisoned local DNS for the Pilot hosts does not matter. `pilotctl`'s own registry commands (`lookup`, the auto-handshake check, `recovery`) follow the daemon: through the proxy when it runs compat (or with an explicit proxy URL), directly when it runs UDP. To skip the UDP probe, pick compat explicitly: `sh -s -- --transport compat`, `pilotctl config --set transport=compat`, or `pilotctl daemon start --transport compat`. - **No root, systemd or launchd needed.** Start the daemon with `pilotctl daemon start` from a shell that has the proxy variables. In a container or VM without systemd the installer also runs as root (the agent user in hosted sandboxes); on a regular host it still refuses root unless `PILOT_ALLOW_ROOT=1`. - **Proxy with credentials:** keep them out of `ps` — export `HTTPS_PROXY` / `PILOT_PROXY`, or `pilotctl daemon start --proxy http://user:pass@host:port` (pilotctl hands a URL with credentials to the daemon in its environment, never on its command line). For a systemd/launchd service, which does not see your shell's variables, save it: `pilotctl config --set proxy=http://user:pass@host:port` (config.json is 0600). -- **Precedence** for transport and proxy: command-line flag, then `$PILOT_TRANSPORT` / `$PILOT_PROXY`, then `config.json` (`transport`, `proxy`), then the default (`auto` from pilotctl, `udp` for a bare `pilot-daemon`; proxy `auto`). `-proxy` accepts `auto`, `off` (also `none`, `direct`) or an `http://` / `https://` URL; anything else is an error. Loopback targets (local webhooks, sidecars) are never sent to a proxy. +- **Proxy credentials that rotate** (Meta Muse rotates the credentials in `HTTPS_PROXY` every few minutes): a long-running daemon would keep the ones it started with, and new connections would start failing with `407 Proxy Authentication Required` while old tunnels stay up ("node online, apps broken"). Give the daemon a command that prints the current proxy URL — `proxy_cmd` in config.json, `$PILOT_PROXY_CMD` or `-proxy-cmd` — and it re-runs it every 60s and whenever the proxy answers 407, retrying with the fresh credentials; no restart needed. The installer saves `bash -c 'printf %s "$https_proxy"'` (a fresh shell sees the current value) in a Linux container/VM without systemd whose `HTTPS_PROXY` carries credentials; elsewhere: `pilotctl config --set proxy_cmd="bash -c 'printf %s \"\$https_proxy\"'"`. Without it, restart the daemon from a fresh shell when it starts failing. +- **Precedence** for transport and proxy: command-line flag, then `$PILOT_TRANSPORT` / `$PILOT_PROXY` / `$PILOT_PROXY_CMD`, then `config.json` (`transport`, `proxy`, `proxy_cmd`), then the default (`auto` from pilotctl and the installed systemd/launchd services, via `PILOT_TRANSPORT_DEFAULT=auto`; `udp` for a bare `pilot-daemon`; proxy `auto`). `auto` is never saved in config.json, so reinstalling an older release (`--version`, `pilotctl update --pin`) leaves nothing it cannot read; if you saved it yourself, both rewrite it to `udp` for a daemon that predates it. `-proxy` accepts `auto`, `off` (also `none`, `direct`) or an `http://` / `https://` URL; anything else is an error. Loopback targets (local webhooks, sidecars) are never sent to a proxy. When the registry dial goes through a proxy (compat, or an explicit proxy URL in any transport), the default raw-TCP registry is replaced by `registry.pilotprotocol.network:443` over TLS, since CONNECT proxies carry port 443 only. - **No CA bundle in the sandbox?** Point Go at one with `SSL_CERT_FILE=/path/to/ca-certificates.crt` (or `SSL_CERT_DIR`) — `pilotctl daemon start` forwards both. The registry can instead be pinned: `PILOT_REGISTRY_FINGERPRINT=` (or `pilotctl config --set registry_fingerprint=...`), which selects `registry_trust=pinned`. The WSS beacon has no fingerprint option, so it needs the CA bundle.
@@ -486,6 +487,8 @@ Most daemon flags have an environment variable equivalent. Useful for containeri | `PILOT_BEACON` | `-beacon` | Beacon server address | | `PILOT_TRANSPORT` | `-transport` | Tunnel transport: `udp` (daemon default), `compat` (TLS registry + WSS beacon on TCP 443 only) or `auto` (udp when the beacon answers over UDP, else compat). Beats `config.json` | | `PILOT_PROXY` | `-proxy` | Outbound proxy: `auto` (default; with compat, the `HTTPS_PROXY`/`ALL_PROXY` proxy honoring `NO_PROXY`), `off` (also `none`, `direct`), or `http(s)://[user:pass@]host:port` for every connection except loopback. Beats `config.json` | +| `PILOT_PROXY_CMD` | `-proxy-cmd` | Command (`/bin/sh -c`) printing the current proxy URL, re-run every 60s and on a 407, for proxies that rotate credentials; supplies the URL `-proxy` would use. Beats `config.json` (`proxy_cmd`) | +| `PILOT_TRANSPORT_DEFAULT` | — | Transport when neither `-transport`, `PILOT_TRANSPORT` nor `config.json` sets one (the installed services set `auto`; daemons without `auto` ignore it) | | `PILOT_REGISTRY_TRUST` / `PILOT_REGISTRY_FINGERPRINT` | `-registry-trust` / `-registry-fingerprint` | Pin the TLS registry (hosts without a CA bundle); a fingerprint alone selects pinned trust in compat mode. Beat `config.json` | | `HTTPS_PROXY` / `ALL_PROXY` / `NO_PROXY` | — | Proxy used with `-proxy=auto` in compat mode and by `pilotctl`'s own registry dials; `pilotctl daemon start` forwards them (and `SSL_CERT_FILE` / `SSL_CERT_DIR`) to the daemon | | `PILOT_SOCKET` | `-socket` | Unix socket path | diff --git a/cmd/daemon/main.go b/cmd/daemon/main.go index 752e146e..b075f19d 100644 --- a/cmd/daemon/main.go +++ b/cmd/daemon/main.go @@ -115,6 +115,7 @@ func main() { // credentials. transportMode := flag.String("transport", "", "tunnel transport: 'udp' (the default), 'compat' (registry over TLS and beacon over WSS, TCP 443 only, for UDP-blocked or proxy-only hosts) or 'auto' (udp when the beacon answers over UDP, otherwise compat when TCP 443 is reachable, through the proxy if there is one). Precedence: this flag, $PILOT_TRANSPORT, config.json \"transport\", udp.") proxySpec := flag.String("proxy", "", "outbound proxy for registry, beacon and HTTP connections: 'auto' (the default: with compat, HTTPS_PROXY/ALL_PROXY from the environment, honoring NO_PROXY; nothing with udp), 'off' (also none, no, false, direct), or an http:// or https:// proxy URL, http://[user:pass@]host:port, used for every connection except loopback. Precedence: this flag, $PILOT_PROXY, config.json \"proxy\", auto.") + proxyCmd := flag.String("proxy-cmd", "", "command (run with /bin/sh -c) whose output is the current proxy URL, for egress proxies that rotate their credentials: it supplies the URL -proxy would use (the explicit URL, or with auto the environment's proxy) and is re-run every 60s and whenever the proxy answers 407, so new connections always carry fresh credentials. Example: bash -c 'printf %s \"$https_proxy\"'. Precedence: this flag, $PILOT_PROXY_CMD, config.json \"proxy_cmd\".") compatBeacon := flag.String("compat-beacon", defaultCompatBeacon, "beacon WSS URL for -transport=compat") tlsTrust := flag.String("tls-trust", "system", "TLS trust store for -transport=compat: 'system' (OS trust store; current default while compat mode uses Let's Encrypt certs on beacon.pilotprotocol.network — on a host without a CA bundle set SSL_CERT_FILE or SSL_CERT_DIR) or 'pinned' (Pilot CA root embedded in the daemon binary; will become the default in a future release once production root ships)") showVersion := flag.Bool("version", false, "print version and exit") @@ -187,6 +188,7 @@ func main() { var transportSrc string *transportMode, transportSrc = sources.envOverConfig("transport", *transportMode, "PILOT_TRANSPORT") *proxySpec, _ = sources.envOverConfig("proxy", *proxySpec, "PILOT_PROXY") + *proxyCmd, _ = sources.envOverConfig("proxy-cmd", *proxyCmd, "PILOT_PROXY_CMD") var trustSrc string *registryTrust, trustSrc = sources.envOverConfig("registry-trust", *registryTrust, "PILOT_REGISTRY_TRUST") *registryFingerprint, _ = sources.envOverConfig("registry-fingerprint", *registryFingerprint, "PILOT_REGISTRY_FINGERPRINT") @@ -205,11 +207,30 @@ func main() { log.Fatalf("-transport: %v", err) } if transport == "" { - transport = daemon.TransportUDP + // Nothing chosen: $PILOT_TRANSPORT_DEFAULT (auto in the service + // units install.sh writes), else udp. + transport = defaultTransport() + if strings.TrimSpace(getenv(transportDefaultEnv)) != "" { + transportSrc = transportDefaultEnv + } } if _, err := proxyconf.Normalize(*proxySpec); err != nil { log.Fatalf("-proxy: %v", err) } + // The proxy policy depends on the transport only; each is resolved + // once (running -proxy-cmd once) and shared by the auto probe and the + // daemon. + proxyPolicies := map[string]*proxyconf.Policy{} + policyFor := func(transport string) (*proxyconf.Policy, error) { + if p, ok := proxyPolicies[transport]; ok { + return p, nil + } + p, err := resolveProxyPolicy(*proxySpec, *proxyCmd, transport) + if err == nil { + proxyPolicies[transport] = p + } + return p, err + } reg := registrySettings{ Addr: *registryAddr, @@ -224,7 +245,7 @@ func main() { // -transport=auto: probe once, before anything depends on the mode. if transport == daemon.TransportAuto { mode, reason, err := resolveAutoTransport(reg, *beaconAddr, sources.explicit("beacon") || beaconFromEnv, - *compatBeacon, sources.explicit("compat-beacon"), *proxySpec) + *compatBeacon, sources.explicit("compat-beacon"), policyFor) if err != nil { log.Fatalf("-proxy: %v", err) } @@ -233,14 +254,24 @@ func main() { } *transportMode = transport - // Registry defaults for the transport (see applyRegistryDefaults): - // compat moves the compiled-in raw-TCP registry to - // registry.pilotprotocol.network:443 over TLS, so the daemon really - // uses a single port; an explicit non-default -registry, or an - // explicit -registry-tls=false (the TCP/9000 fallback), is kept. - // -beacon needs no such rule: in compat mode the UDP beacon address is - // only the relay-wrap destination on the WSS pipe and is never dialed. - final := applyRegistryDefaults(transport, reg) + // Outbound proxy: resolved once, after -transport is final, and shared + // by everything that dials out — the registry client, the compat WSS + // beacon, pkg/daemon's own HTTP fetches (via daemon.Config.ProxyPolicy) + // and every plugin HTTP client (via http.DefaultTransport). + proxyPolicy, err := policyFor(transport) + if err != nil { + log.Fatalf("-proxy: %v", err) + } + + // Registry defaults for the transport and proxy (see + // applyRegistryDefaults): compat, or a proxied registry dial, moves + // the compiled-in raw-TCP registry to registry.pilotprotocol.network:443 + // over TLS, so the daemon really uses a single port that a CONNECT + // proxy carries; an explicit non-default -registry, or an explicit + // -registry-tls=false (the TCP/9000 fallback), is kept. -beacon needs + // no such rule: in compat mode the UDP beacon address is only the + // relay-wrap destination on the WSS pipe and is never dialed. + final := applyRegistryDefaults(transport, reg, proxyPolicy.Proxies) if final.Addr != reg.Addr { registryFromEnv = false } @@ -276,14 +307,6 @@ func main() { *noSkillinject = profileOptions.DisableSkillinject *motdFeedURL = profileOptions.MOTDFeedURL - // Outbound proxy: resolved once, after -transport is final, and shared - // by everything that dials out — the registry client, the compat WSS - // beacon, pkg/daemon's own HTTP fetches (via daemon.Config.Proxy) and - // every plugin HTTP client (via http.DefaultTransport). - proxyPolicy, err := resolveProxyPolicy(*proxySpec, *transportMode) - if err != nil { - log.Fatalf("-proxy: %v", err) - } installDefaultTransportProxy(proxyPolicy) slog.Info("outbound network", "transport", *transportMode, "proxy", describeProxy(*proxySpec, *transportMode, proxyPolicy), "transport_from", transportSrc, "registry", *registryAddr, "registry_tls", *registryTLS) @@ -381,7 +404,7 @@ func main() { TransportMode: *transportMode, CompatBeaconURL: *compatBeacon, CompatTLSTrust: *tlsTrust, - Proxy: proxyPolicy, + ProxyPolicy: proxyPolicy, MOTDFeedURL: *motdFeedURL, MOTDInterval: *motdInterval, TelemetryURL: *telemetryURL, diff --git a/cmd/daemon/netflags.go b/cmd/daemon/netflags.go index d72f8d77..d474848f 100644 --- a/cmd/daemon/netflags.go +++ b/cmd/daemon/netflags.go @@ -123,37 +123,51 @@ func compatKeepsRegistry(r registrySettings) bool { return r.AddrExplicit && strings.TrimSpace(r.Addr) != defaultRegistryAddr } -// applyRegistryDefaults adapts the registry to the final transport: +// applyRegistryDefaults adapts the registry to the final transport and +// proxy policy (proxied reports whether a TCP dial of an address goes +// through the proxy; nil: never): // -// - compat: the registry moves to its TLS host name on :443 unless -// compatKeepsRegistry, and TLS is on unless -registry-tls was chosen; -// - either transport: the compat registry address (registry. -// pilotprotocol.network:443) is TLS-only, so TLS is on for it unless +// - the compiled-in raw-TCP registry moves to its TLS host name on :443 +// (compatRegistryAddr) unless compatKeepsRegistry, in compat mode and +// also in udp mode when the registry dial would go through a proxy — +// egress proxies CONNECT to :443 only (pilotctl's registry route +// applies the same rule); +// - TLS is then on unless -registry-tls was chosen; compatRegistryAddr +// is TLS-only, so TLS is on for it in either transport unless // -registry-tls was chosen — an install switched back from compat to // udp keeps working; -// - whenever TLS was turned on here and -registry-trust was not chosen, -// trust is "pinned" when a -registry-fingerprint is configured (the -// fallback for sandboxes without a CA bundle) and "system" otherwise -// (the production registry has a Let's Encrypt certificate). +// - whenever the registry uses TLS in compat mode or on +// compatRegistryAddr and -registry-trust was not chosen, trust is +// "pinned" when a -registry-fingerprint is configured (the fallback +// for sandboxes without a CA bundle) and "system" otherwise (the +// production registry has a Let's Encrypt certificate) — also when +// -registry-tls itself was explicit, as before -proxy existed. // // Choices made in config.json or the environment count as explicit, so a // pinned registry configured there is never overridden. -func applyRegistryDefaults(transport string, r registrySettings) registrySettings { - tlsDefaulted := false - if transport == daemon.TransportCompat { - if !compatKeepsRegistry(r) { - r.Addr = compatRegistryAddr - } +func applyRegistryDefaults(transport string, r registrySettings, proxied func(addr string) bool) registrySettings { + moveRegistry := false + switch { + case compatKeepsRegistry(r): + case transport == daemon.TransportCompat: + moveRegistry = true + case proxied != nil && proxied(r.Addr): + moveRegistry = true + } + if moveRegistry { + r.Addr = compatRegistryAddr if !r.TLSExplicit { r.TLS = true - tlsDefaulted = true } } - if strings.EqualFold(strings.TrimSpace(r.Addr), compatRegistryAddr) && !r.TLSExplicit && !r.TLS { + onCompatAddr := strings.EqualFold(strings.TrimSpace(r.Addr), compatRegistryAddr) + if transport == daemon.TransportCompat && !r.TLSExplicit { r.TLS = true - tlsDefaulted = true } - if tlsDefaulted && !r.TrustExplicit { + if onCompatAddr && !r.TLSExplicit { + r.TLS = true + } + if r.TLS && !r.TrustExplicit && (transport == daemon.TransportCompat || onCompatAddr) { if strings.TrimSpace(r.Fingerprint) != "" { r.Trust = "pinned" } else { @@ -183,24 +197,20 @@ var autoProbe = daemon.SelectTransport // resolveAutoTransport decides -transport=auto (see daemon.SelectTransport) // for this configuration: compat is only considered when it would reach -// the same network (autoCompatBlocker), and its TCP check runs through the -// proxy compat mode would use (-proxy, auto = the environment's). The error -// is a malformed -proxy. -func resolveAutoTransport(reg registrySettings, beacon string, beaconExplicit bool, compatBeacon string, compatBeaconExplicit bool, proxySpec string) (mode, reason string, err error) { +// the same network (autoCompatBlocker), and its beacon check runs through +// the proxy compat mode would use (policyFor(compat): -proxy / -proxy-cmd, +// auto = the environment's). The error is a malformed -proxy. +func resolveAutoTransport(reg registrySettings, beacon string, beaconExplicit bool, compatBeacon string, compatBeaconExplicit bool, policyFor func(transport string) (*proxyconf.Policy, error)) (mode, reason string, err error) { if why := autoCompatBlocker(reg, beacon, beaconExplicit, compatBeaconExplicit); why != "" { return daemon.TransportUDP, why + "; auto stays on udp (pass -transport=compat to force compat)", nil } - policy, err := daemon.ResolveProxy(proxySpec, daemon.TransportCompat) + policy, err := policyFor(daemon.TransportCompat) if err != nil { - if !isAutoProxy(proxySpec) { - return "", "", err - } - slog.Warn("proxy environment is malformed; the compat check dials directly", "err", err) - policy = nil + return "", "", err } var dial func(ctx context.Context, network, addr string) (net.Conn, error) if policy.Enabled() { - dial = proxyconf.DialContext(policy, nil) + dial = policy.DialContext(nil) } mode, reason = autoProbe(context.Background(), daemon.AutoTransportProbe{ BeaconAddr: beacon, @@ -209,3 +219,25 @@ func resolveAutoTransport(reg registrySettings, beacon string, beaconExplicit bo }) return mode, reason, nil } + +// transportDefaultEnv names the transport a daemon uses when neither +// -transport, $PILOT_TRANSPORT nor config.json chooses one. install.sh sets +// it to auto in the service units it writes: unlike -transport=auto on the +// command line or "transport":"auto" in config.json, a daemon that predates +// auto (after a downgrade) ignores it instead of refusing to start. +const transportDefaultEnv = "PILOT_TRANSPORT_DEFAULT" + +// defaultTransport is $PILOT_TRANSPORT_DEFAULT when it names a transport, +// else udp. +func defaultTransport() string { + v := strings.TrimSpace(getenv(transportDefaultEnv)) + if v == "" { + return daemon.TransportUDP + } + t, err := daemon.NormalizeTransport(v) + if err != nil || t == "" { + slog.Warn("ignoring unknown "+transportDefaultEnv+" value", "value", v, "valid", "udp, compat, auto") + return daemon.TransportUDP + } + return t +} diff --git a/cmd/daemon/netflags_test.go b/cmd/daemon/netflags_test.go index fa33b5a2..79ab6f0b 100644 --- a/cmd/daemon/netflags_test.go +++ b/cmd/daemon/netflags_test.go @@ -7,12 +7,15 @@ import ( "flag" "fmt" "net" + "net/http" + "net/http/httptest" "os" "os/exec" "strings" "testing" "time" + "github.com/pilot-protocol/pilotprotocol/internal/proxyconf" "github.com/pilot-protocol/pilotprotocol/pkg/daemon" ) @@ -113,8 +116,57 @@ func TestApplyRegistryDefaults(t *testing.T) { {"udp with the compat registry and explicit -registry-tls=false is left alone", daemon.TransportUDP, registrySettings{Addr: compatRegistryAddr, AddrExplicit: true, TLSExplicit: true, Trust: "pinned"}, registrySettings{Addr: compatRegistryAddr, AddrExplicit: true, TLSExplicit: true, Trust: "pinned"}}, + // compat-explicit-registry-tls-now-pinned-fatal: an explicit + // -registry-tls (flag or config registry_tls) without a trust + // setting defaults trust like main did, instead of falling to the + // flag default "pinned" with no fingerprint (fatal). + {"compat + explicit -registry-tls, no trust: system", daemon.TransportCompat, + registrySettings{Addr: "127.0.0.1:1", AddrExplicit: true, TLS: true, TLSExplicit: true, Trust: "pinned"}, + registrySettings{Addr: "127.0.0.1:1", AddrExplicit: true, TLS: true, TLSExplicit: true, Trust: "system"}}, + {"compat + bare explicit -registry-tls moves the default registry, system trust", daemon.TransportCompat, + registrySettings{Addr: defaultRegistryAddr, TLS: true, TLSExplicit: true, Trust: "pinned"}, + registrySettings{Addr: compatRegistryAddr, TLS: true, TLSExplicit: true, Trust: "system"}}, + {"compat + explicit -registry-tls with a fingerprint pins", daemon.TransportCompat, + registrySettings{Addr: defaultRegistryAddr, TLS: true, TLSExplicit: true, Trust: "pinned", Fingerprint: fp}, + registrySettings{Addr: compatRegistryAddr, TLS: true, TLSExplicit: true, Trust: "pinned", Fingerprint: fp}}, + {"udp + explicit TLS on the compat registry, no trust: system", daemon.TransportUDP, + registrySettings{Addr: compatRegistryAddr, AddrExplicit: true, TLS: true, TLSExplicit: true, Trust: "pinned"}, + registrySettings{Addr: compatRegistryAddr, AddrExplicit: true, TLS: true, TLSExplicit: true, Trust: "system"}}, + {"udp + explicit TLS on a custom registry keeps the pinned default", daemon.TransportUDP, + registrySettings{Addr: "reg.corp:443", AddrExplicit: true, TLS: true, TLSExplicit: true, Trust: "pinned"}, + registrySettings{Addr: "reg.corp:443", AddrExplicit: true, TLS: true, TLSExplicit: true, Trust: "pinned"}}, } { - if got := applyRegistryDefaults(tc.transport, tc.in); got != tc.want { + if got := applyRegistryDefaults(tc.transport, tc.in, nil); got != tc.want { + t.Errorf("%s:\n got %+v\nwant %+v", tc.name, got, tc.want) + } + } +} + +// daemon-proxied-udp-registry-stays-raw-9000: when the registry dial goes +// through a proxy, the compiled-in raw-TCP registry moves to the TLS +// registry on :443 in udp mode too (a CONNECT proxy carries :443 only); +// a custom registry, an explicit -registry-tls=false and an unproxied +// dial are left alone. +func TestApplyRegistryDefaultsProxiedUDP(t *testing.T) { + proxied := func(addr string) bool { return addr != "10.0.0.5:9000" } + never := func(string) bool { return false } + pilotctl := registrySettings{Addr: defaultRegistryAddr, AddrExplicit: true, Trust: "pinned"} + for _, tc := range []struct { + name string + proxied func(string) bool + in, want registrySettings + }{ + {"proxied default registry moves to TLS/443", proxied, pilotctl, + registrySettings{Addr: compatRegistryAddr, AddrExplicit: true, TLS: true, Trust: "system"}}, + {"unproxied default registry stays raw", never, pilotctl, pilotctl}, + {"custom registry is kept", proxied, + registrySettings{Addr: "reg.corp:9000", AddrExplicit: true, Trust: "pinned"}, + registrySettings{Addr: "reg.corp:9000", AddrExplicit: true, Trust: "pinned"}}, + {"explicit -registry-tls=false keeps the raw registry", proxied, + registrySettings{Addr: defaultRegistryAddr, AddrExplicit: true, TLSExplicit: true, Trust: "pinned"}, + registrySettings{Addr: defaultRegistryAddr, AddrExplicit: true, TLSExplicit: true, Trust: "pinned"}}, + } { + if got := applyRegistryDefaults(daemon.TransportUDP, tc.in, tc.proxied); got != tc.want { t.Errorf("%s:\n got %+v\nwant %+v", tc.name, got, tc.want) } } @@ -155,8 +207,11 @@ func TestResolveAutoTransport(t *testing.T) { } t.Cleanup(func() { autoProbe = daemon.SelectTransport }) std := registrySettings{Addr: defaultRegistryAddr, AddrExplicit: true} + spec := func(s string) func(string) (*proxyconf.Policy, error) { + return func(transport string) (*proxyconf.Policy, error) { return resolveProxyPolicy(s, "", transport) } + } - mode, _, err := resolveAutoTransport(std, defaultBeaconAddr, true, defaultCompatBeacon, false, "auto") + mode, _, err := resolveAutoTransport(std, defaultBeaconAddr, true, defaultCompatBeacon, false, spec("auto")) if err != nil || mode != daemon.TransportCompat || len(got) != 1 { t.Fatalf("auto = (%q, %v), probes %d", mode, err, len(got)) } @@ -165,17 +220,17 @@ func TestResolveAutoTransport(t *testing.T) { } t.Setenv("HTTPS_PROXY", "http://muse:s3cret@egress.test:3128") - if _, _, err := resolveAutoTransport(std, defaultBeaconAddr, true, defaultCompatBeacon, false, "auto"); err != nil || got[1].Dial == nil { + if _, _, err := resolveAutoTransport(std, defaultBeaconAddr, true, defaultCompatBeacon, false, spec("auto")); err != nil || got[1].Dial == nil { t.Errorf("with HTTPS_PROXY the compat check does not use the proxy (err %v)", err) } - if _, _, err := resolveAutoTransport(std, defaultBeaconAddr, true, defaultCompatBeacon, false, "off"); err != nil || got[2].Dial != nil { + if _, _, err := resolveAutoTransport(std, defaultBeaconAddr, true, defaultCompatBeacon, false, spec("off")); err != nil || got[2].Dial != nil { t.Errorf("-proxy=off still proxies the compat check (err %v)", err) } - if _, _, err := resolveAutoTransport(std, defaultBeaconAddr, true, defaultCompatBeacon, false, "ftp://x"); err == nil { + if _, _, err := resolveAutoTransport(std, defaultBeaconAddr, true, defaultCompatBeacon, false, spec("ftp://x")); err == nil { t.Error("malformed -proxy accepted") } - mode, reason, err := resolveAutoTransport(registrySettings{Addr: "10.0.0.5:9000", AddrExplicit: true}, defaultBeaconAddr, true, defaultCompatBeacon, false, "auto") + mode, reason, err := resolveAutoTransport(registrySettings{Addr: "10.0.0.5:9000", AddrExplicit: true}, defaultBeaconAddr, true, defaultCompatBeacon, false, spec("auto")) if err != nil || mode != daemon.TransportUDP || len(got) != 3 { t.Errorf("private registry: (%q, %q, %v), probes %d — want udp without probing", mode, reason, err, len(got)) } @@ -191,6 +246,7 @@ func TestHelpNeverPrintsProxyCredentials(t *testing.T) { "HOME=" + t.TempDir(), "PATH=" + os.Getenv("PATH"), "PILOT_PROXY=http://muse:s3cret@egress.test:3128", + "PILOT_PROXY_CMD=echo http://muse:s3cret@egress.test:3128", "PILOT_REGISTRY_FINGERPRINT=" + strings.Repeat("ab", 32), } out, _ := cmd.CombinedOutput() @@ -289,12 +345,27 @@ func TestCompatRegistryTLSFalseKeepsRawRegistry(t *testing.T) { } } +// fakeCompatBeacon is a TLS server that answers a plain GET of the compat +// path the way a live WebSocket beacon does: 426 Upgrade Required. +func fakeCompatBeacon(t *testing.T) string { + t.Helper() + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "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/v1/compat" { + http.NotFound(w, r) + return + } + http.Error(w, "upgrade required", http.StatusUpgradeRequired) + })) + t.Cleanup(srv.Close) + return srv.Listener.Addr().String() +} + // -transport=auto on a UDP-blocked host whose only way out is the proxy: -// the UDP probe gets no answer, the compat beacon is reachable through the +// the UDP probe gets no answer, the compat beacon answers through the // proxy, so the daemon runs compat and registers through the proxy. func TestAutoTransportFallsBackToCompatThroughProxy(t *testing.T) { proxy := newRefusingProxy(t, "muse", "s3cret") - proxy.allow("beacon.pilotprotocol.network:443") + proxy.forward("beacon.pilotprotocol.network:443", fakeCompatBeacon(t)) start := time.Now() targets, logs := runDaemon(t, proxy, daemonRun{ args: []string{ diff --git a/cmd/daemon/proxy.go b/cmd/daemon/proxy.go index bb716010..2a7edadc 100644 --- a/cmd/daemon/proxy.go +++ b/cmd/daemon/proxy.go @@ -3,38 +3,85 @@ package main import ( + "context" "log/slog" "net/http" + "strings" "github.com/pilot-protocol/common/netproxy" "github.com/pilot-protocol/pilotprotocol/internal/proxyconf" "github.com/pilot-protocol/pilotprotocol/pkg/daemon" ) -// resolveProxyPolicy resolves -proxy for the transport (see +// newCommandPolicy is daemon.NewCommandProxyPolicy (a test seam). +var newCommandPolicy = daemon.NewCommandProxyPolicy + +// resolveProxyPolicy resolves -proxy and -proxy-cmd for the transport (see // daemon.ResolveProxy). A malformed -proxy value is an error. Under "auto" // only an unusable HTTPS_PROXY / https_proxy (the variable that names the // TLS proxy) can fail; that costs the proxy, not the daemon: it is logged // and the daemon dials directly, as it did before -proxy existed. Unusable // HTTP_PROXY / ALL_PROXY values are skipped by netproxy and only logged. -func resolveProxyPolicy(spec, transport string) (*netproxy.Resolver, error) { - policy, err := daemon.ResolveProxy(spec, transport) +// +// -proxy-cmd (a command whose stdout is the current proxy URL) makes the +// policy refresh itself, for egress proxies that rotate credentials: it +// supplies the URL wherever -proxy would use one — the explicit URL, or +// with auto (compat only) the environment's proxy, NO_PROXY still +// honored. It is ignored with -proxy=off, and with auto on udp (no proxy). +// A failing first run is logged and the launch environment's proxy (or the +// explicit URL) serves until the command succeeds. +func resolveProxyPolicy(spec, command, transport string) (*proxyconf.Policy, error) { + s, err := proxyconf.Normalize(spec) if err != nil { - if !isAutoProxy(spec) { - return nil, err + return nil, err + } + command = strings.TrimSpace(command) + if command != "" && s == proxyconf.Off { + slog.Warn("ignoring -proxy-cmd: -proxy=off") + command = "" + } + if command != "" && s == proxyconf.Auto && transport != daemon.TransportCompat { + slog.Info("-proxy-cmd not used: -proxy=auto proxies -transport=compat only", "transport", transport) + command = "" + } + if command == "" { + r, err := daemon.ResolveProxy(s, transport) + if err != nil { + if s != proxyconf.Auto { + return nil, err + } + slog.Warn("proxy environment is malformed; dialing directly", "err", err) + return nil, nil } - slog.Warn("proxy environment is malformed; dialing directly", "err", err) - return nil, nil + logProxyWarnings(r) + return proxyconf.Static(r), nil } - for _, w := range policy.Warnings() { - slog.Warn("ignoring unusable proxy environment variable", "err", w) + var fallback *netproxy.Resolver + if s == proxyconf.Auto { + if fallback, err = netproxy.FromEnvironment(); err != nil { + slog.Warn("proxy environment is malformed; waiting for -proxy-cmd", "err", err) + fallback = nil + } + logProxyWarnings(fallback) + } else if fallback, err = netproxy.Explicit(s); err != nil { + return nil, err + } + policy, err := newCommandPolicy(context.Background(), command, fallback, s == proxyconf.Auto) + if err != nil { + slog.Warn("-proxy-cmd failed; using the launch-time proxy until it succeeds", "err", err) } return policy, nil } +func logProxyWarnings(r *netproxy.Resolver) { + for _, w := range r.Warnings() { + slog.Warn("ignoring unusable proxy environment variable", "err", w) + } +} + // describeProxy renders the resolved policy for the startup log line. // Credentials are always redacted. -func describeProxy(spec, transport string, policy *netproxy.Resolver) string { +func describeProxy(spec, transport string, policy *proxyconf.Policy) string { if policy != nil { return policy.String() } @@ -49,10 +96,11 @@ func describeProxy(spec, transport string, policy *netproxy.Resolver) string { // transport of its own — catalogue pins, skillinject, trustedagents, // webhook, enterprise-control clients — uses DefaultTransport, and not all // of them accept an injected client. Loopback targets (a local webhook or -// sidecar) always go direct. nil leaves DefaultTransport alone (net/http's -// own proxy environment handling). Call before any goroutine issues a -// request. -func installDefaultTransportProxy(policy *netproxy.Resolver) { +// sidecar) always go direct. With a refreshed policy (-proxy-cmd), a 407 +// answer refreshes the credentials at once, so the next request succeeds. +// nil leaves DefaultTransport alone (net/http's own proxy environment +// handling). Call before any goroutine issues a request. +func installDefaultTransportProxy(policy *proxyconf.Policy) { if policy == nil { return } @@ -61,7 +109,7 @@ func installDefaultTransportProxy(policy *netproxy.Resolver) { slog.Warn("http.DefaultTransport is not an *http.Transport; plugin HTTP clients do not follow -proxy") return } - tr.Proxy = proxyconf.RequestProxy(policy) + policy.ConfigureTransport(tr) } func isAutoProxy(spec string) bool { diff --git a/cmd/daemon/proxy_refresh_test.go b/cmd/daemon/proxy_refresh_test.go new file mode 100644 index 00000000..2c92020a --- /dev/null +++ b/cmd/daemon/proxy_refresh_test.go @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// muse-proxy-cred-rotation-unhandled: the egress proxy rotates its +// credentials while the daemon runs. With -proxy-cmd ($PILOT_PROXY_CMD) +// the daemon re-reads the proxy URL when the proxy answers 407, retries +// with the new credentials, and keeps using them — without a restart. +func TestProxyCmdFollowsCredentialRotation(t *testing.T) { + proxy := newRefusingProxy(t, "muse", "old-pass") + urlFile := filepath.Join(t.TempDir(), "proxy-url") + write := func(pass string) { + t.Helper() + if err := os.WriteFile(urlFile, []byte(fmt.Sprintf("http://muse:%s@%s\n", pass, proxy.ln.Addr())), 0o600); err != nil { + t.Fatal(err) + } + } + write("old-pass") + d := startDaemon(t, daemonRun{ + env: []string{ + "PILOT_TRANSPORT=compat", + // The launch environment's proxy, stale after the rotation. + "HTTPS_PROXY=" + fmt.Sprintf("http://muse:old-pass@%s", proxy.ln.Addr()), + "PILOT_PROXY_CMD=cat '" + urlFile + "'", + }, + }) + d.waitFor(t, proxy, 30*time.Second, "no registry CONNECT with the launch credentials", func([]string) bool { + return proxy.accepted("muse", "old-pass") > 0 + }) + + // Rotate: the proxy now wants new-pass and answers old-pass with 407. + write("new-pass") + proxy.rotate("muse", "new-pass") + d.waitFor(t, proxy, 30*time.Second, "the daemon never retried with the rotated credentials", func([]string) bool { + return proxy.accepted("muse", "new-pass") >= 2 // the 407 retry, then the next redial + }) + d.stop() + logs := d.out.String() + if !strings.Contains(logs, "proxy rejected the credentials; retrying with refreshed ones") { + t.Errorf("no 407 retry logged:\n%s", logs) + } + for _, secret := range []string{"old-pass", "new-pass"} { + if strings.Contains(logs, secret) { + t.Errorf("daemon output leaks %q:\n%s", secret, logs) + } + } + if !strings.Contains(logs, "refreshed from the proxy command") { + t.Errorf("startup line does not mention the proxy command:\n%s", logs) + } +} + +// A failing -proxy-cmd at startup costs nothing: the launch environment's +// proxy serves until the command succeeds. +func TestProxyCmdFailureFallsBackToEnvironment(t *testing.T) { + proxy := newRefusingProxy(t, "muse", "s3cret") + _, logs := runDaemon(t, proxy, daemonRun{ + env: []string{ + "PILOT_TRANSPORT=compat", + "HTTPS_PROXY=" + fmt.Sprintf("http://muse:s3cret@%s", proxy.ln.Addr()), + "PILOT_PROXY_CMD=exit 3", + }, + await: "CONNECT registry.pilotprotocol.network:443", + }) + if proxy.accepted("muse", "s3cret") == 0 { + t.Error("launch-environment proxy not used after the proxy command failed") + } + if !strings.Contains(logs, "-proxy-cmd failed; using the launch-time proxy") { + t.Errorf("proxy command failure not logged:\n%s", logs) + } +} + +// daemon-proxied-udp-registry-stays-raw-9000: an explicit proxy in udp mode +// sends the registry dial through the proxy, so the compiled-in raw-TCP +// registry moves to registry.pilotprotocol.network:443 over TLS — the +// rule pilotctl's registry route applies — instead of asking a +// CONNECT-443-only proxy for 34.71.57.205:9000. +func TestProxiedUDPRegistryUsesTLS443(t *testing.T) { + proxy := newRefusingProxy(t, "muse", "s3cret") + targets, logs := runDaemon(t, proxy, daemonRun{ + args: []string{ + "--registry", defaultRegistryAddr, + "--beacon", silentUDP(t), + "-transport=udp", + "-proxy", fmt.Sprintf("http://muse:s3cret@%s", proxy.ln.Addr()), + }, + await: "CONNECT registry.pilotprotocol.network:443", + }) + for _, target := range targets { + if strings.Contains(target, "34.71.57.205") { + t.Errorf("proxy was asked for the raw-TCP registry: %q", target) + } + } + if !strings.Contains(logs, "registry_trust=system") { + t.Errorf("TLS registry trust not defaulted to system:\n%s", logs) + } +} + +// compat-explicit-registry-tls-now-pinned-fatal: -transport=compat with an +// explicit -registry-tls but no -registry-trust defaults trust to system +// (as before -proxy existed) instead of exiting on the flag default +// "pinned" without a fingerprint. +func TestCompatExplicitRegistryTLSDefaultsTrust(t *testing.T) { + proxy := newRefusingProxy(t, "muse", "s3cret") + _, logs := runDaemon(t, proxy, daemonRun{ + args: []string{ + "--registry", "reg.pilot.invalid:9443", + "--beacon", defaultBeaconAddr, + "-transport=compat", + "-registry-tls", + }, + env: []string{"HTTPS_PROXY=" + fmt.Sprintf("http://muse:s3cret@%s", proxy.ln.Addr())}, + await: "CONNECT reg.pilot.invalid:9443", + }) + if strings.Contains(logs, "requires RegistryFingerprint") { + t.Errorf("daemon fell to pinned trust without a fingerprint:\n%s", logs) + } +} + +// Service units set PILOT_TRANSPORT_DEFAULT=auto: it applies only when +// nothing else chooses a transport, so config.json still wins. +func TestTransportDefaultEnv(t *testing.T) { + proxy := newRefusingProxy(t, "muse", "s3cret") + proxy.forward("beacon.pilotprotocol.network:443", fakeCompatBeacon(t)) + env := []string{ + "PILOT_TRANSPORT_DEFAULT=auto", + "HTTPS_PROXY=" + fmt.Sprintf("http://muse:s3cret@%s", proxy.ln.Addr()), + } + _, logs := runDaemon(t, proxy, daemonRun{ + args: []string{ + "--registry", defaultRegistryAddr, + "--beacon", silentUDP(t), + "-compat-beacon", defaultCompatBeacon, + "-registry-fingerprint=" + strings.Repeat("00", 32), + }, + env: env, + await: "CONNECT registry.pilotprotocol.network:443", + }) + if !strings.Contains(logs, `msg="transport auto-selected" transport=compat`) || !strings.Contains(logs, "transport_from=PILOT_TRANSPORT_DEFAULT") { + t.Errorf("PILOT_TRANSPORT_DEFAULT=auto not applied:\n%s", logs) + } + + proxy2 := newRefusingProxy(t, "muse", "s3cret") + _, logs = runDaemon(t, proxy2, daemonRun{ + args: []string{ + "--registry", defaultRegistryAddr, + "--beacon", silentUDP(t), + "-registry-fingerprint=" + strings.Repeat("00", 32), + }, + config: `{"transport":"compat"}`, + env: []string{"PILOT_TRANSPORT_DEFAULT=udp", "HTTPS_PROXY=" + fmt.Sprintf("http://muse:s3cret@%s", proxy2.ln.Addr())}, + await: "CONNECT registry.pilotprotocol.network:443", + }) + if !strings.Contains(logs, "transport_from=config") { + t.Errorf("config.json transport lost to PILOT_TRANSPORT_DEFAULT:\n%s", logs) + } +} diff --git a/cmd/daemon/proxy_test.go b/cmd/daemon/proxy_test.go index 02615faf..da641943 100644 --- a/cmd/daemon/proxy_test.go +++ b/cmd/daemon/proxy_test.go @@ -7,6 +7,7 @@ import ( "bytes" "context" "fmt" + "io" "net" "net/http" "os" @@ -45,7 +46,7 @@ func TestResolveProxyPolicy(t *testing.T) { } t.Setenv("HTTPS_PROXY", "http://muse:s3cret@egress.test:3128") - p, err := resolveProxyPolicy("auto", "udp") + p, err := resolveProxyPolicy("auto", "", "udp") if err != nil || p != nil { t.Fatalf("auto/udp = (%v, %v), want (nil, nil)", p, err) } @@ -53,7 +54,7 @@ func TestResolveProxyPolicy(t *testing.T) { t.Errorf("describeProxy(auto/udp) = %q", got) } - p, err = resolveProxyPolicy("auto", "compat") + p, err = resolveProxyPolicy("auto", "", "compat") if err != nil || p == nil || p.Mode() != netproxy.ModeAuto || !p.Enabled() { t.Fatalf("auto/compat = (%v, %v), want an enabled auto policy", p, err) } @@ -61,7 +62,7 @@ func TestResolveProxyPolicy(t *testing.T) { t.Errorf("describeProxy(auto/compat) = %q", got) } - p, err = resolveProxyPolicy("http://ops:hunter2@flag.test:8080", "udp") + p, err = resolveProxyPolicy("http://ops:hunter2@flag.test:8080", "", "udp") if err != nil || p == nil || p.Mode() != netproxy.ModeExplicit { t.Fatalf("explicit/udp = (%v, %v), want an explicit policy", p, err) } @@ -69,20 +70,20 @@ func TestResolveProxyPolicy(t *testing.T) { t.Errorf("describeProxy(explicit) = %q", got) } - p, err = resolveProxyPolicy("off", "compat") + p, err = resolveProxyPolicy("off", "", "compat") if err != nil || p == nil || p.Mode() != netproxy.ModeOff { t.Fatalf("off/compat = (%v, %v), want an off policy", p, err) } // A malformed -proxy URL is fatal (operator typo) ... - if _, err := resolveProxyPolicy("ftp://ops:hunter2@flag.test", "compat"); err == nil { + if _, err := resolveProxyPolicy("ftp://ops:hunter2@flag.test", "", "compat"); err == nil { t.Fatal("malformed -proxy URL accepted") } else if strings.Contains(err.Error(), "hunter2") { t.Fatalf("error leaks credentials: %v", err) } // ... but a malformed environment under auto only costs the proxy. t.Setenv("HTTPS_PROXY", "ftp://muse:s3cret@egress.test") - p, err = resolveProxyPolicy("auto", "compat") + p, err = resolveProxyPolicy("auto", "", "compat") if err != nil || p != nil { t.Fatalf("auto/compat with malformed env = (%v, %v), want (nil, nil)", p, err) } @@ -94,24 +95,48 @@ func TestResolveProxyPolicy(t *testing.T) { // refusingProxy records every request it gets and refuses all of them, so // nothing a daemon under test sends ever leaves the machine. type refusingProxy struct { - ln net.Listener - wantAuth string + ln net.Listener - mu sync.Mutex - targets []string - badAuths int - allowed map[string]bool // CONNECT targets answered 200 (then closed) + mu sync.Mutex + wantAuth string + targets []string + badAuths int + goodAuths map[string]int // Proxy-Authorization value -> accepted requests + forwards map[string]string // CONNECT target -> local address tunnelled to + reply407 bool // answer bad credentials with 407 (rotation) } -// allow makes the proxy accept CONNECT target (answer 200, then close the -// tunnel): enough for a reachability check, useless for anything else. -func (p *refusingProxy) allow(target string) { +// forward makes the proxy tunnel CONNECT target to the local address to. +func (p *refusingProxy) forward(target, to string) { p.mu.Lock() defer p.mu.Unlock() - if p.allowed == nil { - p.allowed = map[string]bool{} + if p.forwards == nil { + p.forwards = map[string]string{} } - p.allowed[target] = true + p.forwards[target] = to +} + +// rotate makes the proxy accept only user:pass from now on, answering +// anything else with 407 Proxy Authentication Required — the way a +// sandbox egress proxy rotates its credentials. +func (p *refusingProxy) rotate(user, pass string) { + p.mu.Lock() + defer p.mu.Unlock() + p.wantAuth = basicAuth(user, pass) + p.reply407 = true +} + +// accepted reports how many requests carried user:pass and were accepted. +func (p *refusingProxy) accepted(user, pass string) int { + p.mu.Lock() + defer p.mu.Unlock() + return p.goodAuths[basicAuth(user, pass)] +} + +func basicAuth(user, pass string) string { + req := &http.Request{Header: http.Header{}} + req.SetBasicAuth(user, pass) + return req.Header.Get("Authorization") } func newRefusingProxy(t *testing.T, user, pass string) *refusingProxy { @@ -120,9 +145,7 @@ func newRefusingProxy(t *testing.T, user, pass string) *refusingProxy { if err != nil { t.Fatalf("proxy listen: %v", err) } - req := &http.Request{Header: http.Header{}} - req.SetBasicAuth(user, pass) - p := &refusingProxy{ln: ln, wantAuth: req.Header.Get("Authorization")} + p := &refusingProxy{ln: ln, wantAuth: basicAuth(user, pass), goodAuths: map[string]int{}} var wg sync.WaitGroup wg.Add(1) go func() { @@ -143,17 +166,39 @@ func newRefusingProxy(t *testing.T, user, pass string) *refusingProxy { } p.mu.Lock() p.targets = append(p.targets, r.Method+" "+r.RequestURI) - authOK := r.Header.Get("Proxy-Authorization") == p.wantAuth - if !authOK { + auth := r.Header.Get("Proxy-Authorization") + authOK := auth == p.wantAuth + if authOK { + p.goodAuths[auth]++ + } else { p.badAuths++ } - ok := authOK && r.Method == http.MethodConnect && p.allowed[r.RequestURI] + reply407 := !authOK && p.reply407 + connect := authOK && r.Method == http.MethodConnect + to := "" + if connect { + to = p.forwards[r.RequestURI] + } p.mu.Unlock() - if ok { + switch { + case reply407: + fmt.Fprint(conn, "HTTP/1.1 407 Proxy Authentication Required\r\nProxy-Authenticate: Basic realm=\"muse\"\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + case to != "": + up, err := net.Dial("tcp", to) + if err != nil { + fmt.Fprint(conn, "HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n") + return + } + defer up.Close() + conn.SetDeadline(time.Time{}) fmt.Fprint(conn, "HTTP/1.1 200 Connection established\r\n\r\n") - return + done := make(chan struct{}, 2) + go func() { _, _ = io.Copy(up, conn); done <- struct{}{} }() + go func() { _, _ = io.Copy(conn, up); done <- struct{}{} }() + <-done + default: + fmt.Fprint(conn, "HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") } - fmt.Fprint(conn, "HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") }() } }() @@ -265,6 +310,51 @@ type daemonRun struct { // proxy refuses everything it is not told to allow, so nothing reaches the // network. func runDaemon(t *testing.T, proxy *refusingProxy, run daemonRun) ([]string, string) { + t.Helper() + d := startDaemon(t, run) + defer d.stop() + d.waitFor(t, proxy, 45*time.Second, "proxy never saw "+strconv.Quote(run.await), func(targets []string) bool { + return contains(targets, run.await) + }) + d.stop() + targets, _ := proxy.snapshot() + return targets, d.out.String() +} + +// runningDaemon is a child process running main(). +type runningDaemon struct { + cmd *exec.Cmd + cancel context.CancelFunc + out *syncBuffer + once sync.Once +} + +func (d *runningDaemon) stop() { + d.once.Do(func() { + d.cancel() + _ = d.cmd.Wait() + }) +} + +// waitFor polls cond with the proxy's targets until it holds, failing the +// test with what after timeout. +func (d *runningDaemon) waitFor(t *testing.T, proxy *refusingProxy, timeout time.Duration, what string, cond func(targets []string) bool) { + t.Helper() + deadline := time.Now().Add(timeout) + for { + targets, _ := proxy.snapshot() + if cond(targets) { + return + } + if time.Now().After(deadline) { + t.Fatalf("%s; proxy saw %q\ndaemon output:\n%s", what, targets, d.out.String()) + } + time.Sleep(50 * time.Millisecond) + } +} + +// startDaemon starts main() in a child process (see daemonRun). +func startDaemon(t *testing.T, run daemonRun) *runningDaemon { t.Helper() home := t.TempDir() sockDir, err := os.MkdirTemp("", "pdm") @@ -299,8 +389,7 @@ func runDaemon(t *testing.T, proxy *refusingProxy, run daemonRun) ([]string, str "-motd-feed-url=", ) - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) cmd := exec.CommandContext(ctx, os.Args[0], args...) cmd.Env = append([]string{ runMainEnv + "=1", @@ -310,32 +399,16 @@ func runDaemon(t *testing.T, proxy *refusingProxy, run daemonRun) ([]string, str "PILOT_NO_SKILLINJECT=1", "PILOT_APPSTORE_ROOT=" + filepath.Join(home, "apps"), }, run.env...) - var out syncBuffer - cmd.Stdout = &out - cmd.Stderr = &out + out := &syncBuffer{} + cmd.Stdout = out + cmd.Stderr = out if err := cmd.Start(); err != nil { - t.Fatalf("start daemon: %v", err) - } - defer func() { cancel() - _ = cmd.Wait() - }() - - deadline := time.Now().Add(45 * time.Second) - for { - targets, _ := proxy.snapshot() - if contains(targets, run.await) { - break - } - if time.Now().After(deadline) { - t.Fatalf("proxy never saw %q; saw %q\ndaemon output:\n%s", run.await, targets, out.String()) - } - time.Sleep(50 * time.Millisecond) + t.Fatalf("start daemon: %v", err) } - cancel() - _ = cmd.Wait() - targets, _ := proxy.snapshot() - return targets, out.String() + d := &runningDaemon{cmd: cmd, cancel: cancel, out: out} + t.Cleanup(d.stop) + return d } func contains(list []string, s string) bool { diff --git a/cmd/pilotctl/daemon_transport.go b/cmd/pilotctl/daemon_transport.go index 26625415..c93184ba 100644 --- a/cmd/pilotctl/daemon_transport.go +++ b/cmd/pilotctl/daemon_transport.go @@ -39,11 +39,36 @@ var daemonForwardEnv = []string{ "HTTP_PROXY", "http_proxy", "ALL_PROXY", "all_proxy", "NO_PROXY", "no_proxy", - "PILOT_PROXY", "PILOT_TRANSPORT", + "PILOT_PROXY", "PILOT_PROXY_CMD", "PILOT_TRANSPORT", "PILOT_REGISTRY_TRUST", "PILOT_REGISTRY_FINGERPRINT", "SSL_CERT_FILE", "SSL_CERT_DIR", } +// clearableConfigKeys are the config.json keys `config --set key=` removes +// instead of storing an empty string. +var clearableConfigKeys = map[string]bool{"transport": true, "proxy": true, "proxy_cmd": true} + +// fitTransportToDaemon keeps config.json usable by the pilot-daemon at bin +// after a downgrade (`pilotctl update --pin `): a daemon that +// predates -transport=auto refuses to start with "transport":"auto" in +// config.json, and the pilotctl installed with it never overrides the +// value. Such a config gets transport=udp, the older daemon's default. It +// returns a note for the user, "" when nothing changed. +func fitTransportToDaemon(bin string) string { + cfg := loadConfig() + if t, _ := cfg["transport"].(string); !strings.EqualFold(strings.TrimSpace(t), "auto") { + return "" + } + if ok, known := daemonSupportsAutoTransport(bin); ok || !known { + return "" + } + cfg["transport"] = "udp" + if err := saveConfig(cfg); err != nil { + return fmt.Sprintf("the installed pilot-daemon does not support transport=auto (config.json); set it with: pilotctl config --set transport=udp (%v)", err) + } + return "the installed pilot-daemon predates transport=auto: config.json transport set to udp (after upgrading: pilotctl config --set transport=)" +} + // normalizeTransport lower-cases and validates a transport: udp, compat or // auto ("" stays ""). func normalizeTransport(v string) (string, error) { diff --git a/cmd/pilotctl/main.go b/cmd/pilotctl/main.go index fc95d008..95721b4f 100644 --- a/cmd/pilotctl/main.go +++ b/cmd/pilotctl/main.go @@ -1083,14 +1083,18 @@ Flags: Environment passed through to the daemon (never scrubbed): HTTPS_PROXY https_proxy HTTP_PROXY http_proxy ALL_PROXY all_proxy - NO_PROXY no_proxy PILOT_PROXY PILOT_TRANSPORT PILOT_REGISTRY_TRUST - PILOT_REGISTRY_FINGERPRINT SSL_CERT_FILE SSL_CERT_DIR + NO_PROXY no_proxy PILOT_PROXY PILOT_PROXY_CMD PILOT_TRANSPORT + PILOT_REGISTRY_TRUST PILOT_REGISTRY_FINGERPRINT SSL_CERT_FILE SSL_CERT_DIR A pilot-daemon too old for --transport, --transport auto or --proxy gets the flag dropped (auto becomes udp) with a warning instead of crashing it. Behind an HTTPS proxy with UDP blocked (e.g. hosted agent sandboxes), plain "pilotctl daemon start" picks compat by itself; to skip the UDP probe: pilotctl config --set transport=compat && pilotctl daemon start +If the proxy rotates its credentials, give the daemon a command that prints +the current proxy URL (install.sh does this in hosted agent sandboxes); the +daemon re-runs it every 60s and whenever the proxy answers 407: + pilotctl config --set proxy_cmd="bash -c 'printf %s \"\$https_proxy\"'" `, "daemon stop": `Usage: pilotctl daemon stop @@ -2155,6 +2159,12 @@ func cmdConfig(args []string) { } cfg := loadConfig() cfg[parts[0]] = value + if value == "" && clearableConfigKeys[parts[0]] { + // Empty clears the key: the default applies again (for + // transport, pilotctl's auto), and nothing is left for an + // older pilot-daemon to trip over. + delete(cfg, parts[0]) + } result := map[string]interface{}{"key": parts[0], "value": value} // Leaving compat: an install made with a pilotctl that predated // --transport pointed the registry at the compat TLS host diff --git a/cmd/pilotctl/registry_dial.go b/cmd/pilotctl/registry_dial.go index 8bce1421..133aa430 100644 --- a/cmd/pilotctl/registry_dial.go +++ b/cmd/pilotctl/registry_dial.go @@ -9,29 +9,37 @@ import ( "os" "strings" + "github.com/pilot-protocol/common/driver" "github.com/pilot-protocol/common/netproxy" registry "github.com/pilot-protocol/common/registry/client" "github.com/pilot-protocol/pilotprotocol/internal/proxyconf" ) -// registryRoute is how pilotctl reaches the registry for its own commands -// (lookup, register, rotate-key, the auto-handshake visibility check, -// recovery, ...). It follows the same network the daemon uses, so nothing -// pilotctl does bypasses an egress proxy: +// registryRoute is one way pilotctl reaches the registry for its own +// commands (lookup, register, rotate-key, the auto-handshake visibility +// check, recovery, ...). planRegistryRoutes lists the routes to try, in +// order, following the rules pilot-daemon applies, so pilotctl dials the +// way the daemon on this host does: // -// - the proxy comes from $PILOT_PROXY, else config.json "proxy", else -// "auto" — the environment's HTTPS_PROXY / ALL_PROXY (honouring -// NO_PROXY). pilotctl is a short-lived client whose HTTP calls already -// follow HTTPS_PROXY, so auto applies whatever the daemon's transport; -// set proxy=off to dial directly. Loopback registries are never -// proxied. -// - the compiled-in raw-TCP registry (34.71.57.205:9000) is replaced by -// registry.pilotprotocol.network:443 over TLS when the registry would -// be proxied (proxies allow CONNECT to :443 only) or the transport is -// compat (UDP-blocked hosts are usually TCP-443-only too). That -// address always uses TLS, verified against the system roots, or -// pinned to registry_fingerprint / $PILOT_REGISTRY_FINGERPRINT when -// one is configured (sandboxes without a CA bundle). +// - proxy: $PILOT_PROXY, else config.json "proxy", else "auto". An +// explicit http(s):// URL proxies every registry dial (except +// loopback), whatever the transport; "off" never proxies. "auto" uses +// the environment's HTTPS_PROXY / ALL_PROXY (honoring NO_PROXY) only +// where the daemon would: when the transport is compat — the running +// daemon's (asked over IPC), else $PILOT_TRANSPORT / config.json. On a +// udp host that merely exports a proxy (corporate hosts, where private +// registries live) the registry is dialed directly, as the daemon +// does; with the transport unknown (no daemon, nothing configured) the +// direct dial comes first and the proxy is only the fallback for the +// production registry. A private raw-TCP registry is never sent to the +// environment's proxy unless the transport is compat. +// - address: the compiled-in raw-TCP registry (34.71.57.205:9000) is +// replaced by registry.pilotprotocol.network:443 over TLS whenever it +// would be proxied (proxies allow CONNECT to :443 only) or the +// transport is compat; a direct raw-TCP attempt falls back to that TLS +// registry. That address always uses TLS, verified against the system +// roots, or pinned to registry_fingerprint / $PILOT_REGISTRY_FINGERPRINT +// when one is configured (sandboxes without a CA bundle). type registryRoute struct { Addr string TLS bool @@ -39,10 +47,6 @@ type registryRoute struct { Proxy *netproxy.Resolver // Switched: Addr replaced the raw-TCP default. Switched bool - // FallbackTLS: Addr is the raw-TCP default dialed directly; on failure - // dialRegistry retries the compat TLS registry (UDP- and TCP/9000- - // blocked hosts with direct TCP 443). - FallbackTLS bool } // proxied reports whether dialing r.Addr goes through a proxy. @@ -52,13 +56,17 @@ func (r registryRoute) proxied() bool { // proxyFor returns the redacted proxy for target, "" for a direct dial. func (r registryRoute) proxyFor(target string) string { - if !r.Proxy.Enabled() { + return proxyFor(r.Proxy, target) +} + +func proxyFor(p *netproxy.Resolver, target string) string { + if !p.Enabled() { return "" } if host, _, err := net.SplitHostPort(target); err == nil && proxyconf.IsLoopbackHost(host) { return "" } - u, err := r.Proxy.ProxyForAddr(target) + u, err := p.ProxyForAddr(target) if err != nil || u == nil { return "" } @@ -91,6 +99,41 @@ func configuredTransport(cfg map[string]interface{}) string { return t } +// runningDaemonTransport asks the daemon on this host which transport it +// runs ("udp" or "compat"); "" when no daemon answers or it predates the +// info field. A test seam. +var runningDaemonTransport = func() string { + d, err := driver.Connect(getSocket()) + if err != nil { + return "" + } + defer d.Close() + info, err := d.Info() + if err != nil { + return "" + } + t, _ := info["transport"].(string) + switch t { + case "udp", "compat": + return t + } + return "" +} + +// effectiveTransportFor is the transport the daemon on this host uses: +// the running daemon's, else an explicitly configured udp or compat, else +// "" (unknown: auto, or nothing configured and no daemon). +func effectiveTransportFor(cfg map[string]interface{}) string { + if t := runningDaemonTransport(); t != "" { + return t + } + switch t := configuredTransport(cfg); t { + case "udp", "compat": + return t + } + return "" +} + // registryFingerprintSetting is $PILOT_REGISTRY_FINGERPRINT, else // config.json "registry_fingerprint" — used only when trust is pinned // ($PILOT_REGISTRY_TRUST / config.json "registry_trust", defaulting to @@ -110,28 +153,87 @@ func registryFingerprintSetting(cfg map[string]interface{}) string { return strings.TrimSpace(fp) } -// planRegistryRoute resolves how to reach addr. The error is an invalid -// proxy setting. -func planRegistryRoute(addr string) (registryRoute, error) { +// planRegistryRoutes returns the routes to reach addr, in the order to try +// them (see registryRoute). The error is an invalid proxy setting. +func planRegistryRoutes(addr string) ([]registryRoute, error) { cfg := loadConfig() - policy, err := proxyconf.Resolve(pilotctlProxySpec(cfg)) + spec, err := proxyconf.Normalize(pilotctlProxySpec(cfg)) if err != nil { - return registryRoute{}, err + return nil, err } - r := registryRoute{Addr: strings.TrimSpace(addr), Proxy: policy} - if r.Addr == productionRegistryAddr { - if configuredTransport(cfg) == "compat" || r.proxyFor(compatRegistryAddr) != "" { - r.Addr = compatRegistryAddr + addr = strings.TrimSpace(addr) + fp := registryFingerprintSetting(cfg) + route := func(a string, p *netproxy.Resolver) registryRoute { + r := registryRoute{Addr: a, Proxy: p} + if strings.EqualFold(a, compatRegistryAddr) { + r.TLS, r.Fingerprint = true, fp + } + if a == compatRegistryAddr && addr == productionRegistryAddr { r.Switched = true - } else { - r.FallbackTLS = true } + return r } - if strings.EqualFold(r.Addr, compatRegistryAddr) { - r.TLS = true - r.Fingerprint = registryFingerprintSetting(cfg) + isDefault := addr == productionRegistryAddr + + var policy *netproxy.Resolver // the proxy that applies, nil = direct + var transport string + switch spec { + case proxyconf.Off: + transport = configuredTransport(cfg) + case proxyconf.Auto: + env, envErr := netproxy.FromEnvironment() + if envErr == nil && !env.Enabled() { + // No proxy in the environment: nothing to decide. + transport = configuredTransport(cfg) + break + } + transport = effectiveTransportFor(cfg) + if transport != "compat" && !(transport == "" && isDefault) { + // udp, or unknown with a private registry: direct only, + // exactly as the daemon dials. The environment's proxy is + // not even parsed, so a malformed one cannot matter. + break + } + if envErr != nil { + return nil, envErr + } + if transport == "compat" { + policy = env + break + } + // Unknown transport, production registry: direct raw TCP first, + // then the TLS registry through the environment's proxy. + tlsRoute := route(compatRegistryAddr, nil) + if proxyFor(env, compatRegistryAddr) != "" { + tlsRoute.Proxy = env + } + return []registryRoute{route(addr, nil), tlsRoute}, nil + default: + explicit, err := proxyconf.Resolve(spec) + if err != nil { + return nil, err + } + policy = explicit + transport = configuredTransport(cfg) } - return r, nil + + if isDefault { + if transport == "compat" || proxyFor(policy, compatRegistryAddr) != "" { + routes := []registryRoute{route(compatRegistryAddr, policy)} + if spec == proxyconf.Auto && proxyFor(policy, compatRegistryAddr) != "" { + routes = append(routes, route(compatRegistryAddr, nil)) + } + return routes, nil + } + return []registryRoute{route(addr, policy), route(compatRegistryAddr, policy)}, nil + } + routes := []registryRoute{route(addr, policy)} + if spec == proxyconf.Auto && proxyFor(policy, addr) != "" { + // compat through the environment's proxy failed: the host may + // still reach the registry directly. + routes = append(routes, route(addr, nil)) + } + return routes, nil } // dial opens the registry connection the route describes. @@ -149,25 +251,27 @@ func (r registryRoute) dial() (*registry.Client, error) { return registry.DialTLS(r.Addr, &tls.Config{MinVersion: tls.VersionTLS12}, opts...) } -// dialRegistry connects to the registry at addr along its registryRoute, -// retrying the compat TLS registry once when a direct dial of the raw-TCP -// default fails. The returned route is the one that was tried last. +// dialRegistry connects to the registry at addr along the first working +// route of planRegistryRoutes. On failure the route and error reported are +// the first proxied attempt's (a proxy refusing the CONNECT is the likely +// cause), else the first attempt's. func dialRegistry(addr string) (*registry.Client, registryRoute, error) { - route, err := planRegistryRoute(addr) + routes, err := planRegistryRoutes(addr) if err != nil { - return nil, route, err + return nil, registryRoute{}, err } - rc, err := route.dial() - if err == nil || !route.FallbackTLS { - return rc, route, err - } - fallback := route - fallback.Addr, fallback.TLS, fallback.Switched, fallback.FallbackTLS = compatRegistryAddr, true, true, false - fallback.Fingerprint = registryFingerprintSetting(loadConfig()) - if rc, ferr := fallback.dial(); ferr == nil { - return rc, fallback, nil + var failed registryRoute + var firstErr error + for i, route := range routes { + rc, err := route.dial() + if err == nil { + return rc, route, nil + } + if i == 0 || (route.proxied() && !failed.proxied()) { + failed, firstErr = route, err + } } - return nil, route, err + return nil, failed, firstErr } // registryDialHint says what to check after a failed registry dial. diff --git a/cmd/pilotctl/updates.go b/cmd/pilotctl/updates.go index 69bc5634..39243328 100644 --- a/cmd/pilotctl/updates.go +++ b/cmd/pilotctl/updates.go @@ -326,15 +326,30 @@ func cmdUpdate(args []string) { u.RunOnce() + // A pinned older release may predate settings config.json holds. + note := "" + if bin := filepath.Join(installDir, "pilot-daemon"); pin != "" { + if _, err := os.Stat(bin); err == nil { + note = fitTransportToDaemon(bin) + } + } + if jsonOutput { - outputOK(map[string]interface{}{ + fields := map[string]interface{}{ "install_dir": installDir, "repo": repo, "pinned": pin != "", - }) + } + if note != "" { + fields["note"] = note + } + outputOK(fields) return } fmt.Printf("Update check complete. Install dir: %s\n", installDir) + if note != "" { + fmt.Printf("Note: %s\n", note) + } // In manual mode (no daemon running), re-run skill install so skills // match the (possibly updated) binaries. diff --git a/cmd/pilotctl/zz_daemon_transport_test.go b/cmd/pilotctl/zz_daemon_transport_test.go index 4db644f0..c033da77 100644 --- a/cmd/pilotctl/zz_daemon_transport_test.go +++ b/cmd/pilotctl/zz_daemon_transport_test.go @@ -20,9 +20,28 @@ func withTransportEnvCleared(t *testing.T) string { for _, k := range daemonForwardEnv { t.Setenv(k, "") } + // Never ask a real daemon on this machine which transport it runs. + stubDaemonTransport(t, "") return tmp } +// stubDaemonTransport makes runningDaemonTransport report transport. +func stubDaemonTransport(t *testing.T, transport string) { + t.Helper() + prev := runningDaemonTransport + runningDaemonTransport = func() string { return transport } + t.Cleanup(func() { runningDaemonTransport = prev }) +} + +// planRegistryRoute is the first route planRegistryRoutes would try. +func planRegistryRoute(addr string) (registryRoute, error) { + routes, err := planRegistryRoutes(addr) + if err != nil { + return registryRoute{}, err + } + return routes[0], nil +} + func argsHasPair(args []string, key, value string) bool { for i := 0; i+1 < len(args); i++ { if args[i] == key && args[i+1] == value { @@ -587,6 +606,7 @@ func TestRegistryDialHintMentionsProxy(t *testing.T) { t.Errorf("no-proxy hint = %q", h) } t.Setenv("HTTPS_PROXY", "http://agent:s3cret@egress:3128") + t.Setenv("PILOT_TRANSPORT", "compat") // the environment's proxy applies to compat route, err = planRegistryRoute("r.example:9000") if err != nil { t.Fatal(err) diff --git a/cmd/pilotctl/zz_proxy_route_test.go b/cmd/pilotctl/zz_proxy_route_test.go index 52218fc8..0aa667e6 100644 --- a/cmd/pilotctl/zz_proxy_route_test.go +++ b/cmd/pilotctl/zz_proxy_route_test.go @@ -163,59 +163,106 @@ func TestCLIDaemonStartCredProxyBeatsConfigAuto(t *testing.T) { } } -func TestPlanRegistryRoute(t *testing.T) { +// routeSpec renders a route for comparison: "addr[/tls][/pin][ via proxy]". +func routeSpec(r registryRoute) string { + s := r.Addr + if r.TLS { + s += "/tls" + } + if r.Fingerprint != "" { + s += "/pin" + } + if p := r.proxyFor(r.Addr); p != "" { + s += " via proxy" + } + return s +} + +func TestPlanRegistryRoutes(t *testing.T) { const fp = "c1f958f6bcff667cf6a08d5066cc031a9086115a7667835877ca62a3019b3da9" + const ( + raw = productionRegistryAddr + tlsReg = compatRegistryAddr + "/tls" + tlsProxy = compatRegistryAddr + "/tls via proxy" + ) for _, tc := range []struct { - name string - addr string - env map[string]string - cfg map[string]interface{} - wantAddr string - wantTLS bool - proxied bool - wantFP string + name string + addr string + env map[string]string + cfg map[string]interface{} + daemon string // transport the running daemon reports + want []string }{ - {name: "plain host: raw default, direct", addr: productionRegistryAddr, - wantAddr: productionRegistryAddr}, - {name: "HTTPS_PROXY: compat TLS registry through the proxy", addr: productionRegistryAddr, - env: map[string]string{"HTTPS_PROXY": "http://u:p@egress.test:3128"}, - wantAddr: compatRegistryAddr, wantTLS: true, proxied: true}, - {name: "config transport=compat, no proxy: compat TLS registry direct", addr: productionRegistryAddr, - cfg: map[string]interface{}{"transport": "compat"}, - wantAddr: compatRegistryAddr, wantTLS: true}, - {name: "PILOT_PROXY=off beats HTTPS_PROXY", addr: productionRegistryAddr, - env: map[string]string{"HTTPS_PROXY": "http://egress.test:3128", "PILOT_PROXY": "off"}, - wantAddr: productionRegistryAddr}, - {name: "config proxy=none", addr: productionRegistryAddr, - env: map[string]string{"HTTPS_PROXY": "http://egress.test:3128"}, - cfg: map[string]interface{}{"proxy": "none"}, - wantAddr: productionRegistryAddr}, - // NO_PROXY exempts the TLS registry's name, so the raw default is - // kept; the raw IP itself is not exempt and still goes via the proxy. - {name: "NO_PROXY exempts the TLS registry name", addr: productionRegistryAddr, - env: map[string]string{"HTTPS_PROXY": "http://egress.test:3128", "NO_PROXY": ".pilotprotocol.network"}, - wantAddr: productionRegistryAddr, proxied: true}, - {name: "NO_PROXY exempting both", addr: productionRegistryAddr, - env: map[string]string{"HTTPS_PROXY": "http://egress.test:3128", "NO_PROXY": ".pilotprotocol.network,34.71.57.205"}, - wantAddr: productionRegistryAddr}, + {name: "plain host: raw default, then the TLS registry", addr: raw, + want: []string{raw, tlsReg}}, + {name: "HTTPS_PROXY, transport unknown: direct first, proxied TLS fallback", addr: raw, + env: map[string]string{"HTTPS_PROXY": "http://u:p@egress.test:3128"}, + want: []string{raw, tlsProxy}}, + {name: "HTTPS_PROXY + config compat: proxied TLS, then direct TLS", addr: raw, + env: map[string]string{"HTTPS_PROXY": "http://u:p@egress.test:3128"}, + cfg: map[string]interface{}{"transport": "compat"}, + want: []string{tlsProxy, tlsReg}}, + {name: "HTTPS_PROXY + running compat daemon (Muse)", addr: raw, + env: map[string]string{"HTTPS_PROXY": "http://u:p@egress.test:3128"}, + cfg: map[string]interface{}{"transport": "auto"}, + daemon: "compat", + want: []string{tlsProxy, tlsReg}}, + {name: "HTTPS_PROXY + running udp daemon: direct like the daemon", addr: raw, + env: map[string]string{"HTTPS_PROXY": "http://u:p@egress.test:3128"}, + daemon: "udp", + want: []string{raw, tlsReg}}, + {name: "running daemon beats config", addr: raw, + env: map[string]string{"HTTPS_PROXY": "http://u:p@egress.test:3128"}, + cfg: map[string]interface{}{"transport": "compat"}, + daemon: "udp", + want: []string{raw, tlsReg}}, + // pilotctl-auto-proxy-regardless-of-transport: a private raw-TCP + // registry on a udp (or unknown) host is dialed directly, as the + // daemon does, even with HTTPS_PROXY exported. + {name: "private registry + HTTPS_PROXY, transport unknown: direct", addr: "10.0.5.120:39000", + env: map[string]string{"HTTPS_PROXY": "http://u:p@egress.test:3128"}, + want: []string{"10.0.5.120:39000"}}, + {name: "private registry + HTTPS_PROXY + udp: direct", addr: "10.0.5.120:39000", + env: map[string]string{"HTTPS_PROXY": "http://u:p@egress.test:3128", "PILOT_TRANSPORT": "udp"}, + want: []string{"10.0.5.120:39000"}}, + {name: "private registry + compat: proxied, then direct", addr: "registry.corp.test:9000", + env: map[string]string{"HTTPS_PROXY": "http://u:p@egress.test:3128", "PILOT_TRANSPORT": "compat"}, + want: []string{"registry.corp.test:9000 via proxy", "registry.corp.test:9000"}}, + {name: "config transport=compat, no proxy: TLS direct", addr: raw, + cfg: map[string]interface{}{"transport": "compat"}, + want: []string{tlsReg}}, + {name: "explicit PILOT_PROXY URL: proxied TLS in any transport", addr: raw, + env: map[string]string{"PILOT_PROXY": "http://u:p@corp.test:3128", "PILOT_TRANSPORT": "udp"}, + want: []string{tlsProxy}}, + {name: "explicit config proxy URL + private registry: proxied", addr: "registry.corp.test:9000", + cfg: map[string]interface{}{"proxy": "http://corp.test:3128"}, + want: []string{"registry.corp.test:9000 via proxy"}}, + {name: "PILOT_PROXY=off beats HTTPS_PROXY", addr: raw, + env: map[string]string{"HTTPS_PROXY": "http://egress.test:3128", "PILOT_PROXY": "off", "PILOT_TRANSPORT": "compat"}, + want: []string{tlsReg}}, + {name: "config proxy=none", addr: raw, + env: map[string]string{"HTTPS_PROXY": "http://egress.test:3128"}, + cfg: map[string]interface{}{"proxy": "none"}, + want: []string{raw, tlsReg}}, + {name: "NO_PROXY exempts the TLS registry name (compat)", addr: raw, + env: map[string]string{"HTTPS_PROXY": "http://egress.test:3128", "NO_PROXY": ".pilotprotocol.network", "PILOT_TRANSPORT": "compat"}, + want: []string{tlsReg}}, {name: "loopback registry never proxied", addr: "127.0.0.1:9000", - env: map[string]string{"HTTPS_PROXY": "http://egress.test:3128"}, - wantAddr: "127.0.0.1:9000"}, - {name: "custom registry proxied as is", addr: "registry.corp.test:9000", - env: map[string]string{"HTTPS_PROXY": "http://egress.test:3128"}, - wantAddr: "registry.corp.test:9000", proxied: true}, - {name: "fingerprint from env pins", addr: productionRegistryAddr, - env: map[string]string{"HTTPS_PROXY": "http://egress.test:3128", "PILOT_REGISTRY_FINGERPRINT": fp}, - wantAddr: compatRegistryAddr, wantTLS: true, proxied: true, wantFP: fp}, + env: map[string]string{"HTTPS_PROXY": "http://egress.test:3128", "PILOT_TRANSPORT": "compat"}, + want: []string{"127.0.0.1:9000"}}, + {name: "fingerprint from env pins", addr: raw, + env: map[string]string{"HTTPS_PROXY": "http://egress.test:3128", "PILOT_REGISTRY_FINGERPRINT": fp, "PILOT_TRANSPORT": "compat"}, + want: []string{compatRegistryAddr + "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/tls/pin via proxy", compatRegistryAddr + "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/tls/pin"}}, {name: "fingerprint from config, trust=system wins", addr: compatRegistryAddr, - cfg: map[string]interface{}{"registry_fingerprint": fp, "registry_trust": "system"}, - wantAddr: compatRegistryAddr, wantTLS: true}, + cfg: map[string]interface{}{"registry_fingerprint": fp, "registry_trust": "system"}, + want: []string{tlsReg}}, {name: "fingerprint from config pins", addr: compatRegistryAddr, - cfg: map[string]interface{}{"registry_fingerprint": fp}, - wantAddr: compatRegistryAddr, wantTLS: true, wantFP: fp}, + cfg: map[string]interface{}{"registry_fingerprint": fp}, + want: []string{compatRegistryAddr + "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/tls/pin"}}, } { t.Run(tc.name, func(t *testing.T) { withTransportEnvCleared(t) + stubDaemonTransport(t, tc.daemon) for _, k := range []string{"HTTPS_PROXY", "NO_PROXY", "PILOT_PROXY", "PILOT_REGISTRY_FINGERPRINT", "PILOT_REGISTRY_TRUST", "PILOT_TRANSPORT"} { t.Setenv(k, tc.env[k]) } @@ -224,23 +271,55 @@ func TestPlanRegistryRoute(t *testing.T) { t.Fatal(err) } } - r, err := planRegistryRoute(tc.addr) + routes, err := planRegistryRoutes(tc.addr) if err != nil { - t.Fatalf("planRegistryRoute: %v", err) + t.Fatalf("planRegistryRoutes: %v", err) + } + var got []string + for _, r := range routes { + got = append(got, routeSpec(r)) } - if r.Addr != tc.wantAddr || r.TLS != tc.wantTLS || r.proxied() != tc.proxied || r.Fingerprint != tc.wantFP { - t.Errorf("route = {Addr:%s TLS:%v proxied:%v FP:%q}, want {%s %v %v %q}", - r.Addr, r.TLS, r.proxied(), r.Fingerprint, tc.wantAddr, tc.wantTLS, tc.proxied, tc.wantFP) + if strings.Join(got, " | ") != strings.Join(tc.want, " | ") { + t.Errorf("routes = %q, want %q", got, tc.want) } }) } t.Run("invalid proxy setting", func(t *testing.T) { withTransportEnvCleared(t) t.Setenv("PILOT_PROXY", "proxy.test:3128") - if _, err := planRegistryRoute(productionRegistryAddr); err == nil { + if _, err := planRegistryRoutes(productionRegistryAddr); err == nil { t.Error("bare host:port proxy accepted") } }) + t.Run("malformed HTTPS_PROXY does not matter on udp", func(t *testing.T) { + withTransportEnvCleared(t) + t.Setenv("HTTPS_PROXY", "ftp://u:p@egress.test") + t.Setenv("PILOT_TRANSPORT", "udp") + if _, err := planRegistryRoutes("10.0.5.120:39000"); err != nil { + t.Errorf("udp + malformed HTTPS_PROXY: %v", err) + } + }) +} + +// pilotctl-auto-proxy-regardless-of-transport, end to end: a LAN registry +// that worked directly on v1.13.9 still works with HTTPS_PROXY exported on a +// udp host, and the proxy sees nothing. +func TestPrivateRegistryDialedDirectlyWithProxyExported(t *testing.T) { + r := newFakeRegistry(t) + r.onOK("lookup", map[string]interface{}{"node_id": float64(1), "address": "0:0000.0000.0001", "public": true}) + proxyURL, targets := connectProxyToLoopback(t, "muse", "s3cret") + env := cliEnvCleared(map[string]string{ + "PILOT_REGISTRY": r.addr(), + "HTTPS_PROXY": proxyURL, + "PILOT_SOCKET": filepath.Join(t.TempDir(), "none.sock"), + }) + stdout, stderr, code := runCLI(t, []string{"--json", "lookup", "1"}, env) + if code != 0 || !strings.Contains(stdout, "0:0000.0000.0001") { + t.Fatalf("pilotctl lookup: exit=%d stdout=%s stderr=%s", code, stdout, stderr) + } + if got := targets(); len(got) != 0 { + t.Fatalf("the proxy was used for a direct-reachable private registry: %q", got) + } } // connectProxyToLoopback is an authenticating CONNECT proxy that routes @@ -309,6 +388,7 @@ func TestRegistryCommandsUseTheProxy(t *testing.T) { withTransportEnvCleared(t) t.Setenv("HTTPS_PROXY", proxyURL) + stubDaemonTransport(t, "compat") // a compat daemon runs: the proxy applies rc, route, err := dialRegistry(regAddr) if err != nil { t.Fatalf("dialRegistry through the proxy: %v", err) @@ -326,8 +406,10 @@ func TestRegistryCommandsUseTheProxy(t *testing.T) { // The CLI path too: `pilotctl lookup` in a child process. stdout, stderr, code := runCLI(t, []string{"--json", "lookup", "99"}, cliEnvCleared(map[string]string{ - "PILOT_REGISTRY": regAddr, - "HTTPS_PROXY": proxyURL, + "PILOT_REGISTRY": regAddr, + "HTTPS_PROXY": proxyURL, + "PILOT_TRANSPORT": "compat", + "PILOT_SOCKET": filepath.Join(t.TempDir(), "none.sock"), })) if code != 0 || !strings.Contains(stdout, "0:0000.0000.0063") { t.Fatalf("pilotctl lookup via proxy: exit=%d stdout=%s stderr=%s", code, stdout, stderr) @@ -383,3 +465,56 @@ time=x level=INFO msg="outbound network" transport=compat proxy="auto: http://** t.Errorf("no log: (%q, %v)", got, auto) } } + +// version-skew-config-transport-auto-bricks-older-daemon: after +// `pilotctl update --pin `, a config.json "transport":"auto" +// that the installed (older) daemon would refuse is rewritten to udp; a +// daemon that knows auto, or any other value, is left alone. +func TestFitTransportToDaemon(t *testing.T) { + withTransportEnvCleared(t) + dir := t.TempDir() + v1139 := writeFakeDaemon(t, append([]string{v1139TransportUsage}, baseDaemonFlags...), filepath.Join(dir, "v1139")) + current := writeFakeDaemon(t, append([]string{autoUsage, "proxy"}, baseDaemonFlags...), filepath.Join(dir, "cur")) + + if err := saveConfig(map[string]interface{}{"transport": "auto", "email": "a@b.c"}); err != nil { + t.Fatal(err) + } + if note := fitTransportToDaemon(current); note != "" { + t.Errorf("current daemon: note %q, want none", note) + } + if got := loadConfig()["transport"]; got != "auto" { + t.Errorf("current daemon: transport = %v, want auto kept", got) + } + if note := fitTransportToDaemon(v1139); !strings.Contains(note, "transport set to udp") { + t.Errorf("v1.13.9 daemon: note %q", note) + } + cfg := loadConfig() + if cfg["transport"] != "udp" || cfg["email"] != "a@b.c" { + t.Errorf("v1.13.9 daemon: config = %v, want transport=udp and the rest kept", cfg) + } + if err := saveConfig(map[string]interface{}{"transport": "compat"}); err != nil { + t.Fatal(err) + } + if note := fitTransportToDaemon(v1139); note != "" || loadConfig()["transport"] != "compat" { + t.Errorf("compat config changed for v1.13.9: note %q", note) + } +} + +// `config --set transport=` removes the key (the default, auto from +// pilotctl, applies again) instead of storing "" for a daemon to read. +func TestConfigSetEmptyClearsTransportKeys(t *testing.T) { + t.Parallel() + home := t.TempDir() + env := cliEnvCleared(map[string]string{"PILOT_HOME": home}) + for _, kv := range []string{"transport=compat", "proxy=off", "proxy_cmd=cat /run/proxy-url", "transport=", "proxy=", "proxy_cmd="} { + if _, stderr, code := runCLI(t, []string{"config", "--set", kv}, env); code != 0 { + t.Fatalf("config --set %s: exit=%d stderr=%s", kv, code, stderr) + } + } + raw, _ := os.ReadFile(filepath.Join(home, ".pilot", "config.json")) + for _, key := range []string{`"transport"`, `"proxy"`, `"proxy_cmd"`} { + if strings.Contains(string(raw), key) { + t.Errorf("config.json still has %s: %s", key, raw) + } + } +} diff --git a/go.mod b/go.mod index d47c91bd..598e0e28 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/pilot-protocol/trustedagents v0.2.6 github.com/pilot-protocol/updater v0.2.4 github.com/pilot-protocol/webhook v0.2.0 + golang.org/x/net v0.58.0 golang.org/x/sys v0.47.0 ) @@ -83,7 +84,6 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.55.0 // indirect golang.org/x/mod v0.38.0 // indirect - golang.org/x/net v0.58.0 // indirect golang.org/x/sync v0.22.0 // indirect golang.org/x/term v0.45.0 // indirect golang.org/x/text v0.41.0 // indirect diff --git a/install.sh b/install.sh index 563d8070..1822836f 100755 --- a/install.sh +++ b/install.sh @@ -12,7 +12,8 @@ set -e # UDP blocked / curl -fsSL https://pilotprotocol.network/install.sh | sh # HTTPS proxy: (nothing extra: transport "auto" picks TLS/WSS over TCP 443 # through $HTTPS_PROXY when UDP does not work; add -# `-s -- --transport compat` to skip the UDP probe) +# `-s -- --transport compat` to skip the UDP probe; proxy +# credentials that rotate: see PILOT_PROXY_CMD below) # Uninstall: curl -fsSL https://pilotprotocol.network/install.sh | sh -s uninstall # # Flags: @@ -23,12 +24,16 @@ set -e # never silently falls back to an unverified source build. # --yes / -y Skip the older-version confirmation prompt. # --no-warn Suppress the older-version warning entirely. -# --transport auto (default for new installs), udp or compat, saved -# as "transport" in ~/.pilot/config.json. auto: UDP when -# the beacon answers over UDP, else compat. compat: TLS/WSS -# over TCP 443 only, through $HTTPS_PROXY/$ALL_PROXY when set -# (CONNECT by hostname) — for UDP-blocked hosts and agent -# sandboxes whose only way out is an HTTPS proxy. +# --transport auto (the default), udp or compat. udp and compat are +# saved as "transport" in ~/.pilot/config.json; auto is +# never saved (it is what `pilotctl daemon start` and the +# service units use when nothing is saved, and a daemon +# that predates auto would refuse it after a downgrade). +# auto: UDP when the beacon answers over UDP, else compat. +# compat: TLS/WSS over TCP 443 only, through +# $HTTPS_PROXY/$ALL_PROXY when set (CONNECT by hostname) — +# for UDP-blocked hosts and agent sandboxes whose only way +# out is an HTTPS proxy. # # Legacy env vars (still honored, lower precedence than flags): # PILOT_RELEASE_TAG=vX.Y.Z Same as --version. @@ -38,6 +43,13 @@ set -e # If omitted headless, the daemon auto-synthesizes a # @nodes.pilotprotocol.network identity. # PILOT_TRANSPORT=compat Same as --transport compat. +# PILOT_PROXY_CMD= Saved as "proxy_cmd": a command printing the +# current proxy URL, for proxies that rotate their +# credentials. In a Linux container/VM without +# systemd whose HTTPS_PROXY carries credentials +# (hosted agent sandboxes such as Meta Muse), the +# installer saves one that reads a fresh shell's +# $https_proxy when none is set. # PILOT_ALLOW_ROOT=1 Install as root on a host with systemd/launchd # (not needed in containers/VMs without systemd). # @@ -998,7 +1010,6 @@ fi # further below. Guarding on the file itself makes the documented opt-outs # reachable at install time instead of only after the fact. Defaults are # unchanged — a host with no config still gets the standard one. -CONFIG_WRITTEN=false if [ "$UPDATING" != true ] && [ ! -f "$PILOT_DIR/config.json" ]; then cat > "$PILOT_DIR/config.json" </dev/null 2>&1 \ + && ! grep -q '"proxy_cmd"' "$PILOT_DIR/config.json" 2>/dev/null; then + case "$PILOT_PROXY_URL" in + *@*) PROXY_CMD_TO_SAVE="$SANDBOX_PROXY_CMD" ;; + esac +fi +if [ -n "$PROXY_CMD_TO_SAVE" ]; then + if [ "$DAEMON_HAS_PROXY_CMD" != true ]; then + echo " Note: this pilot-daemon (${TAG:-source}) predates -proxy-cmd; if the proxy rotates its" + echo " credentials, restart the daemon from a fresh shell when it starts failing." + elif pilot_config_set "proxy_cmd=$PROXY_CMD_TO_SAVE"; then + echo "Proxy credentials: re-read by the daemon via proxy_cmd (${PILOT_DIR}/config.json)" + fi +fi +PROXY_CMD_SAVED=false +if grep -q '"proxy_cmd"' "$PILOT_DIR/config.json" 2>/dev/null; then + PROXY_CMD_SAVED=true fi if [ "$EFFECTIVE_TRANSPORT" = "compat" ]; then @@ -1120,6 +1185,23 @@ else NET_FLAGS="-registry $REGISTRY -beacon $BEACON" fi +# The service units ask for transport auto through PILOT_TRANSPORT_DEFAULT: +# it applies only when neither -transport, $PILOT_TRANSPORT nor config.json +# chooses, and a daemon that predates auto ignores it (a -transport auto +# flag would stop it from starting after a downgrade). +UNIT_ENV="" +PLIST_ENV="" +if [ "$DAEMON_HAS_AUTO" = true ]; then + UNIT_ENV=" +Environment=PILOT_TRANSPORT_DEFAULT=auto" + PLIST_ENV=" EnvironmentVariables + + PILOT_TRANSPORT_DEFAULT + auto + +" +fi + # service_proxy_note UNIT — a service manager starts the daemon with its own # environment, not this shell's, so an HTTPS_PROXY exported here never # reaches it. config.json (0600, read by the daemon itself) does. @@ -1198,7 +1280,7 @@ Wants=network-online.target [Service] Type=simple -User=$(whoami) +User=$(whoami)${UNIT_ENV} ExecStart=${BIN_DIR}/pilot-daemon \\ ${NET_FLAGS} \\ -listen :4000 \\ @@ -1362,7 +1444,7 @@ ${PLIST_NET_ARGS} -listen ${PILOT_DIR}/identity.json -encrypt ${EXTRA_ARGS} - RunAtLoad +${PLIST_ENV} RunAtLoad KeepAlive @@ -1563,6 +1645,9 @@ case "$EFFECTIVE_TRANSPORT" in esac if [ "$EFFECTIVE_TRANSPORT" != "udp" ] && [ -n "$PILOT_PROXY_URL" ]; then echo " Proxy: auto -> $(redact_proxy "$PILOT_PROXY_URL") (from environment)" + if [ "$PROXY_CMD_SAVED" = true ]; then + echo " credentials re-read by the daemon (proxy_cmd): rotation needs no restart" + fi fi echo " Socket: /tmp/pilot.sock" echo " Identity: ${PILOT_DIR}/identity.json" diff --git a/internal/proxyconf/policy.go b/internal/proxyconf/policy.go new file mode 100644 index 00000000..b04e247e --- /dev/null +++ b/internal/proxyconf/policy.go @@ -0,0 +1,525 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package proxyconf + +import ( + "bytes" + "context" + "crypto/tls" + "errors" + "fmt" + "log/slog" + "net" + "net/http" + "net/url" + "os" + "os/exec" + "runtime" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/pilot-protocol/common/netproxy" + "golang.org/x/net/http/httpproxy" +) + +// DefaultRefreshInterval is how often a command-backed Policy re-runs its +// command. Sandboxes that rotate proxy credentials (Meta Muse: every few +// minutes) accept the previous credentials for a while, and a 407 forces +// an immediate refresh anyway. +const DefaultRefreshInterval = 60 * time.Second + +// commandTimeout bounds one run of the proxy command. +const commandTimeout = 10 * time.Second + +// commandOutputLimit caps what is read from the proxy command's stdout. +const commandOutputLimit = 64 << 10 + +// Policy is an outbound proxy policy that can change while the process +// runs. It is either static (a fixed netproxy.Resolver) or backed by a +// command whose stdout is the current proxy URL — for egress proxies whose +// credentials rotate (a long-lived daemon keeps the credentials of its +// launch environment, and the proxy starts answering new CONNECTs with 407 +// Proxy Authentication Required a few minutes later). +// +// A command-backed Policy re-runs the command every refresh interval (Run) +// and whenever the proxy rejects the credentials; its dialers and HTTP +// helpers then retry once with the new URL. Connections already open are +// unaffected: established tunnels survive a rotation. +// +// A nil *Policy never proxies. Loopback targets are never proxied. +// Policies are safe for concurrent use. +type Policy struct { + cur atomic.Pointer[policyState] + + // Command-backed policies only. + command string + run func(ctx context.Context, command string) (string, error) + exempt func(scheme, hostport string) bool + mu sync.Mutex // serializes refreshes +} + +type policyState struct { + resolver *netproxy.Resolver + // raw is the command's last URL ("" for the static fallback). Only + // ever compared, never logged. + raw string + gen uint64 +} + +// Static returns a Policy that always uses r. nil r gives a nil Policy (no +// policy: callers keep their historical behaviour). +func Static(r *netproxy.Resolver) *Policy { + if r == nil { + return nil + } + p := &Policy{} + p.cur.Store(&policyState{resolver: r}) + return p +} + +// NewCommand returns a Policy backed by command, a shell command line +// (/bin/sh -c; cmd /C on Windows) whose stdout is the current proxy URL, +// http://[user:pass@]host[:port] or https://.... The command runs once now. +// Its URL is used for every target except loopback and, with honorNoProxy +// (the -proxy=auto semantics), the NO_PROXY / no_proxy exemptions of this +// process's environment. +// +// When that first run fails, fallback (typically the environment's proxy, +// which was fresh when the process started) is used until a refresh +// succeeds, and the error is returned along with the Policy; the Policy is +// usable either way. +func NewCommand(ctx context.Context, command string, fallback *netproxy.Resolver, honorNoProxy bool) (*Policy, error) { + return newCommandPolicy(ctx, command, fallback, honorNoProxy, runProxyCommand) +} + +func newCommandPolicy(ctx context.Context, command string, fallback *netproxy.Resolver, honorNoProxy bool, run func(context.Context, string) (string, error)) (*Policy, error) { + command = strings.TrimSpace(command) + if command == "" { + return nil, errors.New("empty proxy command") + } + p := &Policy{command: command, run: run} + if honorNoProxy { + p.exempt = noProxyMatcher(noProxyFromEnv()) + } + p.cur.Store(&policyState{resolver: fallback}) + if _, err := p.refresh(ctx, 0, true); err != nil { + return p, err + } + return p, nil +} + +// Refreshable reports whether the Policy re-reads its proxy (it is backed +// by a command). +func (p *Policy) Refreshable() bool { return p != nil && p.command != "" } + +// Resolver returns the current resolver (nil when there is none). +func (p *Policy) Resolver() *netproxy.Resolver { + s := p.state() + if s == nil { + return nil + } + return s.resolver +} + +func (p *Policy) state() *policyState { + if p == nil { + return nil + } + return p.cur.Load() +} + +// Enabled reports whether any target may be proxied. A command-backed +// Policy is always enabled: its next refresh can supply a proxy. +func (p *Policy) Enabled() bool { + if p == nil { + return false + } + return p.Refreshable() || p.Resolver().Enabled() +} + +// Mode reports netproxy.ModeAuto, ModeOff or ModeExplicit for the current +// resolver. +func (p *Policy) Mode() string { return p.Resolver().Mode() } + +// Warnings reports the current resolver's skipped environment variables. +func (p *Policy) Warnings() []error { return p.Resolver().Warnings() } + +// String describes the Policy for logs, credentials redacted. +func (p *Policy) String() string { + if p == nil { + return "none" + } + s := p.Resolver().String() + if p.Resolver() == nil { + s = "none yet" + } + if p.Refreshable() { + s += " (refreshed from the proxy command)" + } + return s +} + +// Refresh re-runs the proxy command now. changed reports whether the proxy +// URL differs from the one in use. A static Policy never changes. On error +// the previous proxy stays in use. +func (p *Policy) Refresh(ctx context.Context) (changed bool, err error) { + if !p.Refreshable() { + return false, nil + } + return p.refresh(ctx, 0, false) +} + +// refresh runs the command unless another refresh already replaced the +// state generation `seen` (seen == 0: always run), and installs the new +// URL when it differs. It reports whether the URL in use is now different +// from generation `seen`. +func (p *Policy) refresh(ctx context.Context, seen uint64, initial bool) (bool, error) { + p.mu.Lock() + defer p.mu.Unlock() + cur := p.cur.Load() + if seen != 0 && cur.gen != seen { + return true, nil // someone refreshed while we waited + } + raw, err := p.run(ctx, p.command) + if err != nil { + return false, err + } + if raw == cur.raw && cur.raw != "" { + return false, nil + } + r, err := netproxy.Explicit(raw) + if err != nil { + return false, fmt.Errorf("proxy command printed an unusable URL: %v", unwrapNetproxy(err)) + } + next := &policyState{resolver: r, raw: raw, gen: cur.gen + 1} + p.cur.Store(next) + if !initial { + old := "none" + if cur.resolver != nil { + old = cur.resolver.String() + } + if old != r.String() { + slog.Info("proxy changed (proxy command)", "proxy", r.String(), "was", old) + } else { + slog.Debug("proxy credentials refreshed (proxy command)", "proxy", r.String()) + } + } + return true, nil +} + +// refreshAfterAuthFailure refreshes after the proxy rejected the +// credentials of generation seen, and reports whether a retry would use a +// different URL. +func (p *Policy) refreshAfterAuthFailure(ctx context.Context, seen uint64) bool { + if !p.Refreshable() { + return false + } + changed, err := p.refresh(ctx, seen, false) + if err != nil { + slog.Warn("proxy rejected the credentials and the proxy command failed; keeping the previous proxy", "err", err) + return false + } + return changed +} + +// Run re-runs the proxy command every interval (DefaultRefreshInterval when +// interval <= 0) until ctx ends. It returns at once for a static Policy. +func (p *Policy) Run(ctx context.Context, interval time.Duration) { + if !p.Refreshable() { + return + } + if interval <= 0 { + interval = DefaultRefreshInterval + } + t := time.NewTicker(interval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + if _, err := p.Refresh(ctx); err != nil && ctx.Err() == nil { + slog.Warn("proxy command failed; keeping the previous proxy", "err", err) + } + } + } +} + +// bypass reports whether target (host, port) goes direct whatever the +// resolver says: loopback always, and for a command-backed Policy the +// NO_PROXY exemptions. +func (p *Policy) bypass(scheme, host, port string) bool { + if IsLoopbackHost(host) { + return true + } + return p != nil && p.exempt != nil && p.exempt(scheme, net.JoinHostPort(host, port)) +} + +// Proxies reports whether a TCP connection to addr ("host:port") would go +// through a proxy. A command-backed Policy proxies every target it does +// not exempt, also before its command first succeeds. +func (p *Policy) Proxies(addr string) bool { + if !p.Enabled() { + return false + } + host, port, err := net.SplitHostPort(addr) + if err != nil || p.bypass("https", host, port) { + return false + } + if p.Refreshable() { + return true + } + u, err := p.Resolver().ProxyForAddr(addr) + return err == nil && u != nil +} + +// DialContext returns a dial function that tunnels through the proxy the +// Policy picks for each target (CONNECT by host name) and dials loopback +// and unproxied targets directly. When the proxy rejects the credentials +// (407) of a command-backed Policy, the command is re-run and the dial is +// retried once with the new URL. proxyTLS configures the TLS session with +// an https:// proxy (nil: system roots); it never applies to the target. +func (p *Policy) DialContext(proxyTLS *tls.Config) func(ctx context.Context, network, addr string) (net.Conn, error) { + var direct net.Dialer + return func(ctx context.Context, network, addr string) (net.Conn, error) { + if host, port, err := net.SplitHostPort(addr); err == nil && p.bypass("https", host, port) { + return direct.DialContext(ctx, network, addr) + } + s := p.state() + var r *netproxy.Resolver + var gen uint64 + if s != nil { + r, gen = s.resolver, s.gen + } + conn, err := (&netproxy.Dialer{Resolver: r, TLSConfig: proxyTLS}).DialContext(ctx, network, addr) + if err == nil || !IsProxyAuthError(err) || !p.refreshAfterAuthFailure(ctx, gen) { + return conn, err + } + slog.Info("proxy rejected the credentials; retrying with refreshed ones", "target", addr) + return (&netproxy.Dialer{Resolver: p.Resolver(), TLSConfig: proxyTLS}).DialContext(ctx, network, addr) + } +} + +// RequestProxy returns an http.Transport.Proxy function that follows the +// Policy's current resolver and never proxies loopback targets. nil for a +// nil Policy (net/http's own environment handling then applies wherever +// the caller leaves Proxy unset). +func (p *Policy) RequestProxy() func(*http.Request) (*url.URL, error) { + if p == nil { + return nil + } + return func(req *http.Request) (*url.URL, error) { + if req == nil || req.URL == nil { + return nil, nil + } + scheme := "https" + port := req.URL.Port() + switch strings.ToLower(req.URL.Scheme) { + case "http", "ws": + scheme = "http" + if port == "" { + port = "80" + } + default: + if port == "" { + port = "443" + } + } + if p.bypass(scheme, req.URL.Hostname(), port) { + return nil, nil + } + return p.Resolver().ProxyForRequest(req) + } +} + +// ErrProxyAuth is returned for requests whose proxy CONNECT was answered +// with 407 Proxy Authentication Required (see ConfigureTransport). +var ErrProxyAuth = errors.New("proxy CONNECT: 407 Proxy Authentication Required") + +// ConfigureTransport routes tr through the Policy: tr.Proxy follows the +// current resolver, and a 407 answer to a CONNECT makes a command-backed +// Policy refresh at once, so the next connection uses the new credentials +// (the failed request returns ErrProxyAuth; RoundTripper retries it). A nil +// Policy leaves tr alone. +func (p *Policy) ConfigureTransport(tr *http.Transport) { + if p == nil || tr == nil { + return + } + tr.Proxy = p.RequestProxy() + if !p.Refreshable() { + return + } + tr.OnProxyConnectResponse = func(ctx context.Context, _ *url.URL, _ *http.Request, resp *http.Response) error { + if resp.StatusCode != http.StatusProxyAuthRequired { + return nil + } + s := p.state() + p.refreshAfterAuthFailure(ctx, s.gen) + return ErrProxyAuth + } +} + +// RoundTripper wraps base (a transport set up with ConfigureTransport) so +// that a request whose CONNECT the proxy rejected with 407 is retried once +// when the refreshed Policy has a different URL. Requests with a body are +// retried only when it can be replayed (GetBody). For a Policy that cannot +// refresh it returns base. +func (p *Policy) RoundTripper(base http.RoundTripper) http.RoundTripper { + if !p.Refreshable() || base == nil { + return base + } + return &retryAuthTransport{p: p, base: base} +} + +type retryAuthTransport struct { + p *Policy + base http.RoundTripper +} + +func (t *retryAuthTransport) RoundTrip(req *http.Request) (*http.Response, error) { + s := t.p.state() + resp, err := t.base.RoundTrip(req) + if err == nil || !IsProxyAuthError(err) { + return resp, err + } + if s.gen == t.p.state().gen && !t.p.refreshAfterAuthFailure(req.Context(), s.gen) { + return resp, err + } + retry := req + if req.Body != nil && req.Body != http.NoBody { + if req.GetBody == nil { + return resp, err + } + body, berr := req.GetBody() + if berr != nil { + return resp, err + } + retry = req.Clone(req.Context()) + retry.Body = body + } + return t.base.RoundTrip(retry) +} + +// IsProxyAuthError reports whether err is a proxy's rejection of the +// credentials: a 407 answer to CONNECT, or the malformed status line some +// sandbox egress proxies send in its place. +func IsProxyAuthError(err error) bool { + if err == nil { + return false + } + var ce *netproxy.ConnectError + if errors.As(err, &ce) { + return ce.StatusCode == http.StatusProxyAuthRequired + } + if errors.Is(err, ErrProxyAuth) { + return true + } + msg := err.Error() + return strings.Contains(msg, "Proxy Authentication Required") || + strings.Contains(msg, "malformed HTTP status code") +} + +// runProxyCommand runs command with the platform shell and returns the +// first line of its stdout, validated as an http(s) proxy URL. Neither the +// output nor stderr is ever included in errors or logs. +func runProxyCommand(ctx context.Context, command string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, commandTimeout) + defer cancel() + // #nosec G204 -- the operator's own -proxy-cmd / PILOT_PROXY_CMD / + // config.json proxy_cmd, run with the daemon's own privileges. + var cmd *exec.Cmd + if runtime.GOOS == "windows" { + cmd = exec.CommandContext(ctx, "cmd", "/C", command) // #nosec G204 + } else { + cmd = exec.CommandContext(ctx, "/bin/sh", "-c", command) // #nosec G204 + } + cmd.Env = commandEnv(os.Environ()) + var out limitedBuffer + cmd.Stdout = &out + cmd.WaitDelay = time.Second + if err := cmd.Run(); err != nil { + if ctx.Err() == context.DeadlineExceeded { + return "", fmt.Errorf("proxy command timed out after %s", commandTimeout) + } + return "", fmt.Errorf("proxy command failed: %v", err) + } + line := strings.TrimSpace(out.String()) + if i := strings.IndexAny(line, "\r\n"); i >= 0 { + line = strings.TrimSpace(line[:i]) + } + if line == "" { + return "", errors.New("proxy command printed nothing") + } + s, err := Normalize(line) + if err != nil { + return "", fmt.Errorf("proxy command output: %v", err) + } + if s == Auto || s == Off { + return "", errors.New("proxy command output: want an http:// or https:// proxy URL") + } + return s, nil +} + +// commandEnv is the proxy command's environment: this process's, minus the +// daemon secrets the command has no business seeing. +func commandEnv(env []string) []string { + out := make([]string, 0, len(env)) + for _, kv := range env { + k, _, _ := strings.Cut(kv, "=") + switch k { + case "PILOT_ADMIN_TOKEN", "PILOT_WEBHOOK_SECRET": + continue + } + out = append(out, kv) + } + return out +} + +type limitedBuffer struct{ bytes.Buffer } + +func (b *limitedBuffer) Write(p []byte) (int, error) { + if room := commandOutputLimit - b.Len(); room > 0 { + if len(p) > room { + b.Buffer.Write(p[:room]) + } else { + b.Buffer.Write(p) + } + } + return len(p), nil +} + +func noProxyFromEnv() string { + for _, k := range []string{"NO_PROXY", "no_proxy"} { + if v := strings.TrimSpace(os.Getenv(k)); v != "" { + return v + } + } + return "" +} + +// noProxyMatcher returns a NO_PROXY matcher (net/http semantics: host +// names match themselves and their subdomains, ".x" and "*.x" subdomains +// only, IPs, CIDRs, optional ":port", "*" everything), nil for an empty +// list. +func noProxyMatcher(list string) func(scheme, hostport string) bool { + if strings.TrimSpace(list) == "" { + return nil + } + entries := strings.FieldsFunc(list, func(r rune) bool { return r == ',' || r == ' ' }) + for i, e := range entries { + if strings.HasPrefix(e, "*.") { + entries[i] = e[1:] + } + } + const sentinel = "http://proxy.invalid" + pf := (&httpproxy.Config{HTTPProxy: sentinel, HTTPSProxy: sentinel, NoProxy: strings.Join(entries, ",")}).ProxyFunc() + return func(scheme, hostport string) bool { + if scheme != "http" { + scheme = "https" + } + u, err := pf(&url.URL{Scheme: scheme, Host: hostport}) + return err == nil && u == nil + } +} diff --git a/internal/proxyconf/policy_test.go b/internal/proxyconf/policy_test.go new file mode 100644 index 00000000..211bcb9d --- /dev/null +++ b/internal/proxyconf/policy_test.go @@ -0,0 +1,501 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package proxyconf + +import ( + "bufio" + "bytes" + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/pilot-protocol/common/netproxy" +) + +// rotatingProxy is a CONNECT proxy that accepts one password at a time +// (407 for any other) and tunnels every accepted CONNECT to upstream. +type rotatingProxy struct { + ln net.Listener + upstream string + + mu sync.Mutex + pass string + oks map[string]int // password -> accepted CONNECTs + n407 int + sawNo int // CONNECTs without credentials +} + +func newRotatingProxy(t *testing.T, pass, upstream string) *rotatingProxy { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + p := &rotatingProxy{ln: ln, upstream: upstream, pass: pass, oks: map[string]int{}} + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go p.serve(c) + } + }() + t.Cleanup(func() { ln.Close() }) + return p +} + +func (p *rotatingProxy) url(pass string) string { + return fmt.Sprintf("http://muse:%s@%s", pass, p.ln.Addr()) +} + +func (p *rotatingProxy) rotate(pass string) { + p.mu.Lock() + p.pass = pass + p.mu.Unlock() +} + +func (p *rotatingProxy) stats() (oks map[string]int, n407 int) { + p.mu.Lock() + defer p.mu.Unlock() + m := map[string]int{} + for k, v := range p.oks { + m[k] = v + } + return m, p.n407 +} + +func (p *rotatingProxy) serve(c net.Conn) { + defer c.Close() + br := bufio.NewReader(c) + req, err := http.ReadRequest(br) + if err != nil { + return + } + req.Header.Set("Authorization", req.Header.Get("Proxy-Authorization")) + user, pass, ok := req.BasicAuth() + p.mu.Lock() + good := ok && user == "muse" && pass == p.pass + switch { + case !ok: + p.sawNo++ + case good: + p.oks[pass]++ + default: + p.n407++ + } + p.mu.Unlock() + if req.Method != http.MethodConnect || !good { + fmt.Fprint(c, "HTTP/1.1 407 Proxy Authentication Required\r\nProxy-Authenticate: Basic realm=\"t\"\r\nContent-Length: 0\r\n\r\n") + return + } + up, err := net.Dial("tcp", p.upstream) + if err != nil { + fmt.Fprint(c, "HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n") + return + } + defer up.Close() + fmt.Fprint(c, "HTTP/1.1 200 Connection established\r\n\r\n") + done := make(chan struct{}, 2) + go func() { _, _ = io.Copy(up, br); done <- struct{}{} }() + go func() { _, _ = io.Copy(c, up); done <- struct{}{} }() + <-done +} + +// fakeSource is a proxy command stand-in: it prints *url, counting runs. +type fakeSource struct { + mu sync.Mutex + url string + err error + runs int +} + +func (f *fakeSource) set(u string, err error) { + f.mu.Lock() + f.url, f.err = u, err + f.mu.Unlock() +} + +func (f *fakeSource) run(context.Context, string) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.runs++ + return f.url, f.err +} + +func (f *fakeSource) count() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.runs +} + +func echoServer(t *testing.T) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go func() { defer c.Close(); _, _ = io.Copy(c, c) }() + } + }() + return ln.Addr().String() +} + +func clearEnv(t *testing.T) { + for _, k := range []string{"HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy", "ALL_PROXY", "all_proxy", "NO_PROXY", "no_proxy"} { + t.Setenv(k, "") + } +} + +func TestStaticAndNilPolicy(t *testing.T) { + var nilPolicy *Policy + if Static(nil) != nil { + t.Fatal("Static(nil) is not nil") + } + if nilPolicy.Enabled() || nilPolicy.Refreshable() || nilPolicy.Resolver() != nil || nilPolicy.RequestProxy() != nil { + t.Fatal("nil policy is not inert") + } + if nilPolicy.String() != "none" || nilPolicy.Proxies("registry.pilot.invalid:443") { + t.Fatal("nil policy describes itself as proxying") + } + if changed, err := nilPolicy.Refresh(context.Background()); changed || err != nil { + t.Fatal("nil policy refreshed") + } + nilPolicy.Run(context.Background(), time.Millisecond) // returns at once + tr := &http.Transport{} + nilPolicy.ConfigureTransport(tr) + if tr.Proxy != nil { + t.Fatal("nil policy configured a transport") + } + + r, _ := netproxy.Explicit("http://u:secret@proxy.test:3128") + p := Static(r) + if !p.Enabled() || p.Refreshable() || p.Mode() != netproxy.ModeExplicit { + t.Fatalf("static policy = enabled %v refreshable %v mode %s", p.Enabled(), p.Refreshable(), p.Mode()) + } + if strings.Contains(p.String(), "secret") { + t.Fatalf("String leaks the password: %s", p.String()) + } + if !p.Proxies("registry.pilot.invalid:443") || p.Proxies("127.0.0.1:9000") || p.Proxies("localhost:1") { + t.Fatal("Proxies ignores the loopback exemption") + } + if rt := p.RoundTripper(http.DefaultTransport); rt != http.DefaultTransport { + t.Fatal("a static policy wraps the round tripper") + } +} + +func TestCommandPolicyRefresh(t *testing.T) { + clearEnv(t) + src := &fakeSource{url: "http://muse:one@proxy.test:3128"} + p, err := newCommandPolicy(context.Background(), "cmd", nil, true, src.run) + if err != nil { + t.Fatal(err) + } + if !p.Refreshable() || !p.Enabled() || p.Mode() != netproxy.ModeExplicit { + t.Fatalf("command policy: refreshable %v enabled %v mode %s", p.Refreshable(), p.Enabled(), p.Mode()) + } + if s := p.String(); strings.Contains(s, "one") || !strings.Contains(s, "refreshed from the proxy command") { + t.Fatalf("String = %q", s) + } + if changed, err := p.Refresh(context.Background()); changed || err != nil { + t.Fatalf("unchanged refresh = (%v, %v)", changed, err) + } + src.set("http://muse:two@proxy.test:3128", nil) + if changed, err := p.Refresh(context.Background()); !changed || err != nil { + t.Fatalf("rotated refresh = (%v, %v)", changed, err) + } + u, _ := p.Resolver().ProxyForAddr("registry.pilot.invalid:443") + if pw, _ := u.User.Password(); pw != "two" { + t.Fatalf("password after refresh = %q, want two", pw) + } + // A failing command keeps the current proxy. + src.set("", errors.New("exit status 1")) + if changed, err := p.Refresh(context.Background()); changed || err == nil { + t.Fatalf("failing refresh = (%v, %v)", changed, err) + } + u, _ = p.Resolver().ProxyForAddr("registry.pilot.invalid:443") + if pw, _ := u.User.Password(); pw != "two" { + t.Fatalf("password after a failed refresh = %q, want two", pw) + } + if _, err := newCommandPolicy(context.Background(), " ", nil, true, src.run); err == nil { + t.Fatal("empty command accepted") + } +} + +// A failing first run serves the fallback until the command succeeds. +func TestCommandPolicyFallback(t *testing.T) { + clearEnv(t) + src := &fakeSource{err: errors.New("exit status 127")} + fallback, _ := netproxy.Explicit("http://muse:launch@proxy.test:3128") + p, err := newCommandPolicy(context.Background(), "cmd", fallback, true, src.run) + if err == nil || p == nil { + t.Fatalf("first run failure = (%v, %v), want a usable policy and the error", p, err) + } + if p.Resolver() != fallback { + t.Fatal("fallback not in use") + } + p2, _ := newCommandPolicy(context.Background(), "cmd", nil, true, src.run) + if !p2.Enabled() || !p2.Proxies("registry.pilot.invalid:443") || p2.String() != "none yet (refreshed from the proxy command)" { + t.Fatalf("no-fallback policy: enabled %v, %q", p2.Enabled(), p2.String()) + } + src.set("http://muse:fresh@proxy.test:3128", nil) + if changed, err := p.Refresh(context.Background()); !changed || err != nil { + t.Fatalf("refresh = (%v, %v)", changed, err) + } +} + +// The heart of muse-proxy-cred-rotation-unhandled: a dial whose CONNECT +// the proxy rejects with 407 re-runs the command and retries once. +func TestCommandPolicyDialRetriesOn407(t *testing.T) { + clearEnv(t) + echo := echoServer(t) + proxy := newRotatingProxy(t, "one", echo) + src := &fakeSource{url: proxy.url("one")} + p, err := newCommandPolicy(context.Background(), "cmd", nil, true, src.run) + if err != nil { + t.Fatal(err) + } + dial := p.DialContext(nil) + ping := func() error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + c, err := dial(ctx, "tcp", "registry.pilot.invalid:443") + if err != nil { + return err + } + defer c.Close() + if _, err := c.Write([]byte("ping")); err != nil { + return err + } + buf := make([]byte, 4) + if _, err := io.ReadFull(c, buf); err != nil || string(buf) != "ping" { + return fmt.Errorf("echo = (%q, %v)", buf, err) + } + return nil + } + if err := ping(); err != nil { + t.Fatalf("dial before the rotation: %v", err) + } + + proxy.rotate("two") + src.set(proxy.url("two"), nil) + runs := src.count() + if err := ping(); err != nil { + t.Fatalf("dial after the rotation: %v", err) + } + oks, n407 := proxy.stats() + if n407 != 1 || oks["two"] != 1 { + t.Fatalf("proxy: 407s %d, accepted %v; want one 407, then the refreshed credentials", n407, oks) + } + if src.count() != runs+1 { + t.Fatalf("command runs = %d, want one refresh", src.count()-runs) + } + + // Credentials that are really wrong: one refresh, no retry (same URL), + // and the 407 error comes back. + proxy.rotate("three") + if err := ping(); err == nil || !IsProxyAuthError(err) { + t.Fatalf("dial with unrefreshable credentials = %v, want the 407", err) + } + if _, n407 := proxy.stats(); n407 != 2 { + t.Fatalf("407s = %d, want 2 (no retry with the same URL)", n407) + } +} + +// NO_PROXY applies to a command policy with -proxy=auto semantics only; +// loopback always goes direct. +func TestCommandPolicyNoProxy(t *testing.T) { + clearEnv(t) + t.Setenv("NO_PROXY", "*.corp.example,10.0.0.0/8") + src := &fakeSource{url: "http://muse:x@proxy.test:3128"} + auto, _ := newCommandPolicy(context.Background(), "cmd", nil, true, src.run) + explicit, _ := newCommandPolicy(context.Background(), "cmd", nil, false, src.run) + for addr, want := range map[string]bool{ + "registry.pilotprotocol.network:443": true, + "git.corp.example:443": false, + "10.1.2.3:9000": false, + "127.0.0.1:9000": false, + } { + if got := auto.Proxies(addr); got != want { + t.Errorf("auto: Proxies(%s) = %v, want %v", addr, got, want) + } + } + if !explicit.Proxies("git.corp.example:443") || explicit.Proxies("localhost:80") { + t.Error("explicit command policy: NO_PROXY applied or loopback proxied") + } + pf := auto.RequestProxy() + for raw, want := range map[string]bool{ + "https://raw.githubusercontent.com/x": true, + "https://git.corp.example/x": false, + "http://127.0.0.1:8080/hook": false, + "ws://10.0.0.7/x": false, + } { + u, _ := url.Parse(raw) + got, err := pf(&http.Request{URL: u}) + if err != nil || (got != nil) != want { + t.Errorf("RequestProxy(%s) = (%v, %v), want proxied=%v", raw, got, err, want) + } + } +} + +// HTTP clients: the transport follows the refreshed URL, a 407 on CONNECT +// refreshes at once, and RoundTripper retries the request (replaying a +// body when it can). +func TestCommandPolicyHTTPRetriesOn407(t *testing.T) { + clearEnv(t) + var hits atomic.Int32 + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + body, _ := io.ReadAll(r.Body) + fmt.Fprintf(w, "%s %s", r.Method, body) + })) + defer srv.Close() + proxy := newRotatingProxy(t, "one", srv.Listener.Addr().String()) + src := &fakeSource{url: proxy.url("one")} + p, err := newCommandPolicy(context.Background(), "cmd", nil, true, src.run) + if err != nil { + t.Fatal(err) + } + tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} // test server + p.ConfigureTransport(tr) + client := &http.Client{Transport: p.RoundTripper(tr), Timeout: 10 * time.Second} + defer tr.CloseIdleConnections() + + get := func(method string, body io.Reader) (string, error) { + req, err := http.NewRequest(method, "https://app.pilot.invalid/api", body) + if err != nil { + return "", err + } + req.Close = true // a new CONNECT per request + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + return string(b), err + } + if got, err := get(http.MethodGet, nil); err != nil || got != "GET " { + t.Fatalf("GET before the rotation = (%q, %v)", got, err) + } + proxy.rotate("two") + src.set(proxy.url("two"), nil) + if got, err := get(http.MethodPost, bytes.NewReader([]byte("payload"))); err != nil || got != "POST payload" { + t.Fatalf("POST after the rotation = (%q, %v)", got, err) + } + oks, n407 := proxy.stats() + if n407 != 1 || oks["two"] != 1 { + t.Fatalf("proxy: 407s %d, accepted %v", n407, oks) + } + + // A body that cannot be replayed is not retried. + proxy.rotate("three") + src.set(proxy.url("three"), nil) + if _, err := get(http.MethodPost, io.NopCloser(strings.NewReader("once"))); err == nil || !IsProxyAuthError(err) { + t.Fatalf("unreplayable POST after a rotation = %v, want the 407", err) + } + // ... but the refresh happened: the next request goes through. + if got, err := get(http.MethodGet, nil); err != nil || got != "GET " { + t.Fatalf("GET after the refresh = (%q, %v)", got, err) + } + if n := hits.Load(); n != 3 { + t.Fatalf("server hits = %d, want 3", n) + } +} + +func TestCommandPolicyRun(t *testing.T) { + clearEnv(t) + src := &fakeSource{url: "http://muse:one@proxy.test:3128"} + p, _ := newCommandPolicy(context.Background(), "cmd", nil, true, src.run) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { p.Run(ctx, 10*time.Millisecond); close(done) }() + src.set("http://muse:two@proxy.test:3128", nil) + deadline := time.Now().Add(3 * time.Second) + for { + u, _ := p.Resolver().ProxyForAddr("x.test:443") + if pw, _ := u.User.Password(); pw == "two" { + break + } + if time.Now().After(deadline) { + t.Fatal("Run never refreshed") + } + time.Sleep(5 * time.Millisecond) + } + cancel() + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("Run did not stop with its context") + } +} + +func TestIsProxyAuthError(t *testing.T) { + for _, tc := range []struct { + err error + want bool + }{ + {nil, false}, + {&netproxy.ConnectError{Target: "x:443", StatusCode: 407}, true}, + {fmt.Errorf("dial: %w", &netproxy.ConnectError{Target: "x:443", StatusCode: 407}), true}, + {&netproxy.ConnectError{Target: "x:443", StatusCode: 403}, false}, + {fmt.Errorf("Get x: %w", ErrProxyAuth), true}, + {errors.New("Proxy Authentication Required"), true}, + {errors.New(`read CONNECT response: malformed HTTP status code "Proxy"`), true}, + {errors.New("connection refused"), false}, + } { + if got := IsProxyAuthError(tc.err); got != tc.want { + t.Errorf("IsProxyAuthError(%v) = %v, want %v", tc.err, got, tc.want) + } + } +} + +// The real command runner: /bin/sh -c, first line of stdout, validated; +// nothing it prints ever shows up in an error. +func TestRunProxyCommand(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("uses /bin/sh") + } + t.Setenv("PILOT_ADMIN_TOKEN", "admin-secret") + ctx := context.Background() + got, err := runProxyCommand(ctx, `printf 'http://muse:%s@proxy.test:3128\nsecond line\n' "${PILOT_ADMIN_TOKEN:-hidden}"; echo noise >&2`) + if err != nil || got != "http://muse:hidden@proxy.test:3128" { + t.Fatalf("runProxyCommand = (%q, %v), want the first line, without the admin token", got, err) + } + for cmd, want := range map[string]string{ + "exit 3": "exit status 3", + "true": "printed nothing", + "echo auto": "want an http:// or https:// proxy URL", + "echo muse:s3cret@proxy.test:3128": "invalid proxy", + "echo socks5://u:s3cret@p:1080": "scheme must be http or https", + "echo s3cret; exit 1": "exit status 1", + } { + _, err := runProxyCommand(ctx, cmd) + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("runProxyCommand(%q) = %v, want an error containing %q", cmd, err, want) + continue + } + if strings.Contains(err.Error(), "s3cret") { + t.Errorf("runProxyCommand(%q) error leaks output: %v", cmd, err) + } + } +} diff --git a/internal/proxyconf/proxyconf.go b/internal/proxyconf/proxyconf.go index 1dd45965..977a180d 100644 --- a/internal/proxyconf/proxyconf.go +++ b/internal/proxyconf/proxyconf.go @@ -139,17 +139,9 @@ func IsLoopbackHost(host string) bool { // RequestProxy returns an http.Transport.Proxy function that follows r but // never proxies loopback targets. nil for a nil resolver (net/http's own // environment handling then applies wherever the caller leaves Proxy -// unset). +// unset). It is Static(r).RequestProxy(). func RequestProxy(r *netproxy.Resolver) func(*http.Request) (*url.URL, error) { - if r == nil { - return nil - } - return func(req *http.Request) (*url.URL, error) { - if req != nil && req.URL != nil && IsLoopbackHost(req.URL.Hostname()) { - return nil, nil - } - return r.ProxyForRequest(req) - } + return Static(r).RequestProxy() } // DialContext returns a dial function that tunnels through the proxy r @@ -157,16 +149,10 @@ func RequestProxy(r *netproxy.Resolver) func(*http.Request) (*url.URL, error) { // dials loopback targets, and targets r does not proxy, directly. // proxyTLS configures the TLS session with an https:// proxy — never the // target's TLS, which the caller runs end to end over the returned conn; -// nil verifies the proxy against the system roots. +// nil verifies the proxy against the system roots. It is +// Static(r).DialContext(proxyTLS). func DialContext(r *netproxy.Resolver, proxyTLS *tls.Config) func(ctx context.Context, network, addr string) (net.Conn, error) { - d := &netproxy.Dialer{Resolver: r, TLSConfig: proxyTLS} - var direct net.Dialer - return func(ctx context.Context, network, addr string) (net.Conn, error) { - if host, _, err := net.SplitHostPort(addr); err == nil && IsLoopbackHost(host) { - return direct.DialContext(ctx, network, addr) - } - return d.DialContext(ctx, network, addr) - } + return Static(r).DialContext(proxyTLS) } // unwrapNetproxy drops netproxy's "netproxy: " prefix for messages that diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index e6b65a6f..b9b380a8 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -224,6 +224,14 @@ type Config struct { // directly and HTTP fetches follow net/http's proxy environment. Proxy *netproxy.Resolver + // ProxyPolicy, when non-nil, replaces Proxy with a policy that can + // change while the daemon runs — NewCommandProxyPolicy re-reads the + // proxy URL from a command, for egress proxies that rotate their + // credentials. Start keeps it refreshed every ProxyRefreshInterval + // (0 = 60s) until Stop. + ProxyPolicy *ProxyPolicy + ProxyRefreshInterval time.Duration + // systemRoots replaces the OS trust store behind the "system" trust // settings (RegistryTrust, CompatTLSTrust). Test seam only: it is // always nil outside this package's tests. @@ -881,10 +889,10 @@ func (d *Daemon) Start() error { // Compat would use the environment's proxy (-proxy=auto) // unless the embedder set a policy; check reachability the // same way. - policy := d.config.Proxy + policy := d.proxyPolicy() if policy == nil { if p, err := ResolveProxy(proxyAutoSpec, TransportCompat); err == nil { - policy = p + policy = StaticProxyPolicy(p) } else { slog.Warn("proxy environment unusable; compat check dials directly", "error", err) } @@ -896,8 +904,8 @@ func (d *Daemon) Start() error { }) if mode == TransportCompat { d.config.TransportMode = TransportCompat - if d.config.Proxy == nil { - d.config.Proxy = policy + if d.config.ProxyPolicy == nil && d.config.Proxy == nil { + d.config.ProxyPolicy = policy } slog.Warn("UDP probe to beacon failed — auto-falling back to compat mode (WSS/443)", "beacon", stunBeacon, @@ -926,6 +934,11 @@ func (d *Daemon) Start() error { return fmt.Errorf("invalid -transport %q: must be 'udp' or 'compat'", d.config.TransportMode) } + // A command-backed proxy policy (rotating proxy credentials) stays + // current for the daemon's lifetime: registry redials, WSS reconnects + // and HTTP fetches always see the latest proxy URL. + d.startProxyRefresh() + var registrationAddr string if d.config.TransportMode == "compat" { registrationAddr = "0.0.0.0:0" // placeholder — relay_only daemons hide this from peers @@ -2860,6 +2873,19 @@ type DaemonInfo struct { BeaconAddr string // active beacon address MOTD string // message-of-the-day active for the current UTC day ("" = none) + + // Transport is the tunnel transport the daemon runs: "udp" or "compat" + // (after -transport=auto was resolved). + Transport string +} + +// transportName is the resolved tunnel transport, "udp" or "compat". Only +// meaningful after Start has resolved it. +func (d *Daemon) transportName() string { + if d.config.TransportMode == TransportCompat { + return TransportCompat + } + return TransportUDP } // Info returns current daemon status. @@ -2952,6 +2978,7 @@ func (d *Daemon) Info() *DaemonInfo { RelayPeerCount: len(d.tunnels.RelayPeerIDs()), BeaconAddr: d.config.BeaconAddr, MOTD: d.currentMOTD(), + Transport: d.transportName(), } } diff --git a/pkg/daemon/ipc.go b/pkg/daemon/ipc.go index 3383d024..14b8a3c8 100644 --- a/pkg/daemon/ipc.go +++ b/pkg/daemon/ipc.go @@ -1186,6 +1186,7 @@ func (s *IPCServer) handleInfo(conn *ipcConn, reqID uint64) { "relay_peer_count": info.RelayPeerCount, "beacon_addr": info.BeaconAddr, "motd": info.MOTD, + "transport": info.Transport, }) if err != nil { s.sendError(conn, reqID, fmt.Sprintf("info marshal: %v", err)) diff --git a/pkg/daemon/proxy.go b/pkg/daemon/proxy.go index 939d7699..08475c23 100644 --- a/pkg/daemon/proxy.go +++ b/pkg/daemon/proxy.go @@ -15,8 +15,27 @@ import ( "github.com/pilot-protocol/pilotprotocol/internal/proxyconf" ) +// ProxyPolicy is an outbound proxy policy that can change while the daemon +// runs: a fixed resolver (StaticProxyPolicy) or one refreshed from a +// command (NewCommandProxyPolicy) for egress proxies that rotate their +// credentials. Config.ProxyPolicy takes precedence over Config.Proxy. +type ProxyPolicy = proxyconf.Policy + +// StaticProxyPolicy wraps a fixed resolver. nil gives nil (no policy). +func StaticProxyPolicy(r *netproxy.Resolver) *ProxyPolicy { return proxyconf.Static(r) } + +// NewCommandProxyPolicy returns a policy whose proxy URL is the stdout of +// command (run with /bin/sh -c), re-run every refresh interval and when the +// proxy answers 407 (see proxyconf.NewCommand). fallback serves until the +// command first succeeds; the error reports a failed first run and is not +// fatal: the policy is usable either way. honorNoProxy applies the +// environment's NO_PROXY (the -proxy=auto semantics). +func NewCommandProxyPolicy(ctx context.Context, command string, fallback *netproxy.Resolver, honorNoProxy bool) (*ProxyPolicy, error) { + return proxyconf.NewCommand(ctx, command, fallback, honorNoProxy) +} + // ResolveProxy turns a -proxy setting into the daemon's outbound proxy -// policy (Config.Proxy) for the given transport mode: +// resolver (Config.Proxy) for the given transport mode: // // - "auto" or "": with transportMode "compat", the proxy from the // environment — HTTPS_PROXY / https_proxy, falling back to ALL_PROXY / @@ -54,19 +73,28 @@ func (d *Daemon) proxyTLSConfig() *tls.Config { return &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: d.config.systemRoots} } +// proxyPolicy is the daemon's outbound proxy policy: Config.ProxyPolicy, +// else Config.Proxy, else nil (no policy). +func (d *Daemon) proxyPolicy() *ProxyPolicy { + if d.config.ProxyPolicy != nil { + return d.config.ProxyPolicy + } + return proxyconf.Static(d.config.Proxy) +} + // proxyDialer returns the dial function for raw TCP connections the daemon // opens itself (the registry, the compat WSS beacon), or nil when there is // no policy or it proxies nothing. Loopback targets are dialed directly. func (d *Daemon) proxyDialer() func(ctx context.Context, network, addr string) (net.Conn, error) { - return d.dialerFor(d.config.Proxy) + return d.dialerFor(d.proxyPolicy()) } // dialerFor is proxyDialer for an arbitrary policy. -func (d *Daemon) dialerFor(policy *netproxy.Resolver) func(ctx context.Context, network, addr string) (net.Conn, error) { +func (d *Daemon) dialerFor(policy *ProxyPolicy) func(ctx context.Context, network, addr string) (net.Conn, error) { if !policy.Enabled() { return nil } - return proxyconf.DialContext(policy, d.proxyTLSConfig()) + return policy.DialContext(d.proxyTLSConfig()) } // registryDialOptions routes every registry connection — the primary, each @@ -84,23 +112,33 @@ func (d *Daemon) registryDialOptions() []registry.DialOption { // the daemon makes itself, or nil when there is no policy. Loopback targets // always go direct. func (d *Daemon) httpProxyFunc() func(*http.Request) (*url.URL, error) { - return proxyconf.RequestProxy(d.config.Proxy) + return d.proxyPolicy().RequestProxy() } // newHTTPClient returns a client for daemon-owned HTTP fetches. Without a // proxy policy it is a plain client on http.DefaultTransport, as before; -// with one, its transport routes through the policy. +// with one, its transport routes through the policy (and, for a refreshed +// policy, retries a request whose CONNECT got 407 once with the new +// credentials). func (d *Daemon) newHTTPClient(timeout time.Duration) *http.Client { client := &http.Client{Timeout: timeout} - if proxy := d.httpProxyFunc(); proxy != nil { + if policy := d.proxyPolicy(); policy != nil { var tr *http.Transport if base, ok := http.DefaultTransport.(*http.Transport); ok { tr = base.Clone() } else { tr = &http.Transport{} } - tr.Proxy = proxy - client.Transport = tr + policy.ConfigureTransport(tr) + client.Transport = policy.RoundTripper(tr) } return client } + +// startProxyRefresh keeps a command-backed proxy policy current for the +// daemon's lifetime (no-op otherwise). +func (d *Daemon) startProxyRefresh() { + if p := d.proxyPolicy(); p.Refreshable() { + go p.Run(d.ctx, d.config.ProxyRefreshInterval) + } +} diff --git a/pkg/daemon/transport_auto.go b/pkg/daemon/transport_auto.go index 4d07021c..52ba986c 100644 --- a/pkg/daemon/transport_auto.go +++ b/pkg/daemon/transport_auto.go @@ -3,11 +3,14 @@ package daemon import ( + "bufio" "context" + "crypto/tls" "crypto/x509" "errors" "fmt" "net" + "net/http" "net/url" "strings" "time" @@ -35,9 +38,10 @@ func NormalizeTransport(v string) (string, error) { return "", fmt.Errorf("invalid transport %q: must be udp, compat or auto", v) } -// defaultCompatCheckTimeout bounds the TCP reachability check of the -// compat beacon in SelectTransport. -const defaultCompatCheckTimeout = 5 * time.Second +// defaultCompatCheckTimeout bounds the compat beacon check in +// SelectTransport: TCP connect (through the proxy: its CONNECT), TLS +// handshake and one HTTP exchange. +const defaultCompatCheckTimeout = 8 * time.Second // AutoTransportProbe configures SelectTransport. type AutoTransportProbe struct { @@ -47,14 +51,14 @@ type AutoTransportProbe struct { // CompatBeaconURL is the WSS beacon compat mode would dial. Empty // means compat is not available and auto always picks udp. CompatBeaconURL string - // Dial opens the compat reachability check, normally through the - // proxy policy compat mode would use. nil dials directly. + // Dial opens the connection for the compat beacon check, normally + // through the proxy policy compat mode would use. nil dials directly. Dial func(ctx context.Context, network, addr string) (net.Conn, error) // UDPTimeout bounds the UDP probe (0 = udpProbeTimeout). A reply // returns as soon as it arrives, so on a working UDP path the probe // costs one round trip. UDPTimeout time.Duration - // TCPTimeout bounds the compat check (0 = 5s). It only runs when the + // TCPTimeout bounds the compat check (0 = 8s). It only runs when the // UDP probe got no answer. TCPTimeout time.Duration } @@ -63,12 +67,17 @@ type AutoTransportProbe struct { // // - udp when the beacon answers a UDP discover — the transport a daemon // would have used anyway, found in one round trip; -// - otherwise compat, when the compat beacon's host:port accepts a TCP -// connection through p.Dial (i.e. through the egress proxy, if there -// is one): the host blocks UDP but can reach TCP 443; +// - otherwise compat, when the compat beacon itself answers through +// p.Dial (i.e. through the egress proxy, if there is one): a TLS +// handshake and a plain GET of the compat path return 426 Upgrade +// Required, which only a live WebSocket endpoint sends. A bare TCP +// connect proves nothing: the production host is an SNI-routing front +// that accepts TCP while the beacon behind it is down, and through a +// proxy it is only the proxy's CONNECT 200; // - otherwise udp, exactly as before -transport=auto existed: nothing // is reachable yet (no network at boot, beacon outage), and a compat -// daemon could not start either. +// daemon could not start either, while a udp daemon starts degraded +// and registers. // // reason is a short, log-ready explanation of the choice. func SelectTransport(ctx context.Context, p AutoTransportProbe) (mode, reason string) { @@ -96,17 +105,70 @@ func SelectTransport(ctx context.Context, p AutoTransportProbe) (mode, reason st } cctx, cancel := context.WithTimeout(ctx, tcpTimeout) defer cancel() - dial := p.Dial + if err := checkCompatBeacon(cctx, p.Dial, p.CompatBeaconURL, target); err != nil { + return TransportUDP, fmt.Sprintf("no UDP answer from beacon %s, and compat beacon %s did not answer (%v)", beacon, p.CompatBeaconURL, err) + } + return TransportCompat, fmt.Sprintf("no UDP answer from beacon %s within %s; compat beacon %s answered", beacon, udpTimeout, p.CompatBeaconURL) +} + +// checkCompatBeacon proves that the WebSocket beacon at rawURL is alive: +// it opens target through dial (nil: direct), runs TLS for wss:// and +// sends a plain GET of the beacon path, which a live WebSocket endpoint +// answers with 426 Upgrade Required. Anything else — a front that accepts +// TCP but has no backend (502/503), a proxy error, a closed connection — +// is an error. +// +// The TLS session is not verified: nothing is sent but a GET of a public +// path, and the result only selects the transport. The compat connection +// itself verifies the beacon with the configured trust (and reports a +// missing CA bundle far more clearly than a failed probe could). +func checkCompatBeacon(ctx context.Context, dial func(ctx context.Context, network, addr string) (net.Conn, error), rawURL, target string) error { + u, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil { + return err + } if dial == nil { var d net.Dialer dial = d.DialContext } - conn, err := dial(cctx, "tcp", target) + conn, err := dial(ctx, "tcp", target) + if err != nil { + return err + } + defer conn.Close() + if dl, ok := ctx.Deadline(); ok { + _ = conn.SetDeadline(dl) + } + switch strings.ToLower(u.Scheme) { + case "wss", "https": + tc := tls.Client(conn, &tls.Config{ + ServerName: u.Hostname(), + MinVersion: tls.VersionTLS12, + // #nosec G402 -- liveness probe only; see the doc comment. + InsecureSkipVerify: true, // lgtm[go/disabled-certificate-check] + }) + if err := tc.HandshakeContext(ctx); err != nil { + return fmt.Errorf("tls: %w", err) + } + conn = tc + } + path := u.EscapedPath() + if path == "" { + path = "/" + } + req := "GET " + path + " HTTP/1.1\r\nHost: " + u.Host + "\r\nUser-Agent: pilot-daemon/transport-auto\r\nConnection: close\r\n\r\n" + if _, err := conn.Write([]byte(req)); err != nil { + return err + } + resp, err := http.ReadResponse(bufio.NewReader(conn), &http.Request{Method: http.MethodGet}) if err != nil { - return TransportUDP, fmt.Sprintf("no UDP answer from beacon %s, and compat beacon %s unreachable (%v)", beacon, target, err) + return err + } + resp.Body.Close() + if resp.StatusCode != http.StatusUpgradeRequired { + return fmt.Errorf("HTTP %d, want 426 from a live beacon", resp.StatusCode) } - conn.Close() - return TransportCompat, fmt.Sprintf("no UDP answer from beacon %s within %s; compat beacon %s reachable over TCP", beacon, udpTimeout, target) + return nil } // probeUDPReachableWithin is probeUDPReachable with a caller-chosen bound, diff --git a/pkg/daemon/zz_proxy_refresh_test.go b/pkg/daemon/zz_proxy_refresh_test.go new file mode 100644 index 00000000..87d0200f --- /dev/null +++ b/pkg/daemon/zz_proxy_refresh_test.go @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package daemon + +import ( + "context" + "crypto/x509" + "net/http" + "os" + "path/filepath" + "testing" + "time" +) + +// muse-proxy-cred-rotation-unhandled, in miniature: a compat daemon behind +// an authenticating CONNECT proxy whose credentials rotate while it runs. +// With a command-backed ProxyPolicy the registry reconnect and the WSS +// beacon reconnect after the rotation get 407 once, re-read the proxy URL +// and succeed — no restart. +func TestCommandProxyPolicyFollowsCredentialRotation(t *testing.T) { + clearProxyEnv(t) + proxy := newProxyTestConnect(t, "muse", "old-pass") + urlFile := filepath.Join(t.TempDir(), "proxy-url") + setURL := func(pass string) { + t.Helper() + if err := os.WriteFile(urlFile, []byte(proxy.url("muse", pass)+"\n"), 0o600); err != nil { + t.Fatal(err) + } + } + setURL("old-pass") + policy, err := NewCommandProxyPolicy(context.Background(), "cat '"+urlFile+"'", nil, true) + if err != nil { + t.Fatalf("NewCommandProxyPolicy: %v", err) + } + + roots := x509.NewCertPool() + reg, regAddr, _ := startProxiedRegistry(t, roots) + beacon, beaconHost := startProxiedBeacon(t, roots, reg.LookupPublicKey) + sockDir, err := os.MkdirTemp("", "pdr") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(sockDir) }) + d := New(Config{ + RegistryAddr: regAddr, + RegistryTLS: true, + RegistryTrust: "system", + TransportMode: "compat", + CompatBeaconURL: "wss://" + beaconHost + "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/v1/compat", + CompatTLSTrust: "system", + ProxyPolicy: policy, + ProxyRefreshInterval: time.Hour, // only the 407 path refreshes here + SocketPath: sockDir + "/s", + IdentityPath: t.TempDir() + "/id.json", + Email: "proxy-rotation@example.test", + Encrypt: true, + DisablePolicyRunner: true, + systemRoots: roots, + }) + if err := d.Start(); err != nil { + t.Fatalf("Start behind the proxy: %v", err) + } + t.Cleanup(func() { _ = d.Stop() }) + for len(beacon.snis) > 0 { + <-beacon.snis + } + + // Rotate: the proxy now rejects old-pass with 407. + setURL("new-pass") + proxy.setAuth("muse", "new-pass") + + if err := d.forceReconnectRegistry(); err != nil { + t.Fatalf("registry reconnect after the rotation: %v", err) + } + if _, err := d.reg().Lookup(d.NodeID()); err != nil { + t.Fatalf("Lookup after the rotation: %v", err) + } + if n := proxy.deniedWith(http.StatusProxyAuthRequired); n < 1 { + t.Fatalf("proxy 407s = %d, want the stale credentials rejected at least once", n) + } + + // The WSS beacon drops; its reconnect carries the new credentials. + beacon.dropAll() + select { + case sni := <-beacon.snis: + if sni != "beacon.pilot.invalid" { + t.Fatalf("beacon SNI = %q", sni) + } + case <-time.After(20 * time.Second): + t.Fatalf("the WSS beacon never reconnected through the rotated proxy (proxy: %v)", proxy.counts()) + } +} + +// Run refreshes a command-backed policy periodically, so connections made +// after a rotation carry the new credentials from the start (no 407). +func TestCommandProxyPolicyPeriodicRefresh(t *testing.T) { + clearProxyEnv(t) + proxy := newProxyTestConnect(t, "muse", "old-pass") + _, regAddr, pin := startProxiedRegistry(t, nil) + urlFile := filepath.Join(t.TempDir(), "proxy-url") + if err := os.WriteFile(urlFile, []byte(proxy.url("muse", "old-pass")), 0o600); err != nil { + t.Fatal(err) + } + policy, err := NewCommandProxyPolicy(context.Background(), "cat '"+urlFile+"'", nil, true) + if err != nil { + t.Fatal(err) + } + d := New(Config{ + RegistryAddr: regAddr, + RegistryTLS: true, + RegistryTrust: "pinned", + RegistryFingerprint: pin, + ProxyPolicy: policy, + ProxyRefreshInterval: 50 * time.Millisecond, + }) + d.startProxyRefresh() + t.Cleanup(d.cancelCtx) // ends the refresh loop (the daemon never started) + + if err := os.WriteFile(urlFile, []byte(proxy.url("muse", "new-pass")), 0o600); err != nil { + t.Fatal(err) + } + proxy.setAuth("muse", "new-pass") + deadline := time.Now().Add(5 * time.Second) + for !proxyURLHasPassword(policy, "new-pass") { + if time.Now().After(deadline) { + t.Fatal("periodic refresh never picked up the rotated URL") + } + time.Sleep(20 * time.Millisecond) + } + rc, err := d.dialRegistryClient() + if err != nil { + t.Fatalf("dialRegistryClient after the refresh: %v", err) + } + rc.Close() + if n := proxy.deniedWith(http.StatusProxyAuthRequired); n != 0 { + t.Errorf("proxy 407s = %d, want 0: the refresh ran before the dial", n) + } +} + +func proxyURLHasPassword(p *ProxyPolicy, pass string) bool { + u, err := p.Resolver().ProxyForAddr("registry.pilot.invalid:443") + if err != nil || u == nil { + return false + } + got, _ := u.User.Password() + return got == pass +} + +// pilotctl asks the running daemon which transport it resolved (its +// registry route mirrors the daemon's): info reports udp or compat. +func TestInfoReportsTransport(t *testing.T) { + t.Parallel() + for mode, want := range map[string]string{"": "udp", "udp": "udp", "compat": "compat"} { + if got := New(Config{TransportMode: mode}).Info().Transport; got != want { + t.Errorf("TransportMode %q: Info().Transport = %q, want %q", mode, got, want) + } + } +} diff --git a/pkg/daemon/zz_proxy_test.go b/pkg/daemon/zz_proxy_test.go index 732e0038..15ad69d1 100644 --- a/pkg/daemon/zz_proxy_test.go +++ b/pkg/daemon/zz_proxy_test.go @@ -183,13 +183,31 @@ func TestResolveProxyRejectsMalformedSettings(t *testing.T) { // *.pilot.invalid (names that never resolve locally) to loopback and // records every request target. type proxyTestConnect struct { - ln net.Listener + ln net.Listener + + mu sync.Mutex wantAuth string + targets []string + denied map[int]int // status -> count of refused requests + live []net.Conn + wg sync.WaitGroup +} - mu sync.Mutex - targets []string - live []net.Conn - wg sync.WaitGroup +// setAuth makes the proxy accept only user:pass from now on (a credential +// rotation); anything else gets 407. +func (p *proxyTestConnect) setAuth(user, pass string) { + req := &http.Request{Header: http.Header{}} + req.SetBasicAuth(user, pass) + p.mu.Lock() + p.wantAuth = req.Header.Get("Authorization") + p.mu.Unlock() +} + +// deniedWith returns how many requests the proxy refused with status. +func (p *proxyTestConnect) deniedWith(status int) int { + p.mu.Lock() + defer p.mu.Unlock() + return p.denied[status] } func newProxyTestConnect(t *testing.T, user, pass string) *proxyTestConnect { @@ -259,15 +277,22 @@ func (p *proxyTestConnect) serve(conn net.Conn) { } p.mu.Lock() p.targets = append(p.targets, req.RequestURI) + wantAuth := p.wantAuth p.mu.Unlock() deny := func(code int) { + p.mu.Lock() + if p.denied == nil { + p.denied = map[int]int{} + } + p.denied[code]++ + p.mu.Unlock() fmt.Fprintf(conn, "HTTP/1.1 %d %s\r\nContent-Length: 0\r\n\r\n", code, http.StatusText(code)) } if req.Method != http.MethodConnect { deny(http.StatusMethodNotAllowed) return } - if req.Header.Get("Proxy-Authorization") != p.wantAuth { + if req.Header.Get("Proxy-Authorization") != wantAuth { deny(http.StatusProxyAuthRequired) return } @@ -367,6 +392,20 @@ type proxiedBeacon struct { srv *httptest.Server authed atomic.Uint32 // node ID of the last authenticated daemon snis chan string + + mu sync.Mutex + conns []*websocket.Conn +} + +// dropAll closes every authenticated WSS connection (a beacon restart). +func (b *proxiedBeacon) dropAll() { + b.mu.Lock() + conns := b.conns + b.conns = nil + b.mu.Unlock() + for _, c := range conns { + _ = c.CloseNow() + } } func startProxiedBeacon(t *testing.T, pool *x509.CertPool, lookup func(uint32) ([]byte, bool)) (*proxiedBeacon, string) { @@ -429,6 +468,9 @@ func (b *proxiedBeacon) handle(w http.ResponseWriter, r *http.Request, lookup fu return } b.authed.Store(reply.NodeID) + b.mu.Lock() + b.conns = append(b.conns, conn) + b.mu.Unlock() for { if _, _, err := conn.Read(r.Context()); err != nil { return diff --git a/pkg/daemon/zz_transport_auto_test.go b/pkg/daemon/zz_transport_auto_test.go index 2d162154..c0473199 100644 --- a/pkg/daemon/zz_transport_auto_test.go +++ b/pkg/daemon/zz_transport_auto_test.go @@ -8,6 +8,7 @@ import ( "errors" "net" "net/http" + "net/http/httptest" "net/url" "strings" "testing" @@ -64,6 +65,22 @@ func tcpListener(t *testing.T) string { return ln.Addr().String() } +// compatBeaconStub is a TLS server that answers a plain GET of the compat +// path with status — 426 Upgrade Required is what a live WebSocket beacon +// sends; a front whose beacon is down answers 502. +func compatBeaconStub(t *testing.T, status int) string { + t.Helper() + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "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/v1/compat" { + http.NotFound(w, r) + return + } + w.WriteHeader(status) + })) + t.Cleanup(srv.Close) + return srv.Listener.Addr().String() +} + // When UDP works, auto is udp — the transport the daemon always used — // found in one round trip, and the compat side is never touched. func TestSelectTransportUDPWorks(t *testing.T) { @@ -90,12 +107,12 @@ func TestSelectTransportUDPWorks(t *testing.T) { } } -// UDP blocked, compat beacon reachable over TCP: compat, within the probe -// bound plus the TCP connect. +// UDP blocked, compat beacon answering (426 to a plain GET): compat, +// within the probe bound plus the local check. func TestSelectTransportUDPBlockedFallsBackToCompat(t *testing.T) { t.Parallel() beacon := udpBeacon(t, false) - compat := tcpListener(t) + compat := compatBeaconStub(t, http.StatusUpgradeRequired) start := time.Now() mode, reason := SelectTransport(context.Background(), AutoTransportProbe{ BeaconAddr: beacon, @@ -147,13 +164,37 @@ func TestSelectTransportNothingReachableStaysUDP(t *testing.T) { } } +// auto-compat-check-tcp-only-fatal-on-beacon-outage: a front that accepts +// TCP (or TLS) while the beacon behind it is down is not a working compat +// path. auto stays on udp, which starts degraded and registers, instead of +// picking compat and exiting on the failed WSS connect. +func TestSelectTransportFrontUpBeaconDownStaysUDP(t *testing.T) { + t.Parallel() + beacon := udpBeacon(t, false) + for name, compat := range map[string]string{ + "TCP accept-and-close": tcpListener(t), + "TLS front, 502": compatBeaconStub(t, http.StatusBadGateway), + "TLS front, 503": compatBeaconStub(t, http.StatusServiceUnavailable), + } { + mode, reason := SelectTransport(context.Background(), AutoTransportProbe{ + BeaconAddr: beacon, + CompatBeaconURL: "wss://" + compat + "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/v1/compat", + UDPTimeout: 100 * time.Millisecond, + TCPTimeout: 2 * time.Second, + }) + if mode != TransportUDP { + t.Errorf("%s: mode = %q (%s), want udp", name, mode, reason) + } + } +} + // Behind an egress proxy the compat check goes through the proxy, by host // name — the Muse case: UDP silently dropped, direct TCP killed. func TestSelectTransportCompatCheckUsesProxy(t *testing.T) { clearProxyEnv(t) beacon := udpBeacon(t, false) proxy := newProxyTestConnect(t, "muse", "s3cret") - target := tcpListener(t) + target := compatBeaconStub(t, http.StatusUpgradeRequired) _, port, _ := net.SplitHostPort(target) policy, err := ResolveProxy(proxy.url("muse", "s3cret"), TransportCompat) if err != nil { From 3a8ac44ad633fc9238de423f1c4e5978f47b9a2d Mon Sep 17 00:00:00 2001 From: Teodor Calin Date: Thu, 24 Sep 2026 03:20:50 +0300 Subject: [PATCH 07/19] fix: installer changes via pilot-protocol/release, proxy-first pilotctl registry dial Fixes the two re-review findings on feat/native-https-proxy. web4-install-sh-edits-not-in-canonical-installer (high) - install.sh here is a synced copy of pilot-protocol/release:install.sh (the script https://pilotprotocol.network/install.sh serves; the canonical-drift job enforces byte equality). The installer changes are now ported onto the canonical script in pilot-protocol/release branch feat/installer-proxy-transport, keeping its managed-node mode (--managed-url, --no-start, PILOT_ENROLLMENT_TOKEN), and this copy is byte-identical to that branch. canonical-drift passes once the release change merges; merge it first. - Rotation no longer depends on the installer: on Linux without systemd, when $HTTPS_PROXY / $https_proxy carries credentials, the proxy setting is auto, no proxy_cmd is configured ($PILOT_PROXY_CMD, config.json / --config file) and the daemon supports -proxy-cmd, `pilotctl daemon start` hands the daemon PILOT_PROXY_CMD=bash -c 'printf %s "${https_proxy:-$HTTPS_PROXY}"'. A node set up by the currently served installer (PILOT_ALLOW_ROOT=1, no proxy_cmd) therefore re-reads rotated credentials too. The ready summary says so (JSON: "proxy_cmd"). - README/CHANGELOG describe both paths. web4-pilotctl-raw-route-defeated-by-muse-guard (low) - With the transport unknown (no daemon answering, nothing configured) and a proxy in the environment, pilotctl now tries the TLS registry through the proxy first and the raw registry directly second. The direct fallback is probed: a peer that sends data or hangs up within 300ms of connecting (a sandbox network guard) is rejected, so the proxied route's error is reported instead of a broken pipe. NO_PROXY exempting the registry keeps direct-first. Tests: route plan table (proxy first, probe flag, NO_PROXY), probedDirectDial (guard that talks, guard that closes, silent registry usable, dial error), dialRegistry against a local guard + authenticating CONNECT proxy + pinned TLS registry (proxy allowed, proxy refusing, raw fallback to a real registry), sandboxProxyCmdFor matrix, CLI daemon start passes/keeps PILOT_PROXY_CMD. Reviewer repro (muse-sim-vm, --network none, guard on 34.71.57.205:9000): CONNECT registry.pilotprotocol.network:443 is now the first attempt and the error names the proxy. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 22 +- README.md | 4 +- cmd/pilotctl/daemon_transport.go | 106 ++++++ cmd/pilotctl/main.go | 21 +- cmd/pilotctl/registry_dial.go | 87 ++++- cmd/pilotctl/zz_proxy_route_test.go | 13 +- cmd/pilotctl/zz_registry_guard_test.go | 462 +++++++++++++++++++++++++ cmd/pilotctl/zz_subprocess_test.go | 7 + install.sh | 346 ++++++++++++++++-- 9 files changed, 1015 insertions(+), 53 deletions(-) create mode 100644 cmd/pilotctl/zz_registry_guard_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a0e5a22..cf9236d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,9 +35,12 @@ Detailed per-release notes are on the connection once with the new credentials. Registry redials, WSS beacon reconnects and every HTTP client (plugins included) follow it, so a sandbox that rotates its proxy credentials every few minutes (Meta Muse) no longer - leaves a node "online with all apps broken" until a restart. The installer - saves such a command in a Linux container/VM without systemd whose - `HTTPS_PROXY` carries credentials. The command's output is never logged. + leaves a node "online with all apps broken" until a restart. In a Linux + container/VM without systemd whose `HTTPS_PROXY` carries credentials, + `pilotctl daemon start` hands the daemon `PILOT_PROXY_CMD=bash -c 'printf + %s "${https_proxy:-$HTTPS_PROXY}"'` itself when no `proxy_cmd` is + configured (so a node set up by any installer gets it), and the installer + saves the same command. The command's output is never logged. - **`-transport=auto`.** UDP when the beacon answers a UDP discover (one round trip), otherwise compat when the compat beacon itself answers over TCP 443 (through the proxy, if any: a TLS GET of the beacon path must return `426 @@ -78,9 +81,12 @@ Detailed per-release notes are on the `$PILOT_TRANSPORT` / config.json — on a udp host that merely exports a proxy they dial directly, as the daemon does, so a private raw-TCP registry keeps working. Proxied or compat dials use `registry.pilotprotocol.network:443` - over TLS; a direct raw-TCP attempt falls back to it (through the - environment's proxy when the transport is unknown). `proxy=off` restores - direct dials. + over TLS; a direct raw-TCP attempt falls back to it. With the transport + unknown (no daemon answering, nothing configured) the production registry + is tried through the environment's proxy first and directly second, and a + direct connection whose peer talks or hangs up before the first request (a + sandbox's network guard) is not used, so the proxy's error is reported + instead of a broken pipe. `proxy=off` restores direct dials. - **`daemon start` forwards the proxy/TLS environment** (`HTTPS_PROXY`, `HTTP_PROXY`, `ALL_PROXY`, `NO_PROXY` in both cases, `PILOT_PROXY`, `PILOT_TRANSPORT`, `PILOT_REGISTRY_TRUST`, `PILOT_REGISTRY_FINGERPRINT`, @@ -88,6 +94,10 @@ Detailed per-release notes are on the and passes through `--compat-beacon`, `--registry-trust`, `--registry-fingerprint` and `--tls-trust`. - **`install.sh --transport `** (or `PILOT_TRANSPORT`). + install.sh here is a copy of `pilot-protocol/release:install.sh`, the script + https://pilotprotocol.network/install.sh serves; these installer changes + reach users through the matching pilot-protocol/release change, which also + keeps the managed-node mode (`--managed-url`). `udp` and `compat` are saved; `auto` (the default) is not — `--transport auto` removes a saved transport. `compat` skips the UDP probe. No `proxy` key is written (the daemon default already uses the environment's proxy). diff --git a/README.md b/README.md index 3d5afdfa..183aa036 100644 --- a/README.md +++ b/README.md @@ -294,11 +294,11 @@ curl -fsSL https://pilotprotocol.network/install.sh | sh pilotctl daemon start ``` -New installs use transport `auto`: the daemon probes the beacon over UDP once at startup (one round trip when UDP works, at most ~1.5s when it does not) and, when there is no answer but the compat beacon answers over TCP 443 (a TLS request that a live beacon answers with `426 Upgrade Required`), runs in **compat** mode — registry over TLS on `registry.pilotprotocol.network:443`, beacon over WSS on `beacon.pilotprotocol.network:443`. When neither answers (no network yet, beacon outage) it stays on UDP, as before. In compat mode the daemon tunnels through `$HTTPS_PROXY` / `$ALL_PROXY` (honoring `$NO_PROXY`), asking the proxy to `CONNECT` by host name, so poisoned local DNS for the Pilot hosts does not matter. `pilotctl`'s own registry commands (`lookup`, the auto-handshake check, `recovery`) follow the daemon: through the proxy when it runs compat (or with an explicit proxy URL), directly when it runs UDP. To skip the UDP probe, pick compat explicitly: `sh -s -- --transport compat`, `pilotctl config --set transport=compat`, or `pilotctl daemon start --transport compat`. +New installs use transport `auto`: the daemon probes the beacon over UDP once at startup (one round trip when UDP works, at most ~1.5s when it does not) and, when there is no answer but the compat beacon answers over TCP 443 (a TLS request that a live beacon answers with `426 Upgrade Required`), runs in **compat** mode — registry over TLS on `registry.pilotprotocol.network:443`, beacon over WSS on `beacon.pilotprotocol.network:443`. When neither answers (no network yet, beacon outage) it stays on UDP, as before. In compat mode the daemon tunnels through `$HTTPS_PROXY` / `$ALL_PROXY` (honoring `$NO_PROXY`), asking the proxy to `CONNECT` by host name, so poisoned local DNS for the Pilot hosts does not matter. `pilotctl`'s own registry commands (`lookup`, the auto-handshake check, `recovery`) follow the daemon: through the proxy when it runs compat (or with an explicit proxy URL), directly when it runs UDP; with no daemon running and no transport configured they try the proxy first and a direct connection second. To skip the UDP probe, pick compat explicitly: `sh -s -- --transport compat`, `pilotctl config --set transport=compat`, or `pilotctl daemon start --transport compat`. - **No root, systemd or launchd needed.** Start the daemon with `pilotctl daemon start` from a shell that has the proxy variables. In a container or VM without systemd the installer also runs as root (the agent user in hosted sandboxes); on a regular host it still refuses root unless `PILOT_ALLOW_ROOT=1`. - **Proxy with credentials:** keep them out of `ps` — export `HTTPS_PROXY` / `PILOT_PROXY`, or `pilotctl daemon start --proxy http://user:pass@host:port` (pilotctl hands a URL with credentials to the daemon in its environment, never on its command line). For a systemd/launchd service, which does not see your shell's variables, save it: `pilotctl config --set proxy=http://user:pass@host:port` (config.json is 0600). -- **Proxy credentials that rotate** (Meta Muse rotates the credentials in `HTTPS_PROXY` every few minutes): a long-running daemon would keep the ones it started with, and new connections would start failing with `407 Proxy Authentication Required` while old tunnels stay up ("node online, apps broken"). Give the daemon a command that prints the current proxy URL — `proxy_cmd` in config.json, `$PILOT_PROXY_CMD` or `-proxy-cmd` — and it re-runs it every 60s and whenever the proxy answers 407, retrying with the fresh credentials; no restart needed. The installer saves `bash -c 'printf %s "$https_proxy"'` (a fresh shell sees the current value) in a Linux container/VM without systemd whose `HTTPS_PROXY` carries credentials; elsewhere: `pilotctl config --set proxy_cmd="bash -c 'printf %s \"\$https_proxy\"'"`. Without it, restart the daemon from a fresh shell when it starts failing. +- **Proxy credentials that rotate** (Meta Muse rotates the credentials in `HTTPS_PROXY` every few minutes): a long-running daemon would keep the ones it started with, and new connections would start failing with `407 Proxy Authentication Required` while old tunnels stay up ("node online, apps broken"). Give the daemon a command that prints the current proxy URL — `proxy_cmd` in config.json, `$PILOT_PROXY_CMD` or `-proxy-cmd` — and it re-runs it every 60s and whenever the proxy answers 407, retrying with the fresh credentials; no restart needed. In a Linux container/VM without systemd whose `HTTPS_PROXY` carries credentials, `pilotctl daemon start` hands the daemon `PILOT_PROXY_CMD=bash -c 'printf %s "${https_proxy:-$HTTPS_PROXY}"'` (a fresh shell sees the current value) whenever no `proxy_cmd` is configured, and the installer saves the same command in config.json; elsewhere: `pilotctl config --set proxy_cmd="bash -c 'printf %s \"\$https_proxy\"'"`. Without it, restart the daemon from a fresh shell when it starts failing. - **Precedence** for transport and proxy: command-line flag, then `$PILOT_TRANSPORT` / `$PILOT_PROXY` / `$PILOT_PROXY_CMD`, then `config.json` (`transport`, `proxy`, `proxy_cmd`), then the default (`auto` from pilotctl and the installed systemd/launchd services, via `PILOT_TRANSPORT_DEFAULT=auto`; `udp` for a bare `pilot-daemon`; proxy `auto`). `auto` is never saved in config.json, so reinstalling an older release (`--version`, `pilotctl update --pin`) leaves nothing it cannot read; if you saved it yourself, both rewrite it to `udp` for a daemon that predates it. `-proxy` accepts `auto`, `off` (also `none`, `direct`) or an `http://` / `https://` URL; anything else is an error. Loopback targets (local webhooks, sidecars) are never sent to a proxy. When the registry dial goes through a proxy (compat, or an explicit proxy URL in any transport), the default raw-TCP registry is replaced by `registry.pilotprotocol.network:443` over TLS, since CONNECT proxies carry port 443 only. - **No CA bundle in the sandbox?** Point Go at one with `SSL_CERT_FILE=/path/to/ca-certificates.crt` (or `SSL_CERT_DIR`) — `pilotctl daemon start` forwards both. The registry can instead be pinned: `PILOT_REGISTRY_FINGERPRINT=` (or `pilotctl config --set registry_fingerprint=...`), which selects `registry_trust=pinned`. The WSS beacon has no fingerprint option, so it needs the CA bundle. diff --git a/cmd/pilotctl/daemon_transport.go b/cmd/pilotctl/daemon_transport.go index c93184ba..280894dc 100644 --- a/cmd/pilotctl/daemon_transport.go +++ b/cmd/pilotctl/daemon_transport.go @@ -5,10 +5,13 @@ package main import ( "bufio" "context" + "encoding/json" "fmt" "os" "os/exec" + "path/filepath" "regexp" + "runtime" "strings" "sync" "time" @@ -203,6 +206,109 @@ func daemonChildEnv(base []string, adminToken, proxyEnv, transport string) []str return env } +// setEnv returns env with key set to value, replacing every existing entry +// for key (a child reads the first of duplicate entries on some paths and +// the last on others). +func setEnv(env []string, key, value string) []string { + out := make([]string, 0, len(env)+1) + for _, kv := range env { + if k, _, _ := strings.Cut(kv, "="); k == key { + continue + } + out = append(out, kv) + } + return append(out, key+"="+value) +} + +// sandboxProxyCmd is the proxy_cmd for hosted agent sandboxes: a fresh bash +// sees the sandbox's current proxy URL, whose credentials rotate (Meta Muse: +// every few minutes). install.sh saves the same command. +const sandboxProxyCmd = `bash -c 'printf %s "${https_proxy:-$HTTPS_PROXY}"'` + +// Test seams for sandboxProxyCmdFor. +var ( + hostGOOS = runtime.GOOS + systemdRunning = func() bool { + _, err := os.Stat("/run/systemd/system") + return err == nil + } + bashAvailable = func() bool { + _, err := exec.LookPath("bash") + return err == nil + } +) + +// sandboxProxyCmdFor returns the $PILOT_PROXY_CMD `daemon start` hands the +// daemon when nothing configures one, "" when none applies. Without it a +// daemon keeps the proxy credentials it was started with, and once a +// sandbox rotates them every new connection fails with 407 ("node online, +// all apps broken"). It applies only when all of these hold: +// +// - Linux without systemd (a container or VM such as a hosted agent +// sandbox, where pilotctl rather than a service manager starts the +// daemon); +// - $HTTPS_PROXY or $https_proxy carries credentials; +// - the proxy setting is auto (an explicit --proxy / $PILOT_PROXY URL is +// left alone: the command would replace it); +// - no proxy_cmd is configured: $PILOT_PROXY_CMD, and "proxy_cmd" in the +// config file the daemon reads (--config, else ~/.pilot/config.json) and +// in pilotctl's own config; +// - bash is installed and the daemon at bin supports -proxy-cmd. +// +// It does not depend on the installer having saved proxy_cmd, so a node +// installed by an older installer gets it too. +func sandboxProxyCmdFor(bin string, plan daemonLaunchPlan, flags map[string]string) string { + if hostGOOS != "linux" || systemdRunning() { + return "" + } + if plan.Proxy != "" && plan.Proxy != proxyconf.Auto { + return "" + } + if !proxyHasCredentials(os.Getenv("HTTPS_PROXY")) && !proxyHasCredentials(os.Getenv("https_proxy")) { + return "" + } + if strings.TrimSpace(os.Getenv("PILOT_PROXY_CMD")) != "" { + return "" + } + if configuredProxyCmd(loadConfig()) != "" || configuredProxyCmd(daemonConfigFile(flags)) != "" { + return "" + } + if !bashAvailable() { + return "" + } + if f := daemonFlags(bin); !f["proxy-cmd"] { + return "" + } + return sandboxProxyCmd +} + +func configuredProxyCmd(cfg map[string]interface{}) string { + s, _ := cfg["proxy_cmd"].(string) + return strings.TrimSpace(s) +} + +// daemonConfigFile reads the config file pilot-daemon loads: --config, else +// $HOME/.pilot/config.json. Empty when it cannot be read. +func daemonConfigFile(flags map[string]string) map[string]interface{} { + path := flagString(flags, "config", "") + if path == "" { + home, err := os.UserHomeDir() + if err != nil { + return nil + } + path = filepath.Join(home, ".pilot", "config.json") + } + b, err := os.ReadFile(path) // #nosec G304 -- the operator's own config path + if err != nil { + return nil + } + var cfg map[string]interface{} + if json.Unmarshal(b, &cfg) != nil { + return nil + } + return cfg +} + // daemonFlagProbeTimeout bounds the `pilot-daemon -help` flag probe. const daemonFlagProbeTimeout = 5 * time.Second diff --git a/cmd/pilotctl/main.go b/cmd/pilotctl/main.go index cb3da0bb..81f16213 100644 --- a/cmd/pilotctl/main.go +++ b/cmd/pilotctl/main.go @@ -1092,9 +1092,13 @@ Behind an HTTPS proxy with UDP blocked (e.g. hosted agent sandboxes), plain "pilotctl daemon start" picks compat by itself; to skip the UDP probe: pilotctl config --set transport=compat && pilotctl daemon start If the proxy rotates its credentials, give the daemon a command that prints -the current proxy URL (install.sh does this in hosted agent sandboxes); the -daemon re-runs it every 60s and whenever the proxy answers 407: +the current proxy URL; the daemon re-runs it every 60s and whenever the proxy +answers 407: pilotctl config --set proxy_cmd="bash -c 'printf %s \"\$https_proxy\"'" +On Linux without systemd (containers, hosted agent sandboxes), when +$HTTPS_PROXY carries credentials and no proxy_cmd is configured, daemon start +passes the daemon PILOT_PROXY_CMD=bash -c 'printf %s "${https_proxy:-$HTTPS_PROXY}"' +by itself. `, "daemon stop": `Usage: pilotctl daemon stop @@ -3124,6 +3128,13 @@ func cmdDaemonStart(args []string) { // -transport=auto when no transport is configured. daemonArgs, proxyEnv, requestedTransport := adaptDaemonArgs(daemonBin, plan) daemonEnv := daemonChildEnv(os.Environ(), plan.AdminToken, proxyEnv, requestedTransport) + // In a sandbox whose proxy credentials rotate, have the daemon re-read + // them (see sandboxProxyCmdFor) even when no installer saved proxy_cmd. + sandboxRefresh := false + if c := sandboxProxyCmdFor(daemonBin, plan, flags); c != "" { + daemonEnv = setEnv(daemonEnv, "PILOT_PROXY_CMD", c) + sandboxRefresh = true + } // --foreground: replace the current process so signal/lifetime // handling matches what the user expects from systemd unit files @@ -3250,6 +3261,9 @@ func cmdDaemonStart(args []string) { if plan.Proxy != "" { fields["proxy"] = redactProxyURL(plan.Proxy) } + if sandboxRefresh { + fields["proxy_cmd"] = sandboxProxyCmd + } outputOK(fields) } else { fmt.Printf("Daemon running (pid %d)\n", pid) @@ -3266,6 +3280,9 @@ func cmdDaemonStart(args []string) { if plan.Proxy != "" { fmt.Printf(" Proxy: %s\n", redactProxyURL(plan.Proxy)) } + if sandboxRefresh { + fmt.Printf(" Proxy credentials: re-read every 60s and on a 407 (%s)\n", sandboxProxyCmd) + } fmt.Printf(" Socket: %s\n", socketPath) fmt.Printf(" Logs: %s\n", pidLogPath) } diff --git a/cmd/pilotctl/registry_dial.go b/cmd/pilotctl/registry_dial.go index 133aa430..22cf1d96 100644 --- a/cmd/pilotctl/registry_dial.go +++ b/cmd/pilotctl/registry_dial.go @@ -3,11 +3,14 @@ package main import ( + "context" "crypto/tls" + "errors" "fmt" "net" "os" "strings" + "time" "github.com/pilot-protocol/common/driver" "github.com/pilot-protocol/common/netproxy" @@ -29,10 +32,15 @@ import ( // daemon's (asked over IPC), else $PILOT_TRANSPORT / config.json. On a // udp host that merely exports a proxy (corporate hosts, where private // registries live) the registry is dialed directly, as the daemon -// does; with the transport unknown (no daemon, nothing configured) the -// direct dial comes first and the proxy is only the fallback for the -// production registry. A private raw-TCP registry is never sent to the -// environment's proxy unless the transport is compat. +// does. With the transport unknown (no daemon answering, nothing +// configured) the production registry is tried through the +// environment's proxy first — a proxy that accepts the CONNECT is the +// stronger signal: sandboxes such as Meta Muse accept a direct TCP +// connection with a network guard that then breaks the first request — +// and directly over raw TCP second, a connection that is only used when +// the peer does not talk first (see probedDirectDial). A private +// raw-TCP registry is never sent to the environment's proxy unless the +// transport is compat. // - address: the compiled-in raw-TCP registry (34.71.57.205:9000) is // replaced by registry.pilotprotocol.network:443 over TLS whenever it // would be proxied (proxies allow CONNECT to :443 only) or the @@ -47,6 +55,9 @@ type registryRoute struct { Proxy *netproxy.Resolver // Switched: Addr replaced the raw-TCP default. Switched bool + // Probe: a direct raw-TCP fallback, used only when the peer does not + // talk before the first request (probedDirectDial). + Probe bool } // proxied reports whether dialing r.Addr goes through a proxy. @@ -201,13 +212,20 @@ func planRegistryRoutes(addr string) ([]registryRoute, error) { policy = env break } - // Unknown transport, production registry: direct raw TCP first, - // then the TLS registry through the environment's proxy. + // Unknown transport, production registry: the TLS registry + // through the environment's proxy first, then direct raw TCP. + // Direct first would lose to a sandbox network guard, which + // accepts the TCP connection and only fails the first request, + // so the proxied route would never be tried. tlsRoute := route(compatRegistryAddr, nil) - if proxyFor(env, compatRegistryAddr) != "" { - tlsRoute.Proxy = env + if proxyFor(env, compatRegistryAddr) == "" { + // NO_PROXY exempts the registry: nothing is proxied. + return []registryRoute{route(addr, nil), tlsRoute}, nil } - return []registryRoute{route(addr, nil), tlsRoute}, nil + tlsRoute.Proxy = env + raw := route(addr, nil) + raw.Probe = true + return []registryRoute{tlsRoute, raw}, nil default: explicit, err := proxyconf.Resolve(spec) if err != nil { @@ -236,11 +254,60 @@ func planRegistryRoutes(addr string) ([]registryRoute, error) { return routes, nil } +// rawDirectDial opens a direct TCP connection (a test seam). +var rawDirectDial = func(ctx context.Context, network, addr string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, network, addr) +} + +// guardProbeWait is how long probedDirectDial watches a fresh connection. +const guardProbeWait = 300 * time.Millisecond + +// errNotRegistry marks a direct connection whose peer is not a registry. +var errNotRegistry = errors.New("not a Pilot registry") + +// probedDirectDial dials addr directly and hands back the connection only +// when the peer stays silent for guardProbeWait, as a registry does until it +// gets a request. A peer that sends first or closes at once — the network +// guard of a sandbox whose only way out is its HTTPS proxy answers direct +// connections with a policy message — fails the dial, so dialRegistry +// reports the proxied route's error rather than a broken pipe from the +// guard. The check costs guardProbeWait, and it only runs on the direct +// fallback after the proxied route failed. +func probedDirectDial(ctx context.Context, network, addr string) (net.Conn, error) { + c, err := rawDirectDial(ctx, network, addr) + if err != nil { + return nil, err + } + if err := c.SetReadDeadline(time.Now().Add(guardProbeWait)); err != nil { + c.Close() + return nil, err + } + var b [1]byte + n, err := c.Read(b[:]) + var ne net.Error + if n == 0 && errors.As(err, &ne) && ne.Timeout() { + if err := c.SetReadDeadline(time.Time{}); err != nil { + c.Close() + return nil, err + } + return c, nil + } + c.Close() + if n > 0 { + return nil, fmt.Errorf("%w at %s: it sent data before any request (a network guard?)", errNotRegistry, addr) + } + return nil, fmt.Errorf("%w at %s: it closed the connection before any request (a network guard?): %v", errNotRegistry, addr, err) +} + // dial opens the registry connection the route describes. func (r registryRoute) dial() (*registry.Client, error) { var opts []registry.DialOption - if r.Proxy.Enabled() { + switch { + case r.Proxy.Enabled(): opts = append(opts, registry.WithDialer(proxyconf.DialContext(r.Proxy, nil))) + case r.Probe && !r.TLS: + opts = append(opts, registry.WithDialer(probedDirectDial)) } if !r.TLS { return registry.Dial(r.Addr, opts...) diff --git a/cmd/pilotctl/zz_proxy_route_test.go b/cmd/pilotctl/zz_proxy_route_test.go index 0aa667e6..6ee3a563 100644 --- a/cmd/pilotctl/zz_proxy_route_test.go +++ b/cmd/pilotctl/zz_proxy_route_test.go @@ -172,6 +172,9 @@ func routeSpec(r registryRoute) string { if r.Fingerprint != "" { s += "/pin" } + if r.Probe { + s += "/probe" + } if p := r.proxyFor(r.Addr); p != "" { s += " via proxy" } @@ -195,9 +198,15 @@ func TestPlanRegistryRoutes(t *testing.T) { }{ {name: "plain host: raw default, then the TLS registry", addr: raw, want: []string{raw, tlsReg}}, - {name: "HTTPS_PROXY, transport unknown: direct first, proxied TLS fallback", addr: raw, + // pilotctl-raw-route-defeated-by-muse-guard: with the transport + // unknown, the proxied TLS registry comes first; the direct raw + // route is the (probed) fallback. + {name: "HTTPS_PROXY, transport unknown: proxied TLS first, direct raw fallback", addr: raw, env: map[string]string{"HTTPS_PROXY": "http://u:p@egress.test:3128"}, - want: []string{raw, tlsProxy}}, + want: []string{tlsProxy, raw + "/probe"}}, + {name: "HTTPS_PROXY, transport unknown, NO_PROXY exempts the registry: direct", addr: raw, + env: map[string]string{"HTTPS_PROXY": "http://u:p@egress.test:3128", "NO_PROXY": ".pilotprotocol.network"}, + want: []string{raw, tlsReg}}, {name: "HTTPS_PROXY + config compat: proxied TLS, then direct TLS", addr: raw, env: map[string]string{"HTTPS_PROXY": "http://u:p@egress.test:3128"}, cfg: map[string]interface{}{"transport": "compat"}, diff --git a/cmd/pilotctl/zz_registry_guard_test.go b/cmd/pilotctl/zz_registry_guard_test.go new file mode 100644 index 00000000..4a692e35 --- /dev/null +++ b/cmd/pilotctl/zz_registry_guard_test.go @@ -0,0 +1,462 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "bufio" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/hex" + "errors" + "fmt" + "io" + "math/big" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +// guardPolicyReply is what a sandbox network guard answers a direct +// connection with (Meta Muse: a policy message, then close). +const guardPolicyReply = "HTTP/1.1 403 Forbidden\r\nContent-Type: text/plain\r\n\r\nDirect egress is blocked by sandbox policy; use HTTPS_PROXY.\n" + +// fakeGuard accepts TCP connections, writes guardPolicyReply (unless quiet) +// and closes them. It counts the connections it saw. +func fakeGuard(t *testing.T, quiet bool) (addr string, conns *atomic.Int32) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + conns = &atomic.Int32{} + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + conns.Add(1) + if !quiet { + _, _ = io.WriteString(c, guardPolicyReply) + } + _ = c.Close() + } + }() + t.Cleanup(func() { ln.Close() }) + return ln.Addr().String(), conns +} + +func TestProbedDirectDial(t *testing.T) { + ctx := context.Background() + + t.Run("guard that talks first", func(t *testing.T) { + addr, _ := fakeGuard(t, false) + _, err := probedDirectDial(ctx, "tcp", addr) + if !errors.Is(err, errNotRegistry) || !strings.Contains(err.Error(), "sent data before any request") { + t.Fatalf("err = %v, want errNotRegistry (sent data)", err) + } + }) + t.Run("guard that closes at once", func(t *testing.T) { + addr, _ := fakeGuard(t, true) + _, err := probedDirectDial(ctx, "tcp", addr) + if !errors.Is(err, errNotRegistry) || !strings.Contains(err.Error(), "closed the connection") { + t.Fatalf("err = %v, want errNotRegistry (closed)", err) + } + }) + t.Run("registry stays silent: connection usable", func(t *testing.T) { + r := newFakeRegistry(t) + start := time.Now() + c, err := probedDirectDial(ctx, "tcp", r.addr()) + if err != nil { + t.Fatalf("probedDirectDial: %v", err) + } + defer c.Close() + if d := time.Since(start); d < guardProbeWait { + t.Errorf("probe returned after %v, before guardProbeWait", d) + } + // The read deadline is cleared: a request/response still works. + if err := writeFrame(c, `{"type":"lookup","node_id":1}`); err != nil { + t.Fatal(err) + } + if _, err := readFrame(c); err != nil { + t.Fatalf("read after probe: %v", err) + } + }) + t.Run("dial error passes through", func(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + addr := ln.Addr().String() + ln.Close() + if _, err := probedDirectDial(ctx, "tcp", addr); err == nil || errors.Is(err, errNotRegistry) { + t.Fatalf("err = %v, want the dial error", err) + } + }) +} + +func writeFrame(w io.Writer, body string) error { + n := len(body) + _, err := w.Write(append([]byte{byte(n >> 24), byte(n >> 16), byte(n >> 8), byte(n)}, body...)) + return err +} + +func readFrame(r io.Reader) ([]byte, error) { + var hdr [4]byte + if _, err := io.ReadFull(r, hdr[:]); err != nil { + return nil, err + } + n := int(hdr[0])<<24 | int(hdr[1])<<16 | int(hdr[2])<<8 | int(hdr[3]) + b := make([]byte, n) + _, err := io.ReadFull(r, b) + return b, err +} + +// newTLSFakeRegistry is newFakeRegistry behind TLS with a self-signed +// certificate; it returns the registry and the certificate's SHA-256 +// fingerprint (for PILOT_REGISTRY_FINGERPRINT). +func newTLSFakeRegistry(t *testing.T) (*fakeRegistry, string) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: compatRegistryHost}, + DNSNames: []string{compatRegistryHost}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(der) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + tl := tls.NewListener(ln, &tls.Config{ + MinVersion: tls.VersionTLS12, + Certificates: []tls.Certificate{{Certificate: [][]byte{der}, PrivateKey: key}}, + }) + r := &fakeRegistry{t: t, ln: tl, handlers: map[string]func(map[string]interface{}) map[string]interface{}{}} + go r.accept() + t.Cleanup(func() { tl.Close() }) + return r, hex.EncodeToString(sum[:]) +} + +const compatRegistryHost = "registry.pilotprotocol.network" + +// mapProxy is an authenticating CONNECT proxy that sends each allowed +// target to a local address and records every CONNECT target. refuse makes +// it answer 403 to everything, like a proxy whose policy denies the host. +type mapProxy struct { + url string + mu sync.Mutex + routes map[string]string + seen []string + refuse atomic.Bool +} + +func newMapProxy(t *testing.T, routes map[string]string) *mapProxy { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + p := &mapProxy{url: "http://muse:s3cret@" + ln.Addr().String(), routes: routes} + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go p.serve(c) + } + }() + t.Cleanup(func() { ln.Close() }) + return p +} + +func (p *mapProxy) serve(c net.Conn) { + defer c.Close() + br := bufio.NewReader(c) + req, err := http.ReadRequest(br) + if err != nil { + return + } + p.mu.Lock() + p.seen = append(p.seen, req.Method+" "+req.RequestURI) + to, ok := p.routes[req.RequestURI] + p.mu.Unlock() + if req.Method != http.MethodConnect || !ok || p.refuse.Load() { + fmt.Fprint(c, "HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\n\r\n") + return + } + up, err := net.DialTimeout("tcp", to, 5*time.Second) + if err != nil { + fmt.Fprint(c, "HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n") + return + } + defer up.Close() + fmt.Fprint(c, "HTTP/1.1 200 Connection established\r\n\r\n") + go func() { _, _ = io.Copy(up, br); up.Close() }() + _, _ = io.Copy(c, up) +} + +func (p *mapProxy) targets() []string { + p.mu.Lock() + defer p.mu.Unlock() + return append([]string(nil), p.seen...) +} + +// stubRawDirectDial sends direct dials of the production raw registry to +// to (never the real 34.71.57.205) and counts them; any other direct dial +// fails the test. +func stubRawDirectDial(t *testing.T, to string) *atomic.Int32 { + t.Helper() + calls := &atomic.Int32{} + prev := rawDirectDial + rawDirectDial = func(ctx context.Context, network, addr string) (net.Conn, error) { + if addr != productionRegistryAddr { + t.Errorf("unexpected direct dial of %s", addr) + return nil, errors.New("unexpected direct dial") + } + calls.Add(1) + var d net.Dialer + return d.DialContext(ctx, network, to) + } + t.Cleanup(func() { rawDirectDial = prev }) + return calls +} + +// pilotctl-raw-route-defeated-by-muse-guard: no daemon running and no +// transport configured, HTTPS_PROXY exported, and a network guard that +// accepts direct TCP to the raw registry. pilotctl must reach the registry +// through the proxy, and must not hand back the guard's connection. +func TestUnknownTransportRegistryDialPrefersProxyOverGuard(t *testing.T) { + reg, fp := newTLSFakeRegistry(t) + reg.onOK("lookup", map[string]interface{}{"node_id": float64(5), "address": "0:0000.0000.0005", "public": true}) + proxy := newMapProxy(t, map[string]string{compatRegistryAddr: reg.addr()}) + guardAddr, guardConns := fakeGuard(t, false) + + setup := func(t *testing.T, rawTo string) *atomic.Int32 { + withTransportEnvCleared(t) // also: no running daemon answers + t.Setenv("HTTPS_PROXY", proxy.url) + t.Setenv("PILOT_REGISTRY_FINGERPRINT", fp) + return stubRawDirectDial(t, rawTo) + } + + t.Run("proxy allows the registry: proxied TLS, guard untouched", func(t *testing.T) { + rawCalls := setup(t, guardAddr) + rc, route, err := dialRegistry(productionRegistryAddr) + if err != nil { + t.Fatalf("dialRegistry: %v", err) + } + defer rc.Close() + if route.Addr != compatRegistryAddr || !route.proxied() || !route.TLS { + t.Errorf("route = %+v, want the proxied TLS registry", route) + } + if _, err := rc.Lookup(5); err != nil { + t.Fatalf("Lookup: %v", err) + } + if n := rawCalls.Load(); n != 0 { + t.Errorf("direct raw dials = %d, want 0 (the proxied route comes first)", n) + } + if n := guardConns.Load(); n != 0 { + t.Errorf("guard saw %d connections, want 0", n) + } + }) + + t.Run("proxy refuses: the guard is detected, the proxy error is reported", func(t *testing.T) { + rawCalls := setup(t, guardAddr) + proxy.refuse.Store(true) + defer proxy.refuse.Store(false) + before := guardConns.Load() + rc, route, err := dialRegistry(productionRegistryAddr) + if err == nil { + rc.Close() + t.Fatal("dialRegistry succeeded against a guard") + } + if !route.proxied() || !strings.Contains(err.Error(), "403") { + t.Errorf("reported route %+v err %v, want the proxied route's 403", route, err) + } + if rawCalls.Load() != 1 || guardConns.Load() == before { + t.Errorf("direct fallback not tried: raw dials=%d guard conns=%d", rawCalls.Load(), guardConns.Load()-before) + } + }) + + t.Run("proxy refuses, registry reachable directly: raw fallback works", func(t *testing.T) { + plain := newFakeRegistry(t) + plain.onOK("lookup", map[string]interface{}{"node_id": float64(5), "address": "0:0000.0000.0005", "public": true}) + rawCalls := setup(t, plain.addr()) + proxy.refuse.Store(true) + defer proxy.refuse.Store(false) + rc, route, err := dialRegistry(productionRegistryAddr) + if err != nil { + t.Fatalf("dialRegistry: %v", err) + } + defer rc.Close() + if route.Addr != productionRegistryAddr || route.proxied() || route.TLS { + t.Errorf("route = %+v, want the direct raw registry", route) + } + if _, err := rc.Lookup(5); err != nil { + t.Fatalf("Lookup over the raw fallback: %v", err) + } + if rawCalls.Load() != 1 { + t.Errorf("raw dials = %d, want 1", rawCalls.Load()) + } + }) +} + +func TestSandboxProxyCmdFor(t *testing.T) { + dir := t.TempDir() + withCmd := writeFakeDaemon(t, append([]string{autoUsage, "proxy", "proxy-cmd"}, baseDaemonFlags...), filepath.Join(dir, "new")) + withoutCmd := writeFakeDaemon(t, append([]string{autoUsage, "proxy"}, baseDaemonFlags...), filepath.Join(dir, "old")) + const creds = "http://muse:s3cret@egress.test:3128" + + for _, tc := range []struct { + name string + goos string + systemd bool + noBash bool + bin string + env map[string]string + cfg string // ~/.pilot/config.json + flags map[string]string + proxy string // plan.Proxy + want bool + }{ + {name: "Muse-like sandbox", env: map[string]string{"HTTPS_PROXY": creds}, want: true}, + {name: "lower-case https_proxy only", env: map[string]string{"https_proxy": creds}, want: true}, + {name: "plan proxy auto", env: map[string]string{"HTTPS_PROXY": creds}, proxy: "auto", want: true}, + {name: "config without proxy_cmd", env: map[string]string{"HTTPS_PROXY": creds}, cfg: `{"transport":"compat"}`, want: true}, + {name: "macOS", goos: "darwin", env: map[string]string{"HTTPS_PROXY": creds}}, + {name: "systemd host", systemd: true, env: map[string]string{"HTTPS_PROXY": creds}}, + {name: "proxy without credentials", env: map[string]string{"HTTPS_PROXY": "http://egress.test:3128"}}, + {name: "ALL_PROXY only", env: map[string]string{"ALL_PROXY": creds}}, + {name: "no proxy", env: map[string]string{}}, + {name: "PILOT_PROXY_CMD set", env: map[string]string{"HTTPS_PROXY": creds, "PILOT_PROXY_CMD": "cat /run/p"}}, + {name: "config proxy_cmd", env: map[string]string{"HTTPS_PROXY": creds}, cfg: `{"proxy_cmd":"cat /run/p"}`}, + {name: "--config file proxy_cmd", env: map[string]string{"HTTPS_PROXY": creds}, flags: map[string]string{"config": "CFGFILE"}}, + {name: "explicit proxy URL", env: map[string]string{"HTTPS_PROXY": creds}, proxy: "http://u:p@corp.test:3128"}, + {name: "proxy off", env: map[string]string{"HTTPS_PROXY": creds}, proxy: "off"}, + {name: "no bash", noBash: true, env: map[string]string{"HTTPS_PROXY": creds}}, + {name: "daemon without -proxy-cmd", bin: withoutCmd, env: map[string]string{"HTTPS_PROXY": creds}}, + } { + t.Run(tc.name, func(t *testing.T) { + home := withTransportEnvCleared(t) + t.Setenv("HOME", home) + for _, k := range []string{"HTTPS_PROXY", "https_proxy", "ALL_PROXY", "PILOT_PROXY_CMD"} { + t.Setenv(k, tc.env[k]) + } + goos := tc.goos + if goos == "" { + goos = "linux" + } + stubSandboxHost(t, goos, tc.systemd, !tc.noBash) + if tc.cfg != "" { + writeTestFile(t, filepath.Join(home, ".pilot", "config.json"), tc.cfg) + } + flags := map[string]string{} + for k, v := range tc.flags { + if v == "CFGFILE" { + v = filepath.Join(t.TempDir(), "custom.json") + writeTestFile(t, v, `{"proxy_cmd":"cat /run/p"}`) + } + flags[k] = v + } + bin := tc.bin + if bin == "" { + bin = withCmd + } + got := sandboxProxyCmdFor(bin, daemonLaunchPlan{Proxy: tc.proxy}, flags) + if (got != "") != tc.want { + t.Fatalf("sandboxProxyCmdFor = %q, want set=%v", got, tc.want) + } + if tc.want && got != sandboxProxyCmd { + t.Errorf("command = %q, want %q", got, sandboxProxyCmd) + } + }) + } +} + +// stubSandboxHost sets the host seams sandboxProxyCmdFor reads. +func stubSandboxHost(t *testing.T, goos string, systemd, bash bool) { + t.Helper() + prevGOOS, prevSystemd, prevBash := hostGOOS, systemdRunning, bashAvailable + hostGOOS = goos + systemdRunning = func() bool { return systemd } + bashAvailable = func() bool { return bash } + t.Cleanup(func() { hostGOOS, systemdRunning, bashAvailable = prevGOOS, prevSystemd, prevBash }) +} + +func writeTestFile(t *testing.T, path, body string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } +} + +// web4-install-sh-edits-not-in-canonical-installer, the binary half: `daemon +// start` in a sandbox hands the daemon PILOT_PROXY_CMD by itself, so rotating +// credentials are re-read even when the installer that set the node up never +// saved proxy_cmd. A configured proxy_cmd is never overridden. +func TestCLIDaemonStartSandboxProxyCmd(t *testing.T) { + t.Parallel() + dir := t.TempDir() + out := filepath.Join(dir, "daemon") + bin := writeFakeDaemon(t, append([]string{autoUsage, "proxy", "proxy-cmd"}, baseDaemonFlags...), out) + home := t.TempDir() + env := cliEnvCleared(map[string]string{ + "PILOT_DAEMON_BIN": bin, + "PILOT_SOCKET": filepath.Join(dir, "pilot.sock"), + "PILOT_HOME": home, + "HTTPS_PROXY": "http://muse:s3cret@egress.test:3128", + "PILOTCTL_TEST_SANDBOX": "1", // Linux, no systemd, bash present + }) + daemonEnv := func() []string { + var got []string + for _, kv := range readLines(t, out+".env") { + if strings.HasPrefix(kv, "PILOT_PROXY_CMD=") { + got = append(got, kv) + } + } + return got + } + + if _, stderr, code := runCLI(t, []string{"daemon", "start", "--foreground"}, env); code != 0 { + t.Fatalf("exit=%d stderr=%s", code, stderr) + } + if got := daemonEnv(); len(got) != 1 || got[0] != "PILOT_PROXY_CMD="+sandboxProxyCmd { + t.Errorf("daemon PILOT_PROXY_CMD = %q, want exactly %q", got, sandboxProxyCmd) + } + + // A proxy_cmd in config.json wins: the daemon reads it itself. + writeTestFile(t, filepath.Join(home, ".pilot", "config.json"), `{"proxy_cmd":"cat /run/proxy-url"}`) + if _, stderr, code := runCLI(t, []string{"daemon", "start", "--foreground"}, env); code != 0 { + t.Fatalf("exit=%d stderr=%s", code, stderr) + } + for _, kv := range daemonEnv() { + if kv != "PILOT_PROXY_CMD=" { + t.Errorf("configured proxy_cmd overridden by %q", kv) + } + } +} diff --git a/cmd/pilotctl/zz_subprocess_test.go b/cmd/pilotctl/zz_subprocess_test.go index d9073adb..2b5e754b 100644 --- a/cmd/pilotctl/zz_subprocess_test.go +++ b/cmd/pilotctl/zz_subprocess_test.go @@ -36,6 +36,13 @@ func TestMain(m *testing.M) { } } os.Args = append([]string{"pilotctl"}, argv...) + if os.Getenv("PILOTCTL_TEST_SANDBOX") == "1" { + // A Linux container without systemd, with bash (see + // sandboxProxyCmdFor), whatever host runs the test. + hostGOOS = "linux" + systemdRunning = func() bool { return false } + bashAvailable = func() bool { return true } + } main() return } diff --git a/install.sh b/install.sh index 1822836f..f6dd4b95 100755 --- a/install.sh +++ b/install.sh @@ -14,6 +14,8 @@ set -e # through $HTTPS_PROXY when UDP does not work; add # `-s -- --transport compat` to skip the UDP probe; proxy # credentials that rotate: see PILOT_PROXY_CMD below) +# Managed node: export PILOT_ENROLLMENT_TOKEN # enter it without putting it in shell history +# sh install.sh --managed-url https://management.pilotprotocol.network # Uninstall: curl -fsSL https://pilotprotocol.network/install.sh | sh -s uninstall # # Flags: @@ -34,6 +36,12 @@ set -e # $HTTPS_PROXY/$ALL_PROXY when set (CONNECT by hostname) — # for UDP-blocked hosts and agent sandboxes whose only way # out is an HTTPS proxy. +# --managed-url +# Install the checksum-pinned core managed runtime, claim +# a one-time hosted identity, and start signed reporting. +# This does not install pilot-mcp or a harness adapter. +# --no-start With --managed-url, install and adopt without starting +# the daemon. The hosted onboarding flow omits this flag. # # Legacy env vars (still honored, lower precedence than flags): # PILOT_RELEASE_TAG=vX.Y.Z Same as --version. @@ -52,6 +60,9 @@ set -e # $https_proxy when none is set. # PILOT_ALLOW_ROOT=1 Install as root on a host with systemd/launchd # (not needed in containers/VMs without systemd). +# PILOT_MANAGEMENT_URL=https://management.example +# Same as --managed-url. Requires the one-time +# PILOT_ENROLLMENT_TOKEN on first adoption. # # Proxies: every download is a curl HTTPS request, so HTTPS_PROXY / https_proxy / # ALL_PROXY / NO_PROXY are honored (curl asks the proxy to CONNECT by hostname — @@ -125,6 +136,7 @@ REGISTRY="${PILOT_REGISTRY:-$DEFAULT_REGISTRY}" BEACON="${PILOT_BEACON:-$DEFAULT_BEACON}" PILOT_DIR="$HOME/.pilot" BIN_DIR="$PILOT_DIR/bin" +MANAGED_CONTROL_PATH="$PILOT_DIR/managed/enterprise-control.json" # validate_safe LABEL VALUE EXTRA — abort if VALUE contains any character # outside [A-Za-z0-9] plus the punctuation in EXTRA. These values are @@ -167,6 +179,8 @@ PILOT_REQUESTED_VERSION="" PILOT_REQUESTED_CHANNEL="" PILOT_YES=0 PILOT_NO_WARN=0 +PILOT_MANAGED_NO_START=0 +PILOT_MANAGEMENT_URL="${PILOT_MANAGEMENT_URL:-}" PILOT_POSITIONAL="" PILOT_REQUESTED_TRANSPORT="" @@ -191,8 +205,15 @@ while [ $# -gt 0 ]; do PILOT_YES=1; shift ;; --no-warn) PILOT_NO_WARN=1; shift ;; + --managed-url|--management-url) + if [ $# -lt 2 ]; then echo "Error: $1 requires an HTTPS origin" >&2; exit 2; fi + PILOT_MANAGEMENT_URL="$2"; shift 2 ;; + --managed-url=*|--management-url=*) + PILOT_MANAGEMENT_URL="${1#*=}"; shift ;; + --no-start) + PILOT_MANAGED_NO_START=1; shift ;; -h|--help) - sed -n '4,33p' "$0" 2>/dev/null || echo "See https://pilotprotocol.network/install.sh" + sed -n '4,71p' "$0" 2>/dev/null || echo "See https://pilotprotocol.network/install.sh" exit 0 ;; --) shift @@ -207,6 +228,70 @@ while [ $# -gt 0 ]; do esac done +PILOT_MANAGED_MODE=0 +MANAGED_TOKEN="" +if [ -n "$PILOT_MANAGEMENT_URL" ]; then + PILOT_MANAGED_MODE=1 + # Accept a cosmetic trailing slash, but require an HTTPS origin with no + # credentials, path, query, or fragment. The value later becomes both a + # manifest URL and the enrollment authority endpoint. + PILOT_MANAGEMENT_URL="${PILOT_MANAGEMENT_URL%/}" + case "$PILOT_MANAGEMENT_URL" in + https://*) ;; + *) echo "Error: --managed-url must be an HTTPS origin." >&2; exit 2 ;; + esac + _managed_host="${PILOT_MANAGEMENT_URL#https://}" + case "$_managed_host" in + ""|*/*|*@*|*\?*|*\#*) + echo "Error: --managed-url must not contain credentials, a path, query, or fragment." >&2 + exit 2 ;; + esac + validate_safe "management host" "$_managed_host" ".:-" + + if [ -n "$PILOT_POSITIONAL" ]; then + echo "Error: --managed-url cannot be combined with an install/uninstall positional command." >&2 + exit 2 + fi + if [ -n "$PILOT_REQUESTED_VERSION" ] || [ -n "${PILOT_RELEASE_TAG:-}" ] \ + || [ -n "$PILOT_REQUESTED_CHANNEL" ] || [ "${PILOT_RC:-}" = "1" ]; then + echo "Error: managed adoption uses the runtime pinned by the management authority; do not combine it with --version or --channel." >&2 + exit 2 + fi + + MANAGED_TOKEN="${PILOT_ENROLLMENT_TOKEN:-}" + # Do not let the bearer secret reach curl, tar, service managers, or any + # other child. It is exported only to the single pilotctl claim process. + unset PILOT_ENROLLMENT_TOKEN + if [ -e "$MANAGED_CONTROL_PATH" ]; then + if [ -L "$MANAGED_CONTROL_PATH" ] || [ ! -f "$MANAGED_CONTROL_PATH" ]; then + echo "Error: the existing managed control attachment is not a regular file." >&2 + exit 1 + fi + if [ -n "$MANAGED_TOKEN" ]; then + echo "Error: this node is already managed; refusing to consume a new enrollment token." >&2 + echo " Re-run without PILOT_ENROLLMENT_TOKEN to repair or update the managed runtime." >&2 + exit 1 + fi + else + if [ -z "$MANAGED_TOKEN" ]; then + echo "Error: PILOT_ENROLLMENT_TOKEN is required for first managed adoption." >&2 + exit 1 + fi + _managed_token_bytes=$(printf '%s' "$MANAGED_TOKEN" | wc -c | tr -d ' ') + if [ "$_managed_token_bytes" -gt 4096 ] \ + || LC_ALL=C printf '%s' "$MANAGED_TOKEN" | grep -q '[[:cntrl:]]'; then + echo "Error: PILOT_ENROLLMENT_TOKEN is invalid." >&2 + exit 1 + fi + fi + # Managed credentials and state created by this process default owner-only. + umask 077 + MANIFEST_URL="${PILOT_MANAGEMENT_URL}/.well-known/pilot-managed-runtime.json" +elif [ "$PILOT_MANAGED_NO_START" = "1" ]; then + echo "Error: --no-start is only valid with --managed-url." >&2 + exit 2 +fi + # Validate channel value early so we fail fast. `edge` is a back-compat alias # for `beta`: the manifest publishes channels.stable and channels.beta only, so # a literal `edge` lookup would resolve empty and (previously) silently fall @@ -252,6 +337,47 @@ if [ "${1:-}" != "uninstall" ] && [ "$(id -u)" = "0" ] && [ -z "${PILOT_ALLOW_RO fi fi +# A managed identity is per node, but the CLI links, service label and daemon +# socket are machine-wide. A different HOME therefore does not make a second +# installation safe. Refuse before downloading, stopping services, replacing +# links or consuming the enrollment token. An existing attachment in this +# same PILOT_DIR remains the ordinary repair/update path handled above. +if [ "$PILOT_MANAGED_MODE" = "1" ] && [ ! -e "$MANAGED_CONTROL_PATH" ] \ + && [ "${PILOT_REPLACE_EXISTING_NODE:-}" != "1" ]; then + _pilot_collision="" + for _pilot_link in /usr/local/bin/pilotctl /usr/local/bin/pilot-daemon; do + if [ -L "$_pilot_link" ]; then + _pilot_target=$(readlink "$_pilot_link" 2>/dev/null || true) + case "$_pilot_target" in + "$BIN_DIR"/*) ;; + */.pilot/bin/*) _pilot_collision="${_pilot_collision}${_pilot_collision:+, }$_pilot_link -> $_pilot_target" ;; + esac + fi + done + _pilot_os=$(uname -s | tr '[:upper:]' '[:lower:]') + if [ "$_pilot_os" = "darwin" ] && command -v launchctl >/dev/null 2>&1; then + _pilot_service=$(launchctl print "gui/$(id -u)/network.pilotprotocol.pilot-daemon" 2>/dev/null || true) + if [ -n "$_pilot_service" ] && ! printf '%s\n' "$_pilot_service" | grep -Fq "$BIN_DIR/pilot-daemon"; then + _pilot_collision="${_pilot_collision}${_pilot_collision:+, }LaunchAgent network.pilotprotocol.pilot-daemon" + fi + elif [ "$_pilot_os" = "linux" ] && command -v systemctl >/dev/null 2>&1; then + _pilot_service=$(systemctl cat pilot-daemon 2>/dev/null || true) + if [ -n "$_pilot_service" ] && ! printf '%s\n' "$_pilot_service" | grep -Fq "$BIN_DIR/pilot-daemon"; then + _pilot_collision="${_pilot_collision}${_pilot_collision:+, }systemd pilot-daemon" + fi + fi + if [ -n "$_pilot_collision" ]; then + MANAGED_TOKEN="" + unset MANAGED_TOKEN + echo "Error: another Pilot node installation already owns machine-wide resources." >&2 + echo " Detected: $_pilot_collision" >&2 + echo " This enrollment token was not consumed and nothing was changed." >&2 + echo " Manage or remove the existing node first; do not run two node identities under one service label." >&2 + exit 1 + fi + _pilot_collision=""; _pilot_target=""; _pilot_service=""; _pilot_os="" +fi + # The transport already saved in config.json, if any ("udp", "compat", # "auto"). A re-run without --transport keeps it, so regenerated service # units stay consistent with it. @@ -336,9 +462,13 @@ manifest_field() { # release's checksums.txt (served from GitHub). manifest_platform_sha256() { _mp_plat="$1"; _mp_file="$2" - sed -n "/\"${_mp_plat}\"[[:space:]]*:[[:space:]]*{/,/}/p" "$_mp_file" \ - | grep '"sha256"' | head -1 \ - | sed -E 's/.*"sha256"[[:space:]]*:[[:space:]]*"([^"]*)".*/\1/' + # The authority is free to emit compact JSON. A line-range parser sees all + # platform objects on that one line and a greedy replacement can therefore + # return the final platform's hash. Collapse whitespace deliberately, then + # constrain the match to this platform's first closing brace. + tr -d '\r\n' < "$_mp_file" \ + | sed -n -E "s/.*\"${_mp_plat}\"[[:space:]]*:[[:space:]]*\\{[^}]*\"sha256\"[[:space:]]*:[[:space:]]*\"([^\"]*)\"[^}]*\\}.*/\\1/p" \ + | head -1 } # version_compare a b emits -1 / 0 / 1 for ab. @@ -616,6 +746,11 @@ HAVE_MANIFEST=0 if fetch_manifest "$MANIFEST_FILE"; then HAVE_MANIFEST=1 fi +if [ "$PILOT_MANAGED_MODE" = "1" ] && [ "$HAVE_MANIFEST" != "1" ]; then + echo "Error: the management authority did not publish a managed-runtime manifest." >&2 + echo " No binary or enrollment state was changed." >&2 + exit 1 +fi if [ -n "$PILOT_REQUESTED_VERSION" ]; then TAG="$PILOT_REQUESTED_VERSION" @@ -650,6 +785,16 @@ else | tr -d '\r' | head -1) fi +if [ "$PILOT_MANAGED_MODE" = "1" ]; then + case "$TAG" in + managed-runtime-v[0-9]*.[0-9]*.[0-9]*) ;; + *) + echo "Error: the management authority published an invalid managed-runtime version." >&2 + exit 1 ;; + esac + validate_safe "managed runtime version" "$TAG" ".-" +fi + # Fail loudly if the user explicitly asked for a released version/channel but it # resolved to nothing. Silently dropping to the unpinned, UNVERIFIED source # build below would give the user a binary they never asked for, with none of @@ -780,6 +925,11 @@ if [ -n "$TAG" ]; then fi tar -xzf "$TMPDIR/$ARCHIVE" -C "$TMPDIR" --strip-components=1 else + if [ "$PILOT_MANAGED_MODE" = "1" ]; then + echo "Error: managed runtime ${TAG} could not be downloaded." >&2 + echo " Refusing to fall back to an unmanaged build." >&2 + exit 1 + fi # Archive download failed. Only the automatic default path may fall # back to a source build; an explicit request already hard-failed # above, so reaching here means no version/channel was pinned. @@ -846,6 +996,17 @@ if [ -z "$STAGED_DAEMON" ] || [ -z "$STAGED_CTL" ]; then echo " Nothing was replaced — your existing install is untouched." >&2 exit 1 fi +if [ "$PILOT_MANAGED_MODE" = "1" ]; then + _managed_probe=$(PILOT_ENROLLMENT_TOKEN='' "$STAGED_CTL" --json enterprise adopt --endpoint "$PILOT_MANAGEMENT_URL" 2>&1 || true) + case "$_managed_probe" in + *PILOT_ENROLLMENT_TOKEN*) ;; + *) + echo "Error: ${TAG} does not contain core managed-adoption support." >&2 + echo " Nothing was replaced and the enrollment token was not consumed." >&2 + exit 1 ;; + esac + _managed_probe="" +fi # gateway is optional: extracted to a sibling repo, no longer ships in # release tarballs (release.yml BINS=daemon/pilotctl/updater) and the # source build only runs when ./cmd/gateway is present in the checkout. @@ -900,7 +1061,10 @@ if [ "$OS" = "linux" ] && [ "$CAN_PRIV" = true ] \ fi ;; esac if [ -n "$_want" ]; then - RESTART_SYSTEMD="${RESTART_SYSTEMD}${RESTART_SYSTEMD:+ }${_svc}" + if { [ "$PILOT_MANAGED_MODE" != "1" ] || [ "$PILOT_MANAGED_NO_START" != "1" ]; } \ + && { [ "$PILOT_MANAGED_MODE" != "1" ] || [ "$_svc" != "pilot-updater" ]; }; then + RESTART_SYSTEMD="${RESTART_SYSTEMD}${RESTART_SYSTEMD:+ }${_svc}" + fi # shellcheck disable=SC2086 # $PILOT_SUDO is "" or "sudo" — intentional split $PILOT_SUDO systemctl stop "$_svc" 2>/dev/null || true echo " Stopped ${_svc} (will restart after upgrade)" @@ -911,7 +1075,10 @@ if [ "$OS" = "darwin" ]; then for _label in network.pilotprotocol.pilot-daemon network.pilotprotocol.pilot-updater; do _lp="$HOME/Library/LaunchAgents/${_label}.plist" if [ -f "$_lp" ] && launchctl list 2>/dev/null | grep -q "$_label"; then - RESTART_LAUNCHD="${RESTART_LAUNCHD}${RESTART_LAUNCHD:+ }${_label}" + if { [ "$PILOT_MANAGED_MODE" != "1" ] || [ "$PILOT_MANAGED_NO_START" != "1" ]; } \ + && { [ "$PILOT_MANAGED_MODE" != "1" ] || [ "$_label" != "network.pilotprotocol.pilot-updater" ]; }; then + RESTART_LAUNCHD="${RESTART_LAUNCHD}${RESTART_LAUNCHD:+ }${_label}" + fi launchctl unload "$_lp" 2>/dev/null || true echo " Unloaded ${_label} (will reload after upgrade)" fi @@ -937,6 +1104,32 @@ install_bin "$STAGED_CTL" "$BIN_DIR/pilotctl" [ -n "$STAGED_GATEWAY" ] && install_bin "$STAGED_GATEWAY" "$BIN_DIR/pilot-gateway" [ -n "$STAGED_UPDATER" ] && install_bin "$STAGED_UPDATER" "$BIN_DIR/pilot-updater" +# --- Optional hosted adoption (core Pilot, not MCP) --- +# +# The one-time token is exposed only to this process. pilotctl validates the +# delegated key, root pin, trust bundle, bootstrap policy, authority origins, +# and owner-only output before atomically installing ~/.pilot/managed. +MANAGED_ADOPTED=0 +if [ "$PILOT_MANAGED_MODE" = "1" ] && [ ! -e "$MANAGED_CONTROL_PATH" ]; then + if ! _managed_result=$(PILOT_ENROLLMENT_TOKEN="$MANAGED_TOKEN" "$BIN_DIR/pilotctl" --json enterprise adopt --endpoint "$PILOT_MANAGEMENT_URL" 2>&1); then + MANAGED_TOKEN="" + unset MANAGED_TOKEN + echo "Error: the hosted authority did not complete managed adoption." >&2 + printf '%s\n' "$_managed_result" >&2 + exit 1 + fi + MANAGED_TOKEN="" + unset MANAGED_TOKEN + _managed_result="" + if [ -L "$MANAGED_CONTROL_PATH" ] || [ ! -f "$MANAGED_CONTROL_PATH" ]; then + echo "Error: managed adoption returned without installing the verified control attachment." >&2 + exit 1 + fi + MANAGED_ADOPTED=1 +fi +MANAGED_TOKEN="" +unset MANAGED_TOKEN + # --- Symlink into /usr/local/bin so NON-INTERACTIVE shells can find pilotctl --- # # This symlink is the only thing that makes `pilotctl` resolve from a @@ -977,7 +1170,7 @@ if [ "$LINK_OK" = true ]; then # shellcheck disable=SC2086 $LINK_SUDO ln -sf "$BIN_DIR/pilot-gateway" "$LINK_DIR/pilot-gateway" 2>/dev/null || true fi - if [ -f "$BIN_DIR/pilot-updater" ]; then + if [ "$PILOT_MANAGED_MODE" != "1" ] && [ -f "$BIN_DIR/pilot-updater" ]; then # shellcheck disable=SC2086 $LINK_SUDO ln -sf "$BIN_DIR/pilot-updater" "$LINK_DIR/pilot-updater" 2>/dev/null || true fi @@ -1111,6 +1304,7 @@ fi # value. PILOT_PROXY_CMD sets it explicitly; otherwise it is saved only in a # Linux container/VM without systemd whose proxy carries credentials, and # never over an existing proxy_cmd. +# shellcheck disable=SC2016 # literal: the fresh bash expands it, not this shell SANDBOX_PROXY_CMD='bash -c '\''printf %s "${https_proxy:-$HTTPS_PROXY}"'\''' PROXY_CMD_TO_SAVE="${PILOT_PROXY_CMD:-}" if [ -z "$PROXY_CMD_TO_SAVE" ] && [ "$OS" = "linux" ] && [ ! -d /run/systemd/system ] \ @@ -1221,7 +1415,13 @@ service_proxy_note() { # only when the file is absent so an operator who later runs `pilotctl update # disable` is never silently re-enabled. Turn off any time with # `pilotctl update disable`. -if [ "$UPDATING" != true ] && [ ! -f "$PILOT_DIR/auto-update.json" ]; then +if [ "$PILOT_MANAGED_MODE" = "1" ]; then + # Stable-channel updater builds do not yet carry the managed control + # client. Pin this authority-selected runtime instead of allowing + # a background downgrade to silently remove enforcement. + printf '{\n "enabled": false,\n "reason": "managed-runtime-pinned-by-authority"\n}\n' > "$PILOT_DIR/auto-update.json" + echo "Auto-updates pinned to the management authority runtime channel" +elif [ "$UPDATING" != true ] && [ ! -f "$PILOT_DIR/auto-update.json" ]; then printf '{\n "enabled": true\n}\n' > "$PILOT_DIR/auto-update.json" echo "Auto-updates ENABLED (opt-out) — disable with: pilotctl update disable" fi @@ -1232,6 +1432,8 @@ fi # what makes re-running the installer a real repair path: a host whose unit was # written by an older, buggy installer gets a correct one without uninstalling. +MANAGED_START_STYLE="" + if [ "$OS" = "linux" ] && command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then if [ "$CAN_PRIV" = true ]; then echo "Setting up systemd service..." @@ -1294,7 +1496,7 @@ RestartSec=5 WantedBy=multi-user.target SVC # Auto-updater service - if [ -f "$BIN_DIR/pilot-updater" ]; then + if [ "$PILOT_MANAGED_MODE" != "1" ] && [ -f "$BIN_DIR/pilot-updater" ]; then # shellcheck disable=SC2086 $PILOT_SUDO tee /etc/systemd/system/pilot-updater.service >/dev/null </dev/null || true + fi echo " Service: pilot-daemon.service" - echo " Service: pilot-updater.service (auto-updates)" + if [ "$PILOT_MANAGED_MODE" != "1" ] && [ -f "$BIN_DIR/pilot-updater" ]; then + echo " Service: pilot-updater.service (auto-updates)" + fi service_proxy_note "pilot-daemon.service" # Auto-enable + start the updater so future releases land without @@ -1335,7 +1545,7 @@ USVC # operator-tunable flags (-public, -hostname, registry overrides) that the # operator may want to set before first start; on a re-run anything that # was already running is restored below. - if [ -f "$BIN_DIR/pilot-updater" ]; then + if [ "$PILOT_MANAGED_MODE" != "1" ] && [ -f "$BIN_DIR/pilot-updater" ]; then # shellcheck disable=SC2086 if $PILOT_SUDO systemctl enable --now pilot-updater; then echo " Started: pilot-updater (auto-updates enabled)" @@ -1356,23 +1566,36 @@ USVC fi done - case " $RESTART_SYSTEMD " in - *" pilot-daemon "*) ;; - *) echo " Start daemon: sudo systemctl enable --now pilot-daemon" ;; - esac + if [ "$PILOT_MANAGED_MODE" = "1" ] && [ "$PILOT_MANAGED_NO_START" != "1" ]; then + # Managed onboarding promises a live signed check-in, so unlike an + # ordinary local install it explicitly enables the daemon now. + # shellcheck disable=SC2086 + $PILOT_SUDO systemctl enable --now pilot-daemon + MANAGED_START_STYLE="systemd" + echo " Started: pilot-daemon (managed reporting enabled)" + elif [ "$PILOT_MANAGED_MODE" != "1" ]; then + case " $RESTART_SYSTEMD " in + *" pilot-daemon "*) ;; + *) echo " Start daemon: sudo systemctl enable --now pilot-daemon" ;; + esac + fi else echo " Skipped systemd setup (run as root or with passwordless sudo to enable)" - echo " Start the daemon without a service manager: pilotctl daemon start" + if [ "$PILOT_MANAGED_MODE" != "1" ]; then + echo " Start the daemon without a service manager: pilotctl daemon start" + fi fi elif [ "$OS" = "linux" ]; then # systemd is not the init system here (container / WSL / CI runner / # hosted agent sandbox). There is no service to install — tell the agent # the portable start path instead of silently leaving it with no daemon. - echo "No systemd detected (container / WSL / CI / sandbox) — start the daemon manually:" - echo " pilotctl daemon start" - if [ "$EFFECTIVE_TRANSPORT" != "udp" ]; then - echo " (transport=${EFFECTIVE_TRANSPORT}; start it from a shell that has HTTPS_PROXY" - echo " set if this host reaches the internet only through a proxy)" + if [ "$PILOT_MANAGED_MODE" != "1" ]; then + echo "No systemd detected (container / WSL / CI / sandbox) — start the daemon manually:" + echo " pilotctl daemon start" + if [ "$EFFECTIVE_TRANSPORT" != "udp" ]; then + echo " (transport=${EFFECTIVE_TRANSPORT}; start it from a shell that has HTTPS_PROXY" + echo " set if this host reaches the internet only through a proxy)" + fi fi fi @@ -1459,7 +1682,7 @@ ${PLIST_ENV} RunAtLoad PLIST # Auto-updater LaunchAgent - if [ -f "$BIN_DIR/pilot-updater" ]; then + if [ "$PILOT_MANAGED_MODE" != "1" ] && [ -f "$BIN_DIR/pilot-updater" ]; then UPLIST="$PLIST_DIR/network.pilotprotocol.pilot-updater.plist" cat > "$UPLIST" < @@ -1488,7 +1711,15 @@ UPLIST fi echo " Service: network.pilotprotocol.pilot-daemon" - echo " Service: network.pilotprotocol.pilot-updater (auto-updates)" + if [ "$PILOT_MANAGED_MODE" = "1" ]; then + _managed_updater_plist="$PLIST_DIR/network.pilotprotocol.pilot-updater.plist" + if [ -f "$_managed_updater_plist" ]; then + launchctl unload -w "$_managed_updater_plist" 2>/dev/null || true + fi + fi + if [ "$PILOT_MANAGED_MODE" != "1" ] && [ -f "$BIN_DIR/pilot-updater" ]; then + echo " Service: network.pilotprotocol.pilot-updater (auto-updates)" + fi service_proxy_note "the launchd agent" # Auto-load the updater LaunchAgent so future releases land without @@ -1501,7 +1732,7 @@ UPLIST # idempotent: any stale running agent is replaced cleanly. -w persists # the load across reboots. The daemon is left as opt-in for the same # reason as the Linux branch. - if [ -f "$BIN_DIR/pilot-updater" ] && [ -f "$UPLIST" ]; then + if [ "$PILOT_MANAGED_MODE" != "1" ] && [ -f "$BIN_DIR/pilot-updater" ] && [ -f "$UPLIST" ]; then launchctl unload "$UPLIST" 2>/dev/null || true launchctl load -w "$UPLIST" echo " Started: pilot-updater (auto-updates enabled)" @@ -1523,13 +1754,42 @@ UPLIST fi done - case " $RESTART_LAUNCHD " in - *" network.pilotprotocol.pilot-daemon "*) ;; - *) - echo " Start daemon: launchctl load -w $PLIST" - echo " Stop daemon: launchctl unload $PLIST" - ;; - esac + if [ "$PILOT_MANAGED_MODE" != "1" ]; then + case " $RESTART_LAUNCHD " in + *" network.pilotprotocol.pilot-daemon "*) ;; + *) + echo " Start daemon: launchctl load -w $PLIST" + echo " Stop daemon: launchctl unload $PLIST" + ;; + esac + fi +fi + +# A managed Connect Agent run is complete only when the newly adopted core +# daemon is locally responsive. The management console independently waits for +# the signed remote report, so this check cannot forge onboarding success. +if [ "$PILOT_MANAGED_MODE" = "1" ]; then + if [ "$PILOT_MANAGED_NO_START" = "1" ]; then + echo "Managed runtime adopted but not started (--no-start)." + elif [ "$MANAGED_START_STYLE" = "systemd" ]; then + _managed_wait=0 + until "$BIN_DIR/pilotctl" daemon status --check >/dev/null 2>&1; do + _managed_wait=$((_managed_wait + 1)) + if [ "$_managed_wait" -ge 30 ]; then + echo "Error: the managed daemon did not become ready under systemd." >&2 + exit 1 + fi + sleep 1 + done + else + # Portable and launchd installs need an explicit restart so a daemon + # that was already running cannot keep the pre-adoption configuration. + if "$BIN_DIR/pilotctl" daemon status --check >/dev/null 2>&1; then + "$BIN_DIR/pilotctl" daemon stop >/dev/null + fi + "$BIN_DIR/pilotctl" daemon start --wait 30s >/dev/null + "$BIN_DIR/pilotctl" daemon status --check >/dev/null + fi fi # --- Add to PATH --- @@ -1601,6 +1861,30 @@ fi # Write version file for the auto-updater [ -n "$TAG" ] && echo "$TAG" > "$BIN_DIR/.pilot-version" +if [ "$PILOT_MANAGED_MODE" = "1" ]; then + echo "" + echo "Managed Pilot node ready:" + echo " Runtime: ${TAG}" + echo " Authority: ${PILOT_MANAGEMENT_URL}" + echo " Control: ${MANAGED_CONTROL_PATH}" + if [ "$MANAGED_ADOPTED" = "1" ]; then + echo " Enrollment: claimed once and installed" + else + echo " Enrollment: existing managed identity preserved" + fi + if [ "$PILOT_MANAGED_NO_START" = "1" ]; then + echo " Daemon: not started (--no-start)" + else + echo " Daemon: running; signed fleet reporting enabled" + fi + echo " MCP: not installed (optional, separate product)" + echo "" + echo "Harness interception is a separate optional attachment step." + echo "The management console will verify the node's signed report before" + echo "it marks onboarding complete." + exit 0 +fi + # --- Upgrade: short summary, skip the first-run onboarding text --- # # Everything above this point (binary swap, unit/plist regeneration, service From 1dd9e8e7a7a440f2f0bdbe1815a2ec64f31bdb78 Mon Sep 17 00:00:00 2001 From: Teodor Calin Date: Thu, 24 Sep 2026 03:23:07 +0300 Subject: [PATCH 08/19] docs(changelog): link the installer change to pilot-protocol/release#49 Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf9236d1..03bfd4c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,8 +96,8 @@ Detailed per-release notes are on the - **`install.sh --transport `** (or `PILOT_TRANSPORT`). install.sh here is a copy of `pilot-protocol/release:install.sh`, the script https://pilotprotocol.network/install.sh serves; these installer changes - reach users through the matching pilot-protocol/release change, which also - keeps the managed-node mode (`--managed-url`). + reach users through pilot-protocol/release#49, which also keeps the + managed-node mode (`--managed-url`). `udp` and `compat` are saved; `auto` (the default) is not — `--transport auto` removes a saved transport. `compat` skips the UDP probe. No `proxy` key is written (the daemon default already uses the environment's proxy). From fbab53b1dfbef6546a4244c2fdde3c3a3782f48c Mon Sep 17 00:00:00 2001 From: Teodor Calin Date: Thu, 24 Sep 2026 05:02:33 +0300 Subject: [PATCH 09/19] feat(proxy): netproxy credential refresh (common v0.5.15); auto stays on the proxy; daemon start names the proxy error Credential refresh now comes from common v0.5.15's netproxy instead of the branch-local proxy policy (internal/proxyconf/policy.go is gone): - -proxy-cmd / $PILOT_PROXY_CMD / config.json proxy_cmd build the resolver with netproxy.WithRefreshCommand. The command runs at startup, again once 60s have passed (lookup-driven), and whenever a proxy answers 407; the rejected connection is then retried once. - Registry (primary, pool, every redial) and the compat WSS beacon (and its reconnects) dial through netproxy.Dialer; daemon-owned HTTP clients use netproxy.RefreshingTransport (proxyconf.RoundTripper, loopback direct); http.DefaultTransport is configured in place so plugin clients take the current credentials and a 407 refreshes them for the next request. - pkg/daemon: Config.Proxy is the only knob (ProxyPolicy, ProxyRefreshInterval, StaticProxyPolicy, NewCommandProxyPolicy removed); ResolveProxy takes netproxy options. - pilotctl daemon start --proxy-cmd (passed as $PILOT_PROXY_CMD, never on argv); pilotctl's own registry commands use $PILOT_PROXY_CMD, config proxy_cmd or the sandbox default too, so a stale HTTPS_PROXY in its environment is refreshed and a 407 is retried. E2E product gap (phase-2 scenario 2c): with a proxy configured and -transport=auto, a proxy error during the compat check (407, 403, garbled answer, proxy unreachable) no longer falls back to udp, which dialed the raw registry directly past the proxy. SelectTransport picks compat and returns the proxy error; the daemon logs it at WARN with a hint, and the registry/compat dial failures end with a hint naming the fix. Fatal startup errors are logged at ERROR (log.Fatalf printed them at INFO). pilotctl daemon start notices a daemon that exits during startup instead of polling until the deadline, and on exit or timeout prints the daemon's last error and last proxy error from its log with a hint, instead of only "did not become ready". gosec: annotate the daemon exec and log read, handle probe Close errors. Tests: in-process rotating-credential CONNECT proxies for proxyconf (dial retry, RoundTripper retry, DefaultTransport refresh), pkg/daemon (registry + WSS reconnect after rotation, timed refresh, MOTD client retry, auto stays compat on 407/403/unreachable), cmd/daemon (subprocess: auto + 407 stays on the proxy, mutation-checked), pilotctl (registry dial follows rotated credentials, proxy-first routes, daemon start failure reporting with fake daemons, and an end-to-end run of the real daemon behind a rejecting proxy). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 40 +- README.md | 6 +- cmd/daemon/main.go | 56 +-- cmd/daemon/netflags.go | 42 +- cmd/daemon/netflags_test.go | 69 +++- cmd/daemon/proxy.go | 92 ++--- cmd/daemon/proxy_refresh_test.go | 7 +- cmd/daemon/proxy_test.go | 61 ++- cmd/pilotctl/daemon_startlog.go | 275 +++++++++++++ cmd/pilotctl/daemon_transport.go | 24 +- cmd/pilotctl/main.go | 61 ++- cmd/pilotctl/registry_dial.go | 60 ++- cmd/pilotctl/zz_proxy_rotation_test.go | 495 +++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +- internal/proxyconf/policy.go | 525 ------------------------- internal/proxyconf/policy_test.go | 501 ----------------------- internal/proxyconf/proxyconf.go | 184 ++++++++- internal/proxyconf/refresh_test.go | 394 +++++++++++++++++++ pkg/daemon/daemon.go | 76 ++-- pkg/daemon/proxy.go | 107 ++--- pkg/daemon/transport_auto.go | 70 +++- pkg/daemon/tunnel.go | 7 +- pkg/daemon/zz_proxy_refresh_test.go | 211 +++++++--- pkg/daemon/zz_proxy_test.go | 40 +- pkg/daemon/zz_transport_auto_test.go | 110 +++++- 26 files changed, 2107 insertions(+), 1412 deletions(-) create mode 100644 cmd/pilotctl/daemon_startlog.go create mode 100644 cmd/pilotctl/zz_proxy_rotation_test.go delete mode 100644 internal/proxyconf/policy.go delete mode 100644 internal/proxyconf/policy_test.go create mode 100644 internal/proxyconf/refresh_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 936dd9ba..b342c538 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,24 +29,37 @@ Detailed per-release notes are on the word is an error instead of a proxy host name. Proxy credentials never appear in logs, errors or `-help`. - **Rotating proxy credentials: `-proxy-cmd` / `$PILOT_PROXY_CMD` / - config.json `proxy_cmd`.** A command whose output is the current proxy URL - (e.g. `bash -c 'printf %s "$https_proxy"'`); the daemon re-runs it every - 60s and whenever the proxy answers a CONNECT with 407, and retries that - connection once with the new credentials. Registry redials, WSS beacon - reconnects and every HTTP client (plugins included) follow it, so a sandbox - that rotates its proxy credentials every few minutes (Meta Muse) no longer + config.json `proxy_cmd`** (`pilotctl daemon start --proxy-cmd`, passed as + `$PILOT_PROXY_CMD`, never on argv). A command whose output is the current + proxy URL (e.g. `bash -c 'printf %s "$https_proxy"'`). It is common + v0.5.15's netproxy refresh (`netproxy.WithRefreshCommand`): the command + runs at startup, again once 60s have passed, and whenever the proxy answers + a CONNECT with 407, after which that connection is retried once with the + new credentials (`netproxy.Dialer` for the registry — primary, pool and + every redial — and the compat WSS beacon and its reconnects; + `netproxy.RefreshingTransport` for the daemon's own HTTP clients; plugin + clients on `http.DefaultTransport` pick up the new credentials on their + next request). Tunnels already open are never touched. A sandbox that + rotates its proxy credentials every few minutes (Meta Muse) no longer leaves a node "online with all apps broken" until a restart. In a Linux container/VM without systemd whose `HTTPS_PROXY` carries credentials, `pilotctl daemon start` hands the daemon `PILOT_PROXY_CMD=bash -c 'printf %s "${https_proxy:-$HTTPS_PROXY}"'` itself when no `proxy_cmd` is configured (so a node set up by any installer gets it), and the installer - saves the same command. The command's output is never logged. + saves the same command; pilotctl's own registry commands use the same + command. The command's output is never logged, and a failing command keeps + the last good proxy URL (at first, the launch environment's). - **`-transport=auto`.** UDP when the beacon answers a UDP discover (one round trip), otherwise compat when the compat beacon itself answers over TCP 443 (through the proxy, if any: a TLS GET of the beacon path must return `426 Upgrade Required` — a front that accepts TCP while the beacon is down does - not count), otherwise udp as before; the decision is logged once - (`transport auto-selected`). It never moves a node with a private registry + not count), otherwise compat too when a proxy is configured and it refuses + the check (407 wrong or stale credentials, 403, a garbled answer, or the + proxy cannot be reached) — udp would dial the registry directly, past the + proxy, which proxy-only sandboxes kill; compat keeps every connection on + the proxy, refreshes the credentials on a 407 and fails naming the proxy's + answer — otherwise udp as before; the decision is logged once + (`transport auto-selected`, at WARN with a hint when the proxy refused). It never moves a node with a private registry or beacon onto the public compat beacon. pilot-daemon's own default stays `udp` (or `$PILOT_TRANSPORT_DEFAULT`, which applies only when nothing else chooses); `pilotctl daemon start` asks for `auto` whenever no transport is @@ -87,6 +100,15 @@ Detailed per-release notes are on the direct connection whose peer talks or hangs up before the first request (a sandbox's network guard) is not used, so the proxy's error is reported instead of a broken pipe. `proxy=off` restores direct dials. +- **`pilotctl daemon start` says why a start failed.** It notices a daemon + that exits during startup at once instead of polling its socket until the + deadline, and both then and on a timeout it prints the daemon's last error + and its last proxy error from the log (for example `last proxy error: proxy + CONNECT registry.pilotprotocol.network:443: 407 Proxy Authentication + Required`), with a hint for it (wrong or rotated credentials → `proxy_cmd`), + instead of only "did not become ready". pilot-daemon logs its fatal + startup errors at ERROR (they came out at INFO), and a registry or compat + beacon dial the proxy refused ends with a hint naming the fix. - **`daemon start` forwards the proxy/TLS environment** (`HTTPS_PROXY`, `HTTP_PROXY`, `ALL_PROXY`, `NO_PROXY` in both cases, `PILOT_PROXY`, `PILOT_TRANSPORT`, `PILOT_REGISTRY_TRUST`, `PILOT_REGISTRY_FINGERPRINT`, diff --git a/README.md b/README.md index 183aa036..f53849d6 100644 --- a/README.md +++ b/README.md @@ -294,11 +294,11 @@ curl -fsSL https://pilotprotocol.network/install.sh | sh pilotctl daemon start ``` -New installs use transport `auto`: the daemon probes the beacon over UDP once at startup (one round trip when UDP works, at most ~1.5s when it does not) and, when there is no answer but the compat beacon answers over TCP 443 (a TLS request that a live beacon answers with `426 Upgrade Required`), runs in **compat** mode — registry over TLS on `registry.pilotprotocol.network:443`, beacon over WSS on `beacon.pilotprotocol.network:443`. When neither answers (no network yet, beacon outage) it stays on UDP, as before. In compat mode the daemon tunnels through `$HTTPS_PROXY` / `$ALL_PROXY` (honoring `$NO_PROXY`), asking the proxy to `CONNECT` by host name, so poisoned local DNS for the Pilot hosts does not matter. `pilotctl`'s own registry commands (`lookup`, the auto-handshake check, `recovery`) follow the daemon: through the proxy when it runs compat (or with an explicit proxy URL), directly when it runs UDP; with no daemon running and no transport configured they try the proxy first and a direct connection second. To skip the UDP probe, pick compat explicitly: `sh -s -- --transport compat`, `pilotctl config --set transport=compat`, or `pilotctl daemon start --transport compat`. +New installs use transport `auto`: the daemon probes the beacon over UDP once at startup (one round trip when UDP works, at most ~1.5s when it does not) and, when there is no answer but the compat beacon answers over TCP 443 (a TLS request that a live beacon answers with `426 Upgrade Required`), runs in **compat** mode — registry over TLS on `registry.pilotprotocol.network:443`, beacon over WSS on `beacon.pilotprotocol.network:443`. When a proxy is configured and it refuses that check (wrong or stale credentials: `407`, a `403`, or the proxy cannot be reached), it runs compat anyway rather than falling back to direct connections past the proxy, and the registry dial then names the proxy's answer; `pilotctl daemon start` prints the daemon's last proxy error with a hint instead of only "did not become ready". When neither answers and no proxy is involved (no network yet, beacon outage) it stays on UDP, as before. In compat mode the daemon tunnels through `$HTTPS_PROXY` / `$ALL_PROXY` (honoring `$NO_PROXY`), asking the proxy to `CONNECT` by host name, so poisoned local DNS for the Pilot hosts does not matter. `pilotctl`'s own registry commands (`lookup`, the auto-handshake check, `recovery`) follow the daemon: through the proxy when it runs compat (or with an explicit proxy URL), directly when it runs UDP; with no daemon running and no transport configured they try the proxy first and a direct connection second. To skip the UDP probe, pick compat explicitly: `sh -s -- --transport compat`, `pilotctl config --set transport=compat`, or `pilotctl daemon start --transport compat`. - **No root, systemd or launchd needed.** Start the daemon with `pilotctl daemon start` from a shell that has the proxy variables. In a container or VM without systemd the installer also runs as root (the agent user in hosted sandboxes); on a regular host it still refuses root unless `PILOT_ALLOW_ROOT=1`. - **Proxy with credentials:** keep them out of `ps` — export `HTTPS_PROXY` / `PILOT_PROXY`, or `pilotctl daemon start --proxy http://user:pass@host:port` (pilotctl hands a URL with credentials to the daemon in its environment, never on its command line). For a systemd/launchd service, which does not see your shell's variables, save it: `pilotctl config --set proxy=http://user:pass@host:port` (config.json is 0600). -- **Proxy credentials that rotate** (Meta Muse rotates the credentials in `HTTPS_PROXY` every few minutes): a long-running daemon would keep the ones it started with, and new connections would start failing with `407 Proxy Authentication Required` while old tunnels stay up ("node online, apps broken"). Give the daemon a command that prints the current proxy URL — `proxy_cmd` in config.json, `$PILOT_PROXY_CMD` or `-proxy-cmd` — and it re-runs it every 60s and whenever the proxy answers 407, retrying with the fresh credentials; no restart needed. In a Linux container/VM without systemd whose `HTTPS_PROXY` carries credentials, `pilotctl daemon start` hands the daemon `PILOT_PROXY_CMD=bash -c 'printf %s "${https_proxy:-$HTTPS_PROXY}"'` (a fresh shell sees the current value) whenever no `proxy_cmd` is configured, and the installer saves the same command in config.json; elsewhere: `pilotctl config --set proxy_cmd="bash -c 'printf %s \"\$https_proxy\"'"`. Without it, restart the daemon from a fresh shell when it starts failing. +- **Proxy credentials that rotate** (Meta Muse rotates the credentials in `HTTPS_PROXY` every few minutes): a long-running daemon would keep the ones it started with, and new connections would start failing with `407 Proxy Authentication Required` while old tunnels stay up ("node online, apps broken"). Give the daemon a command that prints the current proxy URL — `proxy_cmd` in config.json, `$PILOT_PROXY_CMD`, `-proxy-cmd` or `pilotctl daemon start --proxy-cmd` — and it re-runs it once 60s have passed and whenever the proxy answers 407, retrying that connection once with the fresh credentials (registry, compat beacon and HTTP clients alike, via common's `netproxy` refresh); no restart needed. pilotctl's own registry commands use the same command. In a Linux container/VM without systemd whose `HTTPS_PROXY` carries credentials, `pilotctl daemon start` hands the daemon `PILOT_PROXY_CMD=bash -c 'printf %s "${https_proxy:-$HTTPS_PROXY}"'` (a fresh shell sees the current value) whenever no `proxy_cmd` is configured, and the installer saves the same command in config.json; elsewhere: `pilotctl config --set proxy_cmd="bash -c 'printf %s \"\$https_proxy\"'"`. Without it, restart the daemon from a fresh shell when it starts failing. - **Precedence** for transport and proxy: command-line flag, then `$PILOT_TRANSPORT` / `$PILOT_PROXY` / `$PILOT_PROXY_CMD`, then `config.json` (`transport`, `proxy`, `proxy_cmd`), then the default (`auto` from pilotctl and the installed systemd/launchd services, via `PILOT_TRANSPORT_DEFAULT=auto`; `udp` for a bare `pilot-daemon`; proxy `auto`). `auto` is never saved in config.json, so reinstalling an older release (`--version`, `pilotctl update --pin`) leaves nothing it cannot read; if you saved it yourself, both rewrite it to `udp` for a daemon that predates it. `-proxy` accepts `auto`, `off` (also `none`, `direct`) or an `http://` / `https://` URL; anything else is an error. Loopback targets (local webhooks, sidecars) are never sent to a proxy. When the registry dial goes through a proxy (compat, or an explicit proxy URL in any transport), the default raw-TCP registry is replaced by `registry.pilotprotocol.network:443` over TLS, since CONNECT proxies carry port 443 only. - **No CA bundle in the sandbox?** Point Go at one with `SSL_CERT_FILE=/path/to/ca-certificates.crt` (or `SSL_CERT_DIR`) — `pilotctl daemon start` forwards both. The registry can instead be pinned: `PILOT_REGISTRY_FINGERPRINT=` (or `pilotctl config --set registry_fingerprint=...`), which selects `registry_trust=pinned`. The WSS beacon has no fingerprint option, so it needs the CA bundle. @@ -487,7 +487,7 @@ Most daemon flags have an environment variable equivalent. Useful for containeri | `PILOT_BEACON` | `-beacon` | Beacon server address | | `PILOT_TRANSPORT` | `-transport` | Tunnel transport: `udp` (daemon default), `compat` (TLS registry + WSS beacon on TCP 443 only) or `auto` (udp when the beacon answers over UDP, else compat). Beats `config.json` | | `PILOT_PROXY` | `-proxy` | Outbound proxy: `auto` (default; with compat, the `HTTPS_PROXY`/`ALL_PROXY` proxy honoring `NO_PROXY`), `off` (also `none`, `direct`), or `http(s)://[user:pass@]host:port` for every connection except loopback. Beats `config.json` | -| `PILOT_PROXY_CMD` | `-proxy-cmd` | Command (`/bin/sh -c`) printing the current proxy URL, re-run every 60s and on a 407, for proxies that rotate credentials; supplies the URL `-proxy` would use. Beats `config.json` (`proxy_cmd`) | +| `PILOT_PROXY_CMD` | `-proxy-cmd` | Command (`sh -c`) printing the current proxy URL, re-run once 60s have passed and on a 407 (then the connection is retried once), for proxies that rotate credentials; supplies the URL `-proxy` would use. Beats `config.json` (`proxy_cmd`) | | `PILOT_TRANSPORT_DEFAULT` | — | Transport when neither `-transport`, `PILOT_TRANSPORT` nor `config.json` sets one (the installed services set `auto`; daemons without `auto` ignore it) | | `PILOT_REGISTRY_TRUST` / `PILOT_REGISTRY_FINGERPRINT` | `-registry-trust` / `-registry-fingerprint` | Pin the TLS registry (hosts without a CA bundle); a fingerprint alone selects pinned trust in compat mode. Beat `config.json` | | `HTTPS_PROXY` / `ALL_PROXY` / `NO_PROXY` | — | Proxy used with `-proxy=auto` in compat mode and by `pilotctl`'s own registry dials; `pilotctl daemon start` forwards them (and `SSL_CERT_FILE` / `SSL_CERT_DIR`) to the daemon | diff --git a/cmd/daemon/main.go b/cmd/daemon/main.go index b075f19d..da53e34e 100644 --- a/cmd/daemon/main.go +++ b/cmd/daemon/main.go @@ -22,6 +22,7 @@ import ( "github.com/pilot-protocol/common/config" "github.com/pilot-protocol/common/driver" "github.com/pilot-protocol/common/logging" + "github.com/pilot-protocol/common/netproxy" "github.com/pilot-protocol/pilotprotocol/internal/enterprisecontrol" "github.com/pilot-protocol/pilotprotocol/internal/managedsdk/authority" "github.com/pilot-protocol/pilotprotocol/internal/motd" @@ -115,7 +116,7 @@ func main() { // credentials. transportMode := flag.String("transport", "", "tunnel transport: 'udp' (the default), 'compat' (registry over TLS and beacon over WSS, TCP 443 only, for UDP-blocked or proxy-only hosts) or 'auto' (udp when the beacon answers over UDP, otherwise compat when TCP 443 is reachable, through the proxy if there is one). Precedence: this flag, $PILOT_TRANSPORT, config.json \"transport\", udp.") proxySpec := flag.String("proxy", "", "outbound proxy for registry, beacon and HTTP connections: 'auto' (the default: with compat, HTTPS_PROXY/ALL_PROXY from the environment, honoring NO_PROXY; nothing with udp), 'off' (also none, no, false, direct), or an http:// or https:// proxy URL, http://[user:pass@]host:port, used for every connection except loopback. Precedence: this flag, $PILOT_PROXY, config.json \"proxy\", auto.") - proxyCmd := flag.String("proxy-cmd", "", "command (run with /bin/sh -c) whose output is the current proxy URL, for egress proxies that rotate their credentials: it supplies the URL -proxy would use (the explicit URL, or with auto the environment's proxy) and is re-run every 60s and whenever the proxy answers 407, so new connections always carry fresh credentials. Example: bash -c 'printf %s \"$https_proxy\"'. Precedence: this flag, $PILOT_PROXY_CMD, config.json \"proxy_cmd\".") + proxyCmd := flag.String("proxy-cmd", "", "command (run with sh -c) whose output is the current proxy URL, for egress proxies that rotate their credentials: it supplies the URL -proxy would use (the explicit URL, or with auto and compat the environment's proxy) and is re-run once 60s have passed and whenever the proxy answers 407, after which that connection is retried once, so new connections always carry fresh credentials. Example: bash -c 'printf %s \"$https_proxy\"'. Precedence: this flag, $PILOT_PROXY_CMD, config.json \"proxy_cmd\".") compatBeacon := flag.String("compat-beacon", defaultCompatBeacon, "beacon WSS URL for -transport=compat") tlsTrust := flag.String("tls-trust", "system", "TLS trust store for -transport=compat: 'system' (OS trust store; current default while compat mode uses Let's Encrypt certs on beacon.pilotprotocol.network — on a host without a CA bundle set SSL_CERT_FILE or SSL_CERT_DIR) or 'pinned' (Pilot CA root embedded in the daemon binary; will become the default in a future release once production root ships)") showVersion := flag.Bool("version", false, "print version and exit") @@ -204,7 +205,7 @@ func main() { transport, err = daemon.NormalizeTransport(configTransport) } if err != nil { - log.Fatalf("-transport: %v", err) + fatalf("-transport: %v", err) } if transport == "" { // Nothing chosen: $PILOT_TRANSPORT_DEFAULT (auto in the service @@ -215,21 +216,21 @@ func main() { } } if _, err := proxyconf.Normalize(*proxySpec); err != nil { - log.Fatalf("-proxy: %v", err) + fatalf("-proxy: %v", err) } - // The proxy policy depends on the transport only; each is resolved + // The proxy resolver depends on the transport only; each is resolved // once (running -proxy-cmd once) and shared by the auto probe and the - // daemon. - proxyPolicies := map[string]*proxyconf.Policy{} - policyFor := func(transport string) (*proxyconf.Policy, error) { - if p, ok := proxyPolicies[transport]; ok { - return p, nil + // daemon, so the credentials it refreshes stay in one place. + proxyResolvers := map[string]*netproxy.Resolver{} + proxyFor := func(transport string) (*netproxy.Resolver, error) { + if r, ok := proxyResolvers[transport]; ok { + return r, nil } - p, err := resolveProxyPolicy(*proxySpec, *proxyCmd, transport) + r, err := resolveProxy(*proxySpec, *proxyCmd, transport) if err == nil { - proxyPolicies[transport] = p + proxyResolvers[transport] = r } - return p, err + return r, err } reg := registrySettings{ @@ -244,23 +245,28 @@ func main() { // -transport=auto: probe once, before anything depends on the mode. if transport == daemon.TransportAuto { - mode, reason, err := resolveAutoTransport(reg, *beaconAddr, sources.explicit("beacon") || beaconFromEnv, - *compatBeacon, sources.explicit("compat-beacon"), policyFor) + mode, reason, proxyErr, err := resolveAutoTransport(reg, *beaconAddr, sources.explicit("beacon") || beaconFromEnv, + *compatBeacon, sources.explicit("compat-beacon"), proxyFor) if err != nil { - log.Fatalf("-proxy: %v", err) + fatalf("-proxy: %v", err) + } + if proxyErr != nil { + slog.Warn("transport auto-selected", "transport", mode, "reason", reason, + "hint", proxyErrorHint(proxyErr)) + } else { + slog.Info("transport auto-selected", "transport", mode, "reason", reason) } - slog.Info("transport auto-selected", "transport", mode, "reason", reason) transport = mode } *transportMode = transport // Outbound proxy: resolved once, after -transport is final, and shared // by everything that dials out — the registry client, the compat WSS - // beacon, pkg/daemon's own HTTP fetches (via daemon.Config.ProxyPolicy) - // and every plugin HTTP client (via http.DefaultTransport). - proxyPolicy, err := policyFor(transport) + // beacon, pkg/daemon's own HTTP fetches (via daemon.Config.Proxy) and + // every plugin HTTP client (via http.DefaultTransport). + proxyResolver, err := proxyFor(transport) if err != nil { - log.Fatalf("-proxy: %v", err) + fatalf("-proxy: %v", err) } // Registry defaults for the transport and proxy (see @@ -271,7 +277,7 @@ func main() { // -registry-tls=false (the TCP/9000 fallback), is kept. -beacon needs // no such rule: in compat mode the UDP beacon address is only the // relay-wrap destination on the WSS pipe and is never dialed. - final := applyRegistryDefaults(transport, reg, proxyPolicy.Proxies) + final := applyRegistryDefaults(transport, reg, func(addr string) bool { return proxyconf.Proxies(proxyResolver, addr) }) if final.Addr != reg.Addr { registryFromEnv = false } @@ -307,8 +313,8 @@ func main() { *noSkillinject = profileOptions.DisableSkillinject *motdFeedURL = profileOptions.MOTDFeedURL - installDefaultTransportProxy(proxyPolicy) - slog.Info("outbound network", "transport", *transportMode, "proxy", describeProxy(*proxySpec, *transportMode, proxyPolicy), + installDefaultTransportProxy(proxyResolver) + slog.Info("outbound network", "transport", *transportMode, "proxy", describeProxy(*proxySpec, *transportMode, proxyResolver), "transport_from", transportSrc, "registry", *registryAddr, "registry_tls", *registryTLS) // Sandbox: validate all configured file paths are under the confinement @@ -404,7 +410,7 @@ func main() { TransportMode: *transportMode, CompatBeaconURL: *compatBeacon, CompatTLSTrust: *tlsTrust, - ProxyPolicy: proxyPolicy, + Proxy: proxyResolver, MOTDFeedURL: *motdFeedURL, MOTDInterval: *motdInterval, TelemetryURL: *telemetryURL, @@ -604,7 +610,7 @@ func main() { applyNodeIDWhitelist("rekey", *rekeyWhitelist, "PILOT_REKEY_WHITELIST", d.SetRekeyWhitelist, d.SetRekeyWhitelistMatchAll) if err := d.Start(); err != nil { - log.Fatalf("daemon start: %v", err) + fatalf("daemon start: %v", err) } rolloutRefreshCtx, rolloutRefreshCancel := context.WithCancel(context.Background()) diff --git a/cmd/daemon/netflags.go b/cmd/daemon/netflags.go index d474848f..bd17d56a 100644 --- a/cmd/daemon/netflags.go +++ b/cmd/daemon/netflags.go @@ -5,11 +5,13 @@ package main import ( "context" "flag" + "fmt" "log/slog" "net" "os" "strings" + "github.com/pilot-protocol/common/netproxy" "github.com/pilot-protocol/pilotprotocol/internal/proxyconf" "github.com/pilot-protocol/pilotprotocol/pkg/daemon" ) @@ -198,26 +200,37 @@ var autoProbe = daemon.SelectTransport // resolveAutoTransport decides -transport=auto (see daemon.SelectTransport) // for this configuration: compat is only considered when it would reach // the same network (autoCompatBlocker), and its beacon check runs through -// the proxy compat mode would use (policyFor(compat): -proxy / -proxy-cmd, -// auto = the environment's). The error is a malformed -proxy. -func resolveAutoTransport(reg registrySettings, beacon string, beaconExplicit bool, compatBeacon string, compatBeaconExplicit bool, policyFor func(transport string) (*proxyconf.Policy, error)) (mode, reason string, err error) { +// the proxy compat mode would use (proxyFor(compat): -proxy / -proxy-cmd, +// auto = the environment's). proxyErr is the proxy error that kept auto on +// compat (the proxy refused the check); err is a malformed -proxy. +func resolveAutoTransport(reg registrySettings, beacon string, beaconExplicit bool, compatBeacon string, compatBeaconExplicit bool, proxyFor func(transport string) (*netproxy.Resolver, error)) (mode, reason string, proxyErr, err error) { if why := autoCompatBlocker(reg, beacon, beaconExplicit, compatBeaconExplicit); why != "" { - return daemon.TransportUDP, why + "; auto stays on udp (pass -transport=compat to force compat)", nil + return daemon.TransportUDP, why + "; auto stays on udp (pass -transport=compat to force compat)", nil, nil } - policy, err := policyFor(daemon.TransportCompat) + r, err := proxyFor(daemon.TransportCompat) if err != nil { - return "", "", err + return "", "", nil, err } var dial func(ctx context.Context, network, addr string) (net.Conn, error) - if policy.Enabled() { - dial = policy.DialContext(nil) + if r.Enabled() { + dial = proxyconf.DialContext(r, nil) } - mode, reason = autoProbe(context.Background(), daemon.AutoTransportProbe{ + mode, reason, proxyErr = autoProbe(context.Background(), daemon.AutoTransportProbe{ BeaconAddr: beacon, CompatBeaconURL: compatBeacon, Dial: dial, + ProxyFor: func(addr string) string { return proxyconf.ProxyFor(r, addr) }, }) - return mode, reason, nil + return mode, reason, proxyErr, nil +} + +// proxyErrorHint says what to check when the proxy failed the compat +// check. +func proxyErrorHint(err error) string { + if hint := daemon.ProxyRefusalHint(err); hint != "" { + return hint + } + return "the proxy could not be reached or its answer could not be read: check HTTPS_PROXY / -proxy, or pass -transport=udp to bypass it" } // transportDefaultEnv names the transport a daemon uses when neither @@ -241,3 +254,12 @@ func defaultTransport() string { } return t } + +// fatalf logs the message at ERROR and exits 1. log.Fatalf goes through +// slog's default logger at INFO, so a fatal proxy or registry error looked +// like routine output to `pilotctl daemon start`, supervisors and log +// filters. +func fatalf(format string, args ...any) { + slog.Error(fmt.Sprintf(format, args...)) + os.Exit(1) +} diff --git a/cmd/daemon/netflags_test.go b/cmd/daemon/netflags_test.go index 79ab6f0b..a653c86a 100644 --- a/cmd/daemon/netflags_test.go +++ b/cmd/daemon/netflags_test.go @@ -15,7 +15,7 @@ import ( "testing" "time" - "github.com/pilot-protocol/pilotprotocol/internal/proxyconf" + "github.com/pilot-protocol/common/netproxy" "github.com/pilot-protocol/pilotprotocol/pkg/daemon" ) @@ -201,36 +201,43 @@ func TestResolveAutoTransport(t *testing.T) { t.Setenv(k, "") } var got []daemon.AutoTransportProbe - autoProbe = func(ctx context.Context, p daemon.AutoTransportProbe) (string, string) { + autoProbe = func(ctx context.Context, p daemon.AutoTransportProbe) (string, string, error) { got = append(got, p) - return daemon.TransportCompat, "stub" + return daemon.TransportCompat, "stub", nil } t.Cleanup(func() { autoProbe = daemon.SelectTransport }) std := registrySettings{Addr: defaultRegistryAddr, AddrExplicit: true} - spec := func(s string) func(string) (*proxyconf.Policy, error) { - return func(transport string) (*proxyconf.Policy, error) { return resolveProxyPolicy(s, "", transport) } + spec := func(s string) func(string) (*netproxy.Resolver, error) { + return func(transport string) (*netproxy.Resolver, error) { return resolveProxy(s, "", transport) } } + const beaconTarget = "beacon.pilotprotocol.network:443" - mode, _, err := resolveAutoTransport(std, defaultBeaconAddr, true, defaultCompatBeacon, false, spec("auto")) + mode, _, _, err := resolveAutoTransport(std, defaultBeaconAddr, true, defaultCompatBeacon, false, spec("auto")) if err != nil || mode != daemon.TransportCompat || len(got) != 1 { t.Fatalf("auto = (%q, %v), probes %d", mode, err, len(got)) } if got[0].Dial != nil || got[0].BeaconAddr != defaultBeaconAddr || got[0].CompatBeaconURL != defaultCompatBeacon { t.Errorf("probe without a proxy = %+v, want a direct check of the production beacons", got[0]) } + if via := got[0].ProxyFor(beaconTarget); via != "" { + t.Errorf("probe without a proxy reports the proxy %q", via) + } t.Setenv("HTTPS_PROXY", "http://muse:s3cret@egress.test:3128") - if _, _, err := resolveAutoTransport(std, defaultBeaconAddr, true, defaultCompatBeacon, false, spec("auto")); err != nil || got[1].Dial == nil { + if _, _, _, err := resolveAutoTransport(std, defaultBeaconAddr, true, defaultCompatBeacon, false, spec("auto")); err != nil || got[1].Dial == nil { t.Errorf("with HTTPS_PROXY the compat check does not use the proxy (err %v)", err) } - if _, _, err := resolveAutoTransport(std, defaultBeaconAddr, true, defaultCompatBeacon, false, spec("off")); err != nil || got[2].Dial != nil { + if via := got[1].ProxyFor(beaconTarget); via != "http://***@egress.test:3128" { + t.Errorf("ProxyFor(beacon) = %q, want the redacted HTTPS_PROXY", via) + } + if _, _, _, err := resolveAutoTransport(std, defaultBeaconAddr, true, defaultCompatBeacon, false, spec("off")); err != nil || got[2].Dial != nil { t.Errorf("-proxy=off still proxies the compat check (err %v)", err) } - if _, _, err := resolveAutoTransport(std, defaultBeaconAddr, true, defaultCompatBeacon, false, spec("ftp://x")); err == nil { + if _, _, _, err := resolveAutoTransport(std, defaultBeaconAddr, true, defaultCompatBeacon, false, spec("ftp://x")); err == nil { t.Error("malformed -proxy accepted") } - mode, reason, err := resolveAutoTransport(registrySettings{Addr: "10.0.0.5:9000", AddrExplicit: true}, defaultBeaconAddr, true, defaultCompatBeacon, false, spec("auto")) + mode, reason, _, err := resolveAutoTransport(registrySettings{Addr: "10.0.0.5:9000", AddrExplicit: true}, defaultBeaconAddr, true, defaultCompatBeacon, false, spec("auto")) if err != nil || mode != daemon.TransportUDP || len(got) != 3 { t.Errorf("private registry: (%q, %q, %v), probes %d — want udp without probing", mode, reason, err, len(got)) } @@ -395,6 +402,48 @@ func TestAutoTransportFallsBackToCompatThroughProxy(t *testing.T) { t.Logf("auto → compat → registry CONNECT in %s", time.Since(start)) } +// E2E product gap (wrong or stale proxy password, plain `pilotctl daemon +// start` = -transport=auto): UDP gets no answer and the proxy answers the +// compat check with 407. The daemon must not settle on udp and dial the +// raw registry directly (bypass traffic a proxy-only sandbox kills): it +// stays on compat, logs the proxy error at WARN with a hint, and keeps +// dialing the registry through the proxy, naming the 407. +func TestAutoTransportProxyRefusalStaysCompat(t *testing.T) { + proxy := newRefusingProxy(t, "muse", "right-pass") + proxy.rotate("muse", "right-pass") // anything else gets 407 + targets, logs := runDaemon(t, proxy, daemonRun{ + args: []string{ + "--registry", defaultRegistryAddr, + "--beacon", silentUDP(t), + "-compat-beacon", defaultCompatBeacon, + "-transport=auto", + "-registry-fingerprint=" + strings.Repeat("00", 32), + }, + env: []string{"HTTPS_PROXY=" + fmt.Sprintf("http://muse:wrong-pass@%s", proxy.ln.Addr())}, + await: "CONNECT registry.pilotprotocol.network:443", + }) + if !strings.Contains(logs, `level=WARN msg="transport auto-selected" transport=compat`) { + t.Errorf("auto did not stay on compat with a WARN:\n%s", logs) + } + if !strings.Contains(logs, "407 Proxy Authentication Required") || !strings.Contains(logs, "proxy-cmd") { + t.Errorf("the 407 and its hint are not logged:\n%s", logs) + } + if !contains(targets, "CONNECT beacon.pilotprotocol.network:443") { + t.Errorf("compat check did not go through the proxy: %q", targets) + } + for _, target := range targets { + if strings.Contains(target, "34.71.57.205") { + t.Errorf("proxy was asked for the raw-TCP registry: %q", target) + } + } + if strings.Contains(logs, "34.71.57.205:9000") { + t.Errorf("the daemon dialed the raw registry directly:\n%s", logs) + } + if strings.Contains(logs, "wrong-pass") { + t.Errorf("daemon output leaks the proxy password:\n%s", logs) + } +} + // auto never moves a private deployment onto the public compat beacon. func TestAutoTransportKeepsPrivateRegistryOnUDP(t *testing.T) { proxy := newRefusingProxy(t, "muse", "s3cret") diff --git a/cmd/daemon/proxy.go b/cmd/daemon/proxy.go index 2a7edadc..f839e1d0 100644 --- a/cmd/daemon/proxy.go +++ b/cmd/daemon/proxy.go @@ -3,7 +3,6 @@ package main import ( - "context" "log/slog" "net/http" "strings" @@ -13,24 +12,23 @@ import ( "github.com/pilot-protocol/pilotprotocol/pkg/daemon" ) -// newCommandPolicy is daemon.NewCommandProxyPolicy (a test seam). -var newCommandPolicy = daemon.NewCommandProxyPolicy - -// resolveProxyPolicy resolves -proxy and -proxy-cmd for the transport (see +// resolveProxy resolves -proxy and -proxy-cmd for the transport (see // daemon.ResolveProxy). A malformed -proxy value is an error. Under "auto" // only an unusable HTTPS_PROXY / https_proxy (the variable that names the // TLS proxy) can fail; that costs the proxy, not the daemon: it is logged // and the daemon dials directly, as it did before -proxy existed. Unusable // HTTP_PROXY / ALL_PROXY values are skipped by netproxy and only logged. // -// -proxy-cmd (a command whose stdout is the current proxy URL) makes the -// policy refresh itself, for egress proxies that rotate credentials: it +// -proxy-cmd (a command whose output is the current proxy URL) makes the +// resolver follow rotating credentials (netproxy.WithRefreshCommand): it // supplies the URL wherever -proxy would use one — the explicit URL, or // with auto (compat only) the environment's proxy, NO_PROXY still -// honored. It is ignored with -proxy=off, and with auto on udp (no proxy). -// A failing first run is logged and the launch environment's proxy (or the -// explicit URL) serves until the command succeeds. -func resolveProxyPolicy(spec, command, transport string) (*proxyconf.Policy, error) { +// honored — and is re-run every 60s and whenever the proxy answers 407, +// after which the rejected connection is retried once. It is ignored with +// -proxy=off, and with auto on udp (no proxy). A failing run is logged +// once per run of failures, and the last good URL (at first, the launch +// environment's proxy or the explicit URL) stays in use. +func resolveProxy(spec, command, transport string) (*netproxy.Resolver, error) { s, err := proxyconf.Normalize(spec) if err != nil { return nil, err @@ -44,33 +42,24 @@ func resolveProxyPolicy(spec, command, transport string) (*proxyconf.Policy, err slog.Info("-proxy-cmd not used: -proxy=auto proxies -transport=compat only", "transport", transport) command = "" } - if command == "" { - r, err := daemon.ResolveProxy(s, transport) - if err != nil { - if s != proxyconf.Auto { - return nil, err - } - slog.Warn("proxy environment is malformed; dialing directly", "err", err) - return nil, nil - } - logProxyWarnings(r) - return proxyconf.Static(r), nil - } - var fallback *netproxy.Resolver - if s == proxyconf.Auto { - if fallback, err = netproxy.FromEnvironment(); err != nil { - slog.Warn("proxy environment is malformed; waiting for -proxy-cmd", "err", err) - fallback = nil - } - logProxyWarnings(fallback) - } else if fallback, err = netproxy.Explicit(s); err != nil { - return nil, err + var opts []netproxy.Option + if command != "" { + opts = append(opts, + netproxy.WithRefreshCommand(command), + netproxy.WithRefreshErrorHandler(func(err error) { + slog.Warn("-proxy-cmd failed; keeping the last good proxy URL (at first the launch-time proxy)", "err", err) + })) } - policy, err := newCommandPolicy(context.Background(), command, fallback, s == proxyconf.Auto) + r, err := daemon.ResolveProxy(s, transport, opts...) if err != nil { - slog.Warn("-proxy-cmd failed; using the launch-time proxy until it succeeds", "err", err) + if s != proxyconf.Auto { + return nil, err + } + slog.Warn("proxy environment is malformed; dialing directly", "err", err) + return nil, nil } - return policy, nil + logProxyWarnings(r) + return r, nil } func logProxyWarnings(r *netproxy.Resolver) { @@ -79,11 +68,11 @@ func logProxyWarnings(r *netproxy.Resolver) { } } -// describeProxy renders the resolved policy for the startup log line. +// describeProxy renders the resolved proxy for the startup log line. // Credentials are always redacted. -func describeProxy(spec, transport string, policy *proxyconf.Policy) string { - if policy != nil { - return policy.String() +func describeProxy(spec, transport string, r *netproxy.Resolver) string { + if r != nil { + return r.String() } if isAutoProxy(spec) && transport != daemon.TransportCompat { return "none (-proxy=auto applies to -transport=compat only)" @@ -92,16 +81,19 @@ func describeProxy(spec, transport string, policy *proxyconf.Policy) string { } // installDefaultTransportProxy makes net/http's shared DefaultTransport -// follow the policy. Every HTTP client the daemon wires in without a -// transport of its own — catalogue pins, skillinject, trustedagents, -// webhook, enterprise-control clients — uses DefaultTransport, and not all -// of them accept an injected client. Loopback targets (a local webhook or -// sidecar) always go direct. With a refreshed policy (-proxy-cmd), a 407 -// answer refreshes the credentials at once, so the next request succeeds. -// nil leaves DefaultTransport alone (net/http's own proxy environment -// handling). Call before any goroutine issues a request. -func installDefaultTransportProxy(policy *proxyconf.Policy) { - if policy == nil { +// follow the proxy resolver. Every HTTP client the daemon wires in without +// a transport of its own — catalogue pins, skillinject, trustedagents, +// webhook, enterprise-control clients — uses DefaultTransport (or a clone +// of it), and not all of them accept an injected client. Each new +// connection takes the resolver's current settings, so rotated +// credentials (-proxy-cmd) reach these clients too, and a 407 answer +// refreshes them at once so the next request succeeds (see +// proxyconf.ConfigureTransport). Loopback targets (a local webhook or +// sidecar) always go direct. nil leaves DefaultTransport alone (net/http's +// own proxy environment handling). Call before any goroutine issues a +// request. +func installDefaultTransportProxy(r *netproxy.Resolver) { + if r == nil { return } tr, ok := http.DefaultTransport.(*http.Transport) @@ -109,7 +101,7 @@ func installDefaultTransportProxy(policy *proxyconf.Policy) { slog.Warn("http.DefaultTransport is not an *http.Transport; plugin HTTP clients do not follow -proxy") return } - policy.ConfigureTransport(tr) + proxyconf.ConfigureTransport(tr, r) } func isAutoProxy(spec string) bool { diff --git a/cmd/daemon/proxy_refresh_test.go b/cmd/daemon/proxy_refresh_test.go index 2c92020a..6ab30c95 100644 --- a/cmd/daemon/proxy_refresh_test.go +++ b/cmd/daemon/proxy_refresh_test.go @@ -45,15 +45,12 @@ func TestProxyCmdFollowsCredentialRotation(t *testing.T) { }) d.stop() logs := d.out.String() - if !strings.Contains(logs, "proxy rejected the credentials; retrying with refreshed ones") { - t.Errorf("no 407 retry logged:\n%s", logs) - } for _, secret := range []string{"old-pass", "new-pass"} { if strings.Contains(logs, secret) { t.Errorf("daemon output leaks %q:\n%s", secret, logs) } } - if !strings.Contains(logs, "refreshed from the proxy command") { + if !strings.Contains(logs, "credentials refreshed by command") { t.Errorf("startup line does not mention the proxy command:\n%s", logs) } } @@ -73,7 +70,7 @@ func TestProxyCmdFailureFallsBackToEnvironment(t *testing.T) { if proxy.accepted("muse", "s3cret") == 0 { t.Error("launch-environment proxy not used after the proxy command failed") } - if !strings.Contains(logs, "-proxy-cmd failed; using the launch-time proxy") { + if !strings.Contains(logs, "-proxy-cmd failed; keeping the last good proxy URL") { t.Errorf("proxy command failure not logged:\n%s", logs) } } diff --git a/cmd/daemon/proxy_test.go b/cmd/daemon/proxy_test.go index da641943..0146256b 100644 --- a/cmd/daemon/proxy_test.go +++ b/cmd/daemon/proxy_test.go @@ -40,54 +40,73 @@ var proxyEnvVars = []string{ "HTTP_PROXY", "http_proxy", "NO_PROXY", "no_proxy", "REQUEST_METHOD", } -func TestResolveProxyPolicy(t *testing.T) { +func TestResolveProxy(t *testing.T) { for _, k := range proxyEnvVars { t.Setenv(k, "") } t.Setenv("HTTPS_PROXY", "http://muse:s3cret@egress.test:3128") - p, err := resolveProxyPolicy("auto", "", "udp") - if err != nil || p != nil { - t.Fatalf("auto/udp = (%v, %v), want (nil, nil)", p, err) + r, err := resolveProxy("auto", "", "udp") + if err != nil || r != nil { + t.Fatalf("auto/udp = (%v, %v), want (nil, nil)", r, err) } - if got := describeProxy("auto", "udp", p); got != "none (-proxy=auto applies to -transport=compat only)" { + if got := describeProxy("auto", "udp", r); got != "none (-proxy=auto applies to -transport=compat only)" { t.Errorf("describeProxy(auto/udp) = %q", got) } - p, err = resolveProxyPolicy("auto", "", "compat") - if err != nil || p == nil || p.Mode() != netproxy.ModeAuto || !p.Enabled() { - t.Fatalf("auto/compat = (%v, %v), want an enabled auto policy", p, err) + r, err = resolveProxy("auto", "", "compat") + if err != nil || r == nil || r.Mode() != netproxy.ModeAuto || !r.Enabled() { + t.Fatalf("auto/compat = (%v, %v), want an enabled auto resolver", r, err) } - if got := describeProxy("auto", "compat", p); got != "auto: http://***@egress.test:3128" { + if got := describeProxy("auto", "compat", r); got != "auto: http://***@egress.test:3128" { t.Errorf("describeProxy(auto/compat) = %q", got) } - p, err = resolveProxyPolicy("http://ops:hunter2@flag.test:8080", "", "udp") - if err != nil || p == nil || p.Mode() != netproxy.ModeExplicit { - t.Fatalf("explicit/udp = (%v, %v), want an explicit policy", p, err) + r, err = resolveProxy("http://ops:hunter2@flag.test:8080", "", "udp") + if err != nil || r == nil || r.Mode() != netproxy.ModeExplicit { + t.Fatalf("explicit/udp = (%v, %v), want an explicit resolver", r, err) } - if got := describeProxy("http://ops:hunter2@flag.test:8080", "udp", p); got != "http://***@flag.test:8080" { + if got := describeProxy("http://ops:hunter2@flag.test:8080", "udp", r); got != "http://***@flag.test:8080" { t.Errorf("describeProxy(explicit) = %q", got) } - p, err = resolveProxyPolicy("off", "", "compat") - if err != nil || p == nil || p.Mode() != netproxy.ModeOff { - t.Fatalf("off/compat = (%v, %v), want an off policy", p, err) + r, err = resolveProxy("off", "", "compat") + if err != nil || r == nil || r.Mode() != netproxy.ModeOff { + t.Fatalf("off/compat = (%v, %v), want an off resolver", r, err) + } + + // -proxy-cmd supplies the URL (netproxy.WithRefreshCommand) ... + r, err = resolveProxy("auto", "echo http://muse:cmdpw9@cmd.test:3128", "compat") + if err != nil || r == nil || r.Mode() != netproxy.ModeAuto { + t.Fatalf("auto+cmd/compat = (%v, %v)", r, err) + } + if u, _ := r.ProxyForAddr("registry.pilotprotocol.network:443"); u == nil || u.Host != "cmd.test:3128" { + t.Errorf("auto+cmd proxy = %v, want the command's cmd.test:3128", u) + } + if got := describeProxy("auto", "compat", r); strings.Contains(got, "cmdpw9") || !strings.Contains(got, "credentials refreshed by command") { + t.Errorf("describeProxy(auto+cmd) = %q", got) + } + // ... but not with -proxy=off, nor with auto on udp. + if r, err = resolveProxy("off", "echo http://muse:cmdpw9@cmd.test:3128", "compat"); err != nil || r.Mode() != netproxy.ModeOff { + t.Errorf("off+cmd = (%v, %v), want off", r, err) + } + if r, err = resolveProxy("auto", "echo http://muse:cmdpw9@cmd.test:3128", "udp"); err != nil || r != nil { + t.Errorf("auto+cmd/udp = (%v, %v), want no proxy", r, err) } // A malformed -proxy URL is fatal (operator typo) ... - if _, err := resolveProxyPolicy("ftp://ops:hunter2@flag.test", "", "compat"); err == nil { + if _, err := resolveProxy("ftp://ops:hunter2@flag.test", "", "compat"); err == nil { t.Fatal("malformed -proxy URL accepted") } else if strings.Contains(err.Error(), "hunter2") { t.Fatalf("error leaks credentials: %v", err) } // ... but a malformed environment under auto only costs the proxy. t.Setenv("HTTPS_PROXY", "ftp://muse:s3cret@egress.test") - p, err = resolveProxyPolicy("auto", "", "compat") - if err != nil || p != nil { - t.Fatalf("auto/compat with malformed env = (%v, %v), want (nil, nil)", p, err) + r, err = resolveProxy("auto", "", "compat") + if err != nil || r != nil { + t.Fatalf("auto/compat with malformed env = (%v, %v), want (nil, nil)", r, err) } - if got := describeProxy("auto", "compat", p); got != "none" { + if got := describeProxy("auto", "compat", r); got != "none" { t.Errorf("describeProxy(auto/compat, malformed env) = %q, want none", got) } } diff --git a/cmd/pilotctl/daemon_startlog.go b/cmd/pilotctl/daemon_startlog.go new file mode 100644 index 00000000..5ce3a93f --- /dev/null +++ b/cmd/pilotctl/daemon_startlog.go @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "strings" + "time" + + "github.com/pilot-protocol/pilotprotocol/internal/proxyconf" +) + +// startLogTail is how much of the end of a daemon log `daemon start` reads +// to explain a failed start. +const startLogTail = 256 << 10 + +// readLogTail returns the last startLogTail bytes of the log at path, split +// into lines ("" when it cannot be read). The first, possibly partial, +// line of a truncated read is dropped. +func readLogTail(path string) []string { + f, err := os.Open(path) // #nosec G304 -- the daemon log daemon start itself created under the config dir + if err != nil { + return nil + } + defer f.Close() + st, err := f.Stat() + if err != nil { + return nil + } + off := int64(0) + if st.Size() > startLogTail { + off = st.Size() - startLogTail + } + b, err := io.ReadAll(io.NewSectionReader(f, off, st.Size()-off)) + if err != nil { + return nil + } + lines := strings.Split(string(b), "\n") + if off > 0 && len(lines) > 0 { + lines = lines[1:] + } + return lines +} + +// logLevelAtLeastWarn reports whether a slog line (text or JSON) is at +// level WARN or ERROR. +func logLevelAtLeastWarn(line string) bool { + for _, l := range []string{"level=WARN", "level=ERROR", `"level":"WARN"`, `"level":"ERROR"`} { + if strings.Contains(line, l) { + return true + } + } + return false +} + +// lastProxyErrorFromLog returns the most recent proxy error pilot-daemon +// logged — "proxy CONNECT : 407 Proxy Authentication Required", +// "proxy CONNECT : dial proxy ...", as netproxy words them — or "" +// when there is none. WARN and ERROR lines win over INFO ones (a plugin's +// failed fetch, say). netproxy never puts credentials or proxy-supplied +// text in these errors, so they are safe to show. +func lastProxyErrorFromLog(path string) string { + lines := readLogTail(path) + var anyLevel string + for i := len(lines) - 1; i >= 0; i-- { + e := extractProxyError(lines[i]) + if e == "" { + continue + } + if logLevelAtLeastWarn(lines[i]) { + return e + } + if anyLevel == "" { + anyLevel = e + } + } + return anyLevel +} + +// extractProxyError cuts the netproxy error out of one log line: from +// "proxy CONNECT " to the end of the quoted slog value it sits in, an +// unbalanced ")" (an error quoted inside a message) or a "; " (the hint +// pilot-daemon appends). Escaped characters are unescaped. +func extractProxyError(line string) string { + i := strings.Index(line, "proxy CONNECT ") + if i < 0 { + return "" + } + rest := line[i:] + var b strings.Builder + depth := 0 +scan: + for j := 0; j < len(rest); j++ { + c := rest[j] + switch { + case c == '\\' && j+1 < len(rest): + j++ + b.WriteByte(rest[j]) + continue + case c == '"': + break scan + case c == '(': + depth++ + case c == ')': + if depth == 0 { + break scan + } + depth-- + case c == ';' && j+1 < len(rest) && rest[j+1] == ' ': + break scan + } + b.WriteByte(c) + } + return strings.TrimSpace(b.String()) +} + +// lastErrorFromLog returns the message of the last ERROR line (pilot-daemon +// logs its fatal startup error there), else the last non-empty line, cut to +// a readable length. "" for an empty or unreadable log. +func lastErrorFromLog(path string) string { + lines := readLogTail(path) + last := "" + for i := len(lines) - 1; i >= 0; i-- { + line := strings.TrimSpace(lines[i]) + if line == "" { + continue + } + if last == "" { + last = line + } + if strings.Contains(line, "level=ERROR") || strings.Contains(line, `"level":"ERROR"`) { + if msg := slogMessage(line); msg != "" { + return msg + } + return clip(line) + } + } + return clip(last) +} + +// slogMessage returns the msg of a slog text or JSON line, "" if none. +func slogMessage(line string) string { + if strings.HasPrefix(line, "{") { + var rec map[string]any + if json.Unmarshal([]byte(line), &rec) == nil { + if m, ok := rec["msg"].(string); ok { + return clip(m) + } + } + return "" + } + i := strings.Index(line, "msg=") + if i < 0 { + return "" + } + rest := line[i+len("msg="):] + if !strings.HasPrefix(rest, `"`) { + if j := strings.IndexByte(rest, ' '); j >= 0 { + rest = rest[:j] + } + return clip(rest) + } + var b strings.Builder + for j := 1; j < len(rest); j++ { + c := rest[j] + if c == '\\' && j+1 < len(rest) { + j++ + b.WriteByte(rest[j]) + continue + } + if c == '"' { + break + } + b.WriteByte(c) + } + return clip(b.String()) +} + +// clip bounds a log excerpt quoted in an error message. +func clip(s string) string { + const max = 600 + if len(s) > max { + return s[:max] + "…" + } + return s +} + +// daemonExitStatus renders how the daemon process ended. +func daemonExitStatus(err error) string { + var ee *exec.ExitError + if errors.As(err, &ee) { + return ee.ProcessState.String() + } + if err != nil { + return err.Error() + } + return "exit status 0" +} + +// proxyErrorHint says what to do about a proxy error the daemon logged. +func proxyErrorHint(proxyErr string) string { + switch { + case strings.Contains(proxyErr, " 407 "): + return "the proxy rejected the credentials (407): check HTTPS_PROXY (or --proxy / PILOT_PROXY); if the proxy rotates its credentials, give the daemon a command that prints the current proxy URL: --proxy-cmd, or pilotctl config --set proxy_cmd=\"bash -c 'printf %s \\\"\\$https_proxy\\\"'\"" + case strings.Contains(proxyErr, ": dial proxy "): + return "the proxy could not be reached: check HTTPS_PROXY (or --proxy / PILOT_PROXY)" + default: + return "the proxy refused the connection: it must allow CONNECT to " + compatRegistryAddr + " and beacon.pilotprotocol.network:443" + } +} + +// reportDaemonStartFailure ends a `daemon start` whose daemon never became +// ready: it exited (exitStatus != "") or the wait ran out. It shows what +// the daemon's log says went wrong — above all its last proxy error, which +// behind an egress proxy is nearly always the cause — instead of only "did +// not become ready". +func reportDaemonStartFailure(pid int, logPath, exitStatus string, waited time.Duration) { + proxyErr := lastProxyErrorFromLog(logPath) + hint := fmt.Sprintf("check logs: tail -f %s", logPath) + if proxyErr != "" { + hint = proxyErrorHint(proxyErr) + "; full log: " + logPath + } + if exitStatus != "" { + msg := fmt.Sprintf("daemon (pid %d) exited during startup (%s)", pid, exitStatus) + if last := lastErrorFromLog(logPath); last != "" { + msg += ": " + last + } + if proxyErr != "" && !strings.Contains(msg, proxyErr) { + msg += "; last proxy error: " + proxyErr + } + code := "internal" + if proxyErr != "" { + code = "connection_failed" + } + fatalHint(code, hint, "%s", msg) + } + msg := fmt.Sprintf("daemon started (pid %d) but did not become ready within %s", pid, waited) + if proxyErr != "" { + msg += "; last proxy error: " + proxyErr + } + fatalHint("timeout", hint, "%s", msg) +} + +// proxyCmdSource says where the running daemon's proxy refresh command +// comes from, "" when it uses none: pilot-daemon reports one on its +// "outbound network" line ("credentials refreshed by command"), and only +// then does this name the source — the built-in sandbox command, else +// --proxy-cmd, $PILOT_PROXY_CMD or config.json proxy_cmd. A configured +// command is not echoed: it may embed secrets. +func proxyCmdSource(plan daemonLaunchPlan, flags map[string]string, logPath string, sandbox bool) string { + used := false + for _, line := range readLogTail(logPath) { + if strings.Contains(line, "outbound network") && strings.Contains(line, "credentials refreshed by command") { + used = true + } + } + if !used { + return "" + } + switch { + case sandbox: + return sandboxProxyCmd + case plan.ProxyCmd != "": + return "--proxy-cmd" + case strings.TrimSpace(os.Getenv(proxyconf.EnvRefreshCommand)) != "": + return "$" + proxyconf.EnvRefreshCommand + default: + return "config.json proxy_cmd" + } +} diff --git a/cmd/pilotctl/daemon_transport.go b/cmd/pilotctl/daemon_transport.go index 280894dc..4a1093fa 100644 --- a/cmd/pilotctl/daemon_transport.go +++ b/cmd/pilotctl/daemon_transport.go @@ -258,25 +258,35 @@ var ( // It does not depend on the installer having saved proxy_cmd, so a node // installed by an older installer gets it too. func sandboxProxyCmdFor(bin string, plan daemonLaunchPlan, flags map[string]string) string { - if hostGOOS != "linux" || systemdRunning() { + if sandboxRefreshCmd() == "" { return "" } if plan.Proxy != "" && plan.Proxy != proxyconf.Auto { return "" } - if !proxyHasCredentials(os.Getenv("HTTPS_PROXY")) && !proxyHasCredentials(os.Getenv("https_proxy")) { + if plan.ProxyCmd != "" || strings.TrimSpace(os.Getenv(proxyconf.EnvRefreshCommand)) != "" { return "" } - if strings.TrimSpace(os.Getenv("PILOT_PROXY_CMD")) != "" { + if configuredProxyCmd(loadConfig()) != "" || configuredProxyCmd(daemonConfigFile(flags)) != "" { return "" } - if configuredProxyCmd(loadConfig()) != "" || configuredProxyCmd(daemonConfigFile(flags)) != "" { + if f := daemonFlags(bin); !f["proxy-cmd"] { return "" } - if !bashAvailable() { + return sandboxProxyCmd +} + +// sandboxRefreshCmd is sandboxProxyCmd on a host that looks like a hosted +// agent sandbox — Linux without systemd, $HTTPS_PROXY or $https_proxy +// carrying credentials, bash installed — and "" anywhere else. +func sandboxRefreshCmd() string { + if hostGOOS != "linux" || systemdRunning() { return "" } - if f := daemonFlags(bin); !f["proxy-cmd"] { + if !proxyHasCredentials(os.Getenv("HTTPS_PROXY")) && !proxyHasCredentials(os.Getenv("https_proxy")) { + return "" + } + if !bashAvailable() { return "" } return sandboxProxyCmd @@ -512,7 +522,7 @@ var daemonLogTransportRe = regexp.MustCompile(`outbound network"?[ ,]+"?transpor // startup ("" if unknown) — the only place the outcome of -transport=auto // is visible to pilotctl. func transportFromDaemonLog(path string) string { - b, err := os.ReadFile(path) + b, err := os.ReadFile(path) // #nosec G304 G703 -- the per-PID daemon log daemon start itself created under the config dir if err != nil { return "" } diff --git a/cmd/pilotctl/main.go b/cmd/pilotctl/main.go index 81f16213..72865305 100644 --- a/cmd/pilotctl/main.go +++ b/cmd/pilotctl/main.go @@ -1075,6 +1075,10 @@ Flags: off (or none) = never; http(s)://[user:pass@]host:port = every connection except loopback. A URL with credentials is passed via env, never on argv + --proxy-cmd command whose output is the current proxy URL, for + proxies that rotate their credentials (see below). + Precedence: this flag, $PILOT_PROXY_CMD, config + "proxy_cmd". Passed via env, never on argv --compat-beacon beacon WSS URL for compat mode --registry-trust registry TLS trust: pinned or system --registry-fingerprint registry certificate SHA-256 (pins the compat registry; @@ -1089,16 +1093,22 @@ Environment passed through to the daemon (never scrubbed): A pilot-daemon too old for --transport, --transport auto or --proxy gets the flag dropped (auto becomes udp) with a warning instead of crashing it. Behind an HTTPS proxy with UDP blocked (e.g. hosted agent sandboxes), plain -"pilotctl daemon start" picks compat by itself; to skip the UDP probe: +"pilotctl daemon start" picks compat by itself — also when the proxy refuses +the check (a 407, say): auto never falls back to direct connections past a +configured proxy. To skip the UDP probe: pilotctl config --set transport=compat && pilotctl daemon start If the proxy rotates its credentials, give the daemon a command that prints the current proxy URL; the daemon re-runs it every 60s and whenever the proxy -answers 407: +answers 407, and retries that connection once with the new credentials +(pilotctl's own registry commands use the same command): pilotctl config --set proxy_cmd="bash -c 'printf %s \"\$https_proxy\"'" On Linux without systemd (containers, hosted agent sandboxes), when $HTTPS_PROXY carries credentials and no proxy_cmd is configured, daemon start passes the daemon PILOT_PROXY_CMD=bash -c 'printf %s "${https_proxy:-$HTTPS_PROXY}"' by itself. +If the daemon exits during startup or does not become ready in time, daemon +start reports the daemon's last error — the proxy's answer (e.g. "407 Proxy +Authentication Required") when a proxy is involved — with the log path. `, "daemon stop": `Usage: pilotctl daemon stop @@ -2777,6 +2787,9 @@ type daemonLaunchPlan struct { // ProxyEnv is non-empty when Proxy carries credentials: it is handed to // the daemon as $PILOT_PROXY instead of -proxy on argv (PILOT-290). ProxyEnv string + // ProxyCmd is --proxy-cmd: handed to the daemon as $PILOT_PROXY_CMD + // (never on argv), where it beats config.json "proxy_cmd". + ProxyCmd string } // buildDaemonArgs translates pilotctl-style flags into pilot-daemon CLI @@ -2958,6 +2971,7 @@ func planDaemonLaunch(args []string) daemonLaunchPlan { Transport: transport, Proxy: proxy, ProxyEnv: proxyEnv, + ProxyCmd: strings.TrimSpace(flagString(flags, "proxy-cmd", "")), } } @@ -3034,6 +3048,9 @@ func cmdDaemonStart(args []string) { if _, ok := flags["proxy"]; ok && !jsonOutput { fmt.Fprintf(os.Stderr, "warning: --proxy is not applied to the launchd-managed daemon; persist it with: pilotctl config --set proxy=\n") } + if _, ok := flags["proxy-cmd"]; ok && !jsonOutput { + fmt.Fprintf(os.Stderr, "warning: --proxy-cmd is not applied to the launchd-managed daemon; persist it with: pilotctl config --set proxy_cmd=\n") + } if launchdAgentLoaded(label) { fatalHint("already_exists", "stop it first with: pilotctl daemon stop", @@ -3128,11 +3145,22 @@ func cmdDaemonStart(args []string) { // -transport=auto when no transport is configured. daemonArgs, proxyEnv, requestedTransport := adaptDaemonArgs(daemonBin, plan) daemonEnv := daemonChildEnv(os.Environ(), plan.AdminToken, proxyEnv, requestedTransport) + // --proxy-cmd travels as $PILOT_PROXY_CMD (a command can embed + // secrets; argv is world-readable), where it beats config.json. + if plan.ProxyCmd != "" { + if f := daemonFlags(daemonBin); f != nil && !f["proxy-cmd"] { + if !jsonOutput { + fmt.Fprintf(os.Stderr, "warning: %s does not support -proxy-cmd (older pilot-daemon); --proxy-cmd will not be used — upgrade pilot-daemon to use it\n", daemonBin) + } + } else { + daemonEnv = setEnv(daemonEnv, proxyconf.EnvRefreshCommand, plan.ProxyCmd) + } + } // In a sandbox whose proxy credentials rotate, have the daemon re-read // them (see sandboxProxyCmdFor) even when no installer saved proxy_cmd. sandboxRefresh := false if c := sandboxProxyCmdFor(daemonBin, plan, flags); c != "" { - daemonEnv = setEnv(daemonEnv, "PILOT_PROXY_CMD", c) + daemonEnv = setEnv(daemonEnv, proxyconf.EnvRefreshCommand, c) sandboxRefresh = true } @@ -3152,7 +3180,7 @@ func cmdDaemonStart(args []string) { // full env (daemonChildEnv) — it carries PILOT_ADMIN_TOKEN so the // daemon doesn't need the token on its argv (PILOT-290). execArgs := append([]string{daemonBin}, daemonArgs...) - if err := syscall.Exec(daemonBin, execArgs, daemonEnv); err != nil { + if err := syscall.Exec(daemonBin, execArgs, daemonEnv); err != nil { // #nosec G204 G702 -- daemonBin is pilotctl's sibling pilot-daemon or the operator's PILOT_DAEMON_BIN fatalCode("internal", "exec %s: %v", daemonBin, err) } return @@ -3186,6 +3214,10 @@ func cmdDaemonStart(args []string) { if err := proc.Start(); err != nil { fatalCode("internal", "start daemon: %v", err) } + // Notice a daemon that exits during startup (a fatal -proxy, registry + // or compat error) instead of polling its socket until the deadline. + exited := make(chan error, 1) + go func() { exited <- proc.Wait() }() pid := proc.Process.Pid os.WriteFile(pidFilePath(), []byte(strconv.Itoa(pid)), 0600) @@ -3220,7 +3252,15 @@ func cmdDaemonStart(args []string) { deadline := time.Now().Add(waitDur) dots := 0 for time.Now().Before(deadline) { - time.Sleep(200 * time.Millisecond) + select { + case werr := <-exited: + if !jsonOutput { + fmt.Fprintln(os.Stderr) // end the dots line + } + os.Remove(pidFilePath()) + reportDaemonStartFailure(pid, pidLogPath, daemonExitStatus(werr), 0) + case <-time.After(200 * time.Millisecond): + } dots++ if !jsonOutput && dots%5 == 0 { // every second fmt.Fprint(os.Stderr, ".") @@ -3264,6 +3304,9 @@ func cmdDaemonStart(args []string) { if sandboxRefresh { fields["proxy_cmd"] = sandboxProxyCmd } + if src := proxyCmdSource(plan, flags, pidLogPath, sandboxRefresh); src != "" { + fields["proxy_refresh"] = src + } outputOK(fields) } else { fmt.Printf("Daemon running (pid %d)\n", pid) @@ -3280,8 +3323,8 @@ func cmdDaemonStart(args []string) { if plan.Proxy != "" { fmt.Printf(" Proxy: %s\n", redactProxyURL(plan.Proxy)) } - if sandboxRefresh { - fmt.Printf(" Proxy credentials: re-read every 60s and on a 407 (%s)\n", sandboxProxyCmd) + if src := proxyCmdSource(plan, flags, pidLogPath, sandboxRefresh); src != "" { + fmt.Printf(" Proxy credentials: re-read every 60s and on a 407 (%s)\n", src) } fmt.Printf(" Socket: %s\n", socketPath) fmt.Printf(" Logs: %s\n", pidLogPath) @@ -3293,9 +3336,7 @@ func cmdDaemonStart(args []string) { fmt.Fprintln(os.Stderr) // end the dots line } - fatalHint("timeout", - fmt.Sprintf("check logs: tail -f %s", pidLogPath), - "daemon started (pid %d) but did not become ready within %s", pid, waitDur) + reportDaemonStartFailure(pid, pidLogPath, "", waitDur) } func cmdDaemonStop() { diff --git a/cmd/pilotctl/registry_dial.go b/cmd/pilotctl/registry_dial.go index 22cf1d96..016c315b 100644 --- a/cmd/pilotctl/registry_dial.go +++ b/cmd/pilotctl/registry_dial.go @@ -71,17 +71,7 @@ func (r registryRoute) proxyFor(target string) string { } func proxyFor(p *netproxy.Resolver, target string) string { - if !p.Enabled() { - return "" - } - if host, _, err := net.SplitHostPort(target); err == nil && proxyconf.IsLoopbackHost(host) { - return "" - } - u, err := p.ProxyForAddr(target) - if err != nil || u == nil { - return "" - } - return netproxy.Redact(u) + return proxyconf.ProxyFor(p, target) } // pilotctlProxySpec is the proxy setting for pilotctl's own connections: @@ -96,6 +86,35 @@ func pilotctlProxySpec(cfg map[string]interface{}) string { return proxyconf.Auto } +// pilotctlProxyCmd is the command whose output is the current proxy URL, +// for pilotctl's own connections through a proxy that rotates its +// credentials: $PILOT_PROXY_CMD, then config.json "proxy_cmd", then — in a +// sandbox where `daemon start` hands the daemon one (see sandboxRefreshCmd) +// — the same sandbox command. "" when none applies. +func pilotctlProxyCmd(cfg map[string]interface{}) string { + if v := strings.TrimSpace(os.Getenv(proxyconf.EnvRefreshCommand)); v != "" { + return v + } + if v := configuredProxyCmd(cfg); v != "" { + return v + } + return sandboxRefreshCmd() +} + +// proxyResolver builds the resolver for a normalized proxy setting. With a +// refresh command (pilotctlProxyCmd) the command supplies the proxy URL — +// run once now, and again when the proxy answers 407, after which the dial +// is retried once (netproxy) — so a stale $HTTPS_PROXY in pilotctl's own +// environment does not fail its registry commands. A failing command +// leaves the environment's (or the explicit) proxy in use. +func proxyResolver(spec, command string) (*netproxy.Resolver, error) { + var opts []netproxy.Option + if command != "" && spec != proxyconf.Off { + opts = append(opts, netproxy.WithRefreshCommand(command)) + } + return proxyconf.Resolve(spec, opts...) +} + // configuredTransport is $PILOT_TRANSPORT, else config.json "transport", // normalized; "" when neither is set or the value is unknown. func configuredTransport(cfg map[string]interface{}) string { @@ -193,7 +212,7 @@ func planRegistryRoutes(addr string) ([]registryRoute, error) { transport = configuredTransport(cfg) case proxyconf.Auto: env, envErr := netproxy.FromEnvironment() - if envErr == nil && !env.Enabled() { + if envErr == nil && !env.Enabled() && pilotctlProxyCmd(cfg) == "" { // No proxy in the environment: nothing to decide. transport = configuredTransport(cfg) break @@ -208,6 +227,13 @@ func planRegistryRoutes(addr string) ([]registryRoute, error) { if envErr != nil { return nil, envErr } + if cmd := pilotctlProxyCmd(cfg); cmd != "" { + // The proxy's credentials rotate: take the current URL from + // the command (and again on a 407), not only the environment. + if env, envErr = proxyResolver(proxyconf.Auto, cmd); envErr != nil { + return nil, envErr + } + } if transport == "compat" { policy = env break @@ -227,7 +253,7 @@ func planRegistryRoutes(addr string) ([]registryRoute, error) { raw.Probe = true return []registryRoute{tlsRoute, raw}, nil default: - explicit, err := proxyconf.Resolve(spec) + explicit, err := proxyResolver(spec, pilotctlProxyCmd(cfg)) if err != nil { return nil, err } @@ -280,7 +306,7 @@ func probedDirectDial(ctx context.Context, network, addr string) (net.Conn, erro return nil, err } if err := c.SetReadDeadline(time.Now().Add(guardProbeWait)); err != nil { - c.Close() + _ = c.Close() return nil, err } var b [1]byte @@ -288,12 +314,12 @@ func probedDirectDial(ctx context.Context, network, addr string) (net.Conn, erro var ne net.Error if n == 0 && errors.As(err, &ne) && ne.Timeout() { if err := c.SetReadDeadline(time.Time{}); err != nil { - c.Close() + _ = c.Close() return nil, err } return c, nil } - c.Close() + _ = c.Close() if n > 0 { return nil, fmt.Errorf("%w at %s: it sent data before any request (a network guard?)", errNotRegistry, addr) } @@ -344,7 +370,7 @@ func dialRegistry(addr string) (*registry.Client, registryRoute, error) { // registryDialHint says what to check after a failed registry dial. func registryDialHint(route registryRoute) string { if p := route.proxyFor(route.Addr); p != "" { - return fmt.Sprintf("the registry %s is reached through the proxy %s: check that it allows CONNECT to %s and that its credentials are right; if this host can reach the registry directly, set PILOT_PROXY=off (or pilotctl config --set proxy=off)", + return fmt.Sprintf("the registry %s is reached through the proxy %s: check that it allows CONNECT to %s and that its credentials are right (if they rotate, set PILOT_PROXY_CMD or pilotctl config --set proxy_cmd=); if this host can reach the registry directly, set PILOT_PROXY=off (or pilotctl config --set proxy=off)", route.Addr, p, route.Addr) } if route.TLS { diff --git a/cmd/pilotctl/zz_proxy_rotation_test.go b/cmd/pilotctl/zz_proxy_rotation_test.go new file mode 100644 index 00000000..b59e092e --- /dev/null +++ b/cmd/pilotctl/zz_proxy_rotation_test.go @@ -0,0 +1,495 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "syscall" + "testing" + "time" +) + +// rotatingProxy is an in-process CONNECT proxy whose Basic-auth password +// rotates, the way Meta Muse's egress proxy rotates the credentials in +// HTTPS_PROXY: it accepts the current password only, answers anything else +// with 407, and tunnels accepted CONNECTs per routes. +type rotatingProxy struct { + addr string + routes map[string]string + + mu sync.Mutex + pass string + oks map[string]int // password -> accepted CONNECTs + n407 int + seen []string +} + +func newRotatingProxy(t *testing.T, pass string, routes map[string]string) *rotatingProxy { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + p := &rotatingProxy{addr: ln.Addr().String(), routes: routes, pass: pass, oks: map[string]int{}} + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go p.serve(c) + } + }() + t.Cleanup(func() { ln.Close() }) + return p +} + +func (p *rotatingProxy) url(pass string) string { return "http://muse:" + pass + "@" + p.addr } + +func (p *rotatingProxy) rotate(pass string) { + p.mu.Lock() + p.pass = pass + p.mu.Unlock() +} + +func (p *rotatingProxy) stats() (oks map[string]int, n407 int, seen []string) { + p.mu.Lock() + defer p.mu.Unlock() + m := map[string]int{} + for k, v := range p.oks { + m[k] = v + } + return m, p.n407, append([]string(nil), p.seen...) +} + +func (p *rotatingProxy) serve(c net.Conn) { + defer c.Close() + br := bufio.NewReader(c) + req, err := http.ReadRequest(br) + if err != nil { + return + } + req.Header.Set("Authorization", req.Header.Get("Proxy-Authorization")) + user, pass, ok := req.BasicAuth() + p.mu.Lock() + p.seen = append(p.seen, req.Method+" "+req.RequestURI) + good := ok && user == "muse" && pass == p.pass + if good { + p.oks[pass]++ + } else { + p.n407++ + } + to := p.routes[req.RequestURI] + p.mu.Unlock() + switch { + case !good: + fmt.Fprint(c, "HTTP/1.1 407 Proxy Authentication Required\r\nProxy-Authenticate: Basic realm=\"muse\"\r\nContent-Length: 0\r\n\r\n") + return + case req.Method != http.MethodConnect || to == "": + fmt.Fprint(c, "HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\n\r\n") + return + } + up, err := net.DialTimeout("tcp", to, 5*time.Second) + if err != nil { + fmt.Fprint(c, "HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n") + return + } + defer up.Close() + fmt.Fprint(c, "HTTP/1.1 200 Connection established\r\n\r\n") + go func() { _, _ = io.Copy(up, br); up.Close() }() + _, _ = io.Copy(c, up) +} + +// pilotctl's own registry commands behind a proxy that rotates its +// credentials: with a proxy command ($PILOT_PROXY_CMD, config proxy_cmd, +// or the sandbox default) the dial takes the command's current URL rather +// than a stale $HTTPS_PROXY, and a 407 mid-way re-runs the command and +// retries once (netproxy). Without one, the 407 is reported as such. +func TestRegistryDialFollowsRotatedProxyCredentials(t *testing.T) { + reg, fp := newTLSFakeRegistry(t) + reg.onOK("lookup", map[string]interface{}{"node_id": float64(5), "address": "0:0000.0000.0005", "public": true}) + proxy := newRotatingProxy(t, "pw-2", map[string]string{compatRegistryAddr: reg.addr()}) + urlFile := filepath.Join(t.TempDir(), "proxy-url") + setURL := func(pass string) { + t.Helper() + if err := os.WriteFile(urlFile, []byte(proxy.url(pass)), 0o600); err != nil { + t.Fatal(err) + } + } + setup := func(t *testing.T, withCmd bool) { + withTransportEnvCleared(t) + t.Setenv("HTTPS_PROXY", proxy.url("pw-1")) // pilotctl's launch environment: stale + // Transport unknown (no daemon, nothing configured): the fallback + // is the probed raw route, which the stub below keeps off the + // network; compat's fallback would dial the real TLS registry. + t.Setenv("PILOT_REGISTRY_FINGERPRINT", fp) + if withCmd { + t.Setenv("PILOT_PROXY_CMD", "cat '"+urlFile+"'") + } + stubRawDirectDial(t, "127.0.0.1:1") // direct raw dials stay local (refused) + } + + t.Run("stale environment, no command: the 407 is reported", func(t *testing.T) { + setup(t, false) + rc, route, err := dialRegistry(productionRegistryAddr) + if err == nil { + rc.Close() + t.Fatal("dial with stale credentials succeeded") + } + if !route.proxied() || !strings.Contains(err.Error(), "407 Proxy Authentication Required") { + t.Fatalf("route %+v err %v, want the proxied route's 407", route, err) + } + if strings.Contains(err.Error(), "pw-1") { + t.Fatalf("error leaks the password: %v", err) + } + if hint := registryDialHint(route); !strings.Contains(hint, "credentials") { + t.Errorf("hint %q does not mention the credentials", hint) + } + }) + + t.Run("command supplies the current credentials", func(t *testing.T) { + setup(t, true) + setURL("pw-2") + _, before, _ := proxy.stats() + rc, route, err := dialRegistry(productionRegistryAddr) + if err != nil { + t.Fatalf("dialRegistry: %v", err) + } + defer rc.Close() + if _, err := rc.Lookup(5); err != nil { + t.Fatalf("Lookup: %v", err) + } + oks, after, _ := proxy.stats() + if !route.proxied() || oks["pw-2"] == 0 || after != before { + t.Fatalf("route %+v, accepted %v, new 407s %d: want the command's pw-2 and no 407", route, oks, after-before) + } + }) + + t.Run("rotation between resolve and dial: 407, refresh, retry", func(t *testing.T) { + setup(t, true) + setURL("pw-2") + proxy.rotate("pw-2") + routes, err := planRegistryRoutes(productionRegistryAddr) + if err != nil || len(routes) == 0 || !routes[0].proxied() { + t.Fatalf("routes = %+v, %v", routes, err) + } + // The proxy rotates after pilotctl read the URL. + proxy.rotate("pw-3") + setURL("pw-3") + _, before, _ := proxy.stats() + rc, err := routes[0].dial() + if err != nil { + t.Fatalf("dial after the rotation: %v", err) + } + defer rc.Close() + if _, err := rc.Lookup(5); err != nil { + t.Fatalf("Lookup after the rotation: %v", err) + } + oks, after, _ := proxy.stats() + if after-before != 1 || oks["pw-3"] == 0 { + t.Fatalf("407s %d, accepted %v: want one 407, then the refreshed pw-3", after-before, oks) + } + }) +} + +// "pilotctl must not try the raw registry directly first when a proxy is +// configured": for every configuration in which the proxy applies to the +// production registry, the first route is the TLS registry through it. +func TestRegistryRoutesProxyFirstWhenProxyConfigured(t *testing.T) { + for _, tc := range []struct { + name string + env map[string]string + cfg map[string]interface{} + daemon string + }{ + {name: "HTTPS_PROXY, no daemon, nothing configured", + env: map[string]string{"HTTPS_PROXY": "http://u:p@egress.test:3128"}}, + {name: "HTTPS_PROXY, config transport=auto, no daemon", + env: map[string]string{"HTTPS_PROXY": "http://u:p@egress.test:3128"}, + cfg: map[string]interface{}{"transport": "auto"}}, + {name: "HTTPS_PROXY, compat daemon", + env: map[string]string{"HTTPS_PROXY": "http://u:p@egress.test:3128"}, + daemon: "compat"}, + {name: "HTTPS_PROXY, config compat", + env: map[string]string{"HTTPS_PROXY": "http://u:p@egress.test:3128"}, + cfg: map[string]interface{}{"transport": "compat"}}, + {name: "explicit PILOT_PROXY, udp", + env: map[string]string{"PILOT_PROXY": "http://u:p@corp.test:3128", "PILOT_TRANSPORT": "udp"}}, + {name: "explicit config proxy, udp daemon", + cfg: map[string]interface{}{"proxy": "http://corp.test:3128"}, + daemon: "udp"}, + {name: "ALL_PROXY only, no daemon", + env: map[string]string{"ALL_PROXY": "http://u:p@egress.test:3128"}}, + } { + t.Run(tc.name, func(t *testing.T) { + withTransportEnvCleared(t) + stubDaemonTransport(t, tc.daemon) + for k, v := range tc.env { + t.Setenv(k, v) + } + if tc.cfg != nil { + if err := saveConfig(tc.cfg); err != nil { + t.Fatal(err) + } + } + routes, err := planRegistryRoutes(productionRegistryAddr) + if err != nil || len(routes) == 0 { + t.Fatalf("planRegistryRoutes = %v, %v", routes, err) + } + if first := routes[0]; !first.proxied() || first.Addr != compatRegistryAddr || !first.TLS { + t.Fatalf("first route = %s, want the TLS registry through the proxy", routeSpec(first)) + } + for _, r := range routes[1:] { + if !r.proxied() && !r.TLS && !r.Probe { + t.Errorf("unprobed direct raw fallback %s behind a configured proxy", routeSpec(r)) + } + } + }) + } +} + +func TestExtractProxyErrorFromDaemonLog(t *testing.T) { + for _, tc := range []struct{ line, want string }{ + {`time=2026-09-24T00:00:00Z level=WARN msg="registry dial failed, retrying" attempt=1 max=10 backoff=500ms error="dial registry TLS pinned: proxy CONNECT registry.pilotprotocol.network:443: 407 Proxy Authentication Required"`, + "proxy CONNECT registry.pilotprotocol.network:443: 407 Proxy Authentication Required"}, + {`{"time":"x","level":"WARN","msg":"registry dial failed, retrying","error":"dial registry TLS: proxy CONNECT registry.pilotprotocol.network:443: 403 Forbidden"}`, + "proxy CONNECT registry.pilotprotocol.network:443: 403 Forbidden"}, + {`time=x level=INFO msg="appstore: catalogue refresh failed (fetch catalogue from https://raw.githubusercontent.com/c.json: Get \"https://raw.githubusercontent.com/c.json\": proxy CONNECT raw.githubusercontent.com:443: 407 Proxy Authentication Required) and no cache; catalogue apps fail closed"`, + "proxy CONNECT raw.githubusercontent.com:443: 407 Proxy Authentication Required"}, + {`time=x level=WARN msg="transport auto-selected" transport=compat reason="no UDP answer from beacon b, and the proxy http://***@p:3128 refused the compat beacon check (proxy CONNECT beacon.pilotprotocol.network:443: 407 Proxy Authentication Required); staying on compat"`, + "proxy CONNECT beacon.pilotprotocol.network:443: 407 Proxy Authentication Required"}, + {`time=x level=ERROR msg="daemon start: registry dial (after 10 attempts): dial registry TLS: proxy CONNECT registry.pilotprotocol.network:443: 407 Proxy Authentication Required; the proxy rejected its credentials (407)"`, + "proxy CONNECT registry.pilotprotocol.network:443: 407 Proxy Authentication Required"}, + {`time=x level=WARN msg="registry dial failed, retrying" error="dial registry TLS: proxy CONNECT registry.pilotprotocol.network:443: dial proxy http://***@10.0.0.1:3128: dial tcp 10.0.0.1:3128: connect: connection refused"`, + "proxy CONNECT registry.pilotprotocol.network:443: dial proxy http://***@10.0.0.1:3128: dial tcp 10.0.0.1:3128: connect: connection refused"}, + {`time=x level=INFO msg="daemon registered" node_id=5`, ""}, + } { + if got := extractProxyError(tc.line); got != tc.want { + t.Errorf("extractProxyError(%s)\n got %q\nwant %q", tc.line, got, tc.want) + } + } + + log := filepath.Join(t.TempDir(), "pilot.log") + writeTestFile(t, log, strings.Join([]string{ + `time=x level=WARN msg="registry dial failed, retrying" error="dial registry TLS: proxy CONNECT registry.pilotprotocol.network:443: 407 Proxy Authentication Required"`, + `time=x level=INFO msg="appstore: catalogue refresh failed (Get \"https://raw.githubusercontent.com/c\": proxy CONNECT raw.githubusercontent.com:443: 403 Forbidden) and no cache"`, + `time=x level=ERROR msg="daemon start: registry dial (after 10 attempts): boom; hint"`, + ``, + }, "\n")) + if got := lastProxyErrorFromLog(log); got != "proxy CONNECT registry.pilotprotocol.network:443: 407 Proxy Authentication Required" { + t.Errorf("lastProxyErrorFromLog = %q, want the WARN line's 407 over the later INFO line", got) + } + if got := lastErrorFromLog(log); got != "daemon start: registry dial (after 10 attempts): boom; hint" { + t.Errorf("lastErrorFromLog = %q", got) + } + if got := lastProxyErrorFromLog(filepath.Join(t.TempDir(), "missing.log")); got != "" { + t.Errorf("missing log = %q", got) + } +} + +// writeStartFailDaemon writes a fake pilot-daemon for the forking `daemon +// start` path: -help lists a current daemon's flags, and a start writes +// logLines to stdout (pilotctl's per-PID log), then either exits with +// exitCode or, with exitCode < 0, stays up without ever serving its socket. +func writeStartFailDaemon(t *testing.T, logLines []string, exitCode int) string { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("shell-script fake daemon") + } + var help strings.Builder + for _, f := range append([]string{autoUsage, "proxy", "proxy-cmd"}, baseDaemonFlags...) { + name, usage, ok := strings.Cut(f, "=") + if !ok { + usage = "description of " + name + } + help.WriteString(" -" + name + " string\n \t" + usage + "\n") + } + var body strings.Builder + for _, l := range logLines { + body.WriteString("printf '%s\\n' " + shellQuote(l) + "\n") + } + tail := "exec sleep 60\n" + if exitCode >= 0 { + tail = "exit " + strconv.Itoa(exitCode) + "\n" + } + script := "#!/bin/sh\nif [ \"$1\" = \"-help\" ]; then\n cat >&2 <<'HELP'\nUsage of pilot-daemon:\n" + help.String() + "HELP\n exit 0\nfi\n" + body.String() + tail + path := filepath.Join(t.TempDir(), "pilot-daemon") + if err := os.WriteFile(path, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + return path +} + +func shellQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" } + +// killPIDFile stops a fake daemon `daemon start` left running. +func killPIDFile(t *testing.T, home string) { + t.Helper() + b, err := os.ReadFile(filepath.Join(home, ".pilot", "pilot.pid")) + if err != nil { + return + } + if pid, err := strconv.Atoi(strings.TrimSpace(string(b))); err == nil && pid > 0 { + _ = syscall.Kill(pid, syscall.SIGKILL) + } +} + +// E2E product gap: `pilotctl daemon start` behind a proxy that rejects the +// credentials used to say only "did not become ready within 30s". It must +// name the daemon's last proxy error (and say what to do about it), and +// it must notice a daemon that exited instead of waiting out the deadline. +func TestCLIDaemonStartReportsDaemonProxyError(t *testing.T) { + const regErr = `time=2026-09-24T00:00:01Z level=WARN msg="registry dial failed, retrying" attempt=1 max=10 backoff=500ms error="dial registry TLS: proxy CONNECT registry.pilotprotocol.network:443: 407 Proxy Authentication Required"` + const want = "proxy CONNECT registry.pilotprotocol.network:443: 407 Proxy Authentication Required" + + t.Run("not ready in time", func(t *testing.T) { + home := t.TempDir() + t.Cleanup(func() { killPIDFile(t, home) }) + bin := writeStartFailDaemon(t, []string{ + `time=2026-09-24T00:00:00Z level=WARN msg="transport auto-selected" transport=compat reason="no UDP answer from beacon b, and the proxy http://***@p:3128 refused the compat beacon check (proxy CONNECT beacon.pilotprotocol.network:443: 407 Proxy Authentication Required); staying on compat"`, + regErr, + }, -1) + env := cliEnvCleared(map[string]string{ + "PILOT_DAEMON_BIN": bin, + "PILOT_SOCKET": filepath.Join(t.TempDir(), "pilot.sock"), + "PILOT_HOME": home, + }) + _, stderr, code := runCLI(t, []string{"daemon", "start", "--wait", "2s"}, env) + if code == 0 { + t.Fatalf("daemon start succeeded against a daemon that never became ready") + } + if !strings.Contains(stderr, "did not become ready within 2s; last proxy error: "+want) { + t.Errorf("stderr lacks the daemon's proxy error:\n%s", stderr) + } + if !strings.Contains(stderr, "proxy_cmd") || !strings.Contains(stderr, "full log:") { + t.Errorf("hint does not say what to do:\n%s", stderr) + } + }) + + t.Run("daemon exits", func(t *testing.T) { + home := t.TempDir() + bin := writeStartFailDaemon(t, []string{ + regErr, + `time=2026-09-24T00:00:48Z level=ERROR msg="daemon start: registry dial (after 10 attempts): dial registry TLS: proxy CONNECT registry.pilotprotocol.network:443: 407 Proxy Authentication Required; the proxy rejected its credentials (407)"`, + }, 1) + env := cliEnvCleared(map[string]string{ + "PILOT_DAEMON_BIN": bin, + "PILOT_SOCKET": filepath.Join(t.TempDir(), "pilot.sock"), + "PILOT_HOME": home, + }) + start := time.Now() + _, stderr, code := runCLI(t, []string{"--json", "daemon", "start", "--wait", "20s"}, env) + if code == 0 { + t.Fatal("daemon start succeeded against a daemon that exited") + } + if elapsed := time.Since(start); elapsed > 10*time.Second { + t.Errorf("daemon start took %s to notice the exit; want well under the 20s wait", elapsed) + } + var res struct{ Code, Message, Hint string } + if err := json.Unmarshal([]byte(strings.TrimSpace(stderr)), &res); err != nil { + t.Fatalf("stderr is not one JSON error: %v\n%s", err, stderr) + } + if res.Code != "connection_failed" || !strings.Contains(res.Message, "exited during startup (exit status 1)") || + !strings.Contains(res.Message, "daemon start: registry dial (after 10 attempts)") || !strings.Contains(res.Message, want) { + t.Errorf("error = %+v", res) + } + if _, err := os.Stat(filepath.Join(home, ".pilot", "pilot.pid")); err == nil { + t.Error("PID file of the exited daemon left behind") + } + }) +} + +// buildRealDaemon compiles ./cmd/daemon for end-to-end tests. +func buildRealDaemon(t *testing.T) string { + t.Helper() + bin := filepath.Join(t.TempDir(), "pilot-daemon") + cmd := exec.Command("go", "build", "-o", bin, "github.com/pilot-protocol/pilotprotocol/cmd/daemon") + cmd.Env = append(os.Environ(), "GOWORK=off", "CGO_ENABLED=0") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("build pilot-daemon: %v\n%s", err, out) + } + return bin +} + +// End to end, the E2E product gap as a user hits it: plain `pilotctl +// daemon start` (transport auto) with the real pilot-daemon, UDP silently +// dropped, and an egress proxy that rejects the credentials. The daemon +// must stay on the proxy (compat; no CONNECT for the raw registry, no +// direct dial) and pilotctl must report the proxy's 407 with a hint, not +// only "did not become ready". Nothing leaves the machine: the in-process +// proxy answers every request with 407. +func TestE2EDaemonStartBehindRejectingProxy(t *testing.T) { + if testing.Short() { + t.Skip("builds and runs the real pilot-daemon") + } + if runtime.GOOS == "windows" { + t.Skip("unix daemon") + } + bin := buildRealDaemon(t) + proxy := newRotatingProxy(t, "right-pass", nil) + udp, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { udp.Close() }) + home := t.TempDir() + t.Cleanup(func() { killPIDFile(t, home) }) + sockDir, err := os.MkdirTemp("", "pe2e") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(sockDir) }) + env := cliEnvCleared(map[string]string{ + "PILOT_DAEMON_BIN": bin, + "PILOT_SOCKET": filepath.Join(sockDir, "s"), + "PILOT_HOME": home, + "HTTPS_PROXY": proxy.url("wrong-pass"), + "PILOT_NO_SKILLINJECT": "1", + "PILOT_APPSTORE_ROOT": filepath.Join(home, "apps"), + }) + _, stderr, code := runCLI(t, []string{"daemon", "start", "--wait", "10s", + "--beacon", udp.LocalAddr().String(), + "--compat-beacon", "wss://beacon.pilotprotocol.network/v1/compat", + "--motd-feed-url", ""}, env) + t.Logf("pilotctl daemon start (exit %d):\n%s", code, stderr) + if code == 0 { + t.Fatal("daemon start succeeded behind a proxy that rejects the credentials") + } + if !strings.Contains(stderr, "last proxy error: proxy CONNECT ") || !strings.Contains(stderr, "407 Proxy Authentication Required") { + t.Errorf("stderr does not name the proxy's 407:\n%s", stderr) + } + if !strings.Contains(stderr, "proxy_cmd") { + t.Errorf("no hint about rotating credentials:\n%s", stderr) + } + if strings.Contains(stderr, "wrong-pass") { + t.Errorf("stderr leaks the proxy password:\n%s", stderr) + } + _, n407, seen := proxy.stats() + if n407 == 0 || !strings.Contains(strings.Join(seen, "\n"), "CONNECT registry.pilotprotocol.network:443") { + t.Errorf("the daemon did not dial the registry through the proxy: %q", seen) + } + for _, s := range seen { + if strings.Contains(s, "34.71.57.205") { + t.Errorf("proxy asked for the raw registry: %q", s) + } + } + logs, _ := os.ReadFile(filepath.Join(home, ".pilot", "pilot.log")) + if strings.Contains(string(logs), "34.71.57.205:9000") { + t.Errorf("the daemon dialed the raw registry directly:\n%s", logs) + } + if !strings.Contains(string(logs), `msg="transport auto-selected" transport=compat`) { + t.Errorf("auto did not stay on compat:\n%s", logs) + } +} diff --git a/go.mod b/go.mod index 598e0e28..9aaedbf8 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/coder/websocket v1.8.15 github.com/pilot-protocol/app-store v1.0.3 github.com/pilot-protocol/beacon v0.2.9 - github.com/pilot-protocol/common v0.5.14 + github.com/pilot-protocol/common v0.5.15 github.com/pilot-protocol/dataexchange v0.2.2 github.com/pilot-protocol/eventstream v0.2.4 github.com/pilot-protocol/handshake v0.2.8 diff --git a/go.sum b/go.sum index 9c976ac4..76434d13 100644 --- a/go.sum +++ b/go.sum @@ -237,8 +237,8 @@ github.com/pilot-protocol/app-store v1.0.3 h1:LfY6iZW6awTPV9nnQn+u/sTG+2UfJPxzS+ github.com/pilot-protocol/app-store v1.0.3/go.mod h1:Sz1vIZ92zNMWEnTJTXNxw7QmlbPRXs73qY/duNC0tgs= github.com/pilot-protocol/beacon v0.2.9 h1:VqXAtRKl4YhZVkDmJzuNhS1bfpK0n9qbGKOGLfVYSyo= github.com/pilot-protocol/beacon v0.2.9/go.mod h1:DE8masXGku/IwfHL8lN/4CUTe5YsxMXFe2fRGJcH7w8= -github.com/pilot-protocol/common v0.5.14 h1:CKLjgRimNlksUe0OePHkOE32aPlemjfdRMBq8X/CDiw= -github.com/pilot-protocol/common v0.5.14/go.mod h1:OTXD84ScrBbKTAhawgFlLw9HtisgNvr83Zs1yDAuG9k= +github.com/pilot-protocol/common v0.5.15 h1:ds6DNWlth97pMcXZ7fMgfy9/s553la3/YuYOkwoXjjE= +github.com/pilot-protocol/common v0.5.15/go.mod h1:OTXD84ScrBbKTAhawgFlLw9HtisgNvr83Zs1yDAuG9k= github.com/pilot-protocol/dataexchange v0.2.2 h1:h1VJFqFCdMDtX1E2E8zxTQPw+9rWnMohXN8b6HpB2R0= github.com/pilot-protocol/dataexchange v0.2.2/go.mod h1:TUj2QtNMZ4oMnOag1j5k5qDxz2457S8uaB2FJif1Ulk= github.com/pilot-protocol/eventstream v0.2.4 h1:SyB64wqo+Qpz0hIAuzHGkI9Npwk5SFSg1aqNFLueUIQ= diff --git a/internal/proxyconf/policy.go b/internal/proxyconf/policy.go deleted file mode 100644 index b04e247e..00000000 --- a/internal/proxyconf/policy.go +++ /dev/null @@ -1,525 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -package proxyconf - -import ( - "bytes" - "context" - "crypto/tls" - "errors" - "fmt" - "log/slog" - "net" - "net/http" - "net/url" - "os" - "os/exec" - "runtime" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/pilot-protocol/common/netproxy" - "golang.org/x/net/http/httpproxy" -) - -// DefaultRefreshInterval is how often a command-backed Policy re-runs its -// command. Sandboxes that rotate proxy credentials (Meta Muse: every few -// minutes) accept the previous credentials for a while, and a 407 forces -// an immediate refresh anyway. -const DefaultRefreshInterval = 60 * time.Second - -// commandTimeout bounds one run of the proxy command. -const commandTimeout = 10 * time.Second - -// commandOutputLimit caps what is read from the proxy command's stdout. -const commandOutputLimit = 64 << 10 - -// Policy is an outbound proxy policy that can change while the process -// runs. It is either static (a fixed netproxy.Resolver) or backed by a -// command whose stdout is the current proxy URL — for egress proxies whose -// credentials rotate (a long-lived daemon keeps the credentials of its -// launch environment, and the proxy starts answering new CONNECTs with 407 -// Proxy Authentication Required a few minutes later). -// -// A command-backed Policy re-runs the command every refresh interval (Run) -// and whenever the proxy rejects the credentials; its dialers and HTTP -// helpers then retry once with the new URL. Connections already open are -// unaffected: established tunnels survive a rotation. -// -// A nil *Policy never proxies. Loopback targets are never proxied. -// Policies are safe for concurrent use. -type Policy struct { - cur atomic.Pointer[policyState] - - // Command-backed policies only. - command string - run func(ctx context.Context, command string) (string, error) - exempt func(scheme, hostport string) bool - mu sync.Mutex // serializes refreshes -} - -type policyState struct { - resolver *netproxy.Resolver - // raw is the command's last URL ("" for the static fallback). Only - // ever compared, never logged. - raw string - gen uint64 -} - -// Static returns a Policy that always uses r. nil r gives a nil Policy (no -// policy: callers keep their historical behaviour). -func Static(r *netproxy.Resolver) *Policy { - if r == nil { - return nil - } - p := &Policy{} - p.cur.Store(&policyState{resolver: r}) - return p -} - -// NewCommand returns a Policy backed by command, a shell command line -// (/bin/sh -c; cmd /C on Windows) whose stdout is the current proxy URL, -// http://[user:pass@]host[:port] or https://.... The command runs once now. -// Its URL is used for every target except loopback and, with honorNoProxy -// (the -proxy=auto semantics), the NO_PROXY / no_proxy exemptions of this -// process's environment. -// -// When that first run fails, fallback (typically the environment's proxy, -// which was fresh when the process started) is used until a refresh -// succeeds, and the error is returned along with the Policy; the Policy is -// usable either way. -func NewCommand(ctx context.Context, command string, fallback *netproxy.Resolver, honorNoProxy bool) (*Policy, error) { - return newCommandPolicy(ctx, command, fallback, honorNoProxy, runProxyCommand) -} - -func newCommandPolicy(ctx context.Context, command string, fallback *netproxy.Resolver, honorNoProxy bool, run func(context.Context, string) (string, error)) (*Policy, error) { - command = strings.TrimSpace(command) - if command == "" { - return nil, errors.New("empty proxy command") - } - p := &Policy{command: command, run: run} - if honorNoProxy { - p.exempt = noProxyMatcher(noProxyFromEnv()) - } - p.cur.Store(&policyState{resolver: fallback}) - if _, err := p.refresh(ctx, 0, true); err != nil { - return p, err - } - return p, nil -} - -// Refreshable reports whether the Policy re-reads its proxy (it is backed -// by a command). -func (p *Policy) Refreshable() bool { return p != nil && p.command != "" } - -// Resolver returns the current resolver (nil when there is none). -func (p *Policy) Resolver() *netproxy.Resolver { - s := p.state() - if s == nil { - return nil - } - return s.resolver -} - -func (p *Policy) state() *policyState { - if p == nil { - return nil - } - return p.cur.Load() -} - -// Enabled reports whether any target may be proxied. A command-backed -// Policy is always enabled: its next refresh can supply a proxy. -func (p *Policy) Enabled() bool { - if p == nil { - return false - } - return p.Refreshable() || p.Resolver().Enabled() -} - -// Mode reports netproxy.ModeAuto, ModeOff or ModeExplicit for the current -// resolver. -func (p *Policy) Mode() string { return p.Resolver().Mode() } - -// Warnings reports the current resolver's skipped environment variables. -func (p *Policy) Warnings() []error { return p.Resolver().Warnings() } - -// String describes the Policy for logs, credentials redacted. -func (p *Policy) String() string { - if p == nil { - return "none" - } - s := p.Resolver().String() - if p.Resolver() == nil { - s = "none yet" - } - if p.Refreshable() { - s += " (refreshed from the proxy command)" - } - return s -} - -// Refresh re-runs the proxy command now. changed reports whether the proxy -// URL differs from the one in use. A static Policy never changes. On error -// the previous proxy stays in use. -func (p *Policy) Refresh(ctx context.Context) (changed bool, err error) { - if !p.Refreshable() { - return false, nil - } - return p.refresh(ctx, 0, false) -} - -// refresh runs the command unless another refresh already replaced the -// state generation `seen` (seen == 0: always run), and installs the new -// URL when it differs. It reports whether the URL in use is now different -// from generation `seen`. -func (p *Policy) refresh(ctx context.Context, seen uint64, initial bool) (bool, error) { - p.mu.Lock() - defer p.mu.Unlock() - cur := p.cur.Load() - if seen != 0 && cur.gen != seen { - return true, nil // someone refreshed while we waited - } - raw, err := p.run(ctx, p.command) - if err != nil { - return false, err - } - if raw == cur.raw && cur.raw != "" { - return false, nil - } - r, err := netproxy.Explicit(raw) - if err != nil { - return false, fmt.Errorf("proxy command printed an unusable URL: %v", unwrapNetproxy(err)) - } - next := &policyState{resolver: r, raw: raw, gen: cur.gen + 1} - p.cur.Store(next) - if !initial { - old := "none" - if cur.resolver != nil { - old = cur.resolver.String() - } - if old != r.String() { - slog.Info("proxy changed (proxy command)", "proxy", r.String(), "was", old) - } else { - slog.Debug("proxy credentials refreshed (proxy command)", "proxy", r.String()) - } - } - return true, nil -} - -// refreshAfterAuthFailure refreshes after the proxy rejected the -// credentials of generation seen, and reports whether a retry would use a -// different URL. -func (p *Policy) refreshAfterAuthFailure(ctx context.Context, seen uint64) bool { - if !p.Refreshable() { - return false - } - changed, err := p.refresh(ctx, seen, false) - if err != nil { - slog.Warn("proxy rejected the credentials and the proxy command failed; keeping the previous proxy", "err", err) - return false - } - return changed -} - -// Run re-runs the proxy command every interval (DefaultRefreshInterval when -// interval <= 0) until ctx ends. It returns at once for a static Policy. -func (p *Policy) Run(ctx context.Context, interval time.Duration) { - if !p.Refreshable() { - return - } - if interval <= 0 { - interval = DefaultRefreshInterval - } - t := time.NewTicker(interval) - defer t.Stop() - for { - select { - case <-ctx.Done(): - return - case <-t.C: - if _, err := p.Refresh(ctx); err != nil && ctx.Err() == nil { - slog.Warn("proxy command failed; keeping the previous proxy", "err", err) - } - } - } -} - -// bypass reports whether target (host, port) goes direct whatever the -// resolver says: loopback always, and for a command-backed Policy the -// NO_PROXY exemptions. -func (p *Policy) bypass(scheme, host, port string) bool { - if IsLoopbackHost(host) { - return true - } - return p != nil && p.exempt != nil && p.exempt(scheme, net.JoinHostPort(host, port)) -} - -// Proxies reports whether a TCP connection to addr ("host:port") would go -// through a proxy. A command-backed Policy proxies every target it does -// not exempt, also before its command first succeeds. -func (p *Policy) Proxies(addr string) bool { - if !p.Enabled() { - return false - } - host, port, err := net.SplitHostPort(addr) - if err != nil || p.bypass("https", host, port) { - return false - } - if p.Refreshable() { - return true - } - u, err := p.Resolver().ProxyForAddr(addr) - return err == nil && u != nil -} - -// DialContext returns a dial function that tunnels through the proxy the -// Policy picks for each target (CONNECT by host name) and dials loopback -// and unproxied targets directly. When the proxy rejects the credentials -// (407) of a command-backed Policy, the command is re-run and the dial is -// retried once with the new URL. proxyTLS configures the TLS session with -// an https:// proxy (nil: system roots); it never applies to the target. -func (p *Policy) DialContext(proxyTLS *tls.Config) func(ctx context.Context, network, addr string) (net.Conn, error) { - var direct net.Dialer - return func(ctx context.Context, network, addr string) (net.Conn, error) { - if host, port, err := net.SplitHostPort(addr); err == nil && p.bypass("https", host, port) { - return direct.DialContext(ctx, network, addr) - } - s := p.state() - var r *netproxy.Resolver - var gen uint64 - if s != nil { - r, gen = s.resolver, s.gen - } - conn, err := (&netproxy.Dialer{Resolver: r, TLSConfig: proxyTLS}).DialContext(ctx, network, addr) - if err == nil || !IsProxyAuthError(err) || !p.refreshAfterAuthFailure(ctx, gen) { - return conn, err - } - slog.Info("proxy rejected the credentials; retrying with refreshed ones", "target", addr) - return (&netproxy.Dialer{Resolver: p.Resolver(), TLSConfig: proxyTLS}).DialContext(ctx, network, addr) - } -} - -// RequestProxy returns an http.Transport.Proxy function that follows the -// Policy's current resolver and never proxies loopback targets. nil for a -// nil Policy (net/http's own environment handling then applies wherever -// the caller leaves Proxy unset). -func (p *Policy) RequestProxy() func(*http.Request) (*url.URL, error) { - if p == nil { - return nil - } - return func(req *http.Request) (*url.URL, error) { - if req == nil || req.URL == nil { - return nil, nil - } - scheme := "https" - port := req.URL.Port() - switch strings.ToLower(req.URL.Scheme) { - case "http", "ws": - scheme = "http" - if port == "" { - port = "80" - } - default: - if port == "" { - port = "443" - } - } - if p.bypass(scheme, req.URL.Hostname(), port) { - return nil, nil - } - return p.Resolver().ProxyForRequest(req) - } -} - -// ErrProxyAuth is returned for requests whose proxy CONNECT was answered -// with 407 Proxy Authentication Required (see ConfigureTransport). -var ErrProxyAuth = errors.New("proxy CONNECT: 407 Proxy Authentication Required") - -// ConfigureTransport routes tr through the Policy: tr.Proxy follows the -// current resolver, and a 407 answer to a CONNECT makes a command-backed -// Policy refresh at once, so the next connection uses the new credentials -// (the failed request returns ErrProxyAuth; RoundTripper retries it). A nil -// Policy leaves tr alone. -func (p *Policy) ConfigureTransport(tr *http.Transport) { - if p == nil || tr == nil { - return - } - tr.Proxy = p.RequestProxy() - if !p.Refreshable() { - return - } - tr.OnProxyConnectResponse = func(ctx context.Context, _ *url.URL, _ *http.Request, resp *http.Response) error { - if resp.StatusCode != http.StatusProxyAuthRequired { - return nil - } - s := p.state() - p.refreshAfterAuthFailure(ctx, s.gen) - return ErrProxyAuth - } -} - -// RoundTripper wraps base (a transport set up with ConfigureTransport) so -// that a request whose CONNECT the proxy rejected with 407 is retried once -// when the refreshed Policy has a different URL. Requests with a body are -// retried only when it can be replayed (GetBody). For a Policy that cannot -// refresh it returns base. -func (p *Policy) RoundTripper(base http.RoundTripper) http.RoundTripper { - if !p.Refreshable() || base == nil { - return base - } - return &retryAuthTransport{p: p, base: base} -} - -type retryAuthTransport struct { - p *Policy - base http.RoundTripper -} - -func (t *retryAuthTransport) RoundTrip(req *http.Request) (*http.Response, error) { - s := t.p.state() - resp, err := t.base.RoundTrip(req) - if err == nil || !IsProxyAuthError(err) { - return resp, err - } - if s.gen == t.p.state().gen && !t.p.refreshAfterAuthFailure(req.Context(), s.gen) { - return resp, err - } - retry := req - if req.Body != nil && req.Body != http.NoBody { - if req.GetBody == nil { - return resp, err - } - body, berr := req.GetBody() - if berr != nil { - return resp, err - } - retry = req.Clone(req.Context()) - retry.Body = body - } - return t.base.RoundTrip(retry) -} - -// IsProxyAuthError reports whether err is a proxy's rejection of the -// credentials: a 407 answer to CONNECT, or the malformed status line some -// sandbox egress proxies send in its place. -func IsProxyAuthError(err error) bool { - if err == nil { - return false - } - var ce *netproxy.ConnectError - if errors.As(err, &ce) { - return ce.StatusCode == http.StatusProxyAuthRequired - } - if errors.Is(err, ErrProxyAuth) { - return true - } - msg := err.Error() - return strings.Contains(msg, "Proxy Authentication Required") || - strings.Contains(msg, "malformed HTTP status code") -} - -// runProxyCommand runs command with the platform shell and returns the -// first line of its stdout, validated as an http(s) proxy URL. Neither the -// output nor stderr is ever included in errors or logs. -func runProxyCommand(ctx context.Context, command string) (string, error) { - ctx, cancel := context.WithTimeout(ctx, commandTimeout) - defer cancel() - // #nosec G204 -- the operator's own -proxy-cmd / PILOT_PROXY_CMD / - // config.json proxy_cmd, run with the daemon's own privileges. - var cmd *exec.Cmd - if runtime.GOOS == "windows" { - cmd = exec.CommandContext(ctx, "cmd", "/C", command) // #nosec G204 - } else { - cmd = exec.CommandContext(ctx, "/bin/sh", "-c", command) // #nosec G204 - } - cmd.Env = commandEnv(os.Environ()) - var out limitedBuffer - cmd.Stdout = &out - cmd.WaitDelay = time.Second - if err := cmd.Run(); err != nil { - if ctx.Err() == context.DeadlineExceeded { - return "", fmt.Errorf("proxy command timed out after %s", commandTimeout) - } - return "", fmt.Errorf("proxy command failed: %v", err) - } - line := strings.TrimSpace(out.String()) - if i := strings.IndexAny(line, "\r\n"); i >= 0 { - line = strings.TrimSpace(line[:i]) - } - if line == "" { - return "", errors.New("proxy command printed nothing") - } - s, err := Normalize(line) - if err != nil { - return "", fmt.Errorf("proxy command output: %v", err) - } - if s == Auto || s == Off { - return "", errors.New("proxy command output: want an http:// or https:// proxy URL") - } - return s, nil -} - -// commandEnv is the proxy command's environment: this process's, minus the -// daemon secrets the command has no business seeing. -func commandEnv(env []string) []string { - out := make([]string, 0, len(env)) - for _, kv := range env { - k, _, _ := strings.Cut(kv, "=") - switch k { - case "PILOT_ADMIN_TOKEN", "PILOT_WEBHOOK_SECRET": - continue - } - out = append(out, kv) - } - return out -} - -type limitedBuffer struct{ bytes.Buffer } - -func (b *limitedBuffer) Write(p []byte) (int, error) { - if room := commandOutputLimit - b.Len(); room > 0 { - if len(p) > room { - b.Buffer.Write(p[:room]) - } else { - b.Buffer.Write(p) - } - } - return len(p), nil -} - -func noProxyFromEnv() string { - for _, k := range []string{"NO_PROXY", "no_proxy"} { - if v := strings.TrimSpace(os.Getenv(k)); v != "" { - return v - } - } - return "" -} - -// noProxyMatcher returns a NO_PROXY matcher (net/http semantics: host -// names match themselves and their subdomains, ".x" and "*.x" subdomains -// only, IPs, CIDRs, optional ":port", "*" everything), nil for an empty -// list. -func noProxyMatcher(list string) func(scheme, hostport string) bool { - if strings.TrimSpace(list) == "" { - return nil - } - entries := strings.FieldsFunc(list, func(r rune) bool { return r == ',' || r == ' ' }) - for i, e := range entries { - if strings.HasPrefix(e, "*.") { - entries[i] = e[1:] - } - } - const sentinel = "http://proxy.invalid" - pf := (&httpproxy.Config{HTTPProxy: sentinel, HTTPSProxy: sentinel, NoProxy: strings.Join(entries, ",")}).ProxyFunc() - return func(scheme, hostport string) bool { - if scheme != "http" { - scheme = "https" - } - u, err := pf(&url.URL{Scheme: scheme, Host: hostport}) - return err == nil && u == nil - } -} diff --git a/internal/proxyconf/policy_test.go b/internal/proxyconf/policy_test.go deleted file mode 100644 index 211bcb9d..00000000 --- a/internal/proxyconf/policy_test.go +++ /dev/null @@ -1,501 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -package proxyconf - -import ( - "bufio" - "bytes" - "context" - "crypto/tls" - "errors" - "fmt" - "io" - "net" - "net/http" - "net/http/httptest" - "net/url" - "runtime" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/pilot-protocol/common/netproxy" -) - -// rotatingProxy is a CONNECT proxy that accepts one password at a time -// (407 for any other) and tunnels every accepted CONNECT to upstream. -type rotatingProxy struct { - ln net.Listener - upstream string - - mu sync.Mutex - pass string - oks map[string]int // password -> accepted CONNECTs - n407 int - sawNo int // CONNECTs without credentials -} - -func newRotatingProxy(t *testing.T, pass, upstream string) *rotatingProxy { - t.Helper() - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - p := &rotatingProxy{ln: ln, upstream: upstream, pass: pass, oks: map[string]int{}} - go func() { - for { - c, err := ln.Accept() - if err != nil { - return - } - go p.serve(c) - } - }() - t.Cleanup(func() { ln.Close() }) - return p -} - -func (p *rotatingProxy) url(pass string) string { - return fmt.Sprintf("http://muse:%s@%s", pass, p.ln.Addr()) -} - -func (p *rotatingProxy) rotate(pass string) { - p.mu.Lock() - p.pass = pass - p.mu.Unlock() -} - -func (p *rotatingProxy) stats() (oks map[string]int, n407 int) { - p.mu.Lock() - defer p.mu.Unlock() - m := map[string]int{} - for k, v := range p.oks { - m[k] = v - } - return m, p.n407 -} - -func (p *rotatingProxy) serve(c net.Conn) { - defer c.Close() - br := bufio.NewReader(c) - req, err := http.ReadRequest(br) - if err != nil { - return - } - req.Header.Set("Authorization", req.Header.Get("Proxy-Authorization")) - user, pass, ok := req.BasicAuth() - p.mu.Lock() - good := ok && user == "muse" && pass == p.pass - switch { - case !ok: - p.sawNo++ - case good: - p.oks[pass]++ - default: - p.n407++ - } - p.mu.Unlock() - if req.Method != http.MethodConnect || !good { - fmt.Fprint(c, "HTTP/1.1 407 Proxy Authentication Required\r\nProxy-Authenticate: Basic realm=\"t\"\r\nContent-Length: 0\r\n\r\n") - return - } - up, err := net.Dial("tcp", p.upstream) - if err != nil { - fmt.Fprint(c, "HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n") - return - } - defer up.Close() - fmt.Fprint(c, "HTTP/1.1 200 Connection established\r\n\r\n") - done := make(chan struct{}, 2) - go func() { _, _ = io.Copy(up, br); done <- struct{}{} }() - go func() { _, _ = io.Copy(c, up); done <- struct{}{} }() - <-done -} - -// fakeSource is a proxy command stand-in: it prints *url, counting runs. -type fakeSource struct { - mu sync.Mutex - url string - err error - runs int -} - -func (f *fakeSource) set(u string, err error) { - f.mu.Lock() - f.url, f.err = u, err - f.mu.Unlock() -} - -func (f *fakeSource) run(context.Context, string) (string, error) { - f.mu.Lock() - defer f.mu.Unlock() - f.runs++ - return f.url, f.err -} - -func (f *fakeSource) count() int { - f.mu.Lock() - defer f.mu.Unlock() - return f.runs -} - -func echoServer(t *testing.T) string { - t.Helper() - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { ln.Close() }) - go func() { - for { - c, err := ln.Accept() - if err != nil { - return - } - go func() { defer c.Close(); _, _ = io.Copy(c, c) }() - } - }() - return ln.Addr().String() -} - -func clearEnv(t *testing.T) { - for _, k := range []string{"HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy", "ALL_PROXY", "all_proxy", "NO_PROXY", "no_proxy"} { - t.Setenv(k, "") - } -} - -func TestStaticAndNilPolicy(t *testing.T) { - var nilPolicy *Policy - if Static(nil) != nil { - t.Fatal("Static(nil) is not nil") - } - if nilPolicy.Enabled() || nilPolicy.Refreshable() || nilPolicy.Resolver() != nil || nilPolicy.RequestProxy() != nil { - t.Fatal("nil policy is not inert") - } - if nilPolicy.String() != "none" || nilPolicy.Proxies("registry.pilot.invalid:443") { - t.Fatal("nil policy describes itself as proxying") - } - if changed, err := nilPolicy.Refresh(context.Background()); changed || err != nil { - t.Fatal("nil policy refreshed") - } - nilPolicy.Run(context.Background(), time.Millisecond) // returns at once - tr := &http.Transport{} - nilPolicy.ConfigureTransport(tr) - if tr.Proxy != nil { - t.Fatal("nil policy configured a transport") - } - - r, _ := netproxy.Explicit("http://u:secret@proxy.test:3128") - p := Static(r) - if !p.Enabled() || p.Refreshable() || p.Mode() != netproxy.ModeExplicit { - t.Fatalf("static policy = enabled %v refreshable %v mode %s", p.Enabled(), p.Refreshable(), p.Mode()) - } - if strings.Contains(p.String(), "secret") { - t.Fatalf("String leaks the password: %s", p.String()) - } - if !p.Proxies("registry.pilot.invalid:443") || p.Proxies("127.0.0.1:9000") || p.Proxies("localhost:1") { - t.Fatal("Proxies ignores the loopback exemption") - } - if rt := p.RoundTripper(http.DefaultTransport); rt != http.DefaultTransport { - t.Fatal("a static policy wraps the round tripper") - } -} - -func TestCommandPolicyRefresh(t *testing.T) { - clearEnv(t) - src := &fakeSource{url: "http://muse:one@proxy.test:3128"} - p, err := newCommandPolicy(context.Background(), "cmd", nil, true, src.run) - if err != nil { - t.Fatal(err) - } - if !p.Refreshable() || !p.Enabled() || p.Mode() != netproxy.ModeExplicit { - t.Fatalf("command policy: refreshable %v enabled %v mode %s", p.Refreshable(), p.Enabled(), p.Mode()) - } - if s := p.String(); strings.Contains(s, "one") || !strings.Contains(s, "refreshed from the proxy command") { - t.Fatalf("String = %q", s) - } - if changed, err := p.Refresh(context.Background()); changed || err != nil { - t.Fatalf("unchanged refresh = (%v, %v)", changed, err) - } - src.set("http://muse:two@proxy.test:3128", nil) - if changed, err := p.Refresh(context.Background()); !changed || err != nil { - t.Fatalf("rotated refresh = (%v, %v)", changed, err) - } - u, _ := p.Resolver().ProxyForAddr("registry.pilot.invalid:443") - if pw, _ := u.User.Password(); pw != "two" { - t.Fatalf("password after refresh = %q, want two", pw) - } - // A failing command keeps the current proxy. - src.set("", errors.New("exit status 1")) - if changed, err := p.Refresh(context.Background()); changed || err == nil { - t.Fatalf("failing refresh = (%v, %v)", changed, err) - } - u, _ = p.Resolver().ProxyForAddr("registry.pilot.invalid:443") - if pw, _ := u.User.Password(); pw != "two" { - t.Fatalf("password after a failed refresh = %q, want two", pw) - } - if _, err := newCommandPolicy(context.Background(), " ", nil, true, src.run); err == nil { - t.Fatal("empty command accepted") - } -} - -// A failing first run serves the fallback until the command succeeds. -func TestCommandPolicyFallback(t *testing.T) { - clearEnv(t) - src := &fakeSource{err: errors.New("exit status 127")} - fallback, _ := netproxy.Explicit("http://muse:launch@proxy.test:3128") - p, err := newCommandPolicy(context.Background(), "cmd", fallback, true, src.run) - if err == nil || p == nil { - t.Fatalf("first run failure = (%v, %v), want a usable policy and the error", p, err) - } - if p.Resolver() != fallback { - t.Fatal("fallback not in use") - } - p2, _ := newCommandPolicy(context.Background(), "cmd", nil, true, src.run) - if !p2.Enabled() || !p2.Proxies("registry.pilot.invalid:443") || p2.String() != "none yet (refreshed from the proxy command)" { - t.Fatalf("no-fallback policy: enabled %v, %q", p2.Enabled(), p2.String()) - } - src.set("http://muse:fresh@proxy.test:3128", nil) - if changed, err := p.Refresh(context.Background()); !changed || err != nil { - t.Fatalf("refresh = (%v, %v)", changed, err) - } -} - -// The heart of muse-proxy-cred-rotation-unhandled: a dial whose CONNECT -// the proxy rejects with 407 re-runs the command and retries once. -func TestCommandPolicyDialRetriesOn407(t *testing.T) { - clearEnv(t) - echo := echoServer(t) - proxy := newRotatingProxy(t, "one", echo) - src := &fakeSource{url: proxy.url("one")} - p, err := newCommandPolicy(context.Background(), "cmd", nil, true, src.run) - if err != nil { - t.Fatal(err) - } - dial := p.DialContext(nil) - ping := func() error { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - c, err := dial(ctx, "tcp", "registry.pilot.invalid:443") - if err != nil { - return err - } - defer c.Close() - if _, err := c.Write([]byte("ping")); err != nil { - return err - } - buf := make([]byte, 4) - if _, err := io.ReadFull(c, buf); err != nil || string(buf) != "ping" { - return fmt.Errorf("echo = (%q, %v)", buf, err) - } - return nil - } - if err := ping(); err != nil { - t.Fatalf("dial before the rotation: %v", err) - } - - proxy.rotate("two") - src.set(proxy.url("two"), nil) - runs := src.count() - if err := ping(); err != nil { - t.Fatalf("dial after the rotation: %v", err) - } - oks, n407 := proxy.stats() - if n407 != 1 || oks["two"] != 1 { - t.Fatalf("proxy: 407s %d, accepted %v; want one 407, then the refreshed credentials", n407, oks) - } - if src.count() != runs+1 { - t.Fatalf("command runs = %d, want one refresh", src.count()-runs) - } - - // Credentials that are really wrong: one refresh, no retry (same URL), - // and the 407 error comes back. - proxy.rotate("three") - if err := ping(); err == nil || !IsProxyAuthError(err) { - t.Fatalf("dial with unrefreshable credentials = %v, want the 407", err) - } - if _, n407 := proxy.stats(); n407 != 2 { - t.Fatalf("407s = %d, want 2 (no retry with the same URL)", n407) - } -} - -// NO_PROXY applies to a command policy with -proxy=auto semantics only; -// loopback always goes direct. -func TestCommandPolicyNoProxy(t *testing.T) { - clearEnv(t) - t.Setenv("NO_PROXY", "*.corp.example,10.0.0.0/8") - src := &fakeSource{url: "http://muse:x@proxy.test:3128"} - auto, _ := newCommandPolicy(context.Background(), "cmd", nil, true, src.run) - explicit, _ := newCommandPolicy(context.Background(), "cmd", nil, false, src.run) - for addr, want := range map[string]bool{ - "registry.pilotprotocol.network:443": true, - "git.corp.example:443": false, - "10.1.2.3:9000": false, - "127.0.0.1:9000": false, - } { - if got := auto.Proxies(addr); got != want { - t.Errorf("auto: Proxies(%s) = %v, want %v", addr, got, want) - } - } - if !explicit.Proxies("git.corp.example:443") || explicit.Proxies("localhost:80") { - t.Error("explicit command policy: NO_PROXY applied or loopback proxied") - } - pf := auto.RequestProxy() - for raw, want := range map[string]bool{ - "https://raw.githubusercontent.com/x": true, - "https://git.corp.example/x": false, - "http://127.0.0.1:8080/hook": false, - "ws://10.0.0.7/x": false, - } { - u, _ := url.Parse(raw) - got, err := pf(&http.Request{URL: u}) - if err != nil || (got != nil) != want { - t.Errorf("RequestProxy(%s) = (%v, %v), want proxied=%v", raw, got, err, want) - } - } -} - -// HTTP clients: the transport follows the refreshed URL, a 407 on CONNECT -// refreshes at once, and RoundTripper retries the request (replaying a -// body when it can). -func TestCommandPolicyHTTPRetriesOn407(t *testing.T) { - clearEnv(t) - var hits atomic.Int32 - srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - hits.Add(1) - body, _ := io.ReadAll(r.Body) - fmt.Fprintf(w, "%s %s", r.Method, body) - })) - defer srv.Close() - proxy := newRotatingProxy(t, "one", srv.Listener.Addr().String()) - src := &fakeSource{url: proxy.url("one")} - p, err := newCommandPolicy(context.Background(), "cmd", nil, true, src.run) - if err != nil { - t.Fatal(err) - } - tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} // test server - p.ConfigureTransport(tr) - client := &http.Client{Transport: p.RoundTripper(tr), Timeout: 10 * time.Second} - defer tr.CloseIdleConnections() - - get := func(method string, body io.Reader) (string, error) { - req, err := http.NewRequest(method, "https://app.pilot.invalid/api", body) - if err != nil { - return "", err - } - req.Close = true // a new CONNECT per request - resp, err := client.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - b, err := io.ReadAll(resp.Body) - return string(b), err - } - if got, err := get(http.MethodGet, nil); err != nil || got != "GET " { - t.Fatalf("GET before the rotation = (%q, %v)", got, err) - } - proxy.rotate("two") - src.set(proxy.url("two"), nil) - if got, err := get(http.MethodPost, bytes.NewReader([]byte("payload"))); err != nil || got != "POST payload" { - t.Fatalf("POST after the rotation = (%q, %v)", got, err) - } - oks, n407 := proxy.stats() - if n407 != 1 || oks["two"] != 1 { - t.Fatalf("proxy: 407s %d, accepted %v", n407, oks) - } - - // A body that cannot be replayed is not retried. - proxy.rotate("three") - src.set(proxy.url("three"), nil) - if _, err := get(http.MethodPost, io.NopCloser(strings.NewReader("once"))); err == nil || !IsProxyAuthError(err) { - t.Fatalf("unreplayable POST after a rotation = %v, want the 407", err) - } - // ... but the refresh happened: the next request goes through. - if got, err := get(http.MethodGet, nil); err != nil || got != "GET " { - t.Fatalf("GET after the refresh = (%q, %v)", got, err) - } - if n := hits.Load(); n != 3 { - t.Fatalf("server hits = %d, want 3", n) - } -} - -func TestCommandPolicyRun(t *testing.T) { - clearEnv(t) - src := &fakeSource{url: "http://muse:one@proxy.test:3128"} - p, _ := newCommandPolicy(context.Background(), "cmd", nil, true, src.run) - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { p.Run(ctx, 10*time.Millisecond); close(done) }() - src.set("http://muse:two@proxy.test:3128", nil) - deadline := time.Now().Add(3 * time.Second) - for { - u, _ := p.Resolver().ProxyForAddr("x.test:443") - if pw, _ := u.User.Password(); pw == "two" { - break - } - if time.Now().After(deadline) { - t.Fatal("Run never refreshed") - } - time.Sleep(5 * time.Millisecond) - } - cancel() - select { - case <-done: - case <-time.After(3 * time.Second): - t.Fatal("Run did not stop with its context") - } -} - -func TestIsProxyAuthError(t *testing.T) { - for _, tc := range []struct { - err error - want bool - }{ - {nil, false}, - {&netproxy.ConnectError{Target: "x:443", StatusCode: 407}, true}, - {fmt.Errorf("dial: %w", &netproxy.ConnectError{Target: "x:443", StatusCode: 407}), true}, - {&netproxy.ConnectError{Target: "x:443", StatusCode: 403}, false}, - {fmt.Errorf("Get x: %w", ErrProxyAuth), true}, - {errors.New("Proxy Authentication Required"), true}, - {errors.New(`read CONNECT response: malformed HTTP status code "Proxy"`), true}, - {errors.New("connection refused"), false}, - } { - if got := IsProxyAuthError(tc.err); got != tc.want { - t.Errorf("IsProxyAuthError(%v) = %v, want %v", tc.err, got, tc.want) - } - } -} - -// The real command runner: /bin/sh -c, first line of stdout, validated; -// nothing it prints ever shows up in an error. -func TestRunProxyCommand(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("uses /bin/sh") - } - t.Setenv("PILOT_ADMIN_TOKEN", "admin-secret") - ctx := context.Background() - got, err := runProxyCommand(ctx, `printf 'http://muse:%s@proxy.test:3128\nsecond line\n' "${PILOT_ADMIN_TOKEN:-hidden}"; echo noise >&2`) - if err != nil || got != "http://muse:hidden@proxy.test:3128" { - t.Fatalf("runProxyCommand = (%q, %v), want the first line, without the admin token", got, err) - } - for cmd, want := range map[string]string{ - "exit 3": "exit status 3", - "true": "printed nothing", - "echo auto": "want an http:// or https:// proxy URL", - "echo muse:s3cret@proxy.test:3128": "invalid proxy", - "echo socks5://u:s3cret@p:1080": "scheme must be http or https", - "echo s3cret; exit 1": "exit status 1", - } { - _, err := runProxyCommand(ctx, cmd) - if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("runProxyCommand(%q) = %v, want an error containing %q", cmd, err, want) - continue - } - if strings.Contains(err.Error(), "s3cret") { - t.Errorf("runProxyCommand(%q) error leaks output: %v", cmd, err) - } - } -} diff --git a/internal/proxyconf/proxyconf.go b/internal/proxyconf/proxyconf.go index 977a180d..c2aa9dc8 100644 --- a/internal/proxyconf/proxyconf.go +++ b/internal/proxyconf/proxyconf.go @@ -17,6 +17,13 @@ // Anything else — a bare word such as "proxy", a host:port without a // scheme, or another scheme — is rejected, so a typo can never turn into a // proxy host name. Errors and display strings never contain credentials. +// +// Rotating credentials are netproxy's business: a Resolver built with +// netproxy.WithRefreshCommand (the -proxy-cmd / $PILOT_PROXY_CMD / +// config.json proxy_cmd setting) re-reads the proxy URL every +// netproxy.DefaultRefreshInterval and whenever a proxy answers 407, and +// netproxy's Dialer and RefreshingTransport retry that connection once with +// the new credentials. This package only adds the loopback rule on top. package proxyconf import ( @@ -38,6 +45,10 @@ const ( Off = netproxy.ModeOff ) +// EnvRefreshCommand is the environment variable holding the proxy refresh +// command (netproxy.EnvRefreshCommand, "PILOT_PROXY_CMD"). +const EnvRefreshCommand = netproxy.EnvRefreshCommand + // offAliases are accepted, case-insensitively, as "off". "none" is what the // daemon's startup log prints when nothing is proxied, so operators copy it. var offAliases = map[string]bool{ @@ -71,20 +82,16 @@ func Normalize(spec string) (string, error) { } // Resolve builds the resolver for a proxy setting (see Normalize): Auto -// reads the environment now, Off never proxies, a URL proxies everything. -// It applies no transport policy. -func Resolve(spec string) (*netproxy.Resolver, error) { +// reads the environment, Off never proxies, a URL proxies everything. It +// applies no transport policy. opts are netproxy's (for example +// netproxy.WithRefreshCommand for rotating credentials); they do not apply +// to Off. With a refresh command, Resolve runs it once before returning. +func Resolve(spec string, opts ...netproxy.Option) (*netproxy.Resolver, error) { s, err := Normalize(spec) if err != nil { return nil, err } - switch s { - case Auto: - return netproxy.FromEnvironment() - case Off: - return netproxy.Off(), nil - } - return netproxy.Explicit(s) + return netproxy.NewResolver(s, opts...) } // HasCredentials reports whether an explicit proxy setting carries userinfo. @@ -136,23 +143,158 @@ func IsLoopbackHost(host string) bool { return ip != nil && ip.IsLoopback() } -// RequestProxy returns an http.Transport.Proxy function that follows r but -// never proxies loopback targets. nil for a nil resolver (net/http's own -// environment handling then applies wherever the caller leaves Proxy -// unset). It is Static(r).RequestProxy(). +// loopbackAddr reports whether addr ("host:port") is on this machine. +func loopbackAddr(addr string) bool { + host, _, err := net.SplitHostPort(addr) + return err == nil && IsLoopbackHost(host) +} + +// ProxyFor returns the proxy (redacted, for logs and hints) a TCP dial of +// addr goes through, "" for a direct dial: loopback targets always, and +// targets r does not proxy (nil r, Off, NO_PROXY under auto). It reads r's +// current settings and never waits for a refresh. +func ProxyFor(r *netproxy.Resolver, addr string) string { + if !r.Enabled() || loopbackAddr(addr) { + return "" + } + u, err := r.ProxyForAddr(addr) + if err != nil || u == nil { + return "" + } + return netproxy.Redact(u) +} + +// Proxies reports whether a TCP dial of addr goes through a proxy (see +// ProxyFor). +func Proxies(r *netproxy.Resolver, addr string) bool { + return ProxyFor(r, addr) != "" +} + +// RequestProxy returns an http.Transport.Proxy function that follows r +// (its current, refreshed settings) but never proxies loopback targets. +// nil for a nil resolver (net/http's own environment handling then applies +// wherever the caller leaves Proxy unset). func RequestProxy(r *netproxy.Resolver) func(*http.Request) (*url.URL, error) { - return Static(r).RequestProxy() + if r == nil { + return nil + } + return func(req *http.Request) (*url.URL, error) { + if req == nil || req.URL == nil || IsLoopbackHost(req.URL.Hostname()) { + return nil, nil + } + return r.ProxyForRequest(req) + } } // DialContext returns a dial function that tunnels through the proxy r // picks for each target (CONNECT by host name, never resolved locally) and -// dials loopback targets, and targets r does not proxy, directly. -// proxyTLS configures the TLS session with an https:// proxy — never the -// target's TLS, which the caller runs end to end over the returned conn; -// nil verifies the proxy against the system roots. It is -// Static(r).DialContext(proxyTLS). +// dials loopback targets, and targets r does not proxy, directly. It is a +// netproxy.Dialer: when the proxy rejects the credentials (407), r +// refreshes (re-running its refresh command, if any) and the dial is +// retried once when that produced different credentials. proxyTLS +// configures the TLS session with an https:// proxy — never the target's +// TLS, which the caller runs end to end over the returned conn; nil +// verifies the proxy against the system roots. func DialContext(r *netproxy.Resolver, proxyTLS *tls.Config) func(ctx context.Context, network, addr string) (net.Conn, error) { - return Static(r).DialContext(proxyTLS) + d := &netproxy.Dialer{Resolver: r, TLSConfig: proxyTLS} + var direct net.Dialer + return func(ctx context.Context, network, addr string) (net.Conn, error) { + if loopbackAddr(addr) { + return direct.DialContext(ctx, network, addr) + } + return d.DialContext(ctx, network, addr) + } +} + +// RoundTripper returns the transport for an HTTP client that follows r: a +// netproxy.RefreshingTransport over a copy of base (http.DefaultTransport's +// settings when nil), so new connections always carry r's current +// credentials and a request whose CONNECT got 407 is retried once after +// the refresh, plus the loopback rule: requests to this machine use a +// direct copy of base. base's own Proxy and OnProxyConnectResponse are +// replaced. A nil r returns base (nil: http.DefaultTransport). +func RoundTripper(r *netproxy.Resolver, base *http.Transport) http.RoundTripper { + if r == nil { + if base == nil { + return http.DefaultTransport + } + return base + } + if base == nil { + if dt, ok := http.DefaultTransport.(*http.Transport); ok { + base = dt + } else { + base = &http.Transport{} + } + } + // Drop what ConfigureTransport may have installed on base (the daemon + // configures http.DefaultTransport in place): RefreshingTransport sets + // its own Proxy, and a CONNECT hook that already turns a 407 into an + // error would hide the rejection from RefreshingTransport's retry. + base = base.Clone() + base.Proxy = nil + base.OnProxyConnectResponse = nil + direct := base.Clone() + return &loopbackSplit{direct: direct, proxied: netproxy.RefreshingTransport(base, r)} +} + +// loopbackSplit sends loopback requests direct and everything else through +// the refreshing proxy transport. +type loopbackSplit struct { + direct *http.Transport + proxied http.RoundTripper +} + +func (t *loopbackSplit) RoundTrip(req *http.Request) (*http.Response, error) { + if req.URL != nil && IsLoopbackHost(req.URL.Hostname()) { + return t.direct.RoundTrip(req) + } + return t.proxied.RoundTrip(req) +} + +// CloseIdleConnections closes both transports' idle connections. +func (t *loopbackSplit) CloseIdleConnections() { + t.direct.CloseIdleConnections() + if c, ok := t.proxied.(interface{ CloseIdleConnections() }); ok { + c.CloseIdleConnections() + } +} + +// ConfigureTransport routes tr (in place) through r, for transports that +// cannot be wrapped — http.DefaultTransport, which plugin HTTP clients use +// or clone: tr.Proxy follows r's current settings (loopback always +// direct), and a 407 answer to a CONNECT refreshes r before the request +// fails, so the next request carries the new credentials. (Wrapped clients, +// see RoundTripper, also retry the failed request.) Every refused CONNECT +// fails with a *netproxy.ConnectError, whose message never quotes the +// proxy's reason phrase. A nil r leaves tr alone. +func ConfigureTransport(tr *http.Transport, r *netproxy.Resolver) { + if tr == nil || r == nil { + return + } + tr.Proxy = RequestProxy(r) + next := tr.OnProxyConnectResponse + tr.OnProxyConnectResponse = func(ctx context.Context, proxyURL *url.URL, connectReq *http.Request, res *http.Response) error { + if next != nil { + if err := next(ctx, proxyURL, connectReq, res); err != nil { + return err + } + } + if res.StatusCode == http.StatusOK { + return nil + } + if res.StatusCode == http.StatusProxyAuthRequired { + _ = r.Refresh(ctx) // failures keep the last good settings; netproxy reports them + } + return &netproxy.ConnectError{Target: connectReq.Host, StatusCode: res.StatusCode} + } +} + +// AuthRejected reports whether err is a proxy's refusal of the credentials +// (407 Proxy Authentication Required on CONNECT). +func AuthRejected(err error) bool { + var ce *netproxy.ConnectError + return errors.As(err, &ce) && ce.StatusCode == http.StatusProxyAuthRequired } // unwrapNetproxy drops netproxy's "netproxy: " prefix for messages that diff --git a/internal/proxyconf/refresh_test.go b/internal/proxyconf/refresh_test.go new file mode 100644 index 00000000..8dcd0a19 --- /dev/null +++ b/internal/proxyconf/refresh_test.go @@ -0,0 +1,394 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package proxyconf + +import ( + "bufio" + "bytes" + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/pilot-protocol/common/netproxy" +) + +// rotatingProxy is a CONNECT proxy that accepts one password at a time +// (407 for any other), like an egress proxy that rotates its credentials, +// and tunnels every accepted CONNECT to upstream. +type rotatingProxy struct { + ln net.Listener + upstream string + + mu sync.Mutex + pass string + oks map[string]int // password -> accepted CONNECTs + n407 int +} + +func newRotatingProxy(t *testing.T, pass, upstream string) *rotatingProxy { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + p := &rotatingProxy{ln: ln, upstream: upstream, pass: pass, oks: map[string]int{}} + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go p.serve(c) + } + }() + t.Cleanup(func() { ln.Close() }) + return p +} + +func (p *rotatingProxy) url(pass string) string { + return fmt.Sprintf("http://muse:%s@%s", pass, p.ln.Addr()) +} + +func (p *rotatingProxy) rotate(pass string) { + p.mu.Lock() + p.pass = pass + p.mu.Unlock() +} + +func (p *rotatingProxy) stats() (oks map[string]int, n407 int) { + p.mu.Lock() + defer p.mu.Unlock() + m := map[string]int{} + for k, v := range p.oks { + m[k] = v + } + return m, p.n407 +} + +func (p *rotatingProxy) serve(c net.Conn) { + defer c.Close() + br := bufio.NewReader(c) + req, err := http.ReadRequest(br) + if err != nil { + return + } + req.Header.Set("Authorization", req.Header.Get("Proxy-Authorization")) + user, pass, ok := req.BasicAuth() + p.mu.Lock() + good := ok && user == "muse" && pass == p.pass + if good { + p.oks[pass]++ + } else { + p.n407++ + } + p.mu.Unlock() + if req.Method != http.MethodConnect || !good { + fmt.Fprint(c, "HTTP/1.1 407 Proxy Authentication Required\r\nProxy-Authenticate: Basic realm=\"t\"\r\nContent-Length: 0\r\n\r\n") + return + } + up, err := net.Dial("tcp", p.upstream) + if err != nil { + fmt.Fprint(c, "HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n") + return + } + defer up.Close() + fmt.Fprint(c, "HTTP/1.1 200 Connection established\r\n\r\n") + done := make(chan struct{}, 2) + go func() { _, _ = io.Copy(up, br); done <- struct{}{} }() + go func() { _, _ = io.Copy(c, up); done <- struct{}{} }() + <-done +} + +// urlFile stands in for the sandbox's rotating HTTPS_PROXY: the refresh +// command prints the file. +type urlFile struct { + t *testing.T + path string + p *rotatingProxy +} + +func newURLFile(t *testing.T, p *rotatingProxy, pass string) *urlFile { + f := &urlFile{t: t, path: filepath.Join(t.TempDir(), "proxy-url"), p: p} + f.set(pass) + return f +} + +func (f *urlFile) set(pass string) { + f.t.Helper() + if err := os.WriteFile(f.path, []byte(f.p.url(pass)+"\n"), 0o600); err != nil { + f.t.Fatal(err) + } +} + +// rotate changes the proxy's password and what the command prints. +func (f *urlFile) rotate(pass string) { + f.set(pass) + f.p.rotate(pass) +} + +func (f *urlFile) command() string { return "cat '" + f.path + "'" } + +func clearEnv(t *testing.T) { + for _, k := range []string{"HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy", "ALL_PROXY", "all_proxy", "NO_PROXY", "no_proxy", "REQUEST_METHOD"} { + t.Setenv(k, "") + } +} + +func echoServer(t *testing.T) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go func() { defer c.Close(); _, _ = io.Copy(c, c) }() + } + }() + return ln.Addr().String() +} + +// Resolve wires netproxy's refresh command in: auto takes the command's URL +// (NO_PROXY still applies), an explicit URL is replaced by it, off ignores +// it, and nothing it prints shows up in String. +func TestResolveWithRefreshCommand(t *testing.T) { + clearEnv(t) + t.Setenv("NO_PROXY", ".corp.example") + cmd := netproxy.WithRefreshCommand("echo http://muse:cmdpw9@cmd.test:3128") + for _, spec := range []string{"auto", "http://muse:flagpw@flag.test:8080"} { + r, err := Resolve(spec, cmd) + if err != nil { + t.Fatalf("Resolve(%q): %v", spec, err) + } + if got := ProxyFor(r, "registry.pilotprotocol.network:443"); got != "http://***@cmd.test:3128" { + t.Errorf("%s: ProxyFor(registry) = %q, want the command's proxy", spec, got) + } + if s := r.String(); strings.Contains(s, "cmdpw9") || !strings.Contains(s, "credentials refreshed by command") { + t.Errorf("%s: String = %q", spec, s) + } + if Proxies(r, "127.0.0.1:9000") || Proxies(r, "localhost:80") { + t.Errorf("%s: loopback proxied", spec) + } + } + auto, _ := Resolve("auto", cmd) + if Proxies(auto, "git.corp.example:443") { + t.Error("auto: NO_PROXY ignored with a refresh command") + } + off, err := Resolve("none", cmd) + if err != nil || off.Mode() != netproxy.ModeOff || off.Enabled() { + t.Fatalf("Resolve(none, cmd) = (%v, %v), want off", off, err) + } + if ProxyFor(nil, "x.test:443") != "" || Proxies(nil, "x.test:443") { + t.Error("nil resolver proxies") + } +} + +// The heart of muse-proxy-cred-rotation-unhandled on the raw-dial path +// (registry, WSS beacon): a dial whose CONNECT gets 407 after a rotation +// re-runs the command and is retried once; credentials that stay wrong +// return the 407 after exactly one retry-less refresh. +func TestDialContextRetriesAfterRotation(t *testing.T) { + clearEnv(t) + proxy := newRotatingProxy(t, "one", echoServer(t)) + urls := newURLFile(t, proxy, "one") + r, err := Resolve("auto", netproxy.WithRefreshCommand(urls.command()), netproxy.WithRefreshInterval(-1)) + if err != nil { + t.Fatal(err) + } + dial := DialContext(r, nil) + ping := func() error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + c, err := dial(ctx, "tcp", "registry.pilot.invalid:443") + if err != nil { + return err + } + defer c.Close() + if _, err := c.Write([]byte("ping")); err != nil { + return err + } + buf := make([]byte, 4) + if _, err := io.ReadFull(c, buf); err != nil || string(buf) != "ping" { + return fmt.Errorf("echo = (%q, %v)", buf, err) + } + return nil + } + if err := ping(); err != nil { + t.Fatalf("dial before the rotation: %v", err) + } + urls.rotate("two") + if err := ping(); err != nil { + t.Fatalf("dial after the rotation: %v", err) + } + oks, n407 := proxy.stats() + if n407 != 1 || oks["two"] != 1 { + t.Fatalf("proxy: 407s %d, accepted %v; want one 407, then the refreshed credentials", n407, oks) + } + + // Credentials that are really wrong: the proxy moves on, the command + // does not. One refresh, no retry (same URL), and the 407 comes back. + proxy.rotate("three") + err = ping() + if !AuthRejected(err) { + t.Fatalf("dial with unrefreshable credentials = %v, want the 407", err) + } + if strings.Contains(err.Error(), "two") { + t.Fatalf("error leaks the password: %v", err) + } + if _, n407 := proxy.stats(); n407 != 2 { + t.Fatalf("407s = %d, want 2 (no retry with the same URL)", n407) + } +} + +// HTTP clients built with RoundTripper retry a request whose CONNECT got +// 407 once the command has new credentials (replaying a body when it can), +// and keep loopback off the proxy. +func TestRoundTripperRetriesAfterRotation(t *testing.T) { + clearEnv(t) + var hits atomic.Int32 + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + body, _ := io.ReadAll(r.Body) + fmt.Fprintf(w, "%s %s", r.Method, body) + })) + defer srv.Close() + proxy := newRotatingProxy(t, "one", srv.Listener.Addr().String()) + urls := newURLFile(t, proxy, "one") + r, err := Resolve("auto", netproxy.WithRefreshCommand(urls.command()), netproxy.WithRefreshInterval(-1)) + if err != nil { + t.Fatal(err) + } + // base as the daemon has it: http.DefaultTransport-like, configured in + // place by ConfigureTransport — RoundTripper must not let that hook + // swallow the 407 before RefreshingTransport sees it. + base := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} // #nosec G402 -- test server + ConfigureTransport(base, r) + rt := RoundTripper(r, base) + client := &http.Client{Transport: rt, Timeout: 10 * time.Second} + defer client.CloseIdleConnections() + do := func(method, target string, body io.Reader) (string, error) { + req, err := http.NewRequest(method, target, body) + if err != nil { + return "", err + } + req.Close = true // a new CONNECT per request + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + return string(b), err + } + const target = "https://app.pilot.invalid/api" + if got, err := do(http.MethodGet, target, nil); err != nil || got != "GET " { + t.Fatalf("GET before the rotation = (%q, %v)", got, err) + } + urls.rotate("two") + if got, err := do(http.MethodPost, target, bytes.NewReader([]byte("payload"))); err != nil || got != "POST payload" { + t.Fatalf("POST after the rotation = (%q, %v)", got, err) + } + if oks, n407 := proxy.stats(); n407 != 1 || oks["two"] != 1 { + t.Fatalf("proxy: 407s %d, accepted %v", n407, oks) + } + + // A body that cannot be replayed is not retried after a refused + // CONNECT ... (net/http never sent it, but without GetBody it cannot + // be rebuilt) ... + urls.rotate("three") + if _, err := do(http.MethodPost, target, io.NopCloser(strings.NewReader("once"))); !AuthRejected(err) { + t.Fatalf("unreplayable POST after a rotation = %v, want the 407", err) + } + // ... but the refresh happened: the next request goes through. + if got, err := do(http.MethodGet, target, nil); err != nil || got != "GET " { + t.Fatalf("GET after the refresh = (%q, %v)", got, err) + } + + // Loopback never goes to the proxy. + local := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fmt.Fprint(w, "local") })) + defer local.Close() + before, _ := proxy.stats() + if got, err := do(http.MethodGet, local.URL, nil); err != nil || got != "local" { + t.Fatalf("GET loopback = (%q, %v)", got, err) + } + if after, _ := proxy.stats(); fmt.Sprint(after) != fmt.Sprint(before) { + t.Fatalf("loopback request reached the proxy: %v -> %v", before, after) + } + if n := hits.Load(); n != 3 { + t.Fatalf("server hits = %d, want 3", n) + } + if RoundTripper(nil, nil) != http.DefaultTransport || RoundTripper(nil, base) != base { + t.Error("RoundTripper(nil) wraps the transport") + } +} + +// ConfigureTransport (http.DefaultTransport, which plugins use or clone): +// connections follow the resolver's current credentials, and a 407 +// refreshes them before the request fails, so the next one succeeds; the +// error never quotes the proxy. +func TestConfigureTransportRefreshesOn407(t *testing.T) { + clearEnv(t) + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fmt.Fprint(w, "ok") })) + defer srv.Close() + proxy := newRotatingProxy(t, "one", srv.Listener.Addr().String()) + urls := newURLFile(t, proxy, "one") + r, err := Resolve("auto", netproxy.WithRefreshCommand(urls.command()), netproxy.WithRefreshInterval(-1)) + if err != nil { + t.Fatal(err) + } + tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} // #nosec G402 -- test server + ConfigureTransport(tr, r) + clone := tr.Clone() // a plugin that cloned DefaultTransport + for name, rt := range map[string]*http.Transport{"configured": tr, "clone": clone} { + client := &http.Client{Transport: rt, Timeout: 10 * time.Second} + get := func() error { + req, _ := http.NewRequest(http.MethodGet, "https://plugin.pilot.invalid/x", nil) + req.Close = true + resp, err := client.Do(req) + if err != nil { + return err + } + resp.Body.Close() + return nil + } + if err := get(); err != nil { + t.Fatalf("%s: GET = %v", name, err) + } + pass := "p-" + name + urls.rotate(pass) + err := get() + var ce *netproxy.ConnectError + if !errors.As(err, &ce) || ce.StatusCode != http.StatusProxyAuthRequired { + t.Fatalf("%s: GET after the rotation = %v, want the 407 as a ConnectError", name, err) + } + if err := get(); err != nil { + t.Fatalf("%s: GET after the refresh = %v", name, err) + } + if oks, _ := proxy.stats(); oks[pass] != 1 { + t.Fatalf("%s: accepted %v, want the refreshed %s once", name, oks, pass) + } + } + ConfigureTransport(nil, r) + plain := &http.Transport{} + ConfigureTransport(plain, nil) + if plain.Proxy != nil || plain.OnProxyConnectResponse != nil { + t.Error("ConfigureTransport(nil resolver) changed the transport") + } +} diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index b01822ad..49cafea1 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -36,6 +36,7 @@ import ( registrywire "github.com/pilot-protocol/common/registry/wire" "github.com/pilot-protocol/pilotprotocol/internal/account" "github.com/pilot-protocol/pilotprotocol/internal/motd" + "github.com/pilot-protocol/pilotprotocol/internal/proxyconf" "github.com/pilot-protocol/pilotprotocol/internal/transport/compat" "github.com/pilot-protocol/pilotprotocol/internal/validate" "github.com/pilot-protocol/pilotprotocol/pkg/daemon/routing" @@ -214,24 +215,20 @@ type Config struct { // behind TLS-intercepting corp proxies). CompatTLSTrust string - // Proxy is the outbound proxy policy, normally built by ResolveProxy - // from -proxy and the transport mode. When non-nil it governs every - // TCP/HTTP connection the daemon opens itself: the registry (primary, - // pool and every reconnect), the compat-mode WSS beacon and the MOTD - // fetch. Targets stay host names end to end — the proxy is asked to - // CONNECT by name and TLS runs through the tunnel to the real server. - // nil keeps the historical behaviour: registry and beacon are dialed - // directly and HTTP fetches follow net/http's proxy environment. + // Proxy is the outbound proxy, normally built by ResolveProxy from + // -proxy, -proxy-cmd and the transport mode. When non-nil it governs + // every TCP/HTTP connection the daemon opens itself: the registry + // (primary, pool and every reconnect), the compat-mode WSS beacon (and + // its reconnects) and the MOTD fetch. Targets stay host names end to + // end — the proxy is asked to CONNECT by name and TLS runs through the + // tunnel to the real server. A resolver built with + // netproxy.WithRefreshCommand follows rotating proxy credentials: every + // new connection uses its current settings, and a CONNECT the proxy + // answers with 407 refreshes them and is retried once. nil keeps the + // historical behaviour: registry and beacon are dialed directly and + // HTTP fetches follow net/http's proxy environment. Proxy *netproxy.Resolver - // ProxyPolicy, when non-nil, replaces Proxy with a policy that can - // change while the daemon runs — NewCommandProxyPolicy re-reads the - // proxy URL from a command, for egress proxies that rotate their - // credentials. Start keeps it refreshed every ProxyRefreshInterval - // (0 = 60s) until Stop. - ProxyPolicy *ProxyPolicy - ProxyRefreshInterval time.Duration - // systemRoots replaces the OS trust store behind the "system" trust // settings (RegistryTrust, CompatTLSTrust). Test seam only: it is // always nil outside this package's tests. @@ -887,31 +884,39 @@ func (d *Daemon) Start() error { } default: // Compat would use the environment's proxy (-proxy=auto) - // unless the embedder set a policy; check reachability the - // same way. - policy := d.proxyPolicy() - if policy == nil { + // unless the embedder set one; check reachability the same + // way. + proxy := d.config.Proxy + if proxy == nil { if p, err := ResolveProxy(proxyAutoSpec, TransportCompat); err == nil { - policy = StaticProxyPolicy(p) + proxy = p } else { slog.Warn("proxy environment unusable; compat check dials directly", "error", err) } } - mode, reason := SelectTransport(context.Background(), AutoTransportProbe{ + mode, reason, proxyErr := SelectTransport(context.Background(), AutoTransportProbe{ BeaconAddr: stunBeacon, CompatBeaconURL: d.config.CompatBeaconURL, - Dial: d.dialerFor(policy), + Dial: d.dialerFor(proxy), + ProxyFor: func(addr string) string { return proxyconf.ProxyFor(proxy, addr) }, }) if mode == TransportCompat { d.config.TransportMode = TransportCompat - if d.config.ProxyPolicy == nil && d.config.Proxy == nil { - d.config.ProxyPolicy = policy + if d.config.Proxy == nil { + d.config.Proxy = proxy + } + if proxyErr != nil { + slog.Warn("UDP probe to beacon failed and the proxy refused the compat check — staying on compat (WSS/443) so nothing bypasses the proxy", + "beacon", stunBeacon, + "compat_beacon", d.config.CompatBeaconURL, + "proxy_error", proxyErr) + } else { + slog.Warn("UDP probe to beacon failed — auto-falling back to compat mode (WSS/443)", + "beacon", stunBeacon, + "compat_beacon", d.config.CompatBeaconURL, + "reason", reason, + "hint", "set PILOT_TRANSPORT=udp to force UDP") } - slog.Warn("UDP probe to beacon failed — auto-falling back to compat mode (WSS/443)", - "beacon", stunBeacon, - "compat_beacon", d.config.CompatBeaconURL, - "reason", reason, - "hint", "set PILOT_TRANSPORT=udp to force UDP") } else { slog.Info("transport auto-selected", "transport", TransportUDP, "reason", reason) } @@ -934,11 +939,6 @@ func (d *Daemon) Start() error { return fmt.Errorf("invalid -transport %q: must be 'udp' or 'compat'", d.config.TransportMode) } - // A command-backed proxy policy (rotating proxy credentials) stays - // current for the daemon's lifetime: registry redials, WSS reconnects - // and HTTP fetches always see the latest proxy URL. - d.startProxyRefresh() - var registrationAddr string if d.config.TransportMode == "compat" { registrationAddr = "0.0.0.0:0" // placeholder — relay_only daemons hide this from peers @@ -1131,6 +1131,9 @@ func (d *Daemon) Start() error { if hint := tlsTrustHint(cerr, "beacon"); hint != "" && d.config.CompatTLSTrust == "system" { return fmt.Errorf("compat connect: %w; %s", cerr, hint) } + if hint := ProxyRefusalHint(cerr); hint != "" { + return fmt.Errorf("compat connect: %w; %s", cerr, hint) + } return fmt.Errorf("compat connect: %w", cerr) } slog.Info("compat mode tunnel up", @@ -2323,6 +2326,9 @@ func (d *Daemon) dialRegistryClient() (*registry.Client, error) { if hint := tlsTrustHint(err, "registry"); hint != "" { return nil, fmt.Errorf("registry dial (after %d attempts): %w; %s", attempt, err, hint) } + if hint := ProxyRefusalHint(err); hint != "" { + return nil, fmt.Errorf("registry dial (after %d attempts): %w; %s", attempt, err, hint) + } return nil, fmt.Errorf("registry dial (after %d attempts): %w", attempt, err) } slog.Warn("registry dial failed, retrying", diff --git a/pkg/daemon/proxy.go b/pkg/daemon/proxy.go index 08475c23..d4428465 100644 --- a/pkg/daemon/proxy.go +++ b/pkg/daemon/proxy.go @@ -7,7 +7,6 @@ import ( "crypto/tls" "net" "net/http" - "net/url" "time" "github.com/pilot-protocol/common/netproxy" @@ -15,50 +14,41 @@ import ( "github.com/pilot-protocol/pilotprotocol/internal/proxyconf" ) -// ProxyPolicy is an outbound proxy policy that can change while the daemon -// runs: a fixed resolver (StaticProxyPolicy) or one refreshed from a -// command (NewCommandProxyPolicy) for egress proxies that rotate their -// credentials. Config.ProxyPolicy takes precedence over Config.Proxy. -type ProxyPolicy = proxyconf.Policy - -// StaticProxyPolicy wraps a fixed resolver. nil gives nil (no policy). -func StaticProxyPolicy(r *netproxy.Resolver) *ProxyPolicy { return proxyconf.Static(r) } - -// NewCommandProxyPolicy returns a policy whose proxy URL is the stdout of -// command (run with /bin/sh -c), re-run every refresh interval and when the -// proxy answers 407 (see proxyconf.NewCommand). fallback serves until the -// command first succeeds; the error reports a failed first run and is not -// fatal: the policy is usable either way. honorNoProxy applies the -// environment's NO_PROXY (the -proxy=auto semantics). -func NewCommandProxyPolicy(ctx context.Context, command string, fallback *netproxy.Resolver, honorNoProxy bool) (*ProxyPolicy, error) { - return proxyconf.NewCommand(ctx, command, fallback, honorNoProxy) -} - // ResolveProxy turns a -proxy setting into the daemon's outbound proxy // resolver (Config.Proxy) for the given transport mode: // // - "auto" or "": with transportMode "compat", the proxy from the // environment — HTTPS_PROXY / https_proxy, falling back to ALL_PROXY / // all_proxy, with NO_PROXY / no_proxy honoured. With any other transport -// it returns nil: no policy, the daemon dials exactly as it always has. -// - "off" (or "none", "no", "false", "direct"): a policy that never +// it returns nil: no proxy, the daemon dials exactly as it always has. +// - "off" (or "none", "no", "false", "direct"): a resolver that never // proxies. Unlike nil, it also stops the daemon's HTTP fetches from // following proxy environment variables. // - "http://[user:pass@]host:port" or "https://...": that proxy for every // outbound TCP/HTTP connection, whatever the transport. // +// opts are passed to netproxy.NewResolver. For an egress proxy that +// rotates its credentials, pass netproxy.WithRefreshCommand (the daemon's +// -proxy-cmd): the command's output then supplies the proxy URL — the +// explicit one, or under auto the environment's (NO_PROXY still honoured) +// — re-read every netproxy.DefaultRefreshInterval and whenever the proxy +// answers 407, after which the rejected connection is retried once. With a +// refresh command, ResolveProxy runs it once before returning; a failing +// run leaves the launch-time proxy in use (the error goes to +// netproxy.WithRefreshErrorHandler). +// // Any other value (a bare word, a host:port without a scheme, another // scheme) is an error. Loopback targets are never proxied, whatever the -// policy. Errors never echo proxy credentials. -func ResolveProxy(spec, transportMode string) (*netproxy.Resolver, error) { +// resolver says. Errors never echo proxy credentials. +func ResolveProxy(spec, transportMode string, opts ...netproxy.Option) (*netproxy.Resolver, error) { s, err := proxyconf.Normalize(spec) if err != nil { return nil, err } - if s == proxyconf.Auto && transportMode != "compat" { + if s == proxyconf.Auto && transportMode != TransportCompat { return nil, nil } - return proxyconf.Resolve(s) + return proxyconf.Resolve(s, opts...) } // proxyAutoSpec is the -proxy default: the environment's proxy, in compat @@ -73,32 +63,25 @@ func (d *Daemon) proxyTLSConfig() *tls.Config { return &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: d.config.systemRoots} } -// proxyPolicy is the daemon's outbound proxy policy: Config.ProxyPolicy, -// else Config.Proxy, else nil (no policy). -func (d *Daemon) proxyPolicy() *ProxyPolicy { - if d.config.ProxyPolicy != nil { - return d.config.ProxyPolicy - } - return proxyconf.Static(d.config.Proxy) -} - // proxyDialer returns the dial function for raw TCP connections the daemon // opens itself (the registry, the compat WSS beacon), or nil when there is -// no policy or it proxies nothing. Loopback targets are dialed directly. +// no proxy resolver or it proxies nothing. Loopback targets are dialed +// directly. It reads the resolver's current settings on every dial, and a +// 407 refreshes them and retries the dial once (netproxy.Dialer). func (d *Daemon) proxyDialer() func(ctx context.Context, network, addr string) (net.Conn, error) { - return d.dialerFor(d.proxyPolicy()) + return d.dialerFor(d.config.Proxy) } -// dialerFor is proxyDialer for an arbitrary policy. -func (d *Daemon) dialerFor(policy *ProxyPolicy) func(ctx context.Context, network, addr string) (net.Conn, error) { - if !policy.Enabled() { +// dialerFor is proxyDialer for an arbitrary resolver. +func (d *Daemon) dialerFor(r *netproxy.Resolver) func(ctx context.Context, network, addr string) (net.Conn, error) { + if !r.Enabled() { return nil } - return policy.DialContext(d.proxyTLSConfig()) + return proxyconf.DialContext(r, d.proxyTLSConfig()) } // registryDialOptions routes every registry connection — the primary, each -// pool member and every reconnect — through the proxy policy. With no policy, +// pool member and every reconnect — through the proxy resolver. With none, // or one that proxies nothing, the client keeps its direct dial. func (d *Daemon) registryDialOptions() []registry.DialOption { dial := d.proxyDialer() @@ -108,37 +91,23 @@ func (d *Daemon) registryDialOptions() []registry.DialOption { return []registry.DialOption{registry.WithDialer(dial)} } -// httpProxyFunc returns the http.Transport.Proxy function for HTTP fetches -// the daemon makes itself, or nil when there is no policy. Loopback targets -// always go direct. -func (d *Daemon) httpProxyFunc() func(*http.Request) (*url.URL, error) { - return d.proxyPolicy().RequestProxy() -} - // newHTTPClient returns a client for daemon-owned HTTP fetches. Without a -// proxy policy it is a plain client on http.DefaultTransport, as before; -// with one, its transport routes through the policy (and, for a refreshed -// policy, retries a request whose CONNECT got 407 once with the new -// credentials). +// proxy resolver it is a plain client on http.DefaultTransport, as before; +// with one, its transport is a netproxy.RefreshingTransport: every new +// connection carries the resolver's current credentials, and a request +// whose CONNECT got 407 is retried once after the refresh. Loopback +// targets always go direct. func (d *Daemon) newHTTPClient(timeout time.Duration) *http.Client { client := &http.Client{Timeout: timeout} - if policy := d.proxyPolicy(); policy != nil { - var tr *http.Transport - if base, ok := http.DefaultTransport.(*http.Transport); ok { - tr = base.Clone() - } else { - tr = &http.Transport{} + if d.config.Proxy != nil { + var base *http.Transport + if d.config.systemRoots != nil { // test seam only + if dt, ok := http.DefaultTransport.(*http.Transport); ok { + base = dt.Clone() + base.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: d.config.systemRoots} + } } - policy.ConfigureTransport(tr) - client.Transport = policy.RoundTripper(tr) + client.Transport = proxyconf.RoundTripper(d.config.Proxy, base) } return client } - -// startProxyRefresh keeps a command-backed proxy policy current for the -// daemon's lifetime (no-op otherwise). -func (d *Daemon) startProxyRefresh() { - if p := d.proxyPolicy(); p.Refreshable() { - go p.Run(d.ctx, d.config.ProxyRefreshInterval) - } -} diff --git a/pkg/daemon/transport_auto.go b/pkg/daemon/transport_auto.go index 52ba986c..4be9da70 100644 --- a/pkg/daemon/transport_auto.go +++ b/pkg/daemon/transport_auto.go @@ -15,6 +15,7 @@ import ( "strings" "time" + "github.com/pilot-protocol/common/netproxy" "github.com/pilot-protocol/pilotprotocol/pkg/daemon/routing" ) @@ -52,8 +53,12 @@ type AutoTransportProbe struct { // means compat is not available and auto always picks udp. CompatBeaconURL string // Dial opens the connection for the compat beacon check, normally - // through the proxy policy compat mode would use. nil dials directly. + // through the proxy compat mode would use. nil dials directly. Dial func(ctx context.Context, network, addr string) (net.Conn, error) + // ProxyFor reports the proxy (redacted) Dial goes through for addr, "" + // when it dials addr directly. nil: never through a proxy. It decides + // what a failed Dial means (see SelectTransport). + ProxyFor func(addr string) string // UDPTimeout bounds the UDP probe (0 = udpProbeTimeout). A reply // returns as soon as it arrives, so on a working UDP path the probe // costs one round trip. @@ -74,30 +79,42 @@ type AutoTransportProbe struct { // connect proves nothing: the production host is an SNI-routing front // that accepts TCP while the beacon behind it is down, and through a // proxy it is only the proxy's CONNECT 200; +// - otherwise compat as well when the check goes through a proxy +// (p.ProxyFor) and fails at the proxy — the proxy refused the CONNECT +// (407 stale or wrong credentials, 403 policy), sent a response that +// could not be parsed, or could not be reached. A proxy is configured +// and UDP got no answer, so the proxy is the way out: udp would dial +// the registry directly, past the proxy, which is exactly what a +// proxy-only sandbox (Meta Muse) kills. In compat every connection +// keeps going through the proxy, a 407 refreshes the credentials and +// retries (with -proxy-cmd, from the command), and a proxy that keeps +// refusing makes the registry dial fail with its error. proxyErr is +// that proxy error, nil otherwise; // - otherwise udp, exactly as before -transport=auto existed: nothing // is reachable yet (no network at boot, beacon outage), and a compat // daemon could not start either, while a udp daemon starts degraded // and registers. // -// reason is a short, log-ready explanation of the choice. -func SelectTransport(ctx context.Context, p AutoTransportProbe) (mode, reason string) { +// reason is a short, log-ready explanation of the choice. Neither it nor +// proxyErr ever contains proxy credentials. +func SelectTransport(ctx context.Context, p AutoTransportProbe) (mode, reason string, proxyErr error) { beacon := firstBeacon(p.BeaconAddr) if beacon == "" { - return TransportUDP, "no UDP beacon configured" + return TransportUDP, "no UDP beacon configured", nil } udpTimeout := p.UDPTimeout if udpTimeout <= 0 { udpTimeout = udpProbeTimeout } if probeUDPReachableWithin(beacon, udpTimeout) { - return TransportUDP, "beacon " + beacon + " answered over UDP" + return TransportUDP, "beacon " + beacon + " answered over UDP", nil } if strings.TrimSpace(p.CompatBeaconURL) == "" { - return TransportUDP, "no UDP answer from beacon " + beacon + ", and no compat beacon configured" + return TransportUDP, "no UDP answer from beacon " + beacon + ", and no compat beacon configured", nil } target, err := compatBeaconHostPort(p.CompatBeaconURL) if err != nil { - return TransportUDP, "no UDP answer from beacon " + beacon + "; compat beacon unusable: " + err.Error() + return TransportUDP, "no UDP answer from beacon " + beacon + "; compat beacon unusable: " + err.Error(), nil } tcpTimeout := p.TCPTimeout if tcpTimeout <= 0 { @@ -106,11 +123,25 @@ func SelectTransport(ctx context.Context, p AutoTransportProbe) (mode, reason st cctx, cancel := context.WithTimeout(ctx, tcpTimeout) defer cancel() if err := checkCompatBeacon(cctx, p.Dial, p.CompatBeaconURL, target); err != nil { - return TransportUDP, fmt.Sprintf("no UDP answer from beacon %s, and compat beacon %s did not answer (%v)", beacon, p.CompatBeaconURL, err) + var de *compatDialError + if errors.As(err, &de) && p.ProxyFor != nil { + if via := p.ProxyFor(target); via != "" { + return TransportCompat, fmt.Sprintf("no UDP answer from beacon %s, and the proxy %s refused the compat beacon check (%v); staying on compat so every connection goes through the proxy (udp would dial the registry directly)", + beacon, via, de.err), de.err + } + } + return TransportUDP, fmt.Sprintf("no UDP answer from beacon %s, and compat beacon %s did not answer (%v)", beacon, p.CompatBeaconURL, err), nil } - return TransportCompat, fmt.Sprintf("no UDP answer from beacon %s within %s; compat beacon %s answered", beacon, udpTimeout, p.CompatBeaconURL) + return TransportCompat, fmt.Sprintf("no UDP answer from beacon %s within %s; compat beacon %s answered", beacon, udpTimeout, p.CompatBeaconURL), nil } +// compatDialError is a compat beacon check that failed while opening the +// connection — through a proxy: reaching the proxy or its CONNECT. +type compatDialError struct{ err error } + +func (e *compatDialError) Error() string { return e.err.Error() } +func (e *compatDialError) Unwrap() error { return e.err } + // checkCompatBeacon proves that the WebSocket beacon at rawURL is alive: // it opens target through dial (nil: direct), runs TLS for wss:// and // sends a plain GET of the beacon path, which a live WebSocket endpoint @@ -133,11 +164,13 @@ func checkCompatBeacon(ctx context.Context, dial func(ctx context.Context, netwo } conn, err := dial(ctx, "tcp", target) if err != nil { - return err + return &compatDialError{err: err} } defer conn.Close() if dl, ok := ctx.Deadline(); ok { - _ = conn.SetDeadline(dl) + if err := conn.SetDeadline(dl); err != nil { + return err + } } switch strings.ToLower(u.Scheme) { case "wss", "https": @@ -164,7 +197,7 @@ func checkCompatBeacon(ctx context.Context, dial func(ctx context.Context, netwo if err != nil { return err } - resp.Body.Close() + _ = resp.Body.Close() // nothing is read from it if resp.StatusCode != http.StatusUpgradeRequired { return fmt.Errorf("HTTP %d, want 426 from a live beacon", resp.StatusCode) } @@ -237,3 +270,16 @@ func tlsTrustHint(err error, what string) string { return "cannot verify the beacon certificate: point SSL_CERT_FILE (or SSL_CERT_DIR) at a CA bundle" } } + +// ProxyRefusalHint explains a connection the egress proxy refused, "" for +// any other error. The ConnectError itself never quotes the proxy. +func ProxyRefusalHint(err error) string { + var ce *netproxy.ConnectError + if !errors.As(err, &ce) { + return "" + } + if ce.StatusCode == http.StatusProxyAuthRequired { + return "the proxy rejected its credentials (407), also after re-reading them: check HTTPS_PROXY / -proxy; if the proxy rotates its credentials, set -proxy-cmd ($PILOT_PROXY_CMD, config.json proxy_cmd) to a command that prints the current proxy URL" + } + return fmt.Sprintf("the proxy refused CONNECT %s (HTTP %d): it must allow CONNECT to the Pilot registry and beacon on port 443", ce.Target, ce.StatusCode) +} diff --git a/pkg/daemon/tunnel.go b/pkg/daemon/tunnel.go index ebfe1a08..d23fdd9d 100644 --- a/pkg/daemon/tunnel.go +++ b/pkg/daemon/tunnel.go @@ -1171,9 +1171,10 @@ func (tm *TunnelManager) Listen(addr string) error { type ConnectCompatConfig struct { BeaconURL string TLSConfig *tls.Config - // DialContext opens the TCP connection for the WSS dial (see - // wss.Config.DialContext), e.g. through the proxy policy. nil dials - // the beacon directly. + // DialContext opens the TCP connection for the WSS dial and every + // reconnect (see wss.Config.DialContext), e.g. through the proxy + // resolver, which refreshes rotated credentials on a 407 and retries. + // nil dials the beacon directly. DialContext func(ctx context.Context, network, addr string) (net.Conn, error) Identity *crypto.Identity NodeID uint32 diff --git a/pkg/daemon/zz_proxy_refresh_test.go b/pkg/daemon/zz_proxy_refresh_test.go index 87d0200f..8550930a 100644 --- a/pkg/daemon/zz_proxy_refresh_test.go +++ b/pkg/daemon/zz_proxy_refresh_test.go @@ -3,34 +3,71 @@ package daemon import ( - "context" + "crypto/tls" "crypto/x509" + "fmt" + "io" + "net" "net/http" + "net/http/httptest" "os" "path/filepath" + "strings" + "sync/atomic" "testing" "time" + + "github.com/pilot-protocol/common/netproxy" ) +// rotatingURLFile is the stand-in for Meta Muse's rotating HTTPS_PROXY: a +// file holding the current proxy URL, read by a refresh command, as a +// fresh `bash -c 'printf %s "$https_proxy"'` reads the rotated value. +type rotatingURLFile struct { + t *testing.T + path string + proxy *proxyTestConnect +} + +func newRotatingURLFile(t *testing.T, proxy *proxyTestConnect, pass string) *rotatingURLFile { + f := &rotatingURLFile{t: t, path: filepath.Join(t.TempDir(), "proxy-url"), proxy: proxy} + f.set(pass) + return f +} + +func (f *rotatingURLFile) set(pass string) { + f.t.Helper() + if err := os.WriteFile(f.path, []byte(f.proxy.url("muse", pass)+"\n"), 0o600); err != nil { + f.t.Fatal(err) + } +} + +// rotate makes the proxy accept only pass (407 for the old credentials) +// and the refresh command print it. +func (f *rotatingURLFile) rotate(pass string) { + f.set(pass) + f.proxy.setAuth("muse", pass) +} + +func (f *rotatingURLFile) command() string { return "cat '" + f.path + "'" } + // muse-proxy-cred-rotation-unhandled, in miniature: a compat daemon behind // an authenticating CONNECT proxy whose credentials rotate while it runs. -// With a command-backed ProxyPolicy the registry reconnect and the WSS -// beacon reconnect after the rotation get 407 once, re-read the proxy URL -// and succeed — no restart. -func TestCommandProxyPolicyFollowsCredentialRotation(t *testing.T) { +// With a resolver built with netproxy.WithRefreshCommand (the daemon's +// -proxy-cmd) the registry reconnect and the WSS beacon reconnect after the +// rotation get 407 once, re-read the proxy URL and succeed — no restart. +func TestProxyRefreshCommandFollowsCredentialRotation(t *testing.T) { clearProxyEnv(t) proxy := newProxyTestConnect(t, "muse", "old-pass") - urlFile := filepath.Join(t.TempDir(), "proxy-url") - setURL := func(pass string) { - t.Helper() - if err := os.WriteFile(urlFile, []byte(proxy.url("muse", pass)+"\n"), 0o600); err != nil { - t.Fatal(err) - } - } - setURL("old-pass") - policy, err := NewCommandProxyPolicy(context.Background(), "cat '"+urlFile+"'", nil, true) + urls := newRotatingURLFile(t, proxy, "old-pass") + resolver, err := ResolveProxy("auto", TransportCompat, + netproxy.WithRefreshCommand(urls.command()), + netproxy.WithRefreshInterval(-1)) // only the 407 path refreshes here if err != nil { - t.Fatalf("NewCommandProxyPolicy: %v", err) + t.Fatalf("ResolveProxy: %v", err) + } + if !strings.Contains(resolver.String(), "credentials refreshed by command") || strings.Contains(resolver.String(), "old-pass") { + t.Fatalf("resolver = %q", resolver.String()) } roots := x509.NewCertPool() @@ -42,20 +79,19 @@ func TestCommandProxyPolicyFollowsCredentialRotation(t *testing.T) { } t.Cleanup(func() { os.RemoveAll(sockDir) }) d := New(Config{ - RegistryAddr: regAddr, - RegistryTLS: true, - RegistryTrust: "system", - TransportMode: "compat", - CompatBeaconURL: "wss://" + beaconHost + "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/v1/compat", - CompatTLSTrust: "system", - ProxyPolicy: policy, - ProxyRefreshInterval: time.Hour, // only the 407 path refreshes here - SocketPath: sockDir + "/s", - IdentityPath: t.TempDir() + "/id.json", - Email: "proxy-rotation@example.test", - Encrypt: true, - DisablePolicyRunner: true, - systemRoots: roots, + RegistryAddr: regAddr, + RegistryTLS: true, + RegistryTrust: "system", + TransportMode: "compat", + CompatBeaconURL: "wss://" + beaconHost + "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/v1/compat", + CompatTLSTrust: "system", + Proxy: resolver, + SocketPath: sockDir + "/s", + IdentityPath: t.TempDir() + "/id.json", + Email: "proxy-rotation@example.test", + Encrypt: true, + DisablePolicyRunner: true, + systemRoots: roots, }) if err := d.Start(); err != nil { t.Fatalf("Start behind the proxy: %v", err) @@ -66,8 +102,7 @@ func TestCommandProxyPolicyFollowsCredentialRotation(t *testing.T) { } // Rotate: the proxy now rejects old-pass with 407. - setURL("new-pass") - proxy.setAuth("muse", "new-pass") + urls.rotate("new-pass") if err := d.forceReconnectRegistry(); err != nil { t.Fatalf("registry reconnect after the rotation: %v", err) @@ -89,41 +124,38 @@ func TestCommandProxyPolicyFollowsCredentialRotation(t *testing.T) { case <-time.After(20 * time.Second): t.Fatalf("the WSS beacon never reconnected through the rotated proxy (proxy: %v)", proxy.counts()) } + if !proxyURLHasPassword(resolver, "new-pass") { + t.Fatal("resolver does not carry the rotated credentials") + } } -// Run refreshes a command-backed policy periodically, so connections made -// after a rotation carry the new credentials from the start (no 407). -func TestCommandProxyPolicyPeriodicRefresh(t *testing.T) { +// The timed refresh: a lookup after the refresh interval re-runs the +// command, so connections made after a rotation carry the new credentials +// from the start (no 407). +func TestProxyRefreshCommandPeriodicRefresh(t *testing.T) { clearProxyEnv(t) proxy := newProxyTestConnect(t, "muse", "old-pass") _, regAddr, pin := startProxiedRegistry(t, nil) - urlFile := filepath.Join(t.TempDir(), "proxy-url") - if err := os.WriteFile(urlFile, []byte(proxy.url("muse", "old-pass")), 0o600); err != nil { - t.Fatal(err) - } - policy, err := NewCommandProxyPolicy(context.Background(), "cat '"+urlFile+"'", nil, true) + urls := newRotatingURLFile(t, proxy, "old-pass") + resolver, err := ResolveProxy("auto", TransportCompat, + netproxy.WithRefreshCommand(urls.command()), + netproxy.WithRefreshInterval(50*time.Millisecond)) if err != nil { t.Fatal(err) } d := New(Config{ - RegistryAddr: regAddr, - RegistryTLS: true, - RegistryTrust: "pinned", - RegistryFingerprint: pin, - ProxyPolicy: policy, - ProxyRefreshInterval: 50 * time.Millisecond, + RegistryAddr: regAddr, + RegistryTLS: true, + RegistryTrust: "pinned", + RegistryFingerprint: pin, + Proxy: resolver, }) - d.startProxyRefresh() - t.Cleanup(d.cancelCtx) // ends the refresh loop (the daemon never started) - if err := os.WriteFile(urlFile, []byte(proxy.url("muse", "new-pass")), 0o600); err != nil { - t.Fatal(err) - } - proxy.setAuth("muse", "new-pass") + urls.rotate("new-pass") deadline := time.Now().Add(5 * time.Second) - for !proxyURLHasPassword(policy, "new-pass") { + for !proxyURLHasPassword(resolver, "new-pass") { // each lookup may start the timed refresh if time.Now().After(deadline) { - t.Fatal("periodic refresh never picked up the rotated URL") + t.Fatal("the timed refresh never picked up the rotated URL") } time.Sleep(20 * time.Millisecond) } @@ -137,8 +169,77 @@ func TestCommandProxyPolicyPeriodicRefresh(t *testing.T) { } } -func proxyURLHasPassword(p *ProxyPolicy, pass string) bool { - u, err := p.Resolver().ProxyForAddr("registry.pilot.invalid:443") +// Daemon-owned HTTP clients (newHTTPClient: the MOTD fetch) sit on a +// netproxy.RefreshingTransport: a request whose CONNECT the proxy rejects +// after a rotation is retried once with the refreshed credentials, and +// loopback targets never go to the proxy. +func TestNewHTTPClientRetriesAfterRotation(t *testing.T) { + clearProxyEnv(t) + var hits atomic.Int32 + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + fmt.Fprint(w, "motd") + })) + roots := x509.NewCertPool() + cert, _, _ := proxyTestCert(t, "motd.pilot.invalid", roots) + srv.TLS = &tls.Config{Certificates: []tls.Certificate{cert}} + srv.StartTLS() + t.Cleanup(srv.Close) + _, port, _ := net.SplitHostPort(srv.Listener.Addr().String()) + + proxy := newProxyTestConnect(t, "muse", "one") + urls := newRotatingURLFile(t, proxy, "one") + resolver, err := ResolveProxy("auto", TransportCompat, + netproxy.WithRefreshCommand(urls.command()), + netproxy.WithRefreshInterval(-1)) + if err != nil { + t.Fatal(err) + } + d := New(Config{Proxy: resolver, systemRoots: roots}) + client := d.newHTTPClient(10 * time.Second) + get := func(url string) (string, error) { + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return "", err + } + req.Close = true // a new CONNECT per request + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + return string(b), err + } + target := "https://motd.pilot.invalid:" + port + "/today" + if got, err := get(target); err != nil || got != "motd" { + t.Fatalf("GET before the rotation = (%q, %v)", got, err) + } + urls.rotate("two") + if got, err := get(target); err != nil || got != "motd" { + t.Fatalf("GET after the rotation = (%q, %v), want the 407 retried with the new credentials", got, err) + } + if n := proxy.deniedWith(http.StatusProxyAuthRequired); n != 1 { + t.Errorf("proxy 407s = %d, want exactly one (then the retry)", n) + } + + // Loopback goes direct, whatever the proxy says. + local := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, "local") })) + t.Cleanup(local.Close) + before := len(proxy.counts()) + if got, err := get(local.URL); err != nil || got != "local" { + t.Fatalf("GET loopback = (%q, %v)", got, err) + } + if len(proxy.counts()) != before { + t.Fatalf("loopback request reached the proxy: %v", proxy.counts()) + } + if n := hits.Load(); n != 2 { + t.Errorf("server hits = %d, want 2", n) + } +} + +func proxyURLHasPassword(r *netproxy.Resolver, pass string) bool { + u, err := r.ProxyForAddr("registry.pilot.invalid:443") if err != nil || u == nil { return false } diff --git a/pkg/daemon/zz_proxy_test.go b/pkg/daemon/zz_proxy_test.go index 15ad69d1..ac077909 100644 --- a/pkg/daemon/zz_proxy_test.go +++ b/pkg/daemon/zz_proxy_test.go @@ -612,32 +612,44 @@ func TestDialRegistryClientPinnedThroughExplicitProxy(t *testing.T) { } } -// Without a policy nothing changes: no dialer option, no HTTP proxy -// override, and the MOTD client stays on http.DefaultTransport. +// Without a proxy resolver nothing changes: no dialer option and the MOTD +// client stays on http.DefaultTransport (net/http's own proxy environment +// handling). func TestNoProxyPolicyKeepsHistoricalDialing(t *testing.T) { - t.Parallel() + t.Setenv("HTTPS_PROXY", "http://env-proxy.test:3128") + t.Setenv("NO_PROXY", "") + t.Setenv("no_proxy", "") d := New(Config{}) if opts := d.registryDialOptions(); opts != nil { - t.Fatalf("registryDialOptions without a policy = %d options, want none", len(opts)) - } - if d.httpProxyFunc() != nil { - t.Fatal("httpProxyFunc without a policy is non-nil") + t.Fatalf("registryDialOptions without a proxy = %d options, want none", len(opts)) } if c := d.newHTTPClient(time.Second); c.Transport != nil { t.Fatalf("HTTP client transport = %T, want nil (http.DefaultTransport)", c.Transport) } + // -proxy=off: no registry dialer, and daemon HTTP fetches stop + // following the proxy environment. + var sawProxy atomic.Bool + front := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + sawProxy.Store(true) // an HTTP request here means it went via the "proxy" + w.WriteHeader(http.StatusTeapot) + })) + t.Cleanup(front.Close) + t.Setenv("HTTP_PROXY", front.URL) + t.Setenv("http_proxy", front.URL) off := New(Config{Proxy: netproxy.Off()}) if opts := off.registryDialOptions(); opts != nil { t.Fatal("-proxy=off still installs a registry dialer") } - c := off.newHTTPClient(time.Second) - tr, ok := c.Transport.(*http.Transport) - if !ok || tr.Proxy == nil { - t.Fatalf("-proxy=off HTTP client transport = %T, want an *http.Transport with the policy's Proxy", c.Transport) + c := off.newHTTPClient(2 * time.Second) + if c.Transport == nil { + t.Fatal("-proxy=off HTTP client uses http.DefaultTransport, which follows the proxy environment") + } + resp, err := c.Get("http://unreachable.pilot.invalid/") + if err == nil { + resp.Body.Close() } - u, err := tr.Proxy(&http.Request{URL: &url.URL{Scheme: "https", Host: "raw.githubusercontent.com"}}) - if err != nil || u != nil { - t.Fatalf("-proxy=off HTTP proxy = (%v, %v), want direct", u, err) + if sawProxy.Load() { + t.Fatal("-proxy=off HTTP client went through the environment's proxy") } } diff --git a/pkg/daemon/zz_transport_auto_test.go b/pkg/daemon/zz_transport_auto_test.go index c0473199..a6bc1f1e 100644 --- a/pkg/daemon/zz_transport_auto_test.go +++ b/pkg/daemon/zz_transport_auto_test.go @@ -16,6 +16,7 @@ import ( "github.com/pilot-protocol/common/netproxy" "github.com/pilot-protocol/common/protocol" + "github.com/pilot-protocol/pilotprotocol/internal/proxyconf" ) // udpBeacon answers BeaconMsgDiscover like a real beacon when answer is @@ -88,7 +89,7 @@ func TestSelectTransportUDPWorks(t *testing.T) { beacon := udpBeacon(t, true) dialed := false start := time.Now() - mode, reason := SelectTransport(context.Background(), AutoTransportProbe{ + mode, reason, _ := SelectTransport(context.Background(), AutoTransportProbe{ BeaconAddr: beacon, CompatBeaconURL: "wss://beacon.pilot.invalid/v1/compat", Dial: func(ctx context.Context, network, addr string) (net.Conn, error) { @@ -114,7 +115,7 @@ func TestSelectTransportUDPBlockedFallsBackToCompat(t *testing.T) { beacon := udpBeacon(t, false) compat := compatBeaconStub(t, http.StatusUpgradeRequired) start := time.Now() - mode, reason := SelectTransport(context.Background(), AutoTransportProbe{ + mode, reason, _ := SelectTransport(context.Background(), AutoTransportProbe{ BeaconAddr: beacon, CompatBeaconURL: "wss://" + compat + "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/v1/compat", UDPTimeout: 300 * time.Millisecond, @@ -142,7 +143,7 @@ func TestSelectTransportNothingReachableStaysUDP(t *testing.T) { } refused := ln.Addr().String() ln.Close() - mode, reason := SelectTransport(context.Background(), AutoTransportProbe{ + mode, reason, _ := SelectTransport(context.Background(), AutoTransportProbe{ BeaconAddr: beacon, CompatBeaconURL: "wss://" + refused + "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/v1/compat", UDPTimeout: 200 * time.Millisecond, @@ -156,7 +157,7 @@ func TestSelectTransportNothingReachableStaysUDP(t *testing.T) { {beacon, ""}, {beacon, "ftp://beacon.pilot.invalid"}, } { - if mode, reason := SelectTransport(context.Background(), AutoTransportProbe{ + if mode, reason, _ := SelectTransport(context.Background(), AutoTransportProbe{ BeaconAddr: tc.beacon, CompatBeaconURL: tc.compat, UDPTimeout: 100 * time.Millisecond, }); mode != TransportUDP { t.Errorf("SelectTransport(%q, %q) = %q (%s), want udp", tc.beacon, tc.compat, mode, reason) @@ -176,7 +177,7 @@ func TestSelectTransportFrontUpBeaconDownStaysUDP(t *testing.T) { "TLS front, 502": compatBeaconStub(t, http.StatusBadGateway), "TLS front, 503": compatBeaconStub(t, http.StatusServiceUnavailable), } { - mode, reason := SelectTransport(context.Background(), AutoTransportProbe{ + mode, reason, _ := SelectTransport(context.Background(), AutoTransportProbe{ BeaconAddr: beacon, CompatBeaconURL: "wss://" + compat + "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/v1/compat", UDPTimeout: 100 * time.Millisecond, @@ -201,7 +202,7 @@ func TestSelectTransportCompatCheckUsesProxy(t *testing.T) { t.Fatal(err) } d := New(Config{Proxy: policy}) - mode, reason := SelectTransport(context.Background(), AutoTransportProbe{ + mode, reason, _ := SelectTransport(context.Background(), AutoTransportProbe{ BeaconAddr: beacon, CompatBeaconURL: "wss://beacon.pilot.invalid:" + port + "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/v1/compat", Dial: d.proxyDialer(), @@ -215,6 +216,101 @@ func TestSelectTransportCompatCheckUsesProxy(t *testing.T) { } } +// E2E product gap (Muse, wrong or stale proxy password): UDP gets no +// answer and the proxy refuses the compat check. auto must not settle on +// udp — a udp daemon dials the raw registry directly, past the proxy, +// which a proxy-only sandbox kills — but stay on compat, report the proxy +// error, and let the registry dial (with refreshed credentials) fail or +// recover through the proxy. +func TestSelectTransportProxyRefusalStaysCompat(t *testing.T) { + clearProxyEnv(t) + beacon := udpBeacon(t, false) + target := compatBeaconStub(t, http.StatusUpgradeRequired) + _, port, _ := net.SplitHostPort(target) + compatURL := "wss://beacon.pilot.invalid:" + port + "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/v1/compat" + + proxy := newProxyTestConnect(t, "muse", "right") + closed, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + deadProxy := "http://muse:s3cret@" + closed.Addr().String() + closed.Close() + + for _, tc := range []struct { + name, proxyURL, compatURL string + wantStatus int // 0: not a ConnectError + }{ + {"407 wrong password", proxy.url("muse", "s3cret"), compatURL, http.StatusProxyAuthRequired}, + {"403 forbidden target", proxy.url("muse", "right"), "wss://beacon.example.test:" + port + "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/v1/compat", http.StatusForbidden}, + {"proxy unreachable", deadProxy, compatURL, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + r, err := ResolveProxy(tc.proxyURL, TransportCompat) + if err != nil { + t.Fatal(err) + } + d := New(Config{Proxy: r}) + mode, reason, proxyErr := SelectTransport(context.Background(), AutoTransportProbe{ + BeaconAddr: beacon, + CompatBeaconURL: tc.compatURL, + Dial: d.proxyDialer(), + ProxyFor: func(addr string) string { return proxyconf.ProxyFor(r, addr) }, + UDPTimeout: 200 * time.Millisecond, + TCPTimeout: 3 * time.Second, + }) + if mode != TransportCompat || proxyErr == nil { + t.Fatalf("SelectTransport = (%q, %q, %v), want compat with the proxy error", mode, reason, proxyErr) + } + if strings.Contains(reason, "s3cret") || strings.Contains(proxyErr.Error(), "s3cret") { + t.Fatalf("reason or error leaks the proxy password: %q / %v", reason, proxyErr) + } + if !strings.Contains(reason, "***@") { + t.Errorf("reason %q does not name the (redacted) proxy", reason) + } + var ce *netproxy.ConnectError + if tc.wantStatus != 0 { + if !errors.As(proxyErr, &ce) || ce.StatusCode != tc.wantStatus { + t.Fatalf("proxyErr = %v, want a %d ConnectError", proxyErr, tc.wantStatus) + } + if hint := ProxyRefusalHint(proxyErr); hint == "" { + t.Errorf("no hint for %v", proxyErr) + } + } + }) + } + + // Without a proxy the same failure (nothing reachable) stays on udp, + // as before. + mode, reason, proxyErr := SelectTransport(context.Background(), AutoTransportProbe{ + BeaconAddr: beacon, + CompatBeaconURL: "wss://" + closed.Addr().String() + "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/v1/compat", + UDPTimeout: 100 * time.Millisecond, + TCPTimeout: time.Second, + }) + if mode != TransportUDP || proxyErr != nil { + t.Fatalf("direct, nothing reachable = (%q, %q, %v), want udp", mode, reason, proxyErr) + } + + // A proxy that tunnels fine to a front whose beacon is down is not a + // proxy error: udp, as before. + front := compatBeaconStub(t, http.StatusBadGateway) + _, frontPort, _ := net.SplitHostPort(front) + r, _ := ResolveProxy(proxy.url("muse", "right"), TransportCompat) + d := New(Config{Proxy: r}) + mode, reason, proxyErr = SelectTransport(context.Background(), AutoTransportProbe{ + BeaconAddr: beacon, + CompatBeaconURL: "wss://beacon.pilot.invalid:" + frontPort + "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/v1/compat", + Dial: d.proxyDialer(), + ProxyFor: func(addr string) string { return proxyconf.ProxyFor(r, addr) }, + UDPTimeout: 100 * time.Millisecond, + TCPTimeout: 3 * time.Second, + }) + if mode != TransportUDP || proxyErr != nil { + t.Fatalf("proxied, beacon down = (%q, %q, %v), want udp", mode, reason, proxyErr) + } +} + func TestNormalizeTransport(t *testing.T) { for in, want := range map[string]string{"": "", "udp": "udp", " COMPAT ": "compat", "Auto": "auto"} { if got, err := NormalizeTransport(in); err != nil || got != want { @@ -267,7 +363,7 @@ func TestExplicitProxyNeverTakesLoopback(t *testing.T) { t.Fatal(err) } d := New(Config{Proxy: policy}) - pf := d.httpProxyFunc() + pf := proxyconf.RequestProxy(d.config.Proxy) for _, target := range []string{"http://127.0.0.1:8080/hook", "http://localhost:5002/analyze"} { u, err := pf(&http.Request{URL: mustURL(t, target)}) if err != nil || u != nil { From ecaccf26348d65a79d71b003b46a5e76ffd0d484 Mon Sep 17 00:00:00 2001 From: Teodor Calin Date: Thu, 24 Sep 2026 05:04:59 +0300 Subject: [PATCH 10/19] fix(pilotctl): --json trusted list; warn when an old daemon will ignore HTTPS_PROXY Two phase-2 E2E onboarding papercuts: - `pilotctl --json trusted list` ignored --json and printed the table. - A new pilotctl paired with a daemon that predates -proxy (v1.13.9) in a shell with HTTPS_PROXY/ALL_PROXY started it silently; the failure then surfaced only as "did not become ready". daemon start now warns that the daemon will not use the proxy (not with --proxy off, and not twice when an explicit --proxy was already dropped). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 6 +++++ cmd/pilotctl/daemon_transport.go | 20 ++++++++++++++ cmd/pilotctl/trusted.go | 12 +++++++++ cmd/pilotctl/zz_proxy_rotation_test.go | 31 +++++++++++++++++++++ cmd/pilotctl/zz_trusted_json_test.go | 37 ++++++++++++++++++++++++++ 5 files changed, 106 insertions(+) create mode 100644 cmd/pilotctl/zz_trusted_json_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index b342c538..809f9fe7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -132,6 +132,12 @@ Detailed per-release notes are on the sandboxes run the agent as root); regular hosts still refuse root. ### Fixed +- **`pilotctl --json trusted list` printed the text table**; it now returns + `{"trusted": [{"hostname", "address", "node_id"}], "count"}`. +- **A new pilotctl starting a pilot-daemon that predates proxy support** + (v1.13.9) from a shell with `$HTTPS_PROXY` / `$ALL_PROXY` now warns that + the daemon will not use the proxy, instead of leaving a bare "did not + become ready" to explain it. - **Downgrading after `transport=auto` was saved no longer bricks the daemon.** A pilot-daemon that predates `auto` exits on `"transport":"auto"` in config.json. Reinstalling an older release with `install.sh --version` and diff --git a/cmd/pilotctl/daemon_transport.go b/cmd/pilotctl/daemon_transport.go index 4a1093fa..e296cab4 100644 --- a/cmd/pilotctl/daemon_transport.go +++ b/cmd/pilotctl/daemon_transport.go @@ -460,18 +460,38 @@ func adaptDaemonArgs(bin string, plan daemonLaunchPlan) (args []string, proxyEnv } if flags != nil && !flags["proxy"] { + asked := false if hasFlag(args, "--proxy") { warn("%s does not support -proxy (older pilot-daemon); not passing -proxy %s — upgrade pilot-daemon to use it", bin, redactProxyURL(flagValue(args, "--proxy"))) args = removeFlag(args, "--proxy") + asked = true } if proxyEnv != "" { warn("%s does not support -proxy (older pilot-daemon); proxy %s will not be used — upgrade pilot-daemon to use it", bin, redactProxyURL(proxyEnv)) proxyEnv = "" + asked = true + } + if v := envProxyVar(); v != "" && !asked && plan.Proxy != proxyconf.Off { + // Nothing was asked for explicitly, but this shell gets out + // through a proxy the daemon cannot use: say so now rather than + // leave a bare "did not become ready" to explain it. + warn("%s predates HTTPS-proxy support (older pilot-daemon) and will not use $%s; where the proxy is the only way out it cannot reach the registry — upgrade pilot-daemon", bin, v) } } return args, proxyEnv, transport } +// envProxyVar names the first proxy variable set in this environment +// that a proxy-aware daemon would use for the registry ("" when none). +func envProxyVar() string { + for _, k := range []string{"HTTPS_PROXY", "https_proxy", "ALL_PROXY", "all_proxy"} { + if strings.TrimSpace(os.Getenv(k)) != "" { + return k + } + } + return "" +} + func hasFlag(args []string, name string) bool { for _, a := range args { if a == name { diff --git a/cmd/pilotctl/trusted.go b/cmd/pilotctl/trusted.go index bd108948..2a4cca5b 100644 --- a/cmd/pilotctl/trusted.go +++ b/cmd/pilotctl/trusted.go @@ -15,6 +15,18 @@ func cmdTrusted(args []string) { "usage: pilotctl trusted list") } agents := trustedagents.All() + if jsonOutput { + list := make([]map[string]interface{}, 0, len(agents)) + for _, a := range agents { + list = append(list, map[string]interface{}{ + "hostname": a.Hostname, + "address": a.Address, + "node_id": a.NodeID, + }) + } + outputOK(map[string]interface{}{"trusted": list, "count": len(list)}) + return + } if len(agents) == 0 { fmt.Println("(no trusted agents — daemon will not auto-accept any handshakes via this path)") return diff --git a/cmd/pilotctl/zz_proxy_rotation_test.go b/cmd/pilotctl/zz_proxy_rotation_test.go index b59e092e..c74b8e68 100644 --- a/cmd/pilotctl/zz_proxy_rotation_test.go +++ b/cmd/pilotctl/zz_proxy_rotation_test.go @@ -493,3 +493,34 @@ func TestE2EDaemonStartBehindRejectingProxy(t *testing.T) { t.Errorf("auto did not stay on compat:\n%s", logs) } } + +// A new pilotctl starting a daemon that predates -proxy (v1.13.9) from a +// shell whose only way out is $HTTPS_PROXY warns at start instead of +// leaving a bare "did not become ready" to explain it (phase-2 E2E +// scenario 3). +func TestCLIDaemonStartWarnsOldDaemonIgnoresProxy(t *testing.T) { + t.Parallel() + dir := t.TempDir() + out := filepath.Join(dir, "daemon") + bin := writeFakeDaemon(t, append([]string{v1139TransportUsage}, baseDaemonFlags...), out) + env := cliEnvCleared(map[string]string{ + "PILOT_DAEMON_BIN": bin, + "PILOT_SOCKET": filepath.Join(dir, "pilot.sock"), + "HTTPS_PROXY": "http://muse:s3cret@egress.test:3128", + }) + _, stderr, code := runCLI(t, []string{"daemon", "start", "--foreground"}, env) + if code != 0 { + t.Fatalf("exit=%d stderr=%s", code, stderr) + } + if !strings.Contains(stderr, "predates HTTPS-proxy support") || !strings.Contains(stderr, "$HTTPS_PROXY") { + t.Errorf("no warning that the daemon ignores the proxy:\n%s", stderr) + } + if strings.Contains(stderr, "s3cret") { + t.Errorf("warning leaks the proxy password:\n%s", stderr) + } + // --proxy off: the operator asked for no proxy; nothing to warn about. + _, stderr, _ = runCLI(t, []string{"daemon", "start", "--foreground", "--proxy", "off"}, env) + if strings.Contains(stderr, "predates HTTPS-proxy support") { + t.Errorf("warned although --proxy off:\n%s", stderr) + } +} diff --git a/cmd/pilotctl/zz_trusted_json_test.go b/cmd/pilotctl/zz_trusted_json_test.go new file mode 100644 index 00000000..2602ff50 --- /dev/null +++ b/cmd/pilotctl/zz_trusted_json_test.go @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/pilot-protocol/trustedagents" +) + +// `pilotctl --json trusted list` printed the text table (phase-2 E2E). +func TestCLITrustedListJSON(t *testing.T) { + t.Parallel() + stdout, stderr, code := runCLI(t, []string{"--json", "trusted", "list"}, cliEnvCleared(nil)) + if code != 0 { + t.Fatalf("exit=%d stderr=%s", code, stderr) + } + var res struct { + Status string `json:"status"` + Data struct { + Trusted []struct { + Hostname string `json:"hostname"` + Address string `json:"address"` + NodeID uint32 `json:"node_id"` + } `json:"trusted"` + Count int `json:"count"` + } `json:"data"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(stdout)), &res); err != nil { + t.Fatalf("stdout is not JSON: %v\n%s", err, stdout) + } + if res.Status != "ok" || res.Data.Count != len(trustedagents.All()) || len(res.Data.Trusted) != res.Data.Count { + t.Fatalf("result = %+v, want every trusted agent", res) + } +} From 904fdce143ca7c9ef29c7ae78c6f2193481c2a60 Mon Sep 17 00:00:00 2001 From: Teodor Calin Date: Thu, 24 Sep 2026 05:06:42 +0300 Subject: [PATCH 11/19] docs(daemon): -transport=auto help says a refusing proxy keeps compat Co-Authored-By: Claude Opus 5.5 (1M context) --- cmd/daemon/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/daemon/main.go b/cmd/daemon/main.go index da53e34e..3c4e0bcb 100644 --- a/cmd/daemon/main.go +++ b/cmd/daemon/main.go @@ -114,7 +114,7 @@ func main() { // variables beat config.json (see flagSources.envOverConfig), and -help // must never print an environment value — PILOT_PROXY can hold proxy // credentials. - transportMode := flag.String("transport", "", "tunnel transport: 'udp' (the default), 'compat' (registry over TLS and beacon over WSS, TCP 443 only, for UDP-blocked or proxy-only hosts) or 'auto' (udp when the beacon answers over UDP, otherwise compat when TCP 443 is reachable, through the proxy if there is one). Precedence: this flag, $PILOT_TRANSPORT, config.json \"transport\", udp.") + transportMode := flag.String("transport", "", "tunnel transport: 'udp' (the default), 'compat' (registry over TLS and beacon over WSS, TCP 443 only, for UDP-blocked or proxy-only hosts) or 'auto' (udp when the beacon answers over UDP, otherwise compat when TCP 443 is reachable, through the proxy if there is one — also when a configured proxy refuses the check, so nothing is dialed past the proxy). Precedence: this flag, $PILOT_TRANSPORT, config.json \"transport\", udp.") proxySpec := flag.String("proxy", "", "outbound proxy for registry, beacon and HTTP connections: 'auto' (the default: with compat, HTTPS_PROXY/ALL_PROXY from the environment, honoring NO_PROXY; nothing with udp), 'off' (also none, no, false, direct), or an http:// or https:// proxy URL, http://[user:pass@]host:port, used for every connection except loopback. Precedence: this flag, $PILOT_PROXY, config.json \"proxy\", auto.") proxyCmd := flag.String("proxy-cmd", "", "command (run with sh -c) whose output is the current proxy URL, for egress proxies that rotate their credentials: it supplies the URL -proxy would use (the explicit URL, or with auto and compat the environment's proxy) and is re-run once 60s have passed and whenever the proxy answers 407, after which that connection is retried once, so new connections always carry fresh credentials. Example: bash -c 'printf %s \"$https_proxy\"'. Precedence: this flag, $PILOT_PROXY_CMD, config.json \"proxy_cmd\".") compatBeacon := flag.String("compat-beacon", defaultCompatBeacon, "beacon WSS URL for -transport=compat") From 32a2de9412a43ceed9ccff930a7d96ce2f56b2bb Mon Sep 17 00:00:00 2001 From: Teodor Calin Date: Thu, 24 Sep 2026 05:27:59 +0300 Subject: [PATCH 12/19] test(daemon): record the beacon's authenticated node before auth_ok TestStartCompatModeThroughConnectProxy failed on ubuntu CI with "beacon authenticated node 0, want 1": the daemon's Start returns as soon as it reads auth_ok, and the fake beacon stored the node only after writing it. Store first. Co-Authored-By: Claude Opus 5.5 (1M context) --- pkg/daemon/zz_proxy_test.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/daemon/zz_proxy_test.go b/pkg/daemon/zz_proxy_test.go index ac077909..e526b1f9 100644 --- a/pkg/daemon/zz_proxy_test.go +++ b/pkg/daemon/zz_proxy_test.go @@ -463,14 +463,17 @@ func (b *proxiedBeacon) handle(w http.ResponseWriter, r *http.Request, lookup fu conn.Close(websocket.StatusPolicyViolation, "auth_fail") return } - ok2, _ := json.Marshal(map[string]string{"type": "auth_ok"}) - if err := conn.Write(ctx, websocket.MessageText, ok2); err != nil { - return - } + // Record the node before answering: the daemon's Start returns as soon + // as it reads auth_ok, and a test checking authed right after Start + // must not race this goroutine (it lost on a loaded CI runner). b.authed.Store(reply.NodeID) b.mu.Lock() b.conns = append(b.conns, conn) b.mu.Unlock() + ok2, _ := json.Marshal(map[string]string{"type": "auth_ok"}) + if err := conn.Write(ctx, websocket.MessageText, ok2); err != nil { + return + } for { if _, _, err := conn.Read(r.Context()); err != nil { return From 5097bc220c218b064c04f41ee2267df5c66356d4 Mon Sep 17 00:00:00 2001 From: Teodor Calin Date: Thu, 24 Sep 2026 07:16:48 +0300 Subject: [PATCH 13/19] fix(proxy): apps follow rotating proxy credentials via a loopback relay; Muse rejection diagnostics; credential-preferring sandbox command Phase-3 review findings on #470: web4-470-apps-inherit-stale-proxy-creds (high). App-store apps are processes the daemon spawns with its environment, so they kept the launch-time HTTPS_PROXY credentials: after one rotation every app that opened a new connection failed ("node online, all apps broken"), also after a respawn. New internal/proxyconf.Relay: a CONNECT proxy on 127.0.0.1: that opens each tunnel upstream with a netproxy.Dialer (current credentials, refresh + one retry on a 407 or a garbled answer), CONNECT only, guarded by its own random credentials, never inside the TLS tunnel. A proxying daemon runs one; with -proxy-cmd it points HTTPS_PROXY/https_proxy (and PILOT_PROXY when it is a URL) at it before any plugin starts, so every app it spawns inherits the relay instead of credentials. The daemon itself never uses the relay: the refresh command now runs in the launch environment (proxyconf.CommandSource, same process-group/timeout/output-cap rules as netproxy's), a refresh that names the relay is refused, and a remote restart re-execs with the launch environment. web4-470-configuretransport-ignores-garbled-rejection. net/http never passes an unparseable CONNECT answer to OnProxyConnectResponse, so http.DefaultTransport clients never refreshed on Muse's rejection form and quoted the proxy's bytes. ConfigureTransport now takes the relay: https requests tunnel through it (refresh + retry, netproxy's wording in errors, the relay's upstream error surfaced to the client); http:// keeps going to the proxy. Doc comments and CHANGELOG corrected. web4-470-muse-rejection-diagnostics-misdirect. proxyconf.CredentialHint / UnreadableConnectReply recognise netproxy's "read CONNECT response: ... (response text withheld)"; daemon.ProxyRefusalHint, the auto-selected WARN and pilotctl's start-failure hint now name the credentials and proxy_cmd for it. No hint suggests -transport=udp unconditionally any more. web4-470-sandbox-cmd-variable-precedence. The sandbox proxy_cmd (pilotctl and install.sh) now prints whichever of $https_proxy / $HTTPS_PROXY carries credentials ($https_proxy when both do, the daemon's order when neither does), so it can never replace a credentialed URL with a bare one. install.sh differs from release#49 by that one line; #49 must take it. Tests: real daemon + app-store supervisor + sideloaded app across garbled rotations (fails with "malformed HTTP status code 4O7" without the relay), relay unit tests, the reviewer's ConfigureTransport repro, CommandSource, hints (daemon subprocess, pkg/daemon, pilotctl CLI), and the sandbox command through bash for 7 variable combinations. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 71 ++- README.md | 2 +- cmd/daemon/main.go | 13 +- cmd/daemon/netflags.go | 9 +- cmd/daemon/proxy.go | 175 ++++++-- cmd/daemon/proxy_relay_test.go | 442 +++++++++++++++++++ cmd/daemon/proxy_test.go | 20 + cmd/pilotctl/daemon_startlog.go | 10 +- cmd/pilotctl/daemon_transport.go | 11 +- cmd/pilotctl/main.go | 16 +- cmd/pilotctl/zz_proxy_muse_hints_test.go | 117 +++++ install.sh | 2 +- internal/proxyconf/command.go | 83 ++++ internal/proxyconf/command_other.go | 14 + internal/proxyconf/command_test.go | 49 +++ internal/proxyconf/command_unix.go | 32 ++ internal/proxyconf/command_unix_test.go | 45 ++ internal/proxyconf/proxyconf.go | 137 +++++- internal/proxyconf/refresh_test.go | 23 +- internal/proxyconf/relay.go | 425 ++++++++++++++++++ internal/proxyconf/relay_test.go | 521 +++++++++++++++++++++++ pkg/daemon/proxy.go | 10 +- pkg/daemon/transport_auto.go | 13 +- pkg/daemon/zz_proxy_garbled_hint_test.go | 62 +++ 24 files changed, 2216 insertions(+), 86 deletions(-) create mode 100644 cmd/daemon/proxy_relay_test.go create mode 100644 cmd/pilotctl/zz_proxy_muse_hints_test.go create mode 100644 internal/proxyconf/command.go create mode 100644 internal/proxyconf/command_other.go create mode 100644 internal/proxyconf/command_test.go create mode 100644 internal/proxyconf/command_unix.go create mode 100644 internal/proxyconf/command_unix_test.go create mode 100644 internal/proxyconf/relay.go create mode 100644 internal/proxyconf/relay_test.go create mode 100644 pkg/daemon/zz_proxy_garbled_hint_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 1caf8a1f..8f79571a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,24 +72,52 @@ Detailed per-release notes are on the - **Rotating proxy credentials: `-proxy-cmd` / `$PILOT_PROXY_CMD` / config.json `proxy_cmd`** (`pilotctl daemon start --proxy-cmd`, passed as `$PILOT_PROXY_CMD`, never on argv). A command whose output is the current - proxy URL (e.g. `bash -c 'printf %s "$https_proxy"'`). It is common - v0.5.15's netproxy refresh (`netproxy.WithRefreshCommand`): the command - runs at startup, again once 60s have passed, and whenever the proxy answers - a CONNECT with 407, after which that connection is retried once with the - new credentials (`netproxy.Dialer` for the registry — primary, pool and - every redial — and the compat WSS beacon and its reconnects; - `netproxy.RefreshingTransport` for the daemon's own HTTP clients; plugin - clients on `http.DefaultTransport` pick up the new credentials on their - next request). Tunnels already open are never touched. A sandbox that - rotates its proxy credentials every few minutes (Meta Muse) no longer - leaves a node "online with all apps broken" until a restart. In a Linux - container/VM without systemd whose `HTTPS_PROXY` carries credentials, - `pilotctl daemon start` hands the daemon `PILOT_PROXY_CMD=bash -c 'printf - %s "${https_proxy:-$HTTPS_PROXY}"'` itself when no `proxy_cmd` is - configured (so a node set up by any installer gets it), and the installer - saves the same command; pilotctl's own registry commands use the same - command. The command's output is never logged, and a failing command keeps - the last good proxy URL (at first, the launch environment's). + proxy URL (e.g. `bash -c 'printf %s "$https_proxy"'`), run with `sh -c` + in the environment the daemon was launched with. It drives common + v0.5.15's netproxy refresh: the command runs at startup, again once 60s + have passed, and whenever the proxy rejects the credentials — a 407, or a + CONNECT answer so garbled it cannot be parsed, which is how Meta Muse's + proxy answers them ("malformed HTTP status code") — after which that + connection is retried once with the new credentials (`netproxy.Dialer` for + the registry — primary, pool and every redial — and the compat WSS beacon + and its reconnects; `netproxy.RefreshingTransport` for the daemon's own + HTTP clients). Tunnels already open are never touched. The command's + output is never logged, and a failing command keeps the last good proxy + URL (at first, the launch environment's). + - **Apps follow the rotation too.** App-store apps are processes the + daemon starts, and they inherit its environment — with the launch-time + credentials, which the proxy stops accepting within minutes: every app + that opened a new connection failed (also after a respawn) while the + node stayed online ("node online, all apps broken"). A daemon that + proxies now runs a CONNECT relay on loopback (random port, its own + random credentials, CONNECT only, never sees inside the TLS tunnels), + which opens each tunnel upstream with the current credentials, refreshes + and retries once on a rejection, and answers a tunnel it cannot open + with a 502/403/504 whose reason says why without proxy text or + credentials. With a refresh command, the apps the daemon starts get + `HTTPS_PROXY` / `https_proxy` (and `$PILOT_PROXY` if it holds a URL) + pointing at the relay, so they never hold the proxy's credentials; + `HTTP_PROXY` is left alone (the relay only tunnels). The daemon's own + refresh command still runs in the launch environment, a refresh that + would name the relay is refused, and a remote restart re-execs with the + launch environment. Without a refresh command the apps' environment is + left as it is. + - **Plugin HTTP clients** (`http.DefaultTransport`: catalogue pins, + skillinject, trustedagents, webhook, telemetry) send their https + requests through the same relay, so a rejected CONNECT — including + Muse's garbled form, which net/http never hands to a hook and quotes + in its error — is refreshed and retried instead of failing until the + next timed refresh. + - In a Linux container/VM without systemd whose `HTTPS_PROXY` or + `https_proxy` carries credentials, `pilotctl daemon start` hands the + daemon `PILOT_PROXY_CMD=bash -c 'case $https_proxy in *@*) printf %s + "$https_proxy";; *) printf %s "${HTTPS_PROXY:-$https_proxy}";; esac'` + itself when no `proxy_cmd` is configured (so a node set up by any + installer gets it), and the installer saves the same command: the + variable that carries credentials wins (`$https_proxy`, which Meta + Muse's guidance reads from a fresh shell, when both do), so the command + never swaps a credentialed proxy URL for one without credentials. + pilotctl's own registry commands use the same command. - **`-transport=auto`.** UDP when the beacon answers a UDP discover (one round trip), otherwise compat when the compat beacon itself answers over TCP 443 (through the proxy, if any: a TLS GET of the beacon path must return `426 @@ -149,7 +177,12 @@ Detailed per-release notes are on the Required`), with a hint for it (wrong or rotated credentials → `proxy_cmd`), instead of only "did not become ready". pilot-daemon logs its fatal startup errors at ERROR (they came out at INFO), and a registry or compat - beacon dial the proxy refused ends with a hint naming the fix. + beacon dial the proxy refused ends with a hint naming the fix. A CONNECT + answer that cannot be parsed (`read CONNECT response: malformed HTTP + status code (response text withheld)`, Meta Muse's answer to wrong or + expired credentials) gets the credentials/`proxy_cmd` hint, and no hint + suggests `-transport=udp` except for a proxy that cannot be reached, and + then only for a host that can reach the internet without it. - **`daemon start` forwards the proxy/TLS environment** (`HTTPS_PROXY`, `HTTP_PROXY`, `ALL_PROXY`, `NO_PROXY` in both cases, `PILOT_PROXY`, `PILOT_TRANSPORT`, `PILOT_REGISTRY_TRUST`, `PILOT_REGISTRY_FINGERPRINT`, diff --git a/README.md b/README.md index f53849d6..fb712397 100644 --- a/README.md +++ b/README.md @@ -298,7 +298,7 @@ New installs use transport `auto`: the daemon probes the beacon over UDP once at - **No root, systemd or launchd needed.** Start the daemon with `pilotctl daemon start` from a shell that has the proxy variables. In a container or VM without systemd the installer also runs as root (the agent user in hosted sandboxes); on a regular host it still refuses root unless `PILOT_ALLOW_ROOT=1`. - **Proxy with credentials:** keep them out of `ps` — export `HTTPS_PROXY` / `PILOT_PROXY`, or `pilotctl daemon start --proxy http://user:pass@host:port` (pilotctl hands a URL with credentials to the daemon in its environment, never on its command line). For a systemd/launchd service, which does not see your shell's variables, save it: `pilotctl config --set proxy=http://user:pass@host:port` (config.json is 0600). -- **Proxy credentials that rotate** (Meta Muse rotates the credentials in `HTTPS_PROXY` every few minutes): a long-running daemon would keep the ones it started with, and new connections would start failing with `407 Proxy Authentication Required` while old tunnels stay up ("node online, apps broken"). Give the daemon a command that prints the current proxy URL — `proxy_cmd` in config.json, `$PILOT_PROXY_CMD`, `-proxy-cmd` or `pilotctl daemon start --proxy-cmd` — and it re-runs it once 60s have passed and whenever the proxy answers 407, retrying that connection once with the fresh credentials (registry, compat beacon and HTTP clients alike, via common's `netproxy` refresh); no restart needed. pilotctl's own registry commands use the same command. In a Linux container/VM without systemd whose `HTTPS_PROXY` carries credentials, `pilotctl daemon start` hands the daemon `PILOT_PROXY_CMD=bash -c 'printf %s "${https_proxy:-$HTTPS_PROXY}"'` (a fresh shell sees the current value) whenever no `proxy_cmd` is configured, and the installer saves the same command in config.json; elsewhere: `pilotctl config --set proxy_cmd="bash -c 'printf %s \"\$https_proxy\"'"`. Without it, restart the daemon from a fresh shell when it starts failing. +- **Proxy credentials that rotate** (Meta Muse rotates the credentials in `HTTPS_PROXY` every few minutes): a long-running daemon would keep the ones it started with, and new connections would start failing with `407 Proxy Authentication Required` while old tunnels stay up ("node online, apps broken"). Give the daemon a command that prints the current proxy URL — `proxy_cmd` in config.json, `$PILOT_PROXY_CMD`, `-proxy-cmd` or `pilotctl daemon start --proxy-cmd` — and it re-runs it once 60s have passed and whenever the proxy rejects the credentials (a 407, or Muse's garbled "malformed HTTP status code" answer), retrying that connection once with the fresh credentials (registry, compat beacon and HTTP clients alike, via common's `netproxy` refresh); no restart needed. The apps the daemon starts reach the proxy through a loopback relay in the daemon that adds the current credentials, so they keep working across rotations too. pilotctl's own registry commands use the same command. In a Linux container/VM without systemd whose `HTTPS_PROXY` or `https_proxy` carries credentials, `pilotctl daemon start` hands the daemon a `PILOT_PROXY_CMD` that prints a fresh bash's `$https_proxy` (or `$HTTPS_PROXY` when only that one carries credentials) whenever no `proxy_cmd` is configured, and the installer saves the same command in config.json; elsewhere: `pilotctl config --set proxy_cmd="bash -c 'printf %s \"\$https_proxy\"'"`. Without it, restart the daemon from a fresh shell when it starts failing. - **Precedence** for transport and proxy: command-line flag, then `$PILOT_TRANSPORT` / `$PILOT_PROXY` / `$PILOT_PROXY_CMD`, then `config.json` (`transport`, `proxy`, `proxy_cmd`), then the default (`auto` from pilotctl and the installed systemd/launchd services, via `PILOT_TRANSPORT_DEFAULT=auto`; `udp` for a bare `pilot-daemon`; proxy `auto`). `auto` is never saved in config.json, so reinstalling an older release (`--version`, `pilotctl update --pin`) leaves nothing it cannot read; if you saved it yourself, both rewrite it to `udp` for a daemon that predates it. `-proxy` accepts `auto`, `off` (also `none`, `direct`) or an `http://` / `https://` URL; anything else is an error. Loopback targets (local webhooks, sidecars) are never sent to a proxy. When the registry dial goes through a proxy (compat, or an explicit proxy URL in any transport), the default raw-TCP registry is replaced by `registry.pilotprotocol.network:443` over TLS, since CONNECT proxies carry port 443 only. - **No CA bundle in the sandbox?** Point Go at one with `SSL_CERT_FILE=/path/to/ca-certificates.crt` (or `SSL_CERT_DIR`) — `pilotctl daemon start` forwards both. The registry can instead be pinned: `PILOT_REGISTRY_FINGERPRINT=` (or `pilotctl config --set registry_fingerprint=...`), which selects `registry_trust=pinned`. The WSS beacon has no fingerprint option, so it needs the CA bundle. diff --git a/cmd/daemon/main.go b/cmd/daemon/main.go index afa78f83..510b5553 100644 --- a/cmd/daemon/main.go +++ b/cmd/daemon/main.go @@ -117,7 +117,7 @@ func main() { // credentials. transportMode := flag.String("transport", "", "tunnel transport: 'udp' (the default), 'compat' (registry over TLS and beacon over WSS, TCP 443 only, for UDP-blocked or proxy-only hosts) or 'auto' (udp when the beacon answers over UDP, otherwise compat when TCP 443 is reachable, through the proxy if there is one — also when a configured proxy refuses the check, so nothing is dialed past the proxy). Precedence: this flag, $PILOT_TRANSPORT, config.json \"transport\", udp.") proxySpec := flag.String("proxy", "", "outbound proxy for registry, beacon and HTTP connections: 'auto' (the default: with compat, HTTPS_PROXY/ALL_PROXY from the environment, honoring NO_PROXY; nothing with udp), 'off' (also none, no, false, direct), or an http:// or https:// proxy URL, http://[user:pass@]host:port, used for every connection except loopback. Precedence: this flag, $PILOT_PROXY, config.json \"proxy\", auto.") - proxyCmd := flag.String("proxy-cmd", "", "command (run with sh -c) whose output is the current proxy URL, for egress proxies that rotate their credentials: it supplies the URL -proxy would use (the explicit URL, or with auto and compat the environment's proxy) and is re-run once 60s have passed and whenever the proxy answers 407, after which that connection is retried once, so new connections always carry fresh credentials. Example: bash -c 'printf %s \"$https_proxy\"'. Precedence: this flag, $PILOT_PROXY_CMD, config.json \"proxy_cmd\".") + proxyCmd := flag.String("proxy-cmd", "", "command (run with sh -c) whose output is the current proxy URL, for egress proxies that rotate their credentials: it supplies the URL -proxy would use (the explicit URL, or with auto and compat the environment's proxy) and is re-run once 60s have passed and whenever the proxy rejects the credentials (407, or a CONNECT answer that cannot be parsed), after which that connection is retried once, so new connections always carry fresh credentials; the apps the daemon starts get HTTPS_PROXY pointing at a loopback relay in the daemon that adds them. Runs in the environment the daemon was started with. Example: bash -c 'printf %s \"$https_proxy\"'. Precedence: this flag, $PILOT_PROXY_CMD, config.json \"proxy_cmd\".") compatBeacon := flag.String("compat-beacon", defaultCompatBeacon, "beacon WSS URL for -transport=compat") tlsTrust := flag.String("tls-trust", "system", "TLS trust store for -transport=compat: 'system' (OS trust store; current default while compat mode uses Let's Encrypt certs on beacon.pilotprotocol.network — on a host without a CA bundle set SSL_CERT_FILE or SSL_CERT_DIR) or 'pinned' (Pilot CA root embedded in the daemon binary; will become the default in a future release once production root ships)") showVersion := flag.Bool("version", false, "print version and exit") @@ -331,7 +331,11 @@ func main() { *noSkillinject = profileOptions.DisableSkillinject *motdFeedURL = profileOptions.MOTDFeedURL - installDefaultTransportProxy(proxyResolver) + // The proxy relay first: DefaultTransport tunnels through it, and with + // -proxy-cmd the apps the app store spawns inherit the environment it + // exports. + proxyRelay := startProxyRelay(proxyResolver, proxyCommandFor(*proxySpec, *proxyCmd, transport, false)) + installDefaultTransportProxy(proxyResolver, proxyRelay) slog.Info("outbound network", "transport", *transportMode, "proxy", describeProxy(*proxySpec, *transportMode, proxyResolver), "transport_from", transportSrc, "registry", *registryAddr, "registry_tls", *registryTLS) @@ -751,6 +755,7 @@ func main() { // matters). A daemon-requested exit leaves here via os.Exit with its // code once the teardown finishes. shutdown(cause, func() { d.Stop() }, rt.StopPlugins, os.Exit) + _ = proxyRelay.Close() // the plugins and apps are stopped if cause.restart { executable, err := os.Executable() if err != nil { @@ -759,7 +764,9 @@ func main() { } slog.Info("restarting daemon after graceful shutdown") // #nosec G204,G702 -- restart re-execs the current OS-resolved daemon directly; signed fleet commands cannot supply a path or arguments. - if err := syscall.Exec(executable, os.Args, os.Environ()); err != nil { + // The launch environment, not os.Environ(): that may point the proxy + // variables at this daemon's proxy relay, which is gone once it execs. + if err := syscall.Exec(executable, os.Args, launchEnvironment); err != nil { slog.Error("remote daemon restart failed", "err", err) } } diff --git a/cmd/daemon/netflags.go b/cmd/daemon/netflags.go index bd17d56a..d0f50d2f 100644 --- a/cmd/daemon/netflags.go +++ b/cmd/daemon/netflags.go @@ -225,12 +225,17 @@ func resolveAutoTransport(reg registrySettings, beacon string, beaconExplicit bo } // proxyErrorHint says what to check when the proxy failed the compat -// check. +// check: a refusal or a rejection of the credentials (also in the garbled +// form Meta Muse's proxy answers them with) per daemon.ProxyRefusalHint, +// else a proxy that could not be reached or dropped the connection. It +// never suggests going around the proxy unconditionally: on a proxy-only +// host (a hosted agent sandbox) -transport=udp dials the registry directly, +// and that traffic is dropped or gets the sandbox killed. func proxyErrorHint(err error) string { if hint := daemon.ProxyRefusalHint(err); hint != "" { return hint } - return "the proxy could not be reached or its answer could not be read: check HTTPS_PROXY / -proxy, or pass -transport=udp to bypass it" + return "the proxy could not be reached or dropped the connection: check HTTPS_PROXY / -proxy; only if this host can reach the internet without the proxy, -proxy=off (or -transport=udp) stops using it" } // transportDefaultEnv names the transport a daemon uses when neither diff --git a/cmd/daemon/proxy.go b/cmd/daemon/proxy.go index f839e1d0..3b2171ea 100644 --- a/cmd/daemon/proxy.go +++ b/cmd/daemon/proxy.go @@ -3,15 +3,59 @@ package main import ( + "context" + "errors" "log/slog" "net/http" + "net/url" + "os" "strings" + "sync/atomic" "github.com/pilot-protocol/common/netproxy" "github.com/pilot-protocol/pilotprotocol/internal/proxyconf" "github.com/pilot-protocol/pilotprotocol/pkg/daemon" ) +// launchEnvironment is the environment the daemon was started with, taken +// before exportAppProxy points the proxy variables at the proxy relay. The +// -proxy-cmd refresh command runs in it, and a remote restart re-execs the +// daemon with it: a new daemon inheriting the relay (which dies with this +// one) as its proxy would have no way out. +var launchEnvironment = os.Environ() + +// activeProxyRelay is the running proxy relay, nil before startProxyRelay +// (and when none runs). +var activeProxyRelay atomic.Pointer[proxyconf.Relay] + +// proxyCommandFor returns the -proxy-cmd the resolver for transport uses, +// "" when it uses none: none is set, -proxy=off (ignored, with a warning +// when warn is set), or -proxy=auto on a transport other than compat +// (auto proxies nothing there). +func proxyCommandFor(spec, command, transport string, warn bool) string { + command = strings.TrimSpace(command) + if command == "" { + return "" + } + s, err := proxyconf.Normalize(spec) + if err != nil { + return "" + } + switch { + case s == proxyconf.Off: + if warn { + slog.Warn("ignoring -proxy-cmd: -proxy=off") + } + return "" + case s == proxyconf.Auto && transport != daemon.TransportCompat: + if warn { + slog.Info("-proxy-cmd not used: -proxy=auto proxies -transport=compat only", "transport", transport) + } + return "" + } + return command +} + // resolveProxy resolves -proxy and -proxy-cmd for the transport (see // daemon.ResolveProxy). A malformed -proxy value is an error. Under "auto" // only an unusable HTTPS_PROXY / https_proxy (the variable that names the @@ -20,11 +64,12 @@ import ( // HTTP_PROXY / ALL_PROXY values are skipped by netproxy and only logged. // // -proxy-cmd (a command whose output is the current proxy URL) makes the -// resolver follow rotating credentials (netproxy.WithRefreshCommand): it -// supplies the URL wherever -proxy would use one — the explicit URL, or -// with auto (compat only) the environment's proxy, NO_PROXY still -// honored — and is re-run every 60s and whenever the proxy answers 407, -// after which the rejected connection is retried once. It is ignored with +// resolver follow rotating credentials (netproxy.WithRefreshFunc over +// proxyRefreshSource): it supplies the URL wherever -proxy would use one — +// the explicit URL, or with auto (compat only) the environment's proxy, +// NO_PROXY still honored — and is re-run every 60s and whenever the proxy +// rejects the credentials (407, or an answer that cannot be parsed), after +// which the rejected connection is retried once. It is ignored with // -proxy=off, and with auto on udp (no proxy). A failing run is logged // once per run of failures, and the last good URL (at first, the launch // environment's proxy or the explicit URL) stays in use. @@ -33,19 +78,11 @@ func resolveProxy(spec, command, transport string) (*netproxy.Resolver, error) { if err != nil { return nil, err } - command = strings.TrimSpace(command) - if command != "" && s == proxyconf.Off { - slog.Warn("ignoring -proxy-cmd: -proxy=off") - command = "" - } - if command != "" && s == proxyconf.Auto && transport != daemon.TransportCompat { - slog.Info("-proxy-cmd not used: -proxy=auto proxies -transport=compat only", "transport", transport) - command = "" - } + command = proxyCommandFor(s, command, transport, true) var opts []netproxy.Option if command != "" { opts = append(opts, - netproxy.WithRefreshCommand(command), + netproxy.WithRefreshFunc(proxyRefreshSource(command)), netproxy.WithRefreshErrorHandler(func(err error) { slog.Warn("-proxy-cmd failed; keeping the last good proxy URL (at first the launch-time proxy)", "err", err) })) @@ -62,6 +99,24 @@ func resolveProxy(spec, command, transport string) (*netproxy.Resolver, error) { return r, nil } +// proxyRefreshSource runs the -proxy-cmd command in the launch environment +// (proxyconf.CommandSource), never in the one exportAppProxy changed, and +// refuses a URL that names the daemon's own proxy relay: the relay would +// then forward to itself. +func proxyRefreshSource(command string) func(ctx context.Context) (string, error) { + run := proxyconf.CommandSource(command, launchEnvironment) + return func(ctx context.Context) (string, error) { + out, err := run(ctx) + if err != nil { + return "", err + } + if activeProxyRelay.Load().Serves(out) { + return "", errors.New("refresh command printed the daemon's own proxy relay; it must print the egress proxy's URL") + } + return out, nil + } +} + func logProxyWarnings(r *netproxy.Resolver) { for _, w := range r.Warnings() { slog.Warn("ignoring unusable proxy environment variable", "err", w) @@ -69,10 +124,12 @@ func logProxyWarnings(r *netproxy.Resolver) { } // describeProxy renders the resolved proxy for the startup log line. -// Credentials are always redacted. +// Credentials are always redacted. The refresh source is -proxy-cmd run by +// proxyRefreshSource, which netproxy only knows as a callback; the line +// says "command", which is what pilotctl daemon start looks for. func describeProxy(spec, transport string, r *netproxy.Resolver) string { if r != nil { - return r.String() + return strings.Replace(r.String(), "(credentials refreshed by callback)", "(credentials refreshed by command)", 1) } if isAutoProxy(spec) && transport != daemon.TransportCompat { return "none (-proxy=auto applies to -transport=compat only)" @@ -86,13 +143,17 @@ func describeProxy(spec, transport string, r *netproxy.Resolver) string { // webhook, enterprise-control clients — uses DefaultTransport (or a clone // of it), and not all of them accept an injected client. Each new // connection takes the resolver's current settings, so rotated -// credentials (-proxy-cmd) reach these clients too, and a 407 answer -// refreshes them at once so the next request succeeds (see -// proxyconf.ConfigureTransport). Loopback targets (a local webhook or -// sidecar) always go direct. nil leaves DefaultTransport alone (net/http's -// own proxy environment handling). Call before any goroutine issues a -// request. -func installDefaultTransportProxy(r *netproxy.Resolver) { +// credentials (-proxy-cmd) reach these clients too. With the proxy relay +// running (relay != nil: whenever the daemon proxies), https requests are +// tunnelled through it, so a rejected CONNECT — also one whose answer +// cannot be parsed, Meta Muse's form — is refreshed (with -proxy-cmd) and +// retried once instead of failing, and no error ever quotes the proxy's +// answer (net/http's own does); without it a 407 still refreshes the +// credentials for the next request (see proxyconf.ConfigureTransport). +// Loopback targets (a local webhook or sidecar) always go direct. nil +// leaves DefaultTransport alone (net/http's own proxy environment +// handling). Call before any goroutine issues a request. +func installDefaultTransportProxy(r *netproxy.Resolver, relay *proxyconf.Relay) { if r == nil { return } @@ -101,7 +162,71 @@ func installDefaultTransportProxy(r *netproxy.Resolver) { slog.Warn("http.DefaultTransport is not an *http.Transport; plugin HTTP clients do not follow -proxy") return } - proxyconf.ConfigureTransport(tr, r) + var via *url.URL + if relay != nil { + via = relay.URL() + } + proxyconf.ConfigureTransport(tr, r, via) +} + +// appProxyVars are the proxy variables exportAppProxy points at the proxy +// relay: the ones HTTPS clients read (Go's net/http prefers the upper-case +// one, curl the lower-case one; both are set). HTTP_PROXY / http_proxy are +// left alone: the relay carries CONNECT tunnels only, and a proxy that +// forwards plain http:// requests keeps receiving them as before. +var appProxyVars = []string{"HTTPS_PROXY", "https_proxy"} + +// startProxyRelay starts the daemon's loopback CONNECT relay +// (proxyconf.Relay) whenever the daemon proxies (r is enabled): each +// tunnel it is asked for is opened upstream with the resolver's current +// credentials, refreshed and retried once when the proxy rejects them. +// http.DefaultTransport sends its https requests through it (see +// installDefaultTransportProxy). +// +// When the credentials are refreshed (command, the -proxy-cmd in effect, +// is set), the processes the daemon starts — app-store apps, spawned with +// the daemon's environment — get it too (exportAppProxy). Their inherited +// proxy URL would otherwise carry the launch-time credentials, which a +// rotating proxy (Meta Muse) stops accepting within minutes: every app +// that opens a new connection then fails, even after a respawn, while the +// daemon stays online ("node online, all apps broken"). Without a refresh +// command the environment they inherit is exactly what the daemon itself +// uses, and is left alone. A relay that cannot start is logged; plugins +// then talk to the proxy directly and apps keep the launch environment, as +// before. +func startProxyRelay(r *netproxy.Resolver, command string) *proxyconf.Relay { + if !r.Enabled() { + return nil + } + relay, err := proxyconf.StartRelay(r, nil) + if err != nil { + slog.Warn("proxy relay not started; apps keep the launch-time proxy credentials", "err", err) + return nil + } + activeProxyRelay.Store(relay) + if command == "" { + slog.Info("proxy relay listening", "addr", relay.Addr()) + return relay + } + exportAppProxy(relay.URL().String()) + slog.Info("proxy relay listening", "addr", relay.Addr(), + "apps", "the apps this daemon starts get HTTPS_PROXY pointing here, so their connections carry the current proxy credentials") + return relay +} + +// exportAppProxy points this process's proxy environment, which the apps +// it starts inherit, at relayURL (it carries the relay's token and is never +// logged): appProxyVars, and $PILOT_PROXY when it holds a proxy URL (a +// pilotctl an app runs reads it first). The daemon itself has read its +// settings already, runs its refresh command in launchEnvironment, and +// re-execs with that on a remote restart. +func exportAppProxy(relayURL string) { + for _, k := range appProxyVars { + _ = os.Setenv(k, relayURL) + } + if s, err := proxyconf.Normalize(os.Getenv("PILOT_PROXY")); err == nil && s != proxyconf.Auto && s != proxyconf.Off { + _ = os.Setenv("PILOT_PROXY", relayURL) + } } func isAutoProxy(spec string) bool { diff --git a/cmd/daemon/proxy_relay_test.go b/cmd/daemon/proxy_relay_test.go new file mode 100644 index 00000000..51a9d902 --- /dev/null +++ b/cmd/daemon/proxy_relay_test.go @@ -0,0 +1,442 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "bufio" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/pilot-protocol/common/netproxy" + "github.com/pilot-protocol/pilotprotocol/internal/proxyconf" +) + +// fakeAppSource is an app-store app that does what the cloud-backed apps +// (plainweb, orthogonal, ...) do: HTTPS requests through the proxy in its +// environment (net/http's ProxyFromEnvironment, read once). It fetches +// $FAKE_APP_TARGET on a new connection every 200ms and appends one line +// per attempt to $FAKE_APP_OUT: " proxy=@ ok|err ...". +const fakeAppSource = `package main + +import ( + "crypto/tls" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" +) + +func main() { + proxy := "none" + if u, err := url.Parse(os.Getenv("HTTPS_PROXY")); err == nil && u.Host != "" { + proxy = u.User.Username() + "@" + u.Host + } + client := &http.Client{Timeout: 20 * time.Second, Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + DisableKeepAlives: true, + }} + out, err := os.OpenFile(os.Getenv("FAKE_APP_OUT"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + os.Exit(3) + } + parent := os.Getppid() + for { + if os.Getppid() != parent { + os.Exit(0) // the daemon is gone: do not outlive the test + } + line := "ok" + resp, err := client.Get(os.Getenv("FAKE_APP_TARGET")) + if err != nil { + line = "err " + strings.ReplaceAll(err.Error(), "\n", " ") + } else { + b, _ := io.ReadAll(resp.Body) + resp.Body.Close() + line = "ok " + string(b) + } + fmt.Fprintf(out, "%d proxy=%s %s\n", time.Now().UnixNano(), proxy, line) + time.Sleep(200 * time.Millisecond) + } +} +` + +// buildFakeApp compiles fakeAppSource (standard library only) to path. +func buildFakeApp(t *testing.T, path string) { + t.Helper() + goBin, err := exec.LookPath("go") + if err != nil { + goBin = filepath.Join(runtime.GOROOT(), "bin", "go") + if _, err := os.Stat(goBin); err != nil { + t.Skip("no go toolchain to build the fake app") + } + } + src := filepath.Join(t.TempDir(), "main.go") + if err := os.WriteFile(src, []byte(fakeAppSource), 0o600); err != nil { + t.Fatal(err) + } + cmd := exec.Command(goBin, "build", "-o", path, src) + cmd.Env = append(os.Environ(), "CGO_ENABLED=0", "GOWORK=off", "GOFLAGS=") + cmd.Dir = filepath.Dir(src) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("build fake app: %v\n%s", err, out) + } +} + +// sideloadApp installs binary as a sideloaded app-store app under root +// (the daemon's PILOT_APPSTORE_ROOT), which the daemon's supervisor +// spawns at startup with the daemon's environment. +func sideloadApp(t *testing.T, root, id, binary string) { + t.Helper() + dir := filepath.Join(root, id) + if err := os.MkdirAll(filepath.Join(dir, "bin"), 0o700); err != nil { + t.Fatal(err) + } + b, err := os.ReadFile(binary) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "bin", "app"), b, 0o700); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(b) + manifest := fmt.Sprintf(`{ + "id": %q, + "manifest_version": 1, + "app_version": "0.1.0", + "protection": "shareable", + "binary": {"runtime": "go", "path": "bin/app", "sha256": %q}, + "exposes": ["relaytest.ping"], + "grants": [{"cap": "audit.log", "target": "*"}], + "store": {"publisher": "ed25519:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", "signature": "sig:unsigned"} +}`, id, hex.EncodeToString(sum[:])) + if err := os.WriteFile(filepath.Join(dir, "manifest.json"), []byte(manifest), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, ".sideloaded"), nil, 0o400); err != nil { + t.Fatal(err) + } +} + +// fakeAppLine is one line of the fake app's log. +type fakeAppLine struct { + at time.Time + proxy string + ok bool + text string +} + +func readFakeApp(t *testing.T, path string) []fakeAppLine { + t.Helper() + f, err := os.Open(path) + if err != nil { + return nil + } + defer f.Close() + var lines []fakeAppLine + sc := bufio.NewScanner(f) + for sc.Scan() { + var ns int64 + var proxy, rest string + fields := strings.SplitN(sc.Text(), " ", 3) + if len(fields) < 3 { + continue + } + fmt.Sscan(fields[0], &ns) + proxy = strings.TrimPrefix(fields[1], "proxy=") + rest = fields[2] + lines = append(lines, fakeAppLine{at: time.Unix(0, ns), proxy: proxy, ok: strings.HasPrefix(rest, "ok "), text: rest}) + } + return lines +} + +// web4-470-apps-inherit-stale-proxy-creds, end to end with the real daemon +// and the real app-store supervisor. The egress proxy rotates its +// credentials and answers the stale ones the way Meta Muse's does (an +// unparseable status line). The daemon has a refresh command; the app it +// spawns only has the environment it inherits. Before the fix the app kept +// the launch-time HTTPS_PROXY and every request after the rotation failed +// ("node online, all apps broken"), also after a respawn. Now the app's +// HTTPS_PROXY is the daemon's loopback relay, and its requests keep +// working across rotations without ever holding the proxy's credentials. +func TestAppsFollowProxyRotationThroughTheRelay(t *testing.T) { + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fmt.Fprint(w, "hello") })) + defer srv.Close() + proxy := newRefusingProxy(t, "muse", "pw-1") + proxy.rotate("muse", "pw-1") // anything else is rejected ... + proxy.garbleRejections() // ... with Muse's garbled status line + proxy.forward("app.relay.test:443", srv.Listener.Addr().String()) + + scratch := t.TempDir() + urlFile := filepath.Join(scratch, "proxy-url") + setURL := func(pass string) { + t.Helper() + if err := os.WriteFile(urlFile, []byte(fmt.Sprintf("http://muse:%s@%s\n", pass, proxy.ln.Addr())), 0o600); err != nil { + t.Fatal(err) + } + } + setURL("pw-1") + appBin := filepath.Join(scratch, "fakeapp") + buildFakeApp(t, appBin) + appOut := filepath.Join(scratch, "fetches.log") + + d := startDaemon(t, daemonRun{ + env: []string{ + "PILOT_TRANSPORT=compat", + "HTTPS_PROXY=" + fmt.Sprintf("http://muse:pw-1@%s", proxy.ln.Addr()), + "PILOT_PROXY_CMD=cat '" + urlFile + "'", + "FAKE_APP_TARGET=https://app.relay.test/", + "FAKE_APP_OUT=" + appOut, + }, + setup: func(home string) { sideloadApp(t, filepath.Join(home, "apps"), "io.sideload.relaytest", appBin) }, + }) + waitApp := func(what string, since time.Time, cond func(ok, failed []fakeAppLine) bool) []fakeAppLine { + t.Helper() + deadline := time.Now().Add(45 * time.Second) + for { + var ok, failed []fakeAppLine + for _, l := range readFakeApp(t, appOut) { + if l.at.Before(since) { + continue + } + if l.ok { + ok = append(ok, l) + } else { + failed = append(failed, l) + } + } + if cond(ok, failed) { + return failed + } + if time.Now().After(deadline) { + t.Fatalf("%s; app since %s: %d ok, failures %v\ndaemon output:\n%s", what, since.Format(time.RFC3339Nano), len(ok), failed, d.out.String()) + } + time.Sleep(100 * time.Millisecond) + } + } + + start := time.Now().Add(-time.Minute) + waitApp("the app never fetched through the proxy", start, func(ok, _ []fakeAppLine) bool { return len(ok) >= 2 }) + for _, l := range readFakeApp(t, appOut) { + if !strings.HasPrefix(l.proxy, proxyconf.RelayUser+"@127.0.0.1:") { + t.Fatalf("the app's HTTPS_PROXY is %q, want the daemon's relay (%s@127.0.0.1:port)", l.proxy, proxyconf.RelayUser) + } + } + + for _, pass := range []string{"pw-2", "pw-3"} { + setURL(pass) + proxy.rotate("muse", pass) + rotated := time.Now() + failed := waitApp("the app never fetched again after the rotation to "+pass, rotated, func(ok, _ []fakeAppLine) bool { return len(ok) >= 3 }) + if len(failed) > 0 { + t.Errorf("after the rotation to %s the app saw failures: %v", pass, failed) + } + if proxy.accepted("muse", pass) == 0 { + t.Fatalf("the proxy never saw the rotated credentials %s", pass) + } + } + d.stop() + logs := d.out.String() + if !strings.Contains(logs, `msg="proxy relay listening"`) || !strings.Contains(logs, "the apps this daemon starts get HTTPS_PROXY") { + t.Errorf("relay start not logged:\n%s", logs) + } + for _, secret := range []string{"pw-1", "pw-2", "pw-3"} { + if strings.Contains(logs, secret) { + t.Errorf("daemon output leaks %q", secret) + } + } + if targets, _ := proxy.snapshot(); !contains(targets, "CONNECT app.relay.test:443") { + t.Errorf("the app's CONNECTs never reached the egress proxy: %q", targets) + } +} + +// Without a refresh command the relay only serves the daemon's own +// http.DefaultTransport: the environment the apps inherit is exactly the +// daemon's own, and stays as valid as the daemon's, so it is left alone. +// Without a proxy there is no relay at all. +func TestRelayExportedOnlyWithProxyCommand(t *testing.T) { + for _, k := range append(append([]string(nil), proxyEnvVars...), "PILOT_PROXY") { + t.Setenv(k, "") + } + t.Cleanup(func() { activeProxyRelay.Store(nil) }) + t.Setenv("HTTPS_PROXY", "http://muse:s3cret@egress.test:3128") + r, err := resolveProxy("auto", "", "compat") + if err != nil { + t.Fatal(err) + } + rl := startProxyRelay(r, "") + if rl == nil { + t.Fatal("no relay for a proxying daemon") + } + rl.Close() + if got := os.Getenv("HTTPS_PROXY"); got != "http://muse:s3cret@egress.test:3128" { + t.Fatalf("HTTPS_PROXY changed to %q without a proxy command", proxyconf.Redact(got)) + } + if rl := startProxyRelay(nil, "cat /dev/null"); rl != nil { + rl.Close() + t.Fatal("relay started without a proxy") + } + off, _ := resolveProxy("off", "", "compat") + if rl := startProxyRelay(off, ""); rl != nil { + rl.Close() + t.Fatal("relay started with -proxy=off") + } + if proxyCommandFor("off", "cmd", "compat", false) != "" || proxyCommandFor("auto", "cmd", "udp", false) != "" || + proxyCommandFor("auto", " cmd ", "compat", false) != "cmd" || proxyCommandFor("http://p.test:1", "cmd", "udp", false) != "cmd" { + t.Error("proxyCommandFor") + } +} + +// With a refresh command the daemon starts the relay and exports it to +// the apps it spawns; its own refresh command keeps running in the launch +// environment, and a refresh that would make the relay the daemon's own +// proxy is refused. +func TestAppRelayEnvironment(t *testing.T) { + for _, k := range append(append([]string(nil), proxyEnvVars...), "PILOT_PROXY", "PROBE_PROXY") { + t.Setenv(k, "") + } + proxy := newRefusingProxy(t, "muse", "s3cret") + launch := fmt.Sprintf("http://muse:s3cret@%s", proxy.ln.Addr()) + t.Setenv("HTTPS_PROXY", launch) + t.Setenv("https_proxy", launch) + t.Setenv("HTTP_PROXY", launch) + t.Setenv("PILOT_PROXY", launch) + saved := launchEnvironment + launchEnvironment = os.Environ() + t.Cleanup(func() { launchEnvironment = saved; activeProxyRelay.Store(nil) }) + + const command = `printf %s "$HTTPS_PROXY"` + r, err := resolveProxy("auto", command, "compat") + if err != nil { + t.Fatal(err) + } + if got := describeProxy("auto", "compat", r); !strings.Contains(got, "(credentials refreshed by command)") || strings.Contains(got, "s3cret") { + t.Fatalf("describeProxy = %q", got) + } + relay := startProxyRelay(r, command) + if relay == nil { + t.Fatal("no relay with a proxy command") + } + defer relay.Close() + want := relay.URL().String() + for _, k := range []string{"HTTPS_PROXY", "https_proxy", "PILOT_PROXY"} { + if got := os.Getenv(k); got != want { + t.Errorf("%s = %q, want the relay URL", k, proxyconf.Redact(got)) + } + } + if got := os.Getenv("HTTP_PROXY"); got != launch { + t.Errorf("HTTP_PROXY = %q, want it left alone (the relay only tunnels)", proxyconf.Redact(got)) + } + + // The daemon's own refresh still sees the launch environment ... + if err := r.Refresh(context.Background()); err != nil { + t.Fatalf("refresh after the relay started: %v", err) + } + if got := proxyconf.ProxyFor(r, "registry.pilotprotocol.network:443"); got != proxyconf.Redact(launch) { + t.Fatalf("the daemon's proxy is now %q, want the egress proxy %q", got, proxyconf.Redact(launch)) + } + // ... and a command that prints the relay (a launch environment that + // already named it, say) is refused: the last good URL stays. + launchEnvironment = os.Environ() + loopy, err := resolveProxy(launch, command, "compat") + if err != nil { + t.Fatal(err) + } + if err := loopy.Refresh(context.Background()); err == nil || !strings.Contains(err.Error(), "own proxy relay") { + t.Fatalf("refresh printing the relay = %v, want refused", err) + } + if got := proxyconf.ProxyFor(loopy, "registry.pilotprotocol.network:443"); got != proxyconf.Redact(launch) { + t.Fatalf("the daemon's proxy is %q after a refresh printed its relay, want %q", got, proxyconf.Redact(launch)) + } + + // PILOT_PROXY is only replaced when it names a proxy URL. + for _, v := range []string{"auto", "off", ""} { + t.Setenv("PILOT_PROXY", v) + exportAppProxy(want) + if got := os.Getenv("PILOT_PROXY"); got != v { + t.Errorf("PILOT_PROXY=%q became %q", v, proxyconf.Redact(got)) + } + } +} + +// web4-470-muse-rejection-diagnostics-misdirect: the hint for a proxy that +// could not be parsed names the credentials and -proxy-cmd, and no hint +// sends a proxy-only host around the proxy unconditionally. +func TestProxyErrorHintGarbledRejection(t *testing.T) { + proxy := newRefusingProxy(t, "muse", "right") + proxy.rotate("muse", "right") + proxy.garbleRejections() + r, err := resolveProxy(fmt.Sprintf("http://muse:wrong@%s", proxy.ln.Addr()), "", "compat") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, dialErr := proxyconf.DialContext(r, nil)(ctx, "tcp", "registry.pilotprotocol.network:443") + if dialErr == nil { + t.Fatal("dial with the wrong password succeeded") + } + hint := proxyErrorHint(dialErr) + if !strings.Contains(hint, "could not be parsed") || !strings.Contains(hint, "-proxy-cmd") || strings.Contains(hint, "udp") { + t.Errorf("garbled rejection hint = %q", hint) + } + unreachable := proxyErrorHint(errors.New("proxy CONNECT registry.pilotprotocol.network:443: dial proxy http://***@127.0.0.1:1: connection refused")) + if !strings.Contains(unreachable, "only if this host can reach the internet without the proxy") { + t.Errorf("unreachable-proxy hint = %q", unreachable) + } + var ce *netproxy.ConnectError + if errors.As(dialErr, &ce) { + t.Fatalf("garbled answer surfaced as a ConnectError: %v", dialErr) + } +} + +// End to end with the real daemon: a wrong password under -transport=auto, +// answered the Muse way. The WARN hint and the registry dial errors name +// the credentials, not -transport=udp. +func TestAutoTransportGarbledRefusalHintsCredentials(t *testing.T) { + proxy := newRefusingProxy(t, "muse", "right-pass") + proxy.rotate("muse", "right-pass") + proxy.garbleRejections() + _, logs := runDaemon(t, proxy, daemonRun{ + args: []string{ + "--registry", defaultRegistryAddr, + "--beacon", silentUDP(t), + "-compat-beacon", defaultCompatBeacon, + "-transport=auto", + "-registry-fingerprint=" + strings.Repeat("00", 32), + }, + env: []string{"HTTPS_PROXY=" + fmt.Sprintf("http://muse:wrong-pass@%s", proxy.ln.Addr())}, + await: "CONNECT registry.pilotprotocol.network:443", + }) + if !strings.Contains(logs, `level=WARN msg="transport auto-selected" transport=compat`) { + t.Fatalf("auto did not stay on compat with a WARN:\n%s", logs) + } + for _, line := range strings.Split(logs, "\n") { + if !strings.Contains(line, "transport auto-selected") { + continue + } + if !strings.Contains(line, "could not be parsed") || !strings.Contains(line, "proxy-cmd") { + t.Errorf("auto-selected WARN lacks the credential hint: %s", line) + } + if strings.Contains(line, "-transport=udp") { + t.Errorf("auto-selected WARN suggests -transport=udp: %s", line) + } + } + if strings.Contains(logs, "4O7") || strings.Contains(logs, "wrong-pass") { + t.Errorf("daemon output quotes the proxy's answer or the password:\n%s", logs) + } +} diff --git a/cmd/daemon/proxy_test.go b/cmd/daemon/proxy_test.go index 0146256b..d22f37dc 100644 --- a/cmd/daemon/proxy_test.go +++ b/cmd/daemon/proxy_test.go @@ -123,6 +123,7 @@ type refusingProxy struct { goodAuths map[string]int // Proxy-Authorization value -> accepted requests forwards map[string]string // CONNECT target -> local address tunnelled to reply407 bool // answer bad credentials with 407 (rotation) + garble bool // ... with an unparseable status line instead (Meta Muse) } // forward makes the proxy tunnel CONNECT target to the local address to. @@ -135,6 +136,14 @@ func (p *refusingProxy) forward(target, to string) { p.forwards[target] = to } +// garbleRejections makes the proxy answer rejected credentials the way +// Meta Muse's does: with a status line net/http cannot parse. +func (p *refusingProxy) garbleRejections() { + p.mu.Lock() + defer p.mu.Unlock() + p.garble = true +} + // rotate makes the proxy accept only user:pass from now on, answering // anything else with 407 Proxy Authentication Required — the way a // sandbox egress proxy rotates its credentials. @@ -193,6 +202,7 @@ func newRefusingProxy(t *testing.T, user, pass string) *refusingProxy { p.badAuths++ } reply407 := !authOK && p.reply407 + garble := p.garble connect := authOK && r.Method == http.MethodConnect to := "" if connect { @@ -200,6 +210,8 @@ func newRefusingProxy(t *testing.T, user, pass string) *refusingProxy { } p.mu.Unlock() switch { + case reply407 && garble: + fmt.Fprint(conn, "HTTP/1.1 4O7 Proxy Authentication Required\r\n\r\n") case reply407: fmt.Fprint(conn, "HTTP/1.1 407 Proxy Authentication Required\r\nProxy-Authenticate: Basic realm=\"muse\"\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") case to != "": @@ -322,6 +334,8 @@ type daemonRun struct { env []string config string await string + // setup, when set, prepares the daemon's $HOME before it starts. + setup func(home string) } // runDaemon starts main() in a child process and returns the proxy's @@ -381,6 +395,9 @@ func startDaemon(t *testing.T, run daemonRun) *runningDaemon { t.Fatal(err) } t.Cleanup(func() { os.RemoveAll(sockDir) }) + if run.setup != nil { + run.setup(home) + } if run.config != "" { if err := os.MkdirAll(filepath.Join(home, ".pilot"), 0o700); err != nil { t.Fatal(err) @@ -421,6 +438,9 @@ func startDaemon(t *testing.T, run daemonRun) *runningDaemon { out := &syncBuffer{} cmd.Stdout = out cmd.Stderr = out + // An app the daemon spawned shares these pipes; one left running + // after a hard stop must not hang Wait (and the test) forever. + cmd.WaitDelay = 5 * time.Second if err := cmd.Start(); err != nil { cancel() t.Fatalf("start daemon: %v", err) diff --git a/cmd/pilotctl/daemon_startlog.go b/cmd/pilotctl/daemon_startlog.go index 5ce3a93f..97c229d0 100644 --- a/cmd/pilotctl/daemon_startlog.go +++ b/cmd/pilotctl/daemon_startlog.go @@ -202,11 +202,19 @@ func daemonExitStatus(err error) string { return "exit status 0" } +// rotateHint is the second half of the credential hints: how to hand the +// daemon rotating credentials. +const rotateHint = "if the proxy rotates its credentials, give the daemon a command that prints the current proxy URL: --proxy-cmd, or pilotctl config --set proxy_cmd=\"bash -c 'printf %s \\\"\\$https_proxy\\\"'\"" + // proxyErrorHint says what to do about a proxy error the daemon logged. func proxyErrorHint(proxyErr string) string { switch { case strings.Contains(proxyErr, " 407 "): - return "the proxy rejected the credentials (407): check HTTPS_PROXY (or --proxy / PILOT_PROXY); if the proxy rotates its credentials, give the daemon a command that prints the current proxy URL: --proxy-cmd, or pilotctl config --set proxy_cmd=\"bash -c 'printf %s \\\"\\$https_proxy\\\"'\"" + return "the proxy rejected the credentials (407): check HTTPS_PROXY (or --proxy / PILOT_PROXY); " + rotateHint + case strings.Contains(proxyErr, "read CONNECT response: ") && strings.Contains(proxyErr, "(response text withheld)"): + // netproxy's report of a CONNECT answer it could not parse — how + // Meta Muse's proxy rejects wrong or expired credentials. + return "the proxy's answer to CONNECT could not be parsed (\"malformed HTTP status code\"), which is how some egress proxies (Meta Muse's) reject wrong or expired credentials: check the credentials in HTTPS_PROXY (or --proxy / PILOT_PROXY); " + rotateHint case strings.Contains(proxyErr, ": dial proxy "): return "the proxy could not be reached: check HTTPS_PROXY (or --proxy / PILOT_PROXY)" default: diff --git a/cmd/pilotctl/daemon_transport.go b/cmd/pilotctl/daemon_transport.go index e296cab4..6f937697 100644 --- a/cmd/pilotctl/daemon_transport.go +++ b/cmd/pilotctl/daemon_transport.go @@ -223,7 +223,16 @@ func setEnv(env []string, key, value string) []string { // sandboxProxyCmd is the proxy_cmd for hosted agent sandboxes: a fresh bash // sees the sandbox's current proxy URL, whose credentials rotate (Meta Muse: // every few minutes). install.sh saves the same command. -const sandboxProxyCmd = `bash -c 'printf %s "${https_proxy:-$HTTPS_PROXY}"'` +// +// It prints whichever of $https_proxy and $HTTPS_PROXY carries credentials, +// $https_proxy when both do (the variable Meta Muse's own guidance reads +// from a fresh shell), and ${HTTPS_PROXY:-$https_proxy} — the daemon's own +// order — when neither does. It is injected (see sandboxRefreshCmd) when +// either variable carries credentials, so it never trades a proxy URL with +// credentials for one without them: with HTTPS_PROXY=http://u:p@proxy and +// https_proxy=http://proxy, a plain ${https_proxy:-$HTTPS_PROXY} would +// drop the credentials the daemon started with. +const sandboxProxyCmd = `bash -c 'case $https_proxy in *@*) printf %s "$https_proxy";; *) printf %s "${HTTPS_PROXY:-$https_proxy}";; esac'` // Test seams for sandboxProxyCmdFor. var ( diff --git a/cmd/pilotctl/main.go b/cmd/pilotctl/main.go index ee2fcba7..5bd62f12 100644 --- a/cmd/pilotctl/main.go +++ b/cmd/pilotctl/main.go @@ -1099,13 +1099,17 @@ configured proxy. To skip the UDP probe: pilotctl config --set transport=compat && pilotctl daemon start If the proxy rotates its credentials, give the daemon a command that prints the current proxy URL; the daemon re-runs it every 60s and whenever the proxy -answers 407, and retries that connection once with the new credentials -(pilotctl's own registry commands use the same command): +rejects the credentials (407, or an answer that cannot be parsed), and +retries that connection once with the new credentials (pilotctl's own +registry commands use the same command): pilotctl config --set proxy_cmd="bash -c 'printf %s \"\$https_proxy\"'" On Linux without systemd (containers, hosted agent sandboxes), when -$HTTPS_PROXY carries credentials and no proxy_cmd is configured, daemon start -passes the daemon PILOT_PROXY_CMD=bash -c 'printf %s "${https_proxy:-$HTTPS_PROXY}"' -by itself. +$HTTPS_PROXY or $https_proxy carries credentials and no proxy_cmd is +configured, daemon start passes the daemon a PILOT_PROXY_CMD by itself that +prints a fresh bash's $https_proxy ($HTTPS_PROXY when only that one carries +credentials). With a proxy command, the apps the daemon starts reach the +proxy through a loopback relay in the daemon that adds the current +credentials, so they keep working across rotations too. If the daemon exits during startup or does not become ready in time, daemon start reports the daemon's last error — the proxy's answer (e.g. "407 Proxy Authentication Required") when a proxy is involved — with the log path. @@ -3334,7 +3338,7 @@ func cmdDaemonStart(args []string) { fmt.Printf(" Proxy: %s\n", redactProxyURL(plan.Proxy)) } if src := proxyCmdSource(plan, flags, pidLogPath, sandboxRefresh); src != "" { - fmt.Printf(" Proxy credentials: re-read every 60s and on a 407 (%s)\n", src) + fmt.Printf(" Proxy credentials: re-read every 60s and when the proxy rejects them, for the daemon and its apps (%s)\n", src) } fmt.Printf(" Socket: %s\n", socketPath) fmt.Printf(" Logs: %s\n", pidLogPath) diff --git a/cmd/pilotctl/zz_proxy_muse_hints_test.go b/cmd/pilotctl/zz_proxy_muse_hints_test.go new file mode 100644 index 00000000..3a218755 --- /dev/null +++ b/cmd/pilotctl/zz_proxy_muse_hints_test.go @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/pilot-protocol/common/netproxy" + "github.com/pilot-protocol/pilotprotocol/internal/proxyconf" +) + +// web4-470-muse-rejection-diagnostics-misdirect: Meta Muse's proxy answers +// wrong or expired credentials with a status line net/http cannot parse, +// which netproxy reports as "read CONNECT response: malformed HTTP status +// code (response text withheld)". `daemon start` must call that a +// credential problem and point at proxy_cmd, not tell the operator that +// the proxy must allow CONNECT to the registry. +func TestCLIDaemonStartGarbledProxyAnswerHint(t *testing.T) { + const garbled = "proxy CONNECT registry.pilotprotocol.network:443 via http://***@127.0.0.1:3151: read CONNECT response: malformed HTTP status code (response text withheld)" + hint := proxyErrorHint(garbled) + if !strings.Contains(hint, "could not be parsed") || !strings.Contains(hint, "proxy_cmd") || strings.Contains(hint, "must allow CONNECT") { + t.Errorf("proxyErrorHint(garbled) = %q", hint) + } + if got := extractProxyError(`time=x level=WARN msg="registry dial failed, retrying" attempt=1 max=10 backoff=500ms error="dial registry TLS pinned: ` + garbled + `"`); got != garbled { + t.Errorf("extractProxyError = %q, want %q", got, garbled) + } + for _, other := range []string{ + "proxy CONNECT registry.pilotprotocol.network:443: 403 Forbidden", + "proxy CONNECT registry.pilotprotocol.network:443 via http://***@p:1: read CONNECT response: EOF", + } { + if h := proxyErrorHint(other); strings.Contains(h, "could not be parsed") { + t.Errorf("proxyErrorHint(%q) = %q, not a garbled answer", other, h) + } + } + + home := t.TempDir() + bin := writeStartFailDaemon(t, []string{ + `time=2026-09-24T06:12:45Z level=WARN msg="transport auto-selected" transport=compat reason="no UDP answer from beacon b, and the proxy http://***@127.0.0.1:3151 refused the compat beacon check (proxy CONNECT beacon.pilotprotocol.network:443 via http://***@127.0.0.1:3151: read CONNECT response: malformed HTTP status code (response text withheld)); staying on compat"`, + `time=2026-09-24T06:12:46Z level=WARN msg="registry dial failed, retrying" attempt=1 max=10 backoff=500ms error="dial registry TLS pinned: ` + garbled + `"`, + `time=2026-09-24T06:13:30Z level=ERROR msg="daemon start: registry dial (after 10 attempts): dial registry TLS pinned: ` + garbled + `; the proxy's answer to CONNECT could not be parsed"`, + }, 1) + env := cliEnvCleared(map[string]string{ + "PILOT_DAEMON_BIN": bin, + "PILOT_SOCKET": filepath.Join(t.TempDir(), "pilot.sock"), + "PILOT_HOME": home, + }) + _, stderr, code := runCLI(t, []string{"--json", "daemon", "start", "--wait", "20s"}, env) + if code == 0 { + t.Fatal("daemon start succeeded against a daemon that exited") + } + var res struct{ Code, Message, Hint string } + if err := json.Unmarshal([]byte(strings.TrimSpace(stderr)), &res); err != nil { + t.Fatalf("stderr is not one JSON error: %v\n%s", err, stderr) + } + if !strings.Contains(res.Message, garbled) { + t.Errorf("message lacks the proxy error: %q", res.Message) + } + if !strings.Contains(res.Hint, "could not be parsed") || !strings.Contains(res.Hint, "proxy_cmd") || strings.Contains(res.Hint, "must allow CONNECT") { + t.Errorf("hint = %q, want the credential hint", res.Hint) + } +} + +// web4-470-sandbox-cmd-variable-precedence: the refresh command pilotctl +// hands the daemon in a sandbox (and install.sh saves) must never trade a +// proxy URL with credentials for one without: whichever of $https_proxy +// and $HTTPS_PROXY carries credentials wins, $https_proxy when both do +// (Meta Muse's guidance reads that one from a fresh shell), and the +// daemon's own order when neither does. Run for real through bash, the +// way the daemon's resolver runs it. +func TestSandboxProxyCmdKeepsTheCredentialedProxy(t *testing.T) { + if _, err := exec.LookPath("bash"); err != nil { + t.Skip("no bash") + } + const ( + upper = "http://corp:s3cret@egress.test:3128" + upperBare = "http://egress.test:3128" + lower = "http://muse:rotated@muse-proxy.test:3128" + lowerBare = "http://other.test:3128" + ) + for _, tc := range []struct { + name, upper, lower, want string + }{ + {"HTTPS_PROXY has credentials, https_proxy names the proxy without", upper, upperBare, upper}, + {"HTTPS_PROXY has credentials, https_proxy another proxy without", upper, lowerBare, upper}, + {"HTTPS_PROXY only", upper, "", upper}, + {"https_proxy has credentials, HTTPS_PROXY without", upperBare, lower, lower}, + {"https_proxy only (Meta Muse)", "", lower, lower}, + {"both with credentials: the fresh shell's https_proxy", upper, lower, lower}, + {"neither with credentials: the daemon's order", upperBare, lowerBare, upperBare}, + } { + t.Run(tc.name, func(t *testing.T) { + cmd := exec.Command("sh", "-c", sandboxProxyCmd) + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HTTPS_PROXY=" + tc.upper, "https_proxy=" + tc.lower} + out, err := cmd.Output() + if err != nil || string(out) != tc.want { + t.Fatalf("sandboxProxyCmd printed (%q, %v), want %q", out, err, tc.want) + } + }) + } + + // What the daemon's resolver makes of it, in the reviewer's case. + withTransportEnvCleared(t) + t.Setenv("HTTPS_PROXY", upper) + t.Setenv("https_proxy", upperBare) + r, err := proxyconf.Resolve("auto", netproxy.WithRefreshCommand(sandboxProxyCmd)) + if err != nil { + t.Fatal(err) + } + if got := proxyconf.ProxyFor(r, "registry.pilotprotocol.network:443"); got != "http://***@egress.test:3128" { + t.Fatalf("daemon proxy with the sandbox command = %q, want the credentialed http://***@egress.test:3128", got) + } +} diff --git a/install.sh b/install.sh index f6dd4b95..38be12ad 100755 --- a/install.sh +++ b/install.sh @@ -1305,7 +1305,7 @@ fi # Linux container/VM without systemd whose proxy carries credentials, and # never over an existing proxy_cmd. # shellcheck disable=SC2016 # literal: the fresh bash expands it, not this shell -SANDBOX_PROXY_CMD='bash -c '\''printf %s "${https_proxy:-$HTTPS_PROXY}"'\''' +SANDBOX_PROXY_CMD='bash -c '\''case $https_proxy in *@*) printf %s "$https_proxy";; *) printf %s "${HTTPS_PROXY:-$https_proxy}";; esac'\''' PROXY_CMD_TO_SAVE="${PILOT_PROXY_CMD:-}" if [ -z "$PROXY_CMD_TO_SAVE" ] && [ "$OS" = "linux" ] && [ ! -d /run/systemd/system ] \ && command -v bash >/dev/null 2>&1 \ diff --git a/internal/proxyconf/command.go b/internal/proxyconf/command.go new file mode 100644 index 00000000..d0a2004b --- /dev/null +++ b/internal/proxyconf/command.go @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package proxyconf + +import ( + "context" + "errors" + "fmt" + "os/exec" + "time" +) + +// maxCommandOutput caps what a refresh command may print (as netproxy's). +const maxCommandOutput = 64 << 10 + +// CommandSource returns a refresh source for netproxy.WithRefreshFunc that +// runs command the way netproxy.WithRefreshCommand does — "sh -c", stdin +// and stderr discarded (stderr could echo credentials, e.g. under set -x), +// at most 64 KiB of output, and on Unix in a process group of its own that +// is killed as a whole when the refresh deadline passes — except that it +// runs in env instead of this process's current environment. +// +// The daemon needs that because it points its own HTTPS_PROXY at its app +// Relay (so the apps it starts inherit the Relay, see Relay) once it is +// running. A refresh command such as bash -c 'printf %s "$https_proxy"' +// must still see what the daemon was launched with — in Meta Muse a fresh +// bash then reads the rotated credentials — and never the Relay, which +// would make the daemon's proxy its own Relay. A nil env runs the command +// in this process's environment, as netproxy does. +// +// The output is never part of an error. +func CommandSource(command string, env []string) func(ctx context.Context) (string, error) { + if env != nil { + env = append(make([]string, 0, len(env)), env...) + } + return func(ctx context.Context) (string, error) { + cmd := exec.CommandContext(ctx, "sh", "-c", command) + if env != nil { + cmd.Env = env + } + var out cappedOutput + cmd.Stdout = &out + cmd.WaitDelay = time.Second // a child left holding stdout cannot hang Wait + ownProcessGroup(cmd) + err := cmd.Run() + switch { + case ctx.Err() != nil: + return "", errors.New("refresh command timed out") + case err != nil: + var ee *exec.ExitError + if errors.As(err, &ee) { + return "", fmt.Errorf("refresh command failed: %s", ee.ProcessState) + } + if errors.Is(err, exec.ErrWaitDelay) { + // sh has exited, but a process it started held stdout + // open until now: end the group. + _ = killProcessGroup(cmd) + return "", errors.New("refresh command left a process holding its output open") + } + return "", fmt.Errorf("refresh command failed to run: %v", err) + case out.overflow: + return "", fmt.Errorf("refresh command printed more than %d bytes", maxCommandOutput) + } + return string(out.buf), nil + } +} + +// cappedOutput keeps the first maxCommandOutput bytes written to it and +// notes whether there were more. +type cappedOutput struct { + buf []byte + overflow bool +} + +func (b *cappedOutput) Write(p []byte) (int, error) { + if room := maxCommandOutput - len(b.buf); len(p) > room { + b.buf = append(b.buf, p[:room]...) + b.overflow = true + return len(p), nil + } + b.buf = append(b.buf, p...) + return len(p), nil +} diff --git a/internal/proxyconf/command_other.go b/internal/proxyconf/command_other.go new file mode 100644 index 00000000..2628d5a4 --- /dev/null +++ b/internal/proxyconf/command_other.go @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +//go:build !unix + +package proxyconf + +import "os/exec" + +// ownProcessGroup leaves cmd as it is: without Unix process groups, a +// timed-out refresh command is killed on its own (exec.Cmd's default). +func ownProcessGroup(*exec.Cmd) {} + +// killProcessGroup is a no-op without Unix process groups. +func killProcessGroup(*exec.Cmd) error { return nil } diff --git a/internal/proxyconf/command_test.go b/internal/proxyconf/command_test.go new file mode 100644 index 00000000..e57f0e4c --- /dev/null +++ b/internal/proxyconf/command_test.go @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package proxyconf + +import ( + "context" + "os" + "strings" + "testing" +) + +// CommandSource runs the command in the environment it was given, not in +// this process's current one: the daemon points its own HTTPS_PROXY at its +// app relay, and the refresh command must keep seeing the launch +// environment (and never print the relay). +func TestCommandSourceUsesGivenEnvironment(t *testing.T) { + t.Setenv("PROBE_PROXY", "http://pilot-relay:tok@127.0.0.1:1") // what the daemon exports later + launch := []string{"PATH=" + os.Getenv("PATH"), "PROBE_PROXY=http://muse:launch@egress.test:3128"} + const cmd = `printf %s "$PROBE_PROXY"` + src := CommandSource(cmd, launch) + launch[1] = "PROBE_PROXY=changed" // CommandSource keeps its own copy + out, err := src(context.Background()) + if err != nil || out != "http://muse:launch@egress.test:3128" { + t.Fatalf("with the launch environment = (%q, %v)", out, err) + } + out, err = CommandSource(cmd, nil)(context.Background()) + if err != nil || out != "http://pilot-relay:tok@127.0.0.1:1" { + t.Fatalf("nil environment = (%q, %v), want this process's", out, err) + } + out, err = CommandSource(cmd, []string{})(context.Background()) + if err != nil || out != "" { + t.Fatalf("empty environment = (%q, %v), want nothing inherited", out, err) + } +} + +// Failures never carry the command's output (it would hold credentials), +// and a command that outlives the deadline is killed with everything it +// started. +func TestCommandSourceFailures(t *testing.T) { + env := []string{"PATH=" + os.Getenv("PATH")} + _, err := CommandSource("echo http://muse:s3cret@proxy.test; echo oops >&2; exit 3", env)(context.Background()) + if err == nil || !strings.Contains(err.Error(), "exit status 3") || strings.Contains(err.Error(), "s3cret") || strings.Contains(err.Error(), "oops") { + t.Fatalf("failing command: %v", err) + } + _, err = CommandSource("head -c 70000 /dev/zero", env)(context.Background()) + if err == nil || !strings.Contains(err.Error(), "more than") { + t.Fatalf("oversized output: %v", err) + } +} diff --git a/internal/proxyconf/command_unix.go b/internal/proxyconf/command_unix.go new file mode 100644 index 00000000..b314c462 --- /dev/null +++ b/internal/proxyconf/command_unix.go @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +//go:build unix + +package proxyconf + +import ( + "errors" + "os" + "os/exec" + "syscall" +) + +// ownProcessGroup makes cmd the leader of a new process group and has its +// context's cancellation SIGKILL that whole group, so nothing a timed-out +// refresh command started outlives it. +func ownProcessGroup(cmd *exec.Cmd) { + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.Cancel = func() error { return killProcessGroup(cmd) } +} + +// killProcessGroup SIGKILLs the process group cmd leads. +func killProcessGroup(cmd *exec.Cmd) error { + if cmd.Process == nil { + return nil + } + err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + if errors.Is(err, syscall.ESRCH) { + return os.ErrProcessDone + } + return err +} diff --git a/internal/proxyconf/command_unix_test.go b/internal/proxyconf/command_unix_test.go new file mode 100644 index 00000000..5a024fe7 --- /dev/null +++ b/internal/proxyconf/command_unix_test.go @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +//go:build unix + +package proxyconf + +import ( + "context" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" +) + +// A refresh command that outlives the deadline is killed with everything +// it started, and the refresh fails promptly. +func TestCommandSourceTimeoutKillsTheGroup(t *testing.T) { + env := []string{"PATH=" + os.Getenv("PATH")} + pidFile := filepath.Join(t.TempDir(), "pid") + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + start := time.Now() + _, err := CommandSource("sleep 30 & echo $! > '"+pidFile+"'; wait", env)(ctx) + if err == nil || !strings.Contains(err.Error(), "timed out") { + t.Fatalf("hung command: %v", err) + } + if d := time.Since(start); d > 5*time.Second { + t.Fatalf("hung command took %v to fail", d) + } + b, err := os.ReadFile(pidFile) + if err != nil { + t.Fatal(err) + } + pid, _ := strconv.Atoi(strings.TrimSpace(string(b))) + deadline := time.Now().Add(5 * time.Second) + for syscall.Kill(pid, 0) == nil { + if time.Now().After(deadline) { + t.Fatalf("the command's background child %d outlived the timeout", pid) + } + time.Sleep(20 * time.Millisecond) + } +} diff --git a/internal/proxyconf/proxyconf.go b/internal/proxyconf/proxyconf.go index c2aa9dc8..71958155 100644 --- a/internal/proxyconf/proxyconf.go +++ b/internal/proxyconf/proxyconf.go @@ -18,12 +18,15 @@ // scheme, or another scheme — is rejected, so a typo can never turn into a // proxy host name. Errors and display strings never contain credentials. // -// Rotating credentials are netproxy's business: a Resolver built with -// netproxy.WithRefreshCommand (the -proxy-cmd / $PILOT_PROXY_CMD / -// config.json proxy_cmd setting) re-reads the proxy URL every -// netproxy.DefaultRefreshInterval and whenever a proxy answers 407, and -// netproxy's Dialer and RefreshingTransport retry that connection once with -// the new credentials. This package only adds the loopback rule on top. +// Rotating credentials are mostly netproxy's business: a Resolver with a +// refresh source (the -proxy-cmd / $PILOT_PROXY_CMD / config.json proxy_cmd +// setting; see CommandSource) re-reads the proxy URL every +// netproxy.DefaultRefreshInterval and whenever a proxy rejects the +// credentials, and netproxy's Dialer and RefreshingTransport retry that +// connection once with the new ones. This package adds the loopback rule on +// top, and the Relay: a loopback CONNECT proxy that gives the processes the +// daemon starts, which cannot re-read anything, the same refreshed +// credentials. package proxyconf import ( @@ -262,17 +265,45 @@ func (t *loopbackSplit) CloseIdleConnections() { // ConfigureTransport routes tr (in place) through r, for transports that // cannot be wrapped — http.DefaultTransport, which plugin HTTP clients use -// or clone: tr.Proxy follows r's current settings (loopback always -// direct), and a 407 answer to a CONNECT refreshes r before the request -// fails, so the next request carries the new credentials. (Wrapped clients, -// see RoundTripper, also retry the failed request.) Every refused CONNECT -// fails with a *netproxy.ConnectError, whose message never quotes the -// proxy's reason phrase. A nil r leaves tr alone. -func ConfigureTransport(tr *http.Transport, r *netproxy.Resolver) { +// or clone. tr.Proxy follows r's current settings, loopback always direct. +// +// relay, when set, is the URL of a Relay for r (Relay.URL): requests that +// net/http tunnels with CONNECT (https://, wss://) then go to the proxy +// through it, and the Relay's netproxy.Dialer handles a rejection of the +// credentials — a 407, or a CONNECT answer so garbled it cannot be parsed +// (Meta Muse's form) — by refreshing r and retrying the tunnel once, so the +// request succeeds, and no proxy-supplied text reaches an error. Plain +// http:// requests, which a proxy forwards rather than tunnels, keep going +// to the proxy r names. The daemon passes its Relay whenever it proxies. +// +// Without a relay, a 407 answer to a CONNECT refreshes r before the +// request fails, so the next request carries the new credentials, and the +// refusal fails with a *netproxy.ConnectError, whose message never quotes +// the proxy's reason phrase. An answer net/http cannot parse fails with +// net/http's own error (which quotes the offending status text) and +// refreshes nothing: net/http never passes it to this hook. (Wrapped +// clients, see RoundTripper, handle both forms and also retry the failed +// request.) A nil r leaves tr alone. +func ConfigureTransport(tr *http.Transport, r *netproxy.Resolver, relay *url.URL) { if tr == nil || r == nil { return } - tr.Proxy = RequestProxy(r) + direct := RequestProxy(r) + relayHost := "" + if relay == nil { + tr.Proxy = direct + } else { + via := *relay + relayHost = via.Host + tr.Proxy = func(req *http.Request) (*url.URL, error) { + u, err := direct(req) + if err != nil || u == nil || !tunnelled(req) { + return u, err + } + v := via + return &v, nil + } + } next := tr.OnProxyConnectResponse tr.OnProxyConnectResponse = func(ctx context.Context, proxyURL *url.URL, connectReq *http.Request, res *http.Response) error { if next != nil { @@ -283,11 +314,59 @@ func ConfigureTransport(tr *http.Transport, r *netproxy.Resolver) { if res.StatusCode == http.StatusOK { return nil } + ce := &netproxy.ConnectError{Target: connectReq.Host, StatusCode: res.StatusCode} + if relayHost != "" && proxyURL != nil && proxyURL.Host == relayHost { + // The relay's own answer: its reason phrase carries why the + // tunnel upstream failed, in netproxy's words. + if detail := relayDetail(res.Status); detail != "" { + return &relayRefusal{ConnectError: ce, detail: detail} + } + return ce + } if res.StatusCode == http.StatusProxyAuthRequired { _ = r.Refresh(ctx) // failures keep the last good settings; netproxy reports them } - return &netproxy.ConnectError{Target: connectReq.Host, StatusCode: res.StatusCode} + return ce + } +} + +// relayRefusal is a Relay's refusal of a CONNECT as ConfigureTransport +// reports it: the netproxy error the Relay met upstream (detail, which +// never holds credentials or proxy-supplied text), and the Relay's own +// status as the ConnectError it unwraps to. +type relayRefusal struct { + *netproxy.ConnectError + detail string +} + +func (e *relayRefusal) Error() string { return e.detail } + +func (e *relayRefusal) Unwrap() error { return e.ConnectError } + +// relayDetail extracts the reason writeRelayStatus put in a Relay's status +// line ("502 Bad Gateway (pilot-daemon proxy relay: )"), "" when +// there is none. It is passed through reasonText again: the status line +// came over a socket. +func relayDetail(status string) string { + const marker = " (" + relayReasonPrefix + i := strings.Index(status, marker) + if i < 0 || !strings.HasSuffix(status, ")") { + return "" } + return reasonText(status[i+len(marker) : len(status)-1]) +} + +// tunnelled reports whether net/http reaches req's target through a +// CONNECT tunnel when it uses a proxy. +func tunnelled(req *http.Request) bool { + if req == nil || req.URL == nil { + return false + } + switch strings.ToLower(req.URL.Scheme) { + case "https", "wss": + return true + } + return false } // AuthRejected reports whether err is a proxy's refusal of the credentials @@ -297,6 +376,34 @@ func AuthRejected(err error) bool { return errors.As(err, &ce) && ce.StatusCode == http.StatusProxyAuthRequired } +// UnreadableConnectReply reports whether err is netproxy's report of a +// CONNECT answer it could not parse. Some egress proxies (Meta Muse's) +// answer wrong or expired credentials this way; HTTP clients show it as +// "malformed HTTP status code". netproxy's Dialer and RefreshingTransport +// treat it as a rejection of the credentials (refresh, retry once); its +// error type is unexported, so this matches the fixed wording netproxy +// gives it ("read CONNECT response: (response text withheld)"). +func UnreadableConnectReply(err error) bool { + if err == nil { + return false + } + msg := err.Error() + return strings.Contains(msg, "read CONNECT response: ") && strings.Contains(msg, "(response text withheld)") +} + +// CredentialHint explains a proxy's rejection of the credentials — a 407, +// or an answer that could not be parsed — for a log line or error; "" for +// any other error. +func CredentialHint(err error) string { + switch { + case AuthRejected(err): + return "the proxy rejected its credentials (407), also after re-reading them: check HTTPS_PROXY / -proxy; if the proxy rotates its credentials, set -proxy-cmd ($PILOT_PROXY_CMD, config.json proxy_cmd) to a command that prints the current proxy URL" + case UnreadableConnectReply(err): + return "the proxy's answer to CONNECT could not be parsed, which is how some egress proxies (Meta Muse's) reject wrong or expired credentials: check the credentials in HTTPS_PROXY / -proxy; if the proxy rotates them, set -proxy-cmd ($PILOT_PROXY_CMD, config.json proxy_cmd) to a command that prints the current proxy URL" + } + return "" +} + // unwrapNetproxy drops netproxy's "netproxy: " prefix for messages that // already say which setting failed. func unwrapNetproxy(err error) string { diff --git a/internal/proxyconf/refresh_test.go b/internal/proxyconf/refresh_test.go index 8dcd0a19..e7ddd0b8 100644 --- a/internal/proxyconf/refresh_test.go +++ b/internal/proxyconf/refresh_test.go @@ -35,6 +35,12 @@ type rotatingProxy struct { pass string oks map[string]int // password -> accepted CONNECTs n407 int + // garble answers rejected credentials with a status line net/http + // cannot parse, the way Meta Muse's proxy does ("malformed HTTP + // status code"), instead of a clean 407. + garble bool + // refuse answers every CONNECT with this status (0: none). + refuse int } func newRotatingProxy(t *testing.T, pass, upstream string) *rotatingProxy { @@ -93,11 +99,20 @@ func (p *rotatingProxy) serve(c net.Conn) { } else { p.n407++ } + garble, refuse := p.garble, p.refuse p.mu.Unlock() if req.Method != http.MethodConnect || !good { + if garble { + fmt.Fprint(c, "HTTP/1.1 4O7 Proxy Authentication Required\r\n\r\n") + return + } fmt.Fprint(c, "HTTP/1.1 407 Proxy Authentication Required\r\nProxy-Authenticate: Basic realm=\"t\"\r\nContent-Length: 0\r\n\r\n") return } + if refuse != 0 { + fmt.Fprintf(c, "HTTP/1.1 %d %s\r\nContent-Length: 0\r\n\r\n", refuse, http.StatusText(refuse)) + return + } up, err := net.Dial("tcp", p.upstream) if err != nil { fmt.Fprint(c, "HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n") @@ -279,7 +294,7 @@ func TestRoundTripperRetriesAfterRotation(t *testing.T) { // place by ConfigureTransport — RoundTripper must not let that hook // swallow the 407 before RefreshingTransport sees it. base := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} // #nosec G402 -- test server - ConfigureTransport(base, r) + ConfigureTransport(base, r, nil) rt := RoundTripper(r, base) client := &http.Client{Transport: rt, Timeout: 10 * time.Second} defer client.CloseIdleConnections() @@ -354,7 +369,7 @@ func TestConfigureTransportRefreshesOn407(t *testing.T) { t.Fatal(err) } tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} // #nosec G402 -- test server - ConfigureTransport(tr, r) + ConfigureTransport(tr, r, nil) clone := tr.Clone() // a plugin that cloned DefaultTransport for name, rt := range map[string]*http.Transport{"configured": tr, "clone": clone} { client := &http.Client{Transport: rt, Timeout: 10 * time.Second} @@ -385,9 +400,9 @@ func TestConfigureTransportRefreshesOn407(t *testing.T) { t.Fatalf("%s: accepted %v, want the refreshed %s once", name, oks, pass) } } - ConfigureTransport(nil, r) + ConfigureTransport(nil, r, nil) plain := &http.Transport{} - ConfigureTransport(plain, nil) + ConfigureTransport(plain, nil, nil) if plain.Proxy != nil || plain.OnProxyConnectResponse != nil { t.Error("ConfigureTransport(nil resolver) changed the transport") } diff --git a/internal/proxyconf/relay.go b/internal/proxyconf/relay.go new file mode 100644 index 00000000..37415d8d --- /dev/null +++ b/internal/proxyconf/relay.go @@ -0,0 +1,425 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package proxyconf + +import ( + "bufio" + "context" + "crypto/rand" + "crypto/subtle" + "crypto/tls" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "io" + "log/slog" + "math" + "net" + "net/http" + "net/url" + "strings" + "sync" + "time" + + "github.com/pilot-protocol/common/netproxy" +) + +// RelayUser is the user name in a Relay's URL. The password is a random +// token, different for every Relay. +const RelayUser = "pilot-relay" + +// Relay timeouts. Variables so tests can shorten them. +var ( + // relayRequestTimeout bounds reading a client's CONNECT request. + relayRequestTimeout = 30 * time.Second + // relayDialTimeout bounds opening the tunnel upstream, one credential + // refresh and its retry included (netproxy.Dialer's own limit is 30s). + relayDialTimeout = 45 * time.Second + // relayLogEvery rate-limits the WARN for failed CONNECTs: an app that + // retries in a tight loop must not flood the daemon log. + relayLogEvery = 10 * time.Second +) + +// Relay is a loopback HTTP CONNECT proxy that forwards every tunnel through +// the egress proxy its Resolver picks, with the Resolver's current +// credentials: each CONNECT it accepts is opened upstream by a +// netproxy.Dialer, which refreshes the Resolver and retries once when the +// proxy rejects the credentials (a 407, or an answer so garbled it cannot +// be parsed, which is how Meta Muse's rejections arrive), and whose errors +// never quote the proxy. +// +// The daemon runs one for two kinds of clients: +// +// - the processes it starts (app-store apps). They inherit the daemon's +// environment, and a proxy URL in that environment keeps the +// credentials the daemon was launched with. Where the proxy rotates +// them (Meta Muse: every few minutes), every new connection such a +// process opens is then rejected, while the daemon itself, which +// re-reads its credentials, stays online ("node online, all apps +// broken"). Pointed at the Relay instead (HTTPS_PROXY=Relay.URL), +// those processes never hold the proxy's credentials at all. +// - http.DefaultTransport (see ConfigureTransport), which net/http runs +// itself: it cannot retry a rejected CONNECT, never sees an answer it +// cannot parse, and quotes such an answer in its error. +// +// The Relay listens on loopback only and accepts only CONNECT (it never +// sees inside a tunnel; TLS stays end to end). A client must send the +// Relay's own credentials (RelayUser and a random token, both in URL), so +// another local user cannot borrow the daemon's proxy credentials through +// it. Loopback targets are dialed directly and targets the Resolver does +// not proxy (NO_PROXY) go direct too, as they do for the daemon. When a +// tunnel cannot be opened the client gets 502 Bad Gateway (403 when the +// proxy refused the target, 504 on a timeout), with a reason phrase that +// says why without any credentials or text from the proxy. +type Relay struct { + ln net.Listener + r *netproxy.Resolver + dial func(ctx context.Context, network, addr string) (net.Conn, error) + token string + url *url.URL + ctx context.Context + cancel context.CancelFunc + + mu sync.Mutex + conns map[net.Conn]struct{} + closed bool + wg sync.WaitGroup + + logMu sync.Mutex + lastWarn time.Time + muted int +} + +// StartRelay starts a Relay for r on a loopback port chosen by the system. +// proxyTLS configures the TLS session with an https:// proxy (nil: the +// system roots), as for DialContext. r must proxy something. +func StartRelay(r *netproxy.Resolver, proxyTLS *tls.Config) (*Relay, error) { + if !r.Enabled() { + return nil, errors.New("proxy relay: no proxy to relay to") + } + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + var err6 error + if ln, err6 = net.Listen("tcp", "[::1]:0"); err6 != nil { + return nil, fmt.Errorf("proxy relay: listen on loopback: %w", err) + } + } + return startRelayOn(ln, r, proxyTLS) +} + +// startRelayOn starts a Relay for r on ln, which it takes over. +func startRelayOn(ln net.Listener, r *netproxy.Resolver, proxyTLS *tls.Config) (*Relay, error) { + raw := make([]byte, 24) + if _, err := rand.Read(raw); err != nil { + ln.Close() + return nil, fmt.Errorf("proxy relay: token: %w", err) + } + token := hex.EncodeToString(raw) + ctx, cancel := context.WithCancel(context.Background()) + rl := &Relay{ + ln: ln, + r: r, + dial: DialContext(r, proxyTLS), + token: token, + url: &url.URL{Scheme: "http", User: url.UserPassword(RelayUser, token), Host: ln.Addr().String()}, + ctx: ctx, + cancel: cancel, + conns: map[net.Conn]struct{}{}, + } + go rl.serve() + return rl, nil +} + +// URL returns the proxy URL clients use, http://pilot-relay:@host:port. +// It carries the Relay's token: hand it to child processes (HTTPS_PROXY), +// never log it. +func (rl *Relay) URL() *url.URL { + u := *rl.url + return &u +} + +// Addr returns the "host:port" the Relay listens on. +func (rl *Relay) Addr() string { return rl.ln.Addr().String() } + +// Serves reports whether proxyURL, a proxy URL as a refresh command prints +// it, names this Relay. The daemon never lets its own proxy become its +// Relay: the Relay would then forward to itself. +func (rl *Relay) Serves(proxyURL string) bool { + if rl == nil { + return false + } + s := strings.TrimSpace(proxyURL) + if i := strings.Index(s, "://"); i >= 0 { + s = s[i+3:] + } + if i := strings.LastIndex(s, "@"); i >= 0 { + s = s[i+1:] + } + if i := strings.IndexAny(s, "/?#"); i >= 0 { + s = s[:i] + } + host, port, err := net.SplitHostPort(s) + if err != nil { + return false + } + lhost, lport, _ := net.SplitHostPort(rl.Addr()) + if port != lport { + return false + } + return strings.EqualFold(strings.Trim(host, "[]"), lhost) || IsLoopbackHost(host) +} + +// Close stops the Relay: it stops accepting, closes every open tunnel and +// waits for their goroutines to finish. +func (rl *Relay) Close() error { + if rl == nil { + return nil + } + rl.mu.Lock() + if rl.closed { + rl.mu.Unlock() + return nil + } + rl.closed = true + err := rl.ln.Close() + for c := range rl.conns { + c.Close() + } + rl.mu.Unlock() + rl.cancel() + rl.wg.Wait() + return err +} + +// track registers c so Close can close it; false once the Relay is closed +// (the caller then closes c itself). With add it also counts a goroutine +// Close waits for. +func (rl *Relay) track(c net.Conn, add bool) bool { + rl.mu.Lock() + defer rl.mu.Unlock() + if rl.closed { + return false + } + rl.conns[c] = struct{}{} + if add { + rl.wg.Add(1) + } + return true +} + +func (rl *Relay) untrack(c net.Conn) { + rl.mu.Lock() + delete(rl.conns, c) + rl.mu.Unlock() + c.Close() +} + +func (rl *Relay) serve() { + backoff := 5 * time.Millisecond + for { + c, err := rl.ln.Accept() + if err != nil { + rl.mu.Lock() + closed := rl.closed + rl.mu.Unlock() + if closed || errors.Is(err, net.ErrClosed) { + return + } + // Out of file descriptors, say: back off and keep serving. + time.Sleep(backoff) + if backoff < time.Second { + backoff *= 2 + } + continue + } + backoff = 5 * time.Millisecond + if !rl.track(c, true) { + c.Close() + return + } + go func() { + defer rl.wg.Done() + defer rl.untrack(c) + rl.handle(c) + }() + } +} + +// relayMaxRequest caps a CONNECT request (line and headers). +const relayMaxRequest = 64 << 10 + +// handle serves one client connection: one CONNECT, then the tunnel. +func (rl *Relay) handle(c net.Conn) { + _ = c.SetReadDeadline(time.Now().Add(relayRequestTimeout)) + limited := &io.LimitedReader{R: c, N: relayMaxRequest} + br := bufio.NewReader(limited) + req, err := http.ReadRequest(br) + if err != nil { + writeRelayStatus(c, http.StatusBadRequest, "") + return + } + limited.N = math.MaxInt64 // the tunnel is not capped + if req.Method != http.MethodConnect { + writeRelayStatus(c, http.StatusMethodNotAllowed, "only CONNECT is relayed", "Allow: CONNECT") + return + } + if !rl.authorized(req) { + writeRelayStatus(c, http.StatusProxyAuthRequired, "", `Proxy-Authenticate: Basic realm="pilot-daemon"`) + return + } + target := req.Host + if target == "" && req.URL != nil { + target = req.URL.Host + } + if host, port, err := net.SplitHostPort(target); err != nil || host == "" || port == "" { + writeRelayStatus(c, http.StatusBadRequest, "CONNECT needs host:port") + return + } + if !loopbackAddr(target) { + // The daemon's proxy settings never name the Relay itself (see + // Serves), but a loop would tie up a connection per hop until the + // dial timeout, so refuse one outright. + if u, err := rl.r.ProxyForAddr(target); err == nil && u != nil && rl.Serves(u.String()) { + writeRelayStatus(c, http.StatusLoopDetected, "the proxy for "+target+" is this relay") + return + } + } + + ctx, cancel := context.WithTimeout(rl.ctx, relayDialTimeout) + up, err := rl.dial(ctx, "tcp", target) + cancel() + if err != nil { + rl.warn(target, err) + writeRelayStatus(c, relayFailureStatus(err), err.Error()) + return + } + if !rl.track(up, false) { + up.Close() + return + } + defer rl.untrack(up) + _ = c.SetReadDeadline(time.Time{}) + _ = c.SetWriteDeadline(time.Now().Add(relayRequestTimeout)) + if _, err := io.WriteString(c, "HTTP/1.1 200 Connection established\r\n\r\n"); err != nil { + return + } + _ = c.SetWriteDeadline(time.Time{}) + pipe(c, br, up) +} + +// authorized reports whether req carries the Relay's credentials. +func (rl *Relay) authorized(req *http.Request) bool { + scheme, cred, ok := strings.Cut(strings.TrimSpace(req.Header.Get("Proxy-Authorization")), " ") + if !ok || !strings.EqualFold(scheme, "Basic") { + return false + } + raw, err := base64.StdEncoding.DecodeString(strings.TrimSpace(cred)) + if err != nil { + return false + } + return subtle.ConstantTimeCompare(raw, []byte(RelayUser+":"+rl.token)) == 1 +} + +// warn logs a failed CONNECT at WARN, at most once per relayLogEvery; the +// ones in between are counted in the next line. The error is netproxy's, +// which never holds credentials or proxy-supplied text. +func (rl *Relay) warn(target string, err error) { + rl.logMu.Lock() + now := time.Now() + if !rl.lastWarn.IsZero() && now.Sub(rl.lastWarn) < relayLogEvery { + rl.muted++ + rl.logMu.Unlock() + return + } + muted := rl.muted + rl.lastWarn, rl.muted = now, 0 + rl.logMu.Unlock() + args := []any{"target", target, "err", err} + if muted > 0 { + args = append(args, "more_failures_not_logged", muted) + } + if hint := CredentialHint(err); hint != "" { + args = append(args, "hint", hint) + } + slog.Warn("proxy relay: could not open a tunnel", args...) +} + +// relayFailureStatus is the status the Relay answers a CONNECT with when +// the tunnel could not be opened. A 407 from the egress proxy becomes 502: +// the client's own credentials (the Relay's) were fine, and a 407 would +// send it looking in the wrong place. +func relayFailureStatus(err error) int { + var ce *netproxy.ConnectError + if errors.As(err, &ce) && ce.StatusCode == http.StatusForbidden { + return http.StatusForbidden + } + if errors.Is(err, context.DeadlineExceeded) { + return http.StatusGatewayTimeout + } + var ne net.Error + if errors.As(err, &ne) && ne.Timeout() { + return http.StatusGatewayTimeout + } + return http.StatusBadGateway +} + +// relayReasonPrefix starts the detail in a Relay's reason phrase. +const relayReasonPrefix = "pilot-daemon proxy relay: " + +// writeRelayStatus answers c with an empty response. detail, when set, +// extends the reason phrase (HTTP clients show it: Go's net/http returns +// it as the error text of a failed CONNECT), reduced to printable ASCII. +func writeRelayStatus(c net.Conn, code int, detail string, headers ...string) { + reason := http.StatusText(code) + if detail = reasonText(detail); detail != "" { + reason += " (" + relayReasonPrefix + detail + ")" + } + var b strings.Builder + fmt.Fprintf(&b, "HTTP/1.1 %d %s\r\n", code, reason) + for _, h := range headers { + b.WriteString(h + "\r\n") + } + b.WriteString("Content-Length: 0\r\nConnection: close\r\n\r\n") + _ = c.SetWriteDeadline(time.Now().Add(5 * time.Second)) + _, _ = io.WriteString(c, b.String()) +} + +// reasonText makes s safe for a reason phrase: printable ASCII only, at +// most 300 bytes. +func reasonText(s string) string { + const max = 300 + var b strings.Builder + for i := 0; i < len(s) && b.Len() < max; i++ { + if c := s[i]; c >= 0x20 && c < 0x7f { + b.WriteByte(c) + } else { + b.WriteByte(' ') + } + } + return strings.TrimSpace(b.String()) +} + +// pipe copies client <-> upstream until both directions are done. When one +// side stops sending, its peer's write side is shut down (or, where that is +// not possible, the peer is closed), so the other direction ends too. +func pipe(client net.Conn, clientIn io.Reader, up net.Conn) { + done := make(chan struct{}) + go func() { + defer close(done) + _, _ = io.Copy(up, clientIn) // clientIn drains what was read with the request first + closeWrite(up) + }() + _, _ = io.Copy(client, up) + closeWrite(client) + <-done +} + +func closeWrite(c net.Conn) { + if cw, ok := c.(interface{ CloseWrite() error }); ok { + if cw.CloseWrite() == nil { + return + } + } + c.Close() +} diff --git a/internal/proxyconf/relay_test.go b/internal/proxyconf/relay_test.go new file mode 100644 index 00000000..0476cad2 --- /dev/null +++ b/internal/proxyconf/relay_test.go @@ -0,0 +1,521 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package proxyconf + +import ( + "bufio" + "context" + "crypto/tls" + "encoding/base64" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/pilot-protocol/common/netproxy" +) + +func (p *rotatingProxy) setGarble(on bool) { + p.mu.Lock() + p.garble = on + p.mu.Unlock() +} + +func (p *rotatingProxy) setRefuse(status int) { + p.mu.Lock() + p.refuse = status + p.mu.Unlock() +} + +// countingCommand is a refresh command that prints the rotating URL file +// and counts its runs. +func countingCommand(t *testing.T, urls *urlFile) (command string, runs func() int) { + t.Helper() + counter := filepath.Join(t.TempDir(), "runs") + command = fmt.Sprintf("echo x >> '%s'; cat '%s'", counter, urls.path) + return command, func() int { + b, _ := os.ReadFile(counter) + return strings.Count(string(b), "x") + } +} + +// startTestRelay starts a Relay for r and stops it with the test. +func startTestRelay(t *testing.T, r *netproxy.Resolver) *Relay { + t.Helper() + rl, err := StartRelay(r, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { rl.Close() }) + return rl +} + +// appClient is an app the daemon started: a plain net/http client whose +// proxy comes from its environment, here the Relay's URL. +func appClient(proxyURL *url.URL) *http.Client { + return &http.Client{ + Timeout: 20 * time.Second, + Transport: &http.Transport{ + Proxy: http.ProxyURL(proxyURL), + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // #nosec G402 -- test server + DisableKeepAlives: true, // a new CONNECT per request + }, + } +} + +func getBody(c *http.Client, target string) (string, error) { + resp, err := c.Get(target) + if err != nil { + return "", err + } + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + return string(b), err +} + +// web4-470-apps-inherit-stale-proxy-creds: an app keeps whatever proxy URL +// it inherited. Pointed at the Relay, it never holds the proxy's +// credentials: after the proxy rotates them — and answers the stale ones +// the way Meta Muse does, with a status line that cannot be parsed — the +// app's next request still succeeds, because the Relay refreshes and +// retries the CONNECT upstream. +func TestRelayFollowsRotationForApps(t *testing.T) { + clearEnv(t) + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, "hello ", r.Host) })) + defer srv.Close() + proxy := newRotatingProxy(t, "one", srv.Listener.Addr().String()) + proxy.setGarble(true) + urls := newURLFile(t, proxy, "one") + command, runs := countingCommand(t, urls) + r, err := Resolve("auto", netproxy.WithRefreshFunc(CommandSource(command, nil)), netproxy.WithRefreshInterval(-1)) + if err != nil { + t.Fatal(err) + } + relay := startTestRelay(t, r) + app := appClient(relay.URL()) + + const target = "https://app.pilot.invalid/x" + if got, err := getBody(app, target); err != nil || got != "hello app.pilot.invalid" { + t.Fatalf("GET before the rotation = (%q, %v)", got, err) + } + before := runs() + for i, pass := range []string{"two", "three", "four"} { + urls.rotate(pass) + if got, err := getBody(app, target); err != nil || got != "hello app.pilot.invalid" { + t.Fatalf("GET after rotation %d = (%q, %v)", i+1, got, err) + } + if oks, _ := proxy.stats(); oks[pass] != 1 { + t.Fatalf("rotation %d: proxy accepted %v, want the refreshed %q once", i+1, oks, pass) + } + } + if n := runs() - before; n != 3 { + t.Errorf("refresh command ran %d times for 3 rotations, want 3", n) + } + if _, n407 := proxy.stats(); n407 != 3 { + t.Errorf("proxy rejected %d CONNECTs, want 3 (one per rotation, then the retry)", n407) + } + + // Credentials the command cannot fix: the app gets a 502 that names + // the problem without any credentials or proxy-supplied text. + proxy.rotate("unknown") + _, err = getBody(app, target) + if err == nil { + t.Fatal("GET with unrefreshable credentials succeeded") + } + msg := err.Error() + for _, want := range []string{"Bad Gateway", "pilot-daemon proxy relay", "malformed HTTP status code", "response text withheld"} { + if !strings.Contains(msg, want) { + t.Errorf("app error %q lacks %q", msg, want) + } + } + for _, secret := range []string{"four", "unknown", "4O7", "muse:"} { + if strings.Contains(msg, secret) { + t.Errorf("app error %q leaks %q", msg, secret) + } + } +} + +// Only CONNECT is relayed, only with the Relay's own credentials, and a +// refused request never reaches the egress proxy. +func TestRelayRequiresItsCredentials(t *testing.T) { + clearEnv(t) + echo := echoServer(t) + proxy := newRotatingProxy(t, "one", echo) + r, err := Resolve(proxy.url("one")) + if err != nil { + t.Fatal(err) + } + relay := startTestRelay(t, r) + if u := relay.URL(); u.User.Username() != RelayUser || !IsLoopbackHost(u.Hostname()) { + t.Fatalf("relay URL %s: want %s@loopback", Redact(u.String()), RelayUser) + } + token, _ := relay.URL().User.Password() + basic := func(user, pass string) string { + return "Basic " + base64.StdEncoding.EncodeToString([]byte(user+":"+pass)) + } + for _, tc := range []struct { + name, request string + want int + }{ + {"no credentials", "CONNECT svc.pilot.invalid:443 HTTP/1.1\r\nHost: svc.pilot.invalid:443\r\n\r\n", 407}, + {"wrong token", "CONNECT svc.pilot.invalid:443 HTTP/1.1\r\nHost: svc.pilot.invalid:443\r\nProxy-Authorization: " + basic(RelayUser, "nope") + "\r\n\r\n", 407}, + {"the proxy's credentials", "CONNECT svc.pilot.invalid:443 HTTP/1.1\r\nHost: svc.pilot.invalid:443\r\nProxy-Authorization: " + basic("muse", "one") + "\r\n\r\n", 407}, + {"not CONNECT", "GET http://svc.pilot.invalid/ HTTP/1.1\r\nHost: svc.pilot.invalid\r\nProxy-Authorization: " + basic(RelayUser, token) + "\r\n\r\n", 405}, + {"no port", "CONNECT svc.pilot.invalid HTTP/1.1\r\nHost: svc.pilot.invalid\r\nProxy-Authorization: " + basic(RelayUser, token) + "\r\n\r\n", 400}, + {"garbage", "hello\r\n\r\n", 400}, + } { + t.Run(tc.name, func(t *testing.T) { + status, _ := rawRelayRequest(t, relay, tc.request) + if status != tc.want { + t.Fatalf("status %d, want %d", status, tc.want) + } + }) + } + if oks, n407 := proxy.stats(); len(oks) != 0 || n407 != 0 { + t.Fatalf("refused requests reached the proxy: accepted %v, 407s %d", oks, n407) + } + + // With the token: tunnelled through the proxy, bytes both ways. + status, conn := rawRelayRequest(t, relay, "CONNECT svc.pilot.invalid:443 HTTP/1.1\r\nHost: svc.pilot.invalid:443\r\nProxy-Authorization: "+basic(RelayUser, token)+"\r\n\r\n") + if status != 200 { + t.Fatalf("authorized CONNECT: status %d", status) + } + if _, err := conn.Write([]byte("ping")); err != nil { + t.Fatal(err) + } + buf := make([]byte, 4) + if _, err := io.ReadFull(conn, buf); err != nil || string(buf) != "ping" { + t.Fatalf("echo through the relay = (%q, %v)", buf, err) + } + if oks, _ := proxy.stats(); oks["one"] != 1 { + t.Fatalf("proxy accepted %v, want one CONNECT with the proxy's credentials", oks) + } +} + +// rawRelayRequest sends request to the Relay and returns the response +// status and the connection (closed with the test). +func rawRelayRequest(t *testing.T, relay *Relay, request string) (int, net.Conn) { + t.Helper() + c, err := net.Dial("tcp", relay.Addr()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { c.Close() }) + c.SetDeadline(time.Now().Add(10 * time.Second)) + if _, err := io.WriteString(c, request); err != nil { + t.Fatal(err) + } + br := bufio.NewReader(c) + resp, err := http.ReadResponse(br, &http.Request{Method: http.MethodConnect}) + if err != nil { + t.Fatalf("read relay response: %v", err) + } + c.SetDeadline(time.Time{}) + if br.Buffered() > 0 { + t.Fatalf("relay sent %d bytes after its response", br.Buffered()) + } + return resp.StatusCode, c +} + +// A proxy that refuses the target is passed on as 403; a proxy that +// cannot be reached as 502; neither with credentials in it. +func TestRelayRefusals(t *testing.T) { + clearEnv(t) + proxy := newRotatingProxy(t, "s3cret", echoServer(t)) + proxy.setRefuse(http.StatusForbidden) + r, err := Resolve(proxy.url("s3cret")) + if err != nil { + t.Fatal(err) + } + app := appClient(startTestRelay(t, r).URL()) + _, err = getBody(app, "https://blocked.pilot.invalid/") + if err == nil || !strings.Contains(err.Error(), "Forbidden") || strings.Contains(err.Error(), "s3cret") { + t.Fatalf("GET via a refusing proxy = %v, want Forbidden without credentials", err) + } + + closed, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + dead := "http://muse:s3cret@" + closed.Addr().String() + closed.Close() + r, err = Resolve(dead) + if err != nil { + t.Fatal(err) + } + app = appClient(startTestRelay(t, r).URL()) + _, err = getBody(app, "https://any.pilot.invalid/") + if err == nil || !strings.Contains(err.Error(), "Bad Gateway") || strings.Contains(err.Error(), "s3cret") { + t.Fatalf("GET via an unreachable proxy = %v, want Bad Gateway without credentials", err) + } + + if _, err := StartRelay(nil, nil); err == nil { + t.Error("StartRelay(nil resolver) started") + } + if _, err := StartRelay(netproxy.Off(), nil); err == nil { + t.Error("StartRelay(off) started") + } +} + +// Serves recognises the Relay in a proxy URL (the daemon's guard against a +// refresh command that prints it), and a CONNECT whose proxy would be the +// Relay itself is refused instead of looping. +func TestRelayServesAndLoopGuard(t *testing.T) { + clearEnv(t) + proxy := newRotatingProxy(t, "one", echoServer(t)) + r, err := Resolve(proxy.url("one")) + if err != nil { + t.Fatal(err) + } + relay := startTestRelay(t, r) + _, port, _ := net.SplitHostPort(relay.Addr()) + for in, want := range map[string]bool{ + relay.URL().String(): true, + "http://x:y@127.0.0.1:" + port: true, + "http://localhost:" + port + "/": true, + " http://127.0.0.1:" + port + "\n": true, + proxy.url("one"): false, + "http://proxy.example:" + port: false, + "not a url": false, + "": false, + "http://a:b/c@127.0.0.1:" + port + "": true, + } { + if got := relay.Serves(in); got != want { + t.Errorf("Serves(%q) = %v, want %v", in, got, want) + } + } + var none *Relay + if none.Serves(relay.URL().String()) || none.Close() != nil { + t.Error("nil Relay") + } + + // A relay whose resolver names the relay itself. + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + selfResolver, err := Resolve("http://muse:one@" + ln.Addr().String()) + if err != nil { + t.Fatal(err) + } + self, err := startRelayOn(ln, selfResolver, nil) + if err != nil { + t.Fatal(err) + } + defer self.Close() + selfToken, _ := self.URL().User.Password() + status, _ := rawRelayRequest(t, self, "CONNECT svc.pilot.invalid:443 HTTP/1.1\r\nHost: svc.pilot.invalid:443\r\nProxy-Authorization: Basic "+ + base64.StdEncoding.EncodeToString([]byte(RelayUser+":"+selfToken))+"\r\n\r\n") + if status != http.StatusLoopDetected { + t.Fatalf("CONNECT through a relay that proxies to itself: status %d, want 508", status) + } +} + +// Close ends open tunnels and returns once their goroutines are done. +func TestRelayCloseEndsTunnels(t *testing.T) { + clearEnv(t) + proxy := newRotatingProxy(t, "one", echoServer(t)) + r, err := Resolve(proxy.url("one")) + if err != nil { + t.Fatal(err) + } + relay, err := StartRelay(r, nil) + if err != nil { + t.Fatal(err) + } + token, _ := relay.URL().User.Password() + status, conn := rawRelayRequest(t, relay, "CONNECT svc.pilot.invalid:443 HTTP/1.1\r\nHost: svc.pilot.invalid:443\r\nProxy-Authorization: Basic "+ + base64.StdEncoding.EncodeToString([]byte(RelayUser+":"+token))+"\r\n\r\n") + if status != 200 { + t.Fatalf("CONNECT: status %d", status) + } + done := make(chan error, 1) + go func() { done <- relay.Close() }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("Close did not return with a tunnel open") + } + conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + if _, err := conn.Read(make([]byte, 1)); err == nil { + t.Fatal("tunnel still open after Close") + } else if ne, ok := err.(net.Error); ok && ne.Timeout() { + t.Fatal("tunnel not closed by Close") + } + if _, err := net.DialTimeout("tcp", relay.Addr(), time.Second); err == nil { + t.Error("relay still accepting after Close") + } + if err := relay.Close(); err != nil { + t.Errorf("second Close = %v", err) + } +} + +// web4-470-configuretransport-ignores-garbled-rejection: plugin clients on +// http.DefaultTransport (configured in place, or cloned from it) behind a +// proxy that answers stale credentials with an unparseable status line. +// With the daemon's Relay, the first request after a rotation already +// succeeds (one refresh, the Relay's retry) and no proxy bytes reach an +// error; plain http:// requests and loopback keep their old routes. +func TestConfigureTransportViaRelayHandlesGarbledRejection(t *testing.T) { + clearEnv(t) + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fmt.Fprint(w, "ok") })) + defer srv.Close() + proxy := newRotatingProxy(t, "one", srv.Listener.Addr().String()) + proxy.setGarble(true) + urls := newURLFile(t, proxy, "one") + command, runs := countingCommand(t, urls) + r, err := Resolve("auto", netproxy.WithRefreshFunc(CommandSource(command, nil)), netproxy.WithRefreshInterval(-1)) + if err != nil { + t.Fatal(err) + } + relay := startTestRelay(t, r) + tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} // #nosec G402 -- test server + ConfigureTransport(tr, r, relay.URL()) + clone := tr.Clone() // a plugin that cloned DefaultTransport + for name, rt := range map[string]*http.Transport{"configured": tr, "clone": clone} { + client := &http.Client{Transport: rt, Timeout: 20 * time.Second} + get := func() error { + req, _ := http.NewRequest(http.MethodGet, "https://plugin.pilot.invalid/x", nil) + req.Close = true + resp, err := client.Do(req) + if err != nil { + return err + } + resp.Body.Close() + return nil + } + if err := get(); err != nil { + t.Fatalf("%s: GET = %v", name, err) + } + pass := "p-" + name + before := runs() + urls.rotate(pass) + if err := get(); err != nil { + t.Fatalf("%s: first GET after the rotation = %v, want success (refresh + retry in the relay)", name, err) + } + if n := runs() - before; n != 1 { + t.Errorf("%s: refresh command ran %d times, want 1", name, n) + } + if oks, _ := proxy.stats(); oks[pass] != 1 { + t.Fatalf("%s: accepted %v, want the refreshed %s once", name, oks, pass) + } + } + + // Credentials nothing can fix: the error is the relay's account of + // what happened upstream, in netproxy's words, never the proxy's. + proxy.rotate("unknown") + req, _ := http.NewRequest(http.MethodGet, "https://plugin.pilot.invalid/x", nil) + req.Close = true + _, err = (&http.Client{Transport: tr, Timeout: 20 * time.Second}).Do(req) + var ce *netproxy.ConnectError + if err == nil || !UnreadableConnectReply(err) || !errors.As(err, &ce) || ce.StatusCode != http.StatusBadGateway { + t.Fatalf("GET with unrefreshable credentials = %v, want the relay's 502 carrying netproxy's report", err) + } + if strings.Contains(err.Error(), "4O7") || strings.Contains(err.Error(), "unknown") || strings.Contains(err.Error(), "pilot-relay") { + t.Fatalf("error leaks proxy text or credentials: %v", err) + } + if CredentialHint(err) == "" { + t.Errorf("no credential hint for %v", err) + } + proxy.rotate("p-clone") + proxy.setRefuse(http.StatusForbidden) + _, err = (&http.Client{Transport: tr, Timeout: 20 * time.Second}).Get("https://blocked.pilot.invalid/") + if !errors.As(err, &ce) || ce.StatusCode != http.StatusForbidden || !strings.Contains(err.Error(), "proxy CONNECT blocked.pilot.invalid:443: 403 Forbidden") { + t.Fatalf("GET of a target the proxy refuses = %v, want its 403", err) + } + proxy.setRefuse(0) + + // The routes: https through the relay, http:// straight to the proxy + // (a relay only tunnels), loopback direct. + for target, want := range map[string]string{ + "https://plugin.pilot.invalid/": relay.Addr(), + "wss://plugin.pilot.invalid/": relay.Addr(), + "http://plugin.pilot.invalid/": proxy.ln.Addr().String(), + "https://127.0.0.1:9/": "", + "http://localhost/": "", + } { + req, _ := http.NewRequest(http.MethodGet, target, nil) + u, err := tr.Proxy(req) + got := "" + if u != nil { + got = u.Host + } + if err != nil || got != want { + t.Errorf("Proxy(%s) = (%v, %v), want host %q", target, u, err, want) + } + } +} + +// Without a Relay, a garbled rejection is net/http's own error — which is +// why the daemon passes its Relay whenever it proxies. This documents the +// limit ConfigureTransport's comment describes. +func TestConfigureTransportWithoutRelayGarbled(t *testing.T) { + clearEnv(t) + proxy := newRotatingProxy(t, "one", echoServer(t)) + proxy.setGarble(true) + r, err := Resolve(proxy.url("stale")) + if err != nil { + t.Fatal(err) + } + tr := &http.Transport{} + ConfigureTransport(tr, r, nil) + _, err = (&http.Client{Transport: tr, Timeout: 10 * time.Second}).Get("https://plugin.pilot.invalid/") + if err == nil || !strings.Contains(err.Error(), "malformed HTTP status code") { + t.Fatalf("GET = %v, want net/http's malformed status error", err) + } +} + +// The credential hints: a 407, and netproxy's report of a CONNECT answer +// it could not parse (Meta Muse's form), each get one that names +// -proxy-cmd; anything else gets none. +func TestCredentialHint(t *testing.T) { + clearEnv(t) + proxy := newRotatingProxy(t, "right", echoServer(t)) + dialErr := func(garble bool) error { + proxy.setGarble(garble) + r, err := Resolve(proxy.url("wrong")) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + c, err := DialContext(r, nil)(ctx, "tcp", "registry.pilot.invalid:443") + if err == nil { + c.Close() + t.Fatal("dial with wrong credentials succeeded") + } + return err + } + garbled := dialErr(true) + if !UnreadableConnectReply(garbled) || AuthRejected(garbled) { + t.Fatalf("garbled rejection %v: UnreadableConnectReply %v, AuthRejected %v", garbled, UnreadableConnectReply(garbled), AuthRejected(garbled)) + } + hint := CredentialHint(garbled) + if !strings.Contains(hint, "could not be parsed") || !strings.Contains(hint, "-proxy-cmd") || strings.Contains(hint, "udp") { + t.Errorf("garbled hint = %q", hint) + } + if strings.Contains(garbled.Error(), "4O7") || strings.Contains(garbled.Error(), "wrong") { + t.Errorf("error quotes the proxy or the password: %v", garbled) + } + rejected := dialErr(false) + if !AuthRejected(rejected) || UnreadableConnectReply(rejected) || !strings.Contains(CredentialHint(rejected), "(407)") { + t.Errorf("407: %v, hint %q", rejected, CredentialHint(rejected)) + } + wrapped := fmt.Errorf("registry dial (after 10 attempts): %w", garbled) + if CredentialHint(wrapped) == "" { + t.Error("no hint for a wrapped garbled rejection") + } + for _, err := range []error{nil, errors.New("dial tcp: connection refused"), errors.New("read CONNECT response: EOF")} { + if h := CredentialHint(err); h != "" { + t.Errorf("CredentialHint(%v) = %q, want none", err, h) + } + } +} diff --git a/pkg/daemon/proxy.go b/pkg/daemon/proxy.go index d4428465..a70d23b9 100644 --- a/pkg/daemon/proxy.go +++ b/pkg/daemon/proxy.go @@ -28,12 +28,14 @@ import ( // outbound TCP/HTTP connection, whatever the transport. // // opts are passed to netproxy.NewResolver. For an egress proxy that -// rotates its credentials, pass netproxy.WithRefreshCommand (the daemon's -// -proxy-cmd): the command's output then supplies the proxy URL — the +// rotates its credentials, pass a refresh source — netproxy.WithRefreshCommand, +// or netproxy.WithRefreshFunc over proxyconf.CommandSource as pilot-daemon +// does for -proxy-cmd: its output then supplies the proxy URL — the // explicit one, or under auto the environment's (NO_PROXY still honoured) // — re-read every netproxy.DefaultRefreshInterval and whenever the proxy -// answers 407, after which the rejected connection is retried once. With a -// refresh command, ResolveProxy runs it once before returning; a failing +// rejects the credentials (407, or a CONNECT answer that cannot be +// parsed), after which the rejected connection is retried once. With a +// refresh source, ResolveProxy runs it once before returning; a failing // run leaves the launch-time proxy in use (the error goes to // netproxy.WithRefreshErrorHandler). // diff --git a/pkg/daemon/transport_auto.go b/pkg/daemon/transport_auto.go index 4be9da70..287e47f7 100644 --- a/pkg/daemon/transport_auto.go +++ b/pkg/daemon/transport_auto.go @@ -16,6 +16,7 @@ import ( "time" "github.com/pilot-protocol/common/netproxy" + "github.com/pilot-protocol/pilotprotocol/internal/proxyconf" "github.com/pilot-protocol/pilotprotocol/pkg/daemon/routing" ) @@ -272,14 +273,18 @@ func tlsTrustHint(err error, what string) string { } // ProxyRefusalHint explains a connection the egress proxy refused, "" for -// any other error. The ConnectError itself never quotes the proxy. +// any other error: a rejection of the credentials (407, or a CONNECT +// answer so garbled it cannot be parsed, which is how Meta Muse's proxy +// rejects them; see proxyconf.CredentialHint), or any other refusal. +// Neither the ConnectError nor netproxy's report of a garbled answer ever +// quotes the proxy. func ProxyRefusalHint(err error) string { + if hint := proxyconf.CredentialHint(err); hint != "" { + return hint + } var ce *netproxy.ConnectError if !errors.As(err, &ce) { return "" } - if ce.StatusCode == http.StatusProxyAuthRequired { - return "the proxy rejected its credentials (407), also after re-reading them: check HTTPS_PROXY / -proxy; if the proxy rotates its credentials, set -proxy-cmd ($PILOT_PROXY_CMD, config.json proxy_cmd) to a command that prints the current proxy URL" - } return fmt.Sprintf("the proxy refused CONNECT %s (HTTP %d): it must allow CONNECT to the Pilot registry and beacon on port 443", ce.Target, ce.StatusCode) } diff --git a/pkg/daemon/zz_proxy_garbled_hint_test.go b/pkg/daemon/zz_proxy_garbled_hint_test.go new file mode 100644 index 00000000..c879a7d8 --- /dev/null +++ b/pkg/daemon/zz_proxy_garbled_hint_test.go @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package daemon + +import ( + "bufio" + "context" + "fmt" + "net" + "net/http" + "strings" + "testing" + "time" +) + +// web4-470-muse-rejection-diagnostics-misdirect: Meta Muse's proxy answers +// wrong or expired credentials with a status line that cannot be parsed. +// The registry and compat beacon dials (and so "registry dial (after N +// attempts)") must still get a hint, and it must be about the credentials. +func TestProxyRefusalHintGarbledAnswer(t *testing.T) { + clearProxyEnv(t) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go func() { + defer c.Close() + if _, err := http.ReadRequest(bufio.NewReader(c)); err == nil { + fmt.Fprint(c, "HTTP/1.1 4O7 Proxy Authentication Required\r\n\r\n") + } + }() + } + }() + r, err := ResolveProxy("http://muse:stale@"+ln.Addr().String(), TransportCompat) + if err != nil { + t.Fatal(err) + } + d := New(Config{Proxy: r}) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, dialErr := d.proxyDialer()(ctx, "tcp", "registry.pilotprotocol.network:443") + if dialErr == nil { + t.Fatal("dial through a proxy rejecting the credentials succeeded") + } + err = fmt.Errorf("registry dial (after 10 attempts): %w", dialErr) + hint := ProxyRefusalHint(err) + for _, want := range []string{"could not be parsed", "credentials", "-proxy-cmd"} { + if !strings.Contains(hint, want) { + t.Errorf("hint %q lacks %q", hint, want) + } + } + if strings.Contains(hint, "udp") || strings.Contains(err.Error(), "4O7") || strings.Contains(err.Error(), "stale") { + t.Errorf("hint %q / error %v", hint, err) + } +} From 4eff684192931a1d70c0aa072c8c379a1035787b Mon Sep 17 00:00:00 2001 From: Teodor Calin Date: Thu, 24 Sep 2026 09:17:10 +0300 Subject: [PATCH 14/19] install.sh: sync with pilot-protocol/release#49 (ae447bc) Byte-identical copy of the canonical installer on pilot-protocol/release feat/installer-proxy-transport, so canonical-drift passes once release#49 merges. Takes this branch's credential-preferring sandbox proxy_cmd, and adds: no raw-IP registry/beacon in config.json for compat or auto behind a proxy, downloads retried once after a proxy credential rotation, proxy_cmd saved for older daemons too, PILOT_PROXY downloads, the v1.13.x proxy warning pointing at the pilot-sandbox recipe, and onboarding text that reads replies from send-message --wait and drops routine appstore --force. Only install.sh changes. Co-Authored-By: Claude Opus 5.5 (1M context) --- install.sh | 431 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 301 insertions(+), 130 deletions(-) diff --git a/install.sh b/install.sh index 38be12ad..81fac81c 100755 --- a/install.sh +++ b/install.sh @@ -57,7 +57,8 @@ set -e # systemd whose HTTPS_PROXY carries credentials # (hosted agent sandboxes such as Meta Muse), the # installer saves one that reads a fresh shell's -# $https_proxy when none is set. +# $https_proxy when none is set. The installer +# also uses it to retry a failed download. # PILOT_ALLOW_ROOT=1 Install as root on a host with systemd/launchd # (not needed in containers/VMs without systemd). # PILOT_MANAGEMENT_URL=https://management.example @@ -66,9 +67,11 @@ set -e # # Proxies: every download is a curl HTTPS request, so HTTPS_PROXY / https_proxy / # ALL_PROXY / NO_PROXY are honored (curl asks the proxy to CONNECT by hostname — -# no local DNS lookup of the target). Nothing here needs UDP, a non-443 port, or -# a direct connection to the registry/beacon. Steps that need root, sudo, -# systemd or launchd are skipped with a message, never fatal. +# no local DNS lookup of the target); a PILOT_PROXY http(s):// URL is used when +# none of those is set. Nothing here needs UDP, a non-443 port, or a direct +# connection to the registry/beacon. Proxy credentials are never printed or +# written to disk. Steps that need root, sudo, systemd or launchd are skipped +# with a message, never fatal. # # WHAT THIS SCRIPT DOES (read before piping to sh): # 1. Detects OS/arch (Linux/Darwin × amd64/arm64) @@ -104,7 +107,9 @@ set -e # Network 9 directory and for receiving identifier-based deliveries. # # WHAT THIS SCRIPT DOES NOT DO: -# - Run as root (refuses if invoked as root; see check at line ~25) +# - Run as root on a host with systemd or launchd (refuses; see the root +# check below). A Linux container/VM without systemd, where the agent is +# root, installs into root's own $HOME/.pilot. # - Send any personal data anywhere (the install script only fetches the # release tarball from GitHub; the daemon registers its public key + a # synthetic or user-supplied email with the rendezvous server, nothing else) @@ -213,7 +218,7 @@ while [ $# -gt 0 ]; do --no-start) PILOT_MANAGED_NO_START=1; shift ;; -h|--help) - sed -n '4,71p' "$0" 2>/dev/null || echo "See https://pilotprotocol.network/install.sh" + sed -n '4,74p' "$0" 2>/dev/null || echo "See https://pilotprotocol.network/install.sh" exit 0 ;; --) shift @@ -326,8 +331,12 @@ set -- $PILOT_POSITIONAL # hosted agent sandboxes such as Meta Muse, where the agent IS root) has no # other user to install for and no system service to protect, so root is # allowed there. +SANDBOX_HOST=false +if [ "$(uname -s)" = "Linux" ] && [ ! -d /run/systemd/system ]; then + SANDBOX_HOST=true +fi if [ "${1:-}" != "uninstall" ] && [ "$(id -u)" = "0" ] && [ -z "${PILOT_ALLOW_ROOT:-}" ]; then - if [ "$(uname -s)" = "Linux" ] && [ ! -d /run/systemd/system ]; then + if [ "$SANDBOX_HOST" = true ]; then echo "Note: installing as root (no systemd: container/VM sandbox) into ${HOME}/.pilot" else echo "Error: refusing to install as root." @@ -396,8 +405,80 @@ EFFECTIVE_TRANSPORT="${TRANSPORT:-${CONFIG_TRANSPORT:-auto}}" # by hostname (no local DNS lookup of the target — which matters where local # DNS for pilotprotocol.network is poisoned). PILOT_PROXY_URL is only used in # messages, and only ever printed redacted: the userinfo of an -# authenticating proxy is a credential. -PILOT_PROXY_URL="${HTTPS_PROXY:-${https_proxy:-${ALL_PROXY:-${all_proxy:-}}}}" +# authenticating proxy is a credential. Nothing in this script writes a proxy +# URL to disk. +# +# PILOT_PROXY is the daemon's own proxy setting; an http(s):// URL there +# carries this run's downloads too when the environment names no proxy +# (exported to this process and its children only). +if [ -z "${https_proxy:-}${HTTPS_PROXY:-}${all_proxy:-}${ALL_PROXY:-}" ]; then + case "${PILOT_PROXY:-}" in + http://*|https://*|HTTP://*|HTTPS://*) + https_proxy="$PILOT_PROXY" + HTTPS_PROXY="$PILOT_PROXY" + export https_proxy HTTPS_PROXY ;; + esac +fi +# The order curl uses for an https:// URL. +PILOT_PROXY_URL="${https_proxy:-${HTTPS_PROXY:-${all_proxy:-${ALL_PROXY:-}}}}" + +# Rotating proxy credentials. Hosted agent sandboxes (Meta Muse) put the proxy +# credentials in HTTPS_PROXY and replace them every few minutes; a process +# keeps the ones it started with, and the proxy answers its next CONNECT with +# 407. A fresh shell sees the current ones. PROXY_REFRESH_CMD prints the +# current proxy URL: $PILOT_PROXY_CMD, else — in a Linux container/VM without +# systemd whose HTTPS_PROXY or https_proxy carries credentials — what a fresh +# bash has: whichever of $https_proxy and $HTTPS_PROXY carries credentials +# ($https_proxy when both do, the variable Meta Muse's guidance reads), else +# ${HTTPS_PROXY:-$https_proxy}, so a URL with credentials is never traded for +# one without (pilotctl uses the same command). It is saved as the daemon's +# proxy_cmd further down, and pcurl uses it here to retry a download once +# after the credentials rotated mid-install. +# shellcheck disable=SC2016 # literal: the fresh bash expands it, not this shell +SANDBOX_PROXY_CMD='bash -c '\''case $https_proxy in *@*) printf %s "$https_proxy";; *) printf %s "${HTTPS_PROXY:-$https_proxy}";; esac'\''' +PROXY_REFRESH_CMD="${PILOT_PROXY_CMD:-}" +if [ -z "$PROXY_REFRESH_CMD" ] && [ "$SANDBOX_HOST" = true ] \ + && command -v bash >/dev/null 2>&1; then + case "${https_proxy:-}${HTTPS_PROXY:-}" in + *@*) PROXY_REFRESH_CMD="$SANDBOX_PROXY_CMD" ;; + esac +fi + +# proxy_refresh — run PROXY_REFRESH_CMD (at most 10s where `timeout` exists) +# and, when it prints an http(s):// URL different from the current one, use +# that for the rest of this run. The URL is never printed; only this process's +# environment changes. Returns 0 when the proxy URL changed. +proxy_refresh() { + [ -n "$PROXY_REFRESH_CMD" ] || return 1 + if command -v timeout >/dev/null 2>&1; then + _pr_url=$(timeout 10 sh -c "$PROXY_REFRESH_CMD" 2>/dev/null /dev/null /dev/null + pcurl -fsSL --max-time 10 "$MANIFEST_URL" -o "$1" 2>/dev/null } # manifest_field "" "" extracts a string field. Supports nested @@ -628,8 +709,10 @@ case "$EFFECTIVE_TRANSPORT" in echo " Transport: compat (TLS + WSS over TCP 443 only)" ;; auto) echo " Transport: auto (UDP when it works, else TLS + WSS over TCP 443)" - echo " Registry: ${REGISTRY}" - echo " Beacon: ${BEACON}" ;; + if [ -z "$PILOT_PROXY_URL" ]; then + echo " Registry: ${REGISTRY}" + echo " Beacon: ${BEACON}" + fi ;; *) echo " Registry: ${REGISTRY}" echo " Beacon: ${BEACON}" ;; @@ -779,7 +862,7 @@ elif [ "${PILOT_RC:-}" = "1" ]; then elif [ "$HAVE_MANIFEST" = "1" ]; then TAG=$(manifest_field "latest_stable" "$MANIFEST_FILE") else - TAG=$(curl -fsSI "/${REPO}/releases/latest/download/${ARCHIVE}" 2>/dev/null \ + TAG=$(pcurl -fsSI "/${REPO}/releases/latest/download/${ARCHIVE}" 2>/dev/null \ | grep -i '^location:' \ | sed -n 's|.*/releases/download/\([^/]*\)/.*|\1|p' \ | tr -d '\r' | head -1) @@ -853,7 +936,7 @@ if [ -n "$TAG" ]; then URL="/${REPO}/releases/download/${TAG}/${ARCHIVE}" CHECKSUMS_URL="/${REPO}/releases/download/${TAG}/checksums.txt" echo "Downloading ${TAG}..." - if curl -fsSL "$URL" -o "$TMPDIR/$ARCHIVE" 2>/dev/null; then + if pcurl -fsSL "$URL" -o "$TMPDIR/$ARCHIVE" 2>/dev/null; then # --- Verify SHA-256 (fail closed) --- # This block NEVER extracts an archive it could not verify. Two # independent anchors are used: @@ -867,7 +950,7 @@ if [ -n "$TAG" ]; then # archive line, or the absence of shasum/sha256sum silently extracted # the archive UNVERIFIED.) EXPECTED_CKS="" - if curl -fsSL "$CHECKSUMS_URL" -o "$TMPDIR/checksums.txt" 2>/dev/null; then + if pcurl -fsSL "$CHECKSUMS_URL" -o "$TMPDIR/checksums.txt" 2>/dev/null; then EXPECTED_CKS=$(grep " ${ARCHIVE}\$" "$TMPDIR/checksums.txt" | awk '{print $1}') fi EXPECTED_MAN="" @@ -1185,49 +1268,19 @@ if [ "$LINK_OK" = true ]; then echo " pilotctl now resolves in non-interactive shells (bash -c, cron, CI, agents)" fi -# --- Fresh install: write config --- -# -# config.json is written ONLY when there isn't one already. A re-run must never -# clobber registry/beacon/consent settings the operator edited by hand. -# -# The UPDATING check alone did not deliver that promise: UPDATING is derived -# purely from `[ -x "$BIN_DIR/pilotctl" ]`, i.e. whether the BINARY exists. A -# host with a hand-edited ~/.pilot/config.json but no binary — binaries removed -# for a clean reinstall, config restored from backup, or a config pre-seeded -# before first install — took the "fresh install" branch and had its config -# silently overwritten. -# -# That also made consent settings impossible to set BEFORE first start: -# pre-seeding {"consent":{...}} or {"skill_inject":{"mode":"disabled"}} was -# erased by this write, and the erase happened before the first skills pass -# further below. Guarding on the file itself makes the documented opt-outs -# reachable at install time instead of only after the fact. Defaults are -# unchanged — a host with no config still gets the standard one. -if [ "$UPDATING" != true ] && [ ! -f "$PILOT_DIR/config.json" ]; then - cat > "$PILOT_DIR/config.json" </dev/null 2>&1 } -# What the installed binaries support. The probes are local (no network). +# The probes are local (no network). DAEMON_HAS_TRANSPORT=false DAEMON_HAS_PROXY=false DAEMON_HAS_AUTO=false @@ -1247,6 +1300,8 @@ if printf '%s\n' "$_daemon_help" | grep -qE '^[[:space:]]+-proxy-cmd([[:space:]] DAEMON_HAS_PROXY_CMD=true fi +# --- Transport: auto, udp or compat --- +# # auto is never saved in config.json. It is already the default wherever # this install starts the daemon — `pilotctl daemon start` asks a daemon # that supports it for auto, and the service units below set @@ -1272,18 +1327,6 @@ if [ "$CONFIG_TRANSPORT" = "auto" ] && [ "$DAEMON_HAS_AUTO" != true ] && [ -z "$ echo " selects; switching it to udp (the daemon's default) so the daemon still starts." fi -if [ -n "$TRANSPORT_TO_SAVE" ]; then - if pilot_config_set "transport=$TRANSPORT_TO_SAVE"; then - echo "Transport set to ${TRANSPORT_TO_SAVE} in ${PILOT_DIR}/config.json" - else - echo " Note: could not save transport=${TRANSPORT_TO_SAVE} — run: pilotctl config --set transport=${TRANSPORT_TO_SAVE}" - fi -elif [ "$TRANSPORT_CLEAR" = true ]; then - if pilot_config_set "transport="; then - echo "Transport: auto (the default; removed \"transport\" from ${PILOT_DIR}/config.json)" - fi -fi - # What the daemon will run: the saved transport, else auto where the # daemon supports it, else its default (udp). if [ -n "$TRANSPORT_TO_SAVE" ]; then @@ -1296,30 +1339,123 @@ else EFFECTIVE_TRANSPORT="udp" fi -# Rotating proxy credentials. Hosted agent sandboxes (Meta Muse) put the -# proxy credentials in HTTPS_PROXY and rotate them every few minutes; a -# long-running daemon keeps the launch-time ones and new connections start -# failing with 407. proxy_cmd makes the daemon re-read the URL (every 60s -# and on a 407) from a command — here a fresh shell, which sees the current -# value. PILOT_PROXY_CMD sets it explicitly; otherwise it is saved only in a -# Linux container/VM without systemd whose proxy carries credentials, and -# never over an existing proxy_cmd. -# shellcheck disable=SC2016 # literal: the fresh bash expands it, not this shell -SANDBOX_PROXY_CMD='bash -c '\''case $https_proxy in *@*) printf %s "$https_proxy";; *) printf %s "${HTTPS_PROXY:-$https_proxy}";; esac'\''' -PROXY_CMD_TO_SAVE="${PILOT_PROXY_CMD:-}" -if [ -z "$PROXY_CMD_TO_SAVE" ] && [ "$OS" = "linux" ] && [ ! -d /run/systemd/system ] \ - && command -v bash >/dev/null 2>&1 \ - && ! grep -q '"proxy_cmd"' "$PILOT_DIR/config.json" 2>/dev/null; then - case "$PILOT_PROXY_URL" in - *@*) PROXY_CMD_TO_SAVE="$SANDBOX_PROXY_CMD" ;; +# pilotctl releases before `daemon start --transport` pass config.json's +# registry (else the raw-TCP default) to the daemon verbatim, and a daemon +# given the raw-TCP registry explicitly stays on it even in compat mode. +# Probed only for compat, and only with a daemon that has -transport (v1.11+): +# every pilotctl since v1.10 prints help for `daemon start --help` instead of +# starting a daemon. +PILOTCTL_HAS_TRANSPORT=false +if [ "$EFFECTIVE_TRANSPORT" = "compat" ] && [ "$DAEMON_HAS_TRANSPORT" = true ] \ + && "$BIN_DIR/pilotctl" daemon start --help 2>&1 | grep -q -- '--transport'; then + PILOTCTL_HAS_TRANSPORT=true +fi + +# The stock raw-TCP registry (34.71.57.205:9000) and UDP beacon are left out +# of what this installer writes when the node runs compat, or auto behind a +# proxy: the daemon then applies the endpoints that fit the transport it +# runs (registry.pilotprotocol.network:443 over TLS in compat mode or through +# a proxy), and an address that a 443-only network or HTTPS proxy never +# carries is not pinned anywhere. Custom PILOT_REGISTRY / PILOT_BEACON values +# are always kept. +STOCK_ENDPOINTS=true +if [ "$DAEMON_HAS_TRANSPORT" = true ]; then + case "$EFFECTIVE_TRANSPORT" in + compat) STOCK_ENDPOINTS=false ;; + auto) if [ -n "$PILOT_PROXY_URL" ]; then STOCK_ENDPOINTS=false; fi ;; esac fi + +# --- Fresh install: write config --- +# +# config.json is written ONLY when there isn't one already. A re-run must never +# clobber registry/beacon/consent settings the operator edited by hand. +# +# The UPDATING check alone did not deliver that promise: UPDATING is derived +# purely from `[ -x "$BIN_DIR/pilotctl" ]`, i.e. whether the BINARY exists. A +# host with a hand-edited ~/.pilot/config.json but no binary — binaries removed +# for a clean reinstall, config restored from backup, or a config pre-seeded +# before first install — took the "fresh install" branch and had its config +# silently overwritten. +# +# That also made consent settings impossible to set BEFORE first start: +# pre-seeding {"consent":{...}} or {"skill_inject":{"mode":"disabled"}} was +# erased by this write, and the erase happened before the first skills pass +# further below. Guarding on the file itself makes the documented opt-outs +# reachable at install time instead of only after the fact. Defaults are +# unchanged — a host with no config still gets the standard one (without the +# stock endpoints in compat mode or behind a proxy, see STOCK_ENDPOINTS). +if [ "$UPDATING" != true ] && [ ! -f "$PILOT_DIR/config.json" ]; then + CONF_REGISTRY="$REGISTRY" + CONF_BEACON="$BEACON" + if [ "$STOCK_ENDPOINTS" != true ]; then + if [ "$REGISTRY" = "$DEFAULT_REGISTRY" ]; then + CONF_REGISTRY="" + # A pilotctl that predates --transport would pass the raw + # default instead: name the compat TLS registry explicitly. + if [ "$EFFECTIVE_TRANSPORT" = "compat" ] && [ "$PILOTCTL_HAS_TRANSPORT" != true ]; then + CONF_REGISTRY="$COMPAT_REGISTRY" + fi + fi + if [ "$BEACON" = "$DEFAULT_BEACON" ]; then CONF_BEACON=""; fi + fi + CONF_NET="" + if [ -n "$CONF_REGISTRY" ]; then + CONF_NET="${CONF_NET} \"registry\": \"${CONF_REGISTRY}\", +" + fi + if [ -n "$CONF_BEACON" ]; then + CONF_NET="${CONF_NET} \"beacon\": \"${CONF_BEACON}\", +" + fi + cat > "$PILOT_DIR/config.json" </dev/null; then + PROXY_CMD_TO_SAVE="$SANDBOX_PROXY_CMD" +fi if [ -n "$PROXY_CMD_TO_SAVE" ]; then - if [ "$DAEMON_HAS_PROXY_CMD" != true ]; then - echo " Note: this pilot-daemon (${TAG:-source}) predates -proxy-cmd; if the proxy rotates its" - echo " credentials, restart the daemon from a fresh shell when it starts failing." - elif pilot_config_set "proxy_cmd=$PROXY_CMD_TO_SAVE"; then - echo "Proxy credentials: re-read by the daemon via proxy_cmd (${PILOT_DIR}/config.json)" + if ! pilot_config_set "proxy_cmd=$PROXY_CMD_TO_SAVE"; then + echo " Note: could not save proxy_cmd in ${PILOT_DIR}/config.json" + elif [ "$DAEMON_HAS_PROXY_CMD" = true ]; then + echo "Proxy credentials: re-read by the daemon via proxy_cmd (${PILOT_DIR}/config.json stores the command, not the credentials)" + else + echo "Proxy credentials: proxy_cmd saved in ${PILOT_DIR}/config.json (the command, not the credentials)." + echo " This pilot-daemon (${TAG:-source}) predates -proxy-cmd and ignores it until upgraded;" + echo " until then, if the proxy rotates its credentials, restart the daemon from a fresh shell." fi fi PROXY_CMD_SAVED=false @@ -1327,29 +1463,32 @@ if grep -q '"proxy_cmd"' "$PILOT_DIR/config.json" 2>/dev/null; then PROXY_CMD_SAVED=true fi +# --- Registry for the transport --- +# +# No "proxy" key is written: the daemon's default, auto, already uses +# $HTTPS_PROXY / $ALL_PROXY where it needs a proxy, and a saved "auto" would +# only get in the way of a proxy passed later with --proxy or $PILOT_PROXY. +CONFIG_REGISTRY=$(sed -n 's/.*"registry"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$PILOT_DIR/config.json" 2>/dev/null | head -n 1) if [ "$EFFECTIVE_TRANSPORT" = "compat" ]; then - # No "proxy" key is written: the daemon's default, auto, already uses - # $HTTPS_PROXY / $ALL_PROXY in compat mode, and a saved "auto" would only - # get in the way of a proxy passed later with --proxy or $PILOT_PROXY. if [ "$DAEMON_HAS_TRANSPORT" != true ]; then echo "" echo " WARNING: this pilot-daemon (${TAG:-source}) predates compat mode (-transport)." echo " It will keep using UDP. Re-run without --version to get the latest release." fi - # pilotctl releases before --transport forward config.json's registry to - # the daemon verbatim, and a daemon given the raw-TCP default explicitly - # stays on it even in compat mode. Point such installs at the compat TLS - # registry directly — only when the file still holds the stock default. - if ! "$BIN_DIR/pilotctl" daemon start --help 2>&1 | grep -q -- '--transport' \ - && grep -q "\"registry\"[[:space:]]*:[[:space:]]*\"${DEFAULT_REGISTRY}\"" "$PILOT_DIR/config.json" 2>/dev/null; then + # A pilotctl that predates --transport passes config.json's registry, + # else the raw-TCP default, to the daemon verbatim: point it at the + # compat TLS registry — only when the file holds the stock default or + # no registry at all. + if [ "$DAEMON_HAS_TRANSPORT" = true ] && [ "$PILOTCTL_HAS_TRANSPORT" != true ] \ + && { [ "$CONFIG_REGISTRY" = "$DEFAULT_REGISTRY" ] || [ -z "$CONFIG_REGISTRY" ]; }; then if pilot_config_set "registry=${COMPAT_REGISTRY}"; then echo " Registry set to ${COMPAT_REGISTRY} for compat mode (this pilotctl" echo " always passes config.json's registry to the daemon). Switching back to" echo " UDP later: re-run this installer with --transport udp" fi fi -elif grep -q "\"registry\"[[:space:]]*:[[:space:]]*\"${COMPAT_REGISTRY}\"" "$PILOT_DIR/config.json" 2>/dev/null; then +elif [ "$CONFIG_REGISTRY" = "$COMPAT_REGISTRY" ]; then # Leaving compat after an install that pointed the registry at the # compat TLS host: a udp daemon needs the raw-TCP registry back. if pilot_config_set "registry=${DEFAULT_REGISTRY}"; then @@ -1357,20 +1496,28 @@ elif grep -q "\"registry\"[[:space:]]*:[[:space:]]*\"${COMPAT_REGISTRY}\"" "$PIL fi fi -if [ "$EFFECTIVE_TRANSPORT" != "udp" ] && [ -n "$PILOT_PROXY_URL" ] && [ "$DAEMON_HAS_PROXY" != true ]; then +# A proxy in the environment that this daemon cannot use: say so, and where +# the proxy is the only way out, point at the recipe that works with it. +PROXY_UNSUPPORTED=false +if [ -n "$PILOT_PROXY_URL" ] && [ "$DAEMON_HAS_PROXY" != true ]; then + PROXY_UNSUPPORTED=true echo "" - echo " WARNING: HTTPS_PROXY is set, but this pilot-daemon (${TAG:-source}) cannot use a" - echo " proxy. Where the proxy is the only way out, the daemon will not come" - echo " online. Install a release whose 'pilot-daemon -help' lists -proxy." + echo " WARNING: a proxy is set ($(redact_proxy "$PILOT_PROXY_URL")), but this pilot-daemon" + echo " (${TAG:-source}) cannot use one. If the proxy is this host's only way" + echo " out (UDP blocked, e.g. an agent sandbox), \`pilotctl daemon start\` will" + echo " not come online with this release. Use the pilot-sandbox recipe:" + echo " https://pilotprotocol.network/learn/install-pilot-skills-in-meta-muse" fi # Network flags for the service units. The transport itself comes from # config.json, which the daemon reads, so `pilotctl config --set transport=` -# applies to the service too. In compat mode the raw-TCP default -# registry/beacon are left off (an older daemon given -registry explicitly -# stays pinned to a port no 443-only network or HTTPS proxy will carry); a -# custom PILOT_REGISTRY / PILOT_BEACON is kept. -if [ "$EFFECTIVE_TRANSPORT" = "compat" ] && [ "$DAEMON_HAS_TRANSPORT" = true ]; then +# applies to the service too. Where the stock endpoints are left out (compat, +# or auto behind a proxy; see STOCK_ENDPOINTS) the daemon's built-in defaults +# apply — the same raw-TCP/UDP endpoints for udp, the TLS registry for +# compat (an older daemon given -registry explicitly stays pinned to a port +# no 443-only network or HTTPS proxy will carry); a custom PILOT_REGISTRY / +# PILOT_BEACON is kept. +if [ "$STOCK_ENDPOINTS" != true ]; then NET_FLAGS="" if [ "$REGISTRY" != "$DEFAULT_REGISTRY" ]; then NET_FLAGS="$NET_FLAGS -registry $REGISTRY"; fi if [ "$BEACON" != "$DEFAULT_BEACON" ]; then NET_FLAGS="$NET_FLAGS -beacon $BEACON"; fi @@ -1399,12 +1546,21 @@ fi # service_proxy_note UNIT — a service manager starts the daemon with its own # environment, not this shell's, so an HTTPS_PROXY exported here never # reaches it. config.json (0600, read by the daemon itself) does. +# This installer never writes the URL itself: with credentials in it, where +# to store them is the operator's call. service_proxy_note() { if [ "$EFFECTIVE_TRANSPORT" != "udp" ] && [ -n "$PILOT_PROXY_URL" ] \ && ! grep -q '"proxy"[[:space:]]*:[[:space:]]*"http' "$PILOT_DIR/config.json" 2>/dev/null; then echo " Note: $1 does not inherit this shell's HTTPS_PROXY. For the service to use" - echo " the proxy, save it in config.json (0600):" - echo " pilotctl config --set proxy=''" + case "$PILOT_PROXY_URL" in + *@*) + echo " the proxy, save it in config.json (0600; this stores its credentials):" + echo " pilotctl config --set proxy=''" + echo " or save a command that prints it: pilotctl config --set proxy_cmd=''" ;; + *) + echo " the proxy, save it in config.json:" + echo " pilotctl config --set proxy='${PILOT_PROXY_URL}'" ;; + esac fi } @@ -1592,7 +1748,9 @@ elif [ "$OS" = "linux" ]; then if [ "$PILOT_MANAGED_MODE" != "1" ]; then echo "No systemd detected (container / WSL / CI / sandbox) — start the daemon manually:" echo " pilotctl daemon start" - if [ "$EFFECTIVE_TRANSPORT" != "udp" ]; then + if [ "$PROXY_UNSUPPORTED" = true ]; then + echo " (proxy-only host: see the WARNING above)" + elif [ "$EFFECTIVE_TRANSPORT" != "udp" ]; then echo " (transport=${EFFECTIVE_TRANSPORT}; start it from a shell that has HTTPS_PROXY" echo " set if this host reaches the internet only through a proxy)" fi @@ -1918,24 +2076,35 @@ echo "" echo "Config: ${PILOT_DIR}/config.json" case "$EFFECTIVE_TRANSPORT" in compat) - echo " Transport: compat (registry ${COMPAT_REGISTRY} over TLS, beacon over WSS)" ;; + _summary_registry="$COMPAT_REGISTRY" + if [ "$REGISTRY" != "$DEFAULT_REGISTRY" ]; then _summary_registry="$REGISTRY"; fi + echo " Transport: compat (registry ${_summary_registry} over TLS, beacon over WSS)" ;; auto) echo " Transport: auto (UDP when it works, else compat over TCP 443)" - echo " Registry: ${REGISTRY}" - echo " Beacon: ${BEACON}" ;; + if [ "$STOCK_ENDPOINTS" = true ]; then + echo " Registry: ${REGISTRY}" + echo " Beacon: ${BEACON}" + else + echo " Registry: picked by the daemon for its transport (${COMPAT_REGISTRY} over TLS via the proxy)" + fi ;; *) echo " Registry: ${REGISTRY}" echo " Beacon: ${BEACON}" ;; esac -if [ "$EFFECTIVE_TRANSPORT" != "udp" ] && [ -n "$PILOT_PROXY_URL" ]; then +if [ "$EFFECTIVE_TRANSPORT" != "udp" ] && [ -n "$PILOT_PROXY_URL" ] && [ "$DAEMON_HAS_PROXY" = true ]; then echo " Proxy: auto -> $(redact_proxy "$PILOT_PROXY_URL") (from environment)" - if [ "$PROXY_CMD_SAVED" = true ]; then + if [ "$PROXY_CMD_SAVED" = true ] && [ "$DAEMON_HAS_PROXY_CMD" = true ]; then echo " credentials re-read by the daemon (proxy_cmd): rotation needs no restart" fi fi echo " Socket: /tmp/pilot.sock" echo " Identity: ${PILOT_DIR}/identity.json" -echo " Email: ${EMAIL}" +if [ -n "$EMAIL" ]; then + echo " Email: ${EMAIL}" +else + echo " Email: not set (the daemon uses @nodes.pilotprotocol.network;" + echo " set a real one any time: pilotctl set-email )" +fi echo "" echo "============================================" echo " GET STARTED" @@ -1948,16 +2117,16 @@ cat <<'PILOT_GET_STARTED' pilotctl daemon status # confirm it's running pilotctl info # node ID, address, peer count, uptime - Reading any reply is always the same two-step idiom — send with - --wait, then print the newest reply. No jq, no extra dependency: + Ask with --wait: the reply comes back in the send's own output. If + it fails or times out there is no reply to read — do not substitute + the newest inbox message, which may answer an older question: ------------------------------------------------------------------ pilotctl send-message --data '' --wait - pilotctl inbox --latest # full body of the newest reply + pilotctl --json send-message --data '' --wait # reply in .data.reply - # Other ways to read the inbox: + # Every reply is also kept in the inbox: pilotctl inbox # 10 newest, one-line previews pilotctl inbox --from pilot-mom --limit 3 - pilotctl --json inbox --latest # machine-readable envelope 1) MOM (pilot-mom) — your default for anything LIVE or fresh. @@ -1965,7 +2134,6 @@ cat <<'PILOT_GET_STARTED' agents + filters and returns a validated, ready-to-run plan. ------------------------------------------------------------------ pilotctl send-message pilot-mom --data 'current weather and air quality for Berlin' --wait - pilotctl inbox --latest # More examples — hand it the whole sentence, don't hunt for agents yourself: pilotctl send-message pilot-mom --data 'summarize this season F1 drivers' --wait @@ -1980,16 +2148,13 @@ cat <<'PILOT_GET_STARTED' ------------------------------------------------------------------ # Browse the whole live catalogue: pilotctl send-message list-agents --data '/data' --wait - pilotctl inbox --latest # Search by keyword: pilotctl send-message list-agents --data '/data {"search":"bitcoin","limit":10}' --wait - pilotctl inbox --latest # Learn a specialist's query schema, then query it with filters: pilotctl send-message --data '/help' --wait pilotctl send-message --data '/data {"":""}' --wait - pilotctl inbox --latest @@ -2003,8 +2168,10 @@ cat <<'PILOT_GET_STARTED' # See an app's full details (methods, source, permissions, pricing): pilotctl appstore view io.pilot.sqlite - # Install it (daemon auto-spawns it; re-run `list` if state != ready): - pilotctl appstore install io.pilot.sqlite --force + # Install it once (daemon auto-spawns it; re-run `list` if state != ready). + # Do not add --force routinely: it reinstalls over the app and can + # delete its saved state (keys). + pilotctl appstore install io.pilot.sqlite pilotctl appstore list # ALWAYS call .help first — lists every method, its params, @@ -2015,13 +2182,13 @@ cat <<'PILOT_GET_STARTED' pilotctl appstore call io.pilot.sqlite sqlite.query '{"sql":"select 1"}' # A few concrete capability examples (install first, then call): - pilotctl appstore install io.pilot.smol --force + pilotctl appstore install io.pilot.smol pilotctl appstore call io.pilot.smol smol.push '{"image":"alpine","net":true}' - pilotctl appstore install io.pilot.bowmark --force + pilotctl appstore install io.pilot.bowmark pilotctl appstore call io.pilot.bowmark bowmark.ask '{"site":"amazon.com","task":"search for a product"}' - pilotctl appstore install io.pilot.orthogonal --force + pilotctl appstore install io.pilot.orthogonal pilotctl appstore call io.pilot.orthogonal orthogonal.search '{"prompt":"work email for a person given name + company"}' Cost: most apps run locally and are free. A few (orthogonal, sixtyfour, @@ -2040,8 +2207,9 @@ cat <<'PILOT_GET_STARTED' pilotctl send-message --data '' # talk, once trust is mutual pilotctl send-file /path/to/file.tar.gz # exchange artifacts - Full operator manual & task→agent/app maps: - ~/.claude/skills/pilotctl/SKILL.md + Full operator manual & task→agent/app maps: the pilotctl skill + (`pilotctl skills` lists where it is installed, e.g. + ~/.claude/skills/pilotctl/SKILL.md). ============================================ PILOT_GET_STARTED echo "" @@ -2061,6 +2229,9 @@ echo " The daemon scans every 15 minutes and injects the Pilot Protocol" echo " skill into installed agent tools. Triggering a first pass right now" echo " so your agents know about Pilot before the daemon is even started:" echo "" +# pilotctl fetches the skills through this process's proxy settings: hand it +# the current credentials if they rotated since the downloads. +proxy_refresh || true if "${BIN_DIR}/pilotctl" skills check 2>&1 | sed 's/^/ /'; then : else From 61154319d053a4ea5de6e2f10b22b9243912ff97 Mon Sep 17 00:00:00 2001 From: Teodor Calin Date: Thu, 24 Sep 2026 10:18:19 +0300 Subject: [PATCH 15/19] install.sh: sync with pilot-protocol/release#49 (67e83fc) Byte-identical to release@67e83fc (review fixes: root refused under sudo, no `pilotctl daemon start` advice on proxy-only hosts with a daemon that cannot use the proxy, the transport stated after the download, --version / --channel beta installable again). Co-Authored-By: Claude Opus 5.5 (1M context) --- install.sh | 253 +++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 207 insertions(+), 46 deletions(-) diff --git a/install.sh b/install.sh index 81fac81c..4e9b10b6 100755 --- a/install.sh +++ b/install.sh @@ -9,11 +9,13 @@ set -e # Install: curl -fsSL https://pilotprotocol.network/install.sh | sh # Pin a version: curl -fsSL https://pilotprotocol.network/install.sh | sh -s -- --version v1.13.6 # Beta channel: curl -fsSL https://pilotprotocol.network/install.sh | sh -s -- --channel beta -# UDP blocked / curl -fsSL https://pilotprotocol.network/install.sh | sh -# HTTPS proxy: (nothing extra: transport "auto" picks TLS/WSS over TCP 443 -# through $HTTPS_PROXY when UDP does not work; add -# `-s -- --transport compat` to skip the UDP probe; proxy -# credentials that rotate: see PILOT_PROXY_CMD below) +# UDP blocked / curl -fsSL https://pilotprotocol.network/install.sh | sh -s -- --transport compat +# HTTPS proxy: (with a release that has transport "auto" nothing extra is +# needed: auto picks TLS/WSS over TCP 443, through +# $HTTPS_PROXY when set, where UDP does not work. Releases +# before auto stay on UDP unless given --transport compat, and +# use no proxy: the installer prints what to do instead. +# Proxy credentials that rotate: see PILOT_PROXY_CMD below) # Managed node: export PILOT_ENROLLMENT_TOKEN # enter it without putting it in shell history # sh install.sh --managed-url https://management.pilotprotocol.network # Uninstall: curl -fsSL https://pilotprotocol.network/install.sh | sh -s uninstall @@ -59,8 +61,9 @@ set -e # installer saves one that reads a fresh shell's # $https_proxy when none is set. The installer # also uses it to retry a failed download. -# PILOT_ALLOW_ROOT=1 Install as root on a host with systemd/launchd -# (not needed in containers/VMs without systemd). +# PILOT_ALLOW_ROOT=1 Install as root on a host with systemd/launchd, +# or under sudo/doas (not needed where the agent +# itself is root in a container/VM without systemd). # PILOT_MANAGEMENT_URL=https://management.example # Same as --managed-url. Requires the one-time # PILOT_ENROLLMENT_TOKEN on first adoption. @@ -79,7 +82,8 @@ set -e # 3. Downloads the release tarball + checksums.txt from that release # 4. *** Verifies SHA-256 of the tarball against checksums.txt AND the signed # manifest (aborts on mismatch OR if it cannot verify — never extracts -# an unverified archive) *** +# an unverified archive; the manifest hashes the release it describes, +# so a --version pin or beta tag is checked against checksums.txt) *** # 5. Extracts binaries to ~/.pilot/bin (per-user, NOT system-wide) # 6. Adds ~/.pilot/bin to PATH in your shell profiles (~/.profile, ~/.bashrc, # ~/.zshenv, ~/.zshrc, ~/.bash_profile when it already exists) @@ -107,9 +111,9 @@ set -e # Network 9 directory and for receiving identifier-based deliveries. # # WHAT THIS SCRIPT DOES NOT DO: -# - Run as root on a host with systemd or launchd (refuses; see the root -# check below). A Linux container/VM without systemd, where the agent is -# root, installs into root's own $HOME/.pilot. +# - Run as root on a host with systemd or launchd, or under sudo (refuses; +# see the root check below). A Linux container/VM without systemd, where +# the agent itself is root, installs into root's own $HOME/.pilot. # - Send any personal data anywhere (the install script only fetches the # release tarball from GitHub; the daemon registers its public key + a # synthetic or user-supplied email with the rendezvous server, nothing else) @@ -218,7 +222,9 @@ while [ $# -gt 0 ]; do --no-start) PILOT_MANAGED_NO_START=1; shift ;; -h|--help) - sed -n '4,74p' "$0" 2>/dev/null || echo "See https://pilotprotocol.network/install.sh" + # The usage header: from line 4 up to "WHAT THIS SCRIPT DOES". + awk 'NR >= 4 { if (/^# WHAT THIS SCRIPT DOES/) exit; print }' "$0" 2>/dev/null \ + || echo "See https://pilotprotocol.network/install.sh" exit 0 ;; --) shift @@ -330,13 +336,37 @@ set -- $PILOT_POSITIONAL # home, not /root. A Linux container or VM without systemd (CI runners, # hosted agent sandboxes such as Meta Muse, where the agent IS root) has no # other user to install for and no system service to protect, so root is -# allowed there. +# allowed there — but only when root is who runs it. Root reached through +# sudo/doas/pkexec is a regular user's install (`curl ... | sudo sh` on WSL, +# OpenRC/runit hosts, dev containers): it would land in /root/.pilot (0700, +# with the /usr/local/bin links pointing into it) or, with sudo -E, in a +# root-owned ~/.pilot, and that user could run neither. Refused everywhere. SANDBOX_HOST=false if [ "$(uname -s)" = "Linux" ] && [ ! -d /run/systemd/system ]; then SANDBOX_HOST=true fi +ELEVATED_FROM="" +if [ -n "${SUDO_UID:-}" ]; then + if [ "$SUDO_UID" != "0" ]; then + ELEVATED_FROM="sudo for ${SUDO_USER:-uid $SUDO_UID}" + fi +elif [ -n "${SUDO_USER:-}" ] && [ "$SUDO_USER" != "root" ]; then + ELEVATED_FROM="sudo for $SUDO_USER" +elif [ -n "${DOAS_USER:-}" ] && [ "$DOAS_USER" != "root" ]; then + ELEVATED_FROM="doas for $DOAS_USER" +elif [ -n "${PKEXEC_UID:-}" ] && [ "$PKEXEC_UID" != "0" ]; then + ELEVATED_FROM="pkexec for uid $PKEXEC_UID" +fi if [ "${1:-}" != "uninstall" ] && [ "$(id -u)" = "0" ] && [ -z "${PILOT_ALLOW_ROOT:-}" ]; then - if [ "$SANDBOX_HOST" = true ]; then + if [ -n "$ELEVATED_FROM" ]; then + echo "Error: refusing to install as root: this runs under ${ELEVATED_FROM}." + echo " The node would be installed for root, into ${HOME}/.pilot, and that" + echo " user could not use it. Run the installer as that user, without sudo or doas:" + echo " curl -fsSL https://pilotprotocol.network/install.sh | sh" + echo " It uses sudo itself only where needed (never with a password prompt)." + echo " Set PILOT_ALLOW_ROOT=1 to install for root anyway (not recommended)." + exit 1 + elif [ "$SANDBOX_HOST" = true ]; then echo "Note: installing as root (no systemd: container/VM sandbox) into ${HOME}/.pilot" else echo "Error: refusing to install as root." @@ -542,16 +572,36 @@ manifest_field() { # independent integrity anchor (served from pilotprotocol.network) alongside the # release's checksums.txt (served from GitHub). manifest_platform_sha256() { - _mp_plat="$1"; _mp_file="$2" + manifest_platform_field "$1" sha256 "$2" +} + +# manifest_platform_field "-" "" "" extracts one string +# field (sha256, url) of that platform's object; empty when absent. +manifest_platform_field() { + _mp_plat="$1"; _mp_key="$2"; _mp_file="$3" # The authority is free to emit compact JSON. A line-range parser sees all # platform objects on that one line and a greedy replacement can therefore # return the final platform's hash. Collapse whitespace deliberately, then # constrain the match to this platform's first closing brace. tr -d '\r\n' < "$_mp_file" \ - | sed -n -E "s/.*\"${_mp_plat}\"[[:space:]]*:[[:space:]]*\\{[^}]*\"sha256\"[[:space:]]*:[[:space:]]*\"([^\"]*)\"[^}]*\\}.*/\\1/p" \ + | sed -n -E "s/.*\"${_mp_plat}\"[[:space:]]*:[[:space:]]*\\{[^}]*\"${_mp_key}\"[[:space:]]*:[[:space:]]*\"([^\"]*)\"[^}]*\\}.*/\\1/p" \ | head -1 } +# manifest_describes_tag "" "-" "" — whether the +# manifest's per-platform entry is the archive of . The manifest carries +# hashes for one release only: the platform url names it +# (…/releases/download//…), and a manifest without urls (the managed +# runtime's) describes its latest_stable. Any other tag — a --version pin, the +# beta channel — has no second anchor, so its hash must not be compared. +manifest_describes_tag() { + [ "$1" = "$(manifest_field "latest_stable" "$3")" ] && return 0 + case "$(manifest_platform_field "$2" url "$3")" in + */download/"$1"/*) return 0 ;; + esac + return 1 +} + # version_compare a b emits -1 / 0 / 1 for ab. # Honors semver: a prerelease tag ("X.Y.Z-rcN") is LOWER than the same base # without it ("X.Y.Z"). Plain `sort -V` gets this backwards on hyphenated @@ -704,16 +754,13 @@ echo " Pilot Protocol" echo " The network stack for AI agents." echo "" echo " Platform: ${OS}/${ARCH}" +# auto is not announced here: whether the release being installed has it is +# known only after the download (releases before it run udp). The summary at +# the end states the transport the installed daemon will actually use. case "$EFFECTIVE_TRANSPORT" in compat) echo " Transport: compat (TLS + WSS over TCP 443 only)" ;; - auto) - echo " Transport: auto (UDP when it works, else TLS + WSS over TCP 443)" - if [ -z "$PILOT_PROXY_URL" ]; then - echo " Registry: ${REGISTRY}" - echo " Beacon: ${BEACON}" - fi ;; - *) + udp) echo " Registry: ${REGISTRY}" echo " Beacon: ${BEACON}" ;; esac @@ -953,8 +1000,11 @@ if [ -n "$TAG" ]; then if pcurl -fsSL "$CHECKSUMS_URL" -o "$TMPDIR/checksums.txt" 2>/dev/null; then EXPECTED_CKS=$(grep " ${ARCHIVE}\$" "$TMPDIR/checksums.txt" | awk '{print $1}') fi + # The manifest hashes only the release it describes (see + # manifest_describes_tag): for any other tag it is no anchor at all. EXPECTED_MAN="" - if [ "$HAVE_MANIFEST" = "1" ]; then + if [ "$HAVE_MANIFEST" = "1" ] \ + && manifest_describes_tag "$TAG" "${OS}-${ARCH}" "$MANIFEST_FILE"; then EXPECTED_MAN=$(manifest_platform_sha256 "${OS}-${ARCH}" "$MANIFEST_FILE") fi @@ -1454,8 +1504,12 @@ if [ -n "$PROXY_CMD_TO_SAVE" ]; then echo "Proxy credentials: re-read by the daemon via proxy_cmd (${PILOT_DIR}/config.json stores the command, not the credentials)" else echo "Proxy credentials: proxy_cmd saved in ${PILOT_DIR}/config.json (the command, not the credentials)." - echo " This pilot-daemon (${TAG:-source}) predates -proxy-cmd and ignores it until upgraded;" - echo " until then, if the proxy rotates its credentials, restart the daemon from a fresh shell." + if [ "$DAEMON_HAS_PROXY" = true ]; then + echo " This pilot-daemon (${TAG:-source}) predates -proxy-cmd and ignores it until upgraded;" + echo " until then, if the proxy rotates its credentials, restart the daemon from a fresh shell." + else + echo " This pilot-daemon (${TAG:-source}) predates -proxy-cmd and ignores it until upgraded." + fi fi fi PROXY_CMD_SAVED=false @@ -1498,16 +1552,68 @@ fi # A proxy in the environment that this daemon cannot use: say so, and where # the proxy is the only way out, point at the recipe that works with it. +# Such a daemon dials the registry and beacon directly, around the proxy. +# PROXY_ONLY: the proxy is known to be the way out — $PILOT_PROXY_CMD is set +# (rotating credentials), or a credential-bearing proxy in a Linux +# container/VM without systemd (a hosted agent sandbox such as Meta Muse). +# There every "start the daemon" instruction below is replaced by the +# recipe; elsewhere it is kept, with the condition spelled out. +SANDBOX_RECIPE_URL="https://pilotprotocol.network/learn/install-pilot-skills-in-meta-muse" PROXY_UNSUPPORTED=false +PROXY_ONLY=false if [ -n "$PILOT_PROXY_URL" ] && [ "$DAEMON_HAS_PROXY" != true ]; then PROXY_UNSUPPORTED=true + if [ -n "${PILOT_PROXY_CMD:-}" ]; then + PROXY_ONLY=true + elif [ "$SANDBOX_HOST" = true ]; then + case "${https_proxy:-}${HTTPS_PROXY:-}${PILOT_PROXY_URL}" in + *@*) PROXY_ONLY=true ;; + esac + fi echo "" - echo " WARNING: a proxy is set ($(redact_proxy "$PILOT_PROXY_URL")), but this pilot-daemon" - echo " (${TAG:-source}) cannot use one. If the proxy is this host's only way" - echo " out (UDP blocked, e.g. an agent sandbox), \`pilotctl daemon start\` will" - echo " not come online with this release. Use the pilot-sandbox recipe:" - echo " https://pilotprotocol.network/learn/install-pilot-skills-in-meta-muse" -fi + if [ "$PROXY_ONLY" = true ]; then + echo " WARNING: this host goes out through a proxy ($(redact_proxy "$PILOT_PROXY_URL")), and this" + echo " pilot-daemon (${TAG:-source}) cannot use one: \`pilotctl daemon start\` would dial" + echo " the registry directly, around the proxy, and never come online." + echo " Do not run it here. Bring the node online with the pilot-sandbox" + echo " recipe (step 3): ${SANDBOX_RECIPE_URL}" + else + echo " WARNING: a proxy is set ($(redact_proxy "$PILOT_PROXY_URL")), but this pilot-daemon" + echo " (${TAG:-source}) cannot use one: it dials the registry directly. If the" + echo " proxy is this host's only way out (UDP blocked, e.g. an agent sandbox)," + echo " \`pilotctl daemon start\` will not come online with this release: use the" + echo " pilot-sandbox recipe (step 3) instead: ${SANDBOX_RECIPE_URL}" + fi +fi + +# UDP_ONLY_HINT: the daemon runs udp only because this release predates auto +# (nothing chose udp), so on a UDP-blocked host it will not fall back to TCP +# 443 by itself. Said in the summary, with the re-run that selects compat +# (which also points an older pilotctl at the TLS registry). Not where the +# proxy warning above already applies: compat would not use the proxy either. +UDP_ONLY_HINT=false +if [ "$EFFECTIVE_TRANSPORT" = "udp" ] && [ "$DAEMON_HAS_AUTO" != true ] \ + && [ "$DAEMON_HAS_TRANSPORT" = true ] && [ "$PROXY_UNSUPPORTED" != true ] \ + && [ "$TRANSPORT" != "udp" ] && [ "$CONFIG_TRANSPORT" != "udp" ]; then + UDP_ONLY_HINT=true +fi + +# start_hint PREFIX COMMAND [NAME] — print how to start the daemon: COMMAND, +# except where this daemon cannot use the proxy (the WARNING above). On a +# proxy-only host (PROXY_ONLY) COMMAND is not printed as something to run: +# only that NAME (default: COMMAND) must not be run there, and the recipe. +start_hint() { + if [ "$PROXY_ONLY" = true ]; then + echo "${1}Do not run \`${3:-$2}\` on this host (see the WARNING above)." + echo "${1}Use the pilot-sandbox recipe (step 3): ${SANDBOX_RECIPE_URL}" + elif [ "$PROXY_UNSUPPORTED" = true ]; then + echo "${1}${2}" + echo "${1}(if the proxy is this host's only way out, use the pilot-sandbox recipe" + echo "${1} instead, see the WARNING above)" + else + echo "${1}${2}" + fi +} # Network flags for the service units. The transport itself comes from # config.json, which the daemon reads, so `pilotctl config --set transport=` @@ -1732,13 +1838,24 @@ USVC elif [ "$PILOT_MANAGED_MODE" != "1" ]; then case " $RESTART_SYSTEMD " in *" pilot-daemon "*) ;; - *) echo " Start daemon: sudo systemctl enable --now pilot-daemon" ;; + *) + if [ "$PROXY_UNSUPPORTED" = true ]; then + echo " Start daemon:" + start_hint " " "sudo systemctl enable --now pilot-daemon" + else + echo " Start daemon: sudo systemctl enable --now pilot-daemon" + fi ;; esac fi else echo " Skipped systemd setup (run as root or with passwordless sudo to enable)" if [ "$PILOT_MANAGED_MODE" != "1" ]; then - echo " Start the daemon without a service manager: pilotctl daemon start" + if [ "$PROXY_UNSUPPORTED" = true ]; then + echo " Start the daemon without a service manager:" + start_hint " " "pilotctl daemon start" + else + echo " Start the daemon without a service manager: pilotctl daemon start" + fi fi fi elif [ "$OS" = "linux" ]; then @@ -1746,11 +1863,13 @@ elif [ "$OS" = "linux" ]; then # hosted agent sandbox). There is no service to install — tell the agent # the portable start path instead of silently leaving it with no daemon. if [ "$PILOT_MANAGED_MODE" != "1" ]; then - echo "No systemd detected (container / WSL / CI / sandbox) — start the daemon manually:" - echo " pilotctl daemon start" - if [ "$PROXY_UNSUPPORTED" = true ]; then - echo " (proxy-only host: see the WARNING above)" - elif [ "$EFFECTIVE_TRANSPORT" != "udp" ]; then + if [ "$PROXY_ONLY" = true ]; then + echo "No systemd detected (container / WSL / CI / sandbox):" + else + echo "No systemd detected (container / WSL / CI / sandbox) — start the daemon manually:" + fi + start_hint " " "pilotctl daemon start" + if [ "$PROXY_UNSUPPORTED" != true ] && [ "$EFFECTIVE_TRANSPORT" != "udp" ]; then echo " (transport=${EFFECTIVE_TRANSPORT}; start it from a shell that has HTTPS_PROXY" echo " set if this host reaches the internet only through a proxy)" fi @@ -2043,6 +2162,36 @@ if [ "$PILOT_MANAGED_MODE" = "1" ]; then exit 0 fi +# transport_line — the transport the installed daemon will run, stated once +# the binaries are known (the banner at the top does not guess), plus what to +# do on a UDP-blocked host when this release will not fall back by itself. +transport_line() { + case "$EFFECTIVE_TRANSPORT" in + compat) + if [ "$DAEMON_HAS_TRANSPORT" != true ]; then + echo " Transport: udp (compat is saved, but this pilot-daemon, ${TAG:-source}," + echo " predates it; see the WARNING above)" + return 0 + fi + _tl_registry="$COMPAT_REGISTRY" + if [ "$REGISTRY" != "$DEFAULT_REGISTRY" ]; then _tl_registry="$REGISTRY"; fi + echo " Transport: compat (registry ${_tl_registry} over TLS, beacon over WSS)" ;; + auto) + echo " Transport: auto (UDP when it works, else compat over TCP 443)" ;; + *) + if [ "$DAEMON_HAS_AUTO" != true ]; then + echo " Transport: udp (this pilot-daemon, ${TAG:-source}, predates auto and never" + echo " falls back to TCP 443 by itself)" + else + echo " Transport: udp" + fi ;; + esac + if [ "$UDP_ONLY_HINT" = true ]; then + echo " UDP blocked on this host? Use TLS + WSS over TCP 443 instead:" + echo " curl -fsSL https://pilotprotocol.network/install.sh | sh -s -- --transport compat" + fi +} + # --- Upgrade: short summary, skip the first-run onboarding text --- # # Everything above this point (binary swap, unit/plist regeneration, service @@ -2056,11 +2205,12 @@ if [ "$UPDATING" = true ]; then echo " pilotctl ${BIN_DIR}/pilotctl" [ -f "$BIN_DIR/pilot-gateway" ] && echo " pilot-gateway ${BIN_DIR}/pilot-gateway" [ -f "$BIN_DIR/pilot-updater" ] && echo " pilot-updater ${BIN_DIR}/pilot-updater" + transport_line echo "" if [ -z "$RESTART_SYSTEMD" ] && [ -z "$RESTART_LAUNCHD" ]; then echo "No managed service was running. If you run the daemon yourself," echo "restart it to pick up the new version:" - echo " pilotctl daemon stop && pilotctl daemon start" + start_hint " " "pilotctl daemon stop && pilotctl daemon start" "pilotctl daemon start" echo "" fi exit 0 @@ -2074,13 +2224,10 @@ echo " pilotctl ${BIN_DIR}/pilotctl" [ -f "$BIN_DIR/pilot-updater" ] && echo " pilot-updater ${BIN_DIR}/pilot-updater (auto-updates in background)" echo "" echo "Config: ${PILOT_DIR}/config.json" +transport_line case "$EFFECTIVE_TRANSPORT" in - compat) - _summary_registry="$COMPAT_REGISTRY" - if [ "$REGISTRY" != "$DEFAULT_REGISTRY" ]; then _summary_registry="$REGISTRY"; fi - echo " Transport: compat (registry ${_summary_registry} over TLS, beacon over WSS)" ;; + compat) ;; auto) - echo " Transport: auto (UDP when it works, else compat over TCP 443)" if [ "$STOCK_ENDPOINTS" = true ]; then echo " Registry: ${REGISTRY}" echo " Beacon: ${BEACON}" @@ -2112,8 +2259,22 @@ echo "" echo " 0) Put pilotctl on your PATH and bring the node online." echo " ------------------------------------------------------------------" echo " export PATH=\"${BIN_DIR}:\$PATH\" # only needed in THIS shell, before you open a new one" +# The start command itself only where this daemon can come online with it +# (see PROXY_UNSUPPORTED / PROXY_ONLY): a proxy-only host gets the recipe. +if [ "$PROXY_ONLY" = true ]; then + echo " # Do NOT run \`pilotctl daemon start\` on this host: this pilot-daemon (${TAG:-source})" + echo " # cannot use the proxy and would dial the registry directly. Bring the node" + echo " # online with the pilot-sandbox recipe (step 3), then check it here:" + echo " # ${SANDBOX_RECIPE_URL}" +elif [ "$PROXY_UNSUPPORTED" = true ]; then + echo " # This pilot-daemon (${TAG:-source}) cannot use the proxy (see the WARNING above). If the" + echo " # proxy is this host's only way out, skip the next line and use the pilot-sandbox" + echo " # recipe (step 3) instead: ${SANDBOX_RECIPE_URL}" + echo " pilotctl daemon start --hostname my-agent # blocks until registered; email already saved" +else + echo " pilotctl daemon start --hostname my-agent # blocks until registered; email already saved" +fi cat <<'PILOT_GET_STARTED' - pilotctl daemon start --hostname my-agent # blocks until registered; email already saved pilotctl daemon status # confirm it's running pilotctl info # node ID, address, peer count, uptime From df090a8a4f85e7b5cd32572ccf1cec89e068bacf Mon Sep 17 00:00:00 2001 From: Teodor Calin Date: Thu, 24 Sep 2026 12:03:30 +0300 Subject: [PATCH 16/19] fix(proxy): hints that fit a daemon already on proxy_cmd; gosec; install.sh sync with release#49 (c36ad9b) - pilotctl daemon start: when the daemon re-reads its credentials with a proxy command and the proxy still rejects them, the hint names the command in use and says to check what it prints from a fresh shell, instead of telling the operator to set one (E2E scenario 3: the 407 hint said "give the daemon a command that prints the current proxy URL" with config.json proxy_cmd in place). The daemon's own CredentialHint is worded for both. - TestInstallerSavesTheSandboxProxyCmd: install.sh's SANDBOX_PROXY_CMD must evaluate to pilotctl's sandboxProxyCmd. A saved proxy_cmd turns off the one `daemon start` hands the daemon, so the two must never drift (web4-470 review: the installer kept the pre-change command). - gosec: #nosec G204 with the reason on proxyconf.CommandSource (the operator's proxy command run with sh -c is the function's purpose), and the new G104 warnings (unchecked Close/Remove) in relay.go and pilotctl. - install.sh: byte-identical to pilot-protocol/release#49 c36ad9b (sandbox proxy_cmd only for credentials a fresh shell sees and never over an explicit PILOT_PROXY; restart advice stops the running daemon before the sandbox recipe; macOS LaunchAgent line conditioned and never sent to the Linux-only recipe; truthful --transport auto note for pre-auto daemons). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 9 ++- cmd/pilotctl/daemon_startlog.go | 26 +++++-- cmd/pilotctl/main.go | 6 +- cmd/pilotctl/zz_proxy_muse_hints_test.go | 67 +++++++++++++++++- install.sh | 90 +++++++++++++++++------- internal/proxyconf/command.go | 3 + internal/proxyconf/proxyconf.go | 10 ++- internal/proxyconf/relay.go | 12 ++-- 8 files changed, 178 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f79571a..b1bff3c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -229,10 +229,17 @@ Detailed per-release notes are on the `~/.pilot/update-state.json`. ### Fixed +- **Proxy credential hints no longer send an operator who already set + `proxy_cmd` off to set it.** When the daemon re-reads its credentials with + a proxy command and the proxy still rejects them (407, or Meta Muse's + garbled answer), `pilotctl daemon start` names the command in use and says + to check what it prints from a fresh shell; the daemon's own hint covers + both cases. The installer's sandbox `proxy_cmd` is now checked in CI to be + the command `pilotctl daemon start` hands the daemon (they must not drift). - **`pilotctl --json trusted list` printed the text table**; it now returns `{"trusted": [{"hostname", "address", "node_id"}], "count"}`. - **A new pilotctl starting a pilot-daemon that predates proxy support** - (v1.13.9) from a shell with `$HTTPS_PROXY` / `$ALL_PROXY` now warns that + (v1.13.10 and earlier) from a shell with `$HTTPS_PROXY` / `$ALL_PROXY` now warns that the daemon will not use the proxy, instead of leaving a bare "did not become ready" to explain it. - **Downgrading after `transport=auto` was saved no longer bricks the daemon.** diff --git a/cmd/pilotctl/daemon_startlog.go b/cmd/pilotctl/daemon_startlog.go index 97c229d0..cd058179 100644 --- a/cmd/pilotctl/daemon_startlog.go +++ b/cmd/pilotctl/daemon_startlog.go @@ -206,15 +206,29 @@ func daemonExitStatus(err error) string { // daemon rotating credentials. const rotateHint = "if the proxy rotates its credentials, give the daemon a command that prints the current proxy URL: --proxy-cmd, or pilotctl config --set proxy_cmd=\"bash -c 'printf %s \\\"\\$https_proxy\\\"'\"" +// refreshingHint replaces rotateHint when the daemon already re-reads its +// credentials with a proxy command (src names it, see proxyCmdSource): +// telling the user to set one would send them after the wrong fix. The +// command itself printed credentials the proxy refused. +func refreshingHint(src string) string { + return "the daemon already re-reads them with " + src + " (every 60s and on each rejection), so that command printed credentials the proxy refused: run it from a fresh shell and check that it prints a proxy URL with the current credentials" +} + // proxyErrorHint says what to do about a proxy error the daemon logged. -func proxyErrorHint(proxyErr string) string { +// refreshSrc is the proxy command the daemon re-reads its credentials +// with (proxyCmdSource), "" when it has none. +func proxyErrorHint(proxyErr, refreshSrc string) string { + credentials := rotateHint + if refreshSrc != "" { + credentials = refreshingHint(refreshSrc) + } switch { case strings.Contains(proxyErr, " 407 "): - return "the proxy rejected the credentials (407): check HTTPS_PROXY (or --proxy / PILOT_PROXY); " + rotateHint + return "the proxy rejected the credentials (407): check HTTPS_PROXY (or --proxy / PILOT_PROXY); " + credentials case strings.Contains(proxyErr, "read CONNECT response: ") && strings.Contains(proxyErr, "(response text withheld)"): // netproxy's report of a CONNECT answer it could not parse — how // Meta Muse's proxy rejects wrong or expired credentials. - return "the proxy's answer to CONNECT could not be parsed (\"malformed HTTP status code\"), which is how some egress proxies (Meta Muse's) reject wrong or expired credentials: check the credentials in HTTPS_PROXY (or --proxy / PILOT_PROXY); " + rotateHint + return "the proxy's answer to CONNECT could not be parsed (\"malformed HTTP status code\"), which is how some egress proxies (Meta Muse's) reject wrong or expired credentials: check the credentials in HTTPS_PROXY (or --proxy / PILOT_PROXY); " + credentials case strings.Contains(proxyErr, ": dial proxy "): return "the proxy could not be reached: check HTTPS_PROXY (or --proxy / PILOT_PROXY)" default: @@ -226,12 +240,12 @@ func proxyErrorHint(proxyErr string) string { // ready: it exited (exitStatus != "") or the wait ran out. It shows what // the daemon's log says went wrong — above all its last proxy error, which // behind an egress proxy is nearly always the cause — instead of only "did -// not become ready". -func reportDaemonStartFailure(pid int, logPath, exitStatus string, waited time.Duration) { +// not become ready". refreshSrc is as for proxyErrorHint. +func reportDaemonStartFailure(pid int, logPath, exitStatus string, waited time.Duration, refreshSrc string) { proxyErr := lastProxyErrorFromLog(logPath) hint := fmt.Sprintf("check logs: tail -f %s", logPath) if proxyErr != "" { - hint = proxyErrorHint(proxyErr) + "; full log: " + logPath + hint = proxyErrorHint(proxyErr, refreshSrc) + "; full log: " + logPath } if exitStatus != "" { msg := fmt.Sprintf("daemon (pid %d) exited during startup (%s)", pid, exitStatus) diff --git a/cmd/pilotctl/main.go b/cmd/pilotctl/main.go index 5bd62f12..7968cd98 100644 --- a/cmd/pilotctl/main.go +++ b/cmd/pilotctl/main.go @@ -3271,8 +3271,8 @@ func cmdDaemonStart(args []string) { if !jsonOutput { fmt.Fprintln(os.Stderr) // end the dots line } - os.Remove(pidFilePath()) - reportDaemonStartFailure(pid, pidLogPath, daemonExitStatus(werr), 0) + _ = os.Remove(pidFilePath()) + reportDaemonStartFailure(pid, pidLogPath, daemonExitStatus(werr), 0, proxyCmdSource(plan, flags, pidLogPath, sandboxRefresh)) case <-time.After(200 * time.Millisecond): } dots++ @@ -3350,7 +3350,7 @@ func cmdDaemonStart(args []string) { fmt.Fprintln(os.Stderr) // end the dots line } - reportDaemonStartFailure(pid, pidLogPath, "", waitDur) + reportDaemonStartFailure(pid, pidLogPath, "", waitDur, proxyCmdSource(plan, flags, pidLogPath, sandboxRefresh)) } func cmdDaemonStop() { diff --git a/cmd/pilotctl/zz_proxy_muse_hints_test.go b/cmd/pilotctl/zz_proxy_muse_hints_test.go index 3a218755..512fd7f0 100644 --- a/cmd/pilotctl/zz_proxy_muse_hints_test.go +++ b/cmd/pilotctl/zz_proxy_muse_hints_test.go @@ -22,7 +22,7 @@ import ( // the proxy must allow CONNECT to the registry. func TestCLIDaemonStartGarbledProxyAnswerHint(t *testing.T) { const garbled = "proxy CONNECT registry.pilotprotocol.network:443 via http://***@127.0.0.1:3151: read CONNECT response: malformed HTTP status code (response text withheld)" - hint := proxyErrorHint(garbled) + hint := proxyErrorHint(garbled, "") if !strings.Contains(hint, "could not be parsed") || !strings.Contains(hint, "proxy_cmd") || strings.Contains(hint, "must allow CONNECT") { t.Errorf("proxyErrorHint(garbled) = %q", hint) } @@ -33,7 +33,7 @@ func TestCLIDaemonStartGarbledProxyAnswerHint(t *testing.T) { "proxy CONNECT registry.pilotprotocol.network:443: 403 Forbidden", "proxy CONNECT registry.pilotprotocol.network:443 via http://***@p:1: read CONNECT response: EOF", } { - if h := proxyErrorHint(other); strings.Contains(h, "could not be parsed") { + if h := proxyErrorHint(other, ""); strings.Contains(h, "could not be parsed") { t.Errorf("proxyErrorHint(%q) = %q, not a garbled answer", other, h) } } @@ -115,3 +115,66 @@ func TestSandboxProxyCmdKeepsTheCredentialedProxy(t *testing.T) { t.Fatalf("daemon proxy with the sandbox command = %q, want the credentialed http://***@egress.test:3128", got) } } + +// When the daemon already re-reads its credentials with a proxy command, +// a rejected credential means that command printed stale or wrong ones: +// the hint must say so and not tell the operator to set one (E2E scenario +// 3: a 407 with config.json proxy_cmd in place was answered with "give the +// daemon a command that prints the current proxy URL"). +func TestProxyErrorHintWithProxyCommandInUse(t *testing.T) { + for _, proxyErr := range []string{ + "proxy CONNECT registry.pilotprotocol.network:443: 407 Proxy Authentication Required", + "proxy CONNECT registry.pilotprotocol.network:443 via http://***@127.0.0.1:3151: read CONNECT response: malformed HTTP status code (response text withheld)", + } { + without := proxyErrorHint(proxyErr, "") + if !strings.Contains(without, rotateHint) { + t.Errorf("no proxy command: hint %q lacks the proxy_cmd advice", without) + } + with := proxyErrorHint(proxyErr, "config.json proxy_cmd") + if strings.Contains(with, rotateHint) || strings.Contains(with, "give the daemon a command") { + t.Errorf("proxy command in use: hint %q still says to set one", with) + } + if !strings.Contains(with, "already re-reads them with config.json proxy_cmd") || !strings.Contains(with, "fresh shell") { + t.Errorf("proxy command in use: hint %q does not name it or say how to check it", with) + } + } + if h := proxyErrorHint("proxy CONNECT registry.pilotprotocol.network:443: 403 Forbidden", "config.json proxy_cmd"); strings.Contains(h, "re-reads") { + t.Errorf("a 403 is not a credential problem: %q", h) + } +} + +// install.sh saves SANDBOX_PROXY_CMD as config.json proxy_cmd on sandbox +// hosts, and a saved proxy_cmd turns off the one `daemon start` would hand +// the daemon (sandboxProxyCmdFor). The two must be the same command, or a +// node the installer set up runs an older one (web4-470 review: the +// installer kept `printf %s "${https_proxy:-$HTTPS_PROXY}"`, which drops +// credentials that only HTTPS_PROXY carries). +func TestInstallerSavesTheSandboxProxyCmd(t *testing.T) { + sh, err := exec.LookPath("sh") + if err != nil { + t.Skip("no sh") + } + src, err := os.ReadFile(filepath.Join("..", "..", "install.sh")) + if err != nil { + t.Fatal(err) + } + var assign string + for _, line := range strings.Split(string(src), "\n") { + if strings.HasPrefix(line, "SANDBOX_PROXY_CMD=") { + if assign != "" { + t.Fatal("install.sh assigns SANDBOX_PROXY_CMD more than once") + } + assign = line + } + } + if assign == "" { + t.Fatal("install.sh has no SANDBOX_PROXY_CMD= line") + } + out, err := exec.Command(sh, "-c", assign+`; printf %s "$SANDBOX_PROXY_CMD"`).Output() + if err != nil { + t.Fatalf("evaluating %q: %v", assign, err) + } + if string(out) != sandboxProxyCmd { + t.Errorf("install.sh SANDBOX_PROXY_CMD = %q\npilotctl sandboxProxyCmd = %q", out, sandboxProxyCmd) + } +} diff --git a/install.sh b/install.sh index 4e9b10b6..7dc168ca 100755 --- a/install.sh +++ b/install.sh @@ -438,6 +438,16 @@ EFFECTIVE_TRANSPORT="${TRANSPORT:-${CONFIG_TRANSPORT:-auto}}" # authenticating proxy is a credential. Nothing in this script writes a proxy # URL to disk. # +# ENV_PROXY_CREDS: the proxy environment itself carries credentials — read +# before PILOT_PROXY is copied into it below for this run's downloads. Only +# then can the sandbox proxy command (SANDBOX_PROXY_CMD), which prints what a +# fresh shell's $https_proxy / $HTTPS_PROXY hold, print them: credentials +# that arrive in PILOT_PROXY never reach a fresh shell. +ENV_PROXY_CREDS=false +case "${https_proxy:-}${HTTPS_PROXY:-}" in + *@*) ENV_PROXY_CREDS=true ;; +esac + # PILOT_PROXY is the daemon's own proxy setting; an http(s):// URL there # carries this run's downloads too when the environment names no proxy # (exported to this process and its children only). @@ -457,20 +467,23 @@ PILOT_PROXY_URL="${https_proxy:-${HTTPS_PROXY:-${all_proxy:-${ALL_PROXY:-}}}}" # keeps the ones it started with, and the proxy answers its next CONNECT with # 407. A fresh shell sees the current ones. PROXY_REFRESH_CMD prints the # current proxy URL: $PILOT_PROXY_CMD, else — in a Linux container/VM without -# systemd whose HTTPS_PROXY or https_proxy carries credentials — what a fresh -# bash has: whichever of $https_proxy and $HTTPS_PROXY carries credentials -# ($https_proxy when both do, the variable Meta Muse's guidance reads), else +# systemd whose HTTPS_PROXY or https_proxy carries credentials (ENV_PROXY_CREDS) +# and no explicit PILOT_PROXY is set — what a fresh bash has: whichever of +# $https_proxy and $HTTPS_PROXY carries credentials ($https_proxy when both do, the variable Meta Muse's guidance reads), else # ${HTTPS_PROXY:-$https_proxy}, so a URL with credentials is never traded for # one without (pilotctl uses the same command). It is saved as the daemon's # proxy_cmd further down, and pcurl uses it here to retry a download once # after the credentials rotated mid-install. # shellcheck disable=SC2016 # literal: the fresh bash expands it, not this shell SANDBOX_PROXY_CMD='bash -c '\''case $https_proxy in *@*) printf %s "$https_proxy";; *) printf %s "${HTTPS_PROXY:-$https_proxy}";; esac'\''' +# An explicit PILOT_PROXY is left alone, as pilotctl leaves it: pilot-daemon +# runs a proxy command in place of the URL it would use, so the sandbox +# command would replace that URL with the environment's proxy. PROXY_REFRESH_CMD="${PILOT_PROXY_CMD:-}" if [ -z "$PROXY_REFRESH_CMD" ] && [ "$SANDBOX_HOST" = true ] \ - && command -v bash >/dev/null 2>&1; then - case "${https_proxy:-}${HTTPS_PROXY:-}" in - *@*) PROXY_REFRESH_CMD="$SANDBOX_PROXY_CMD" ;; + && [ "$ENV_PROXY_CREDS" = true ] && command -v bash >/dev/null 2>&1; then + case "${PILOT_PROXY:-}" in + ""|auto|AUTO|Auto) PROXY_REFRESH_CMD="$SANDBOX_PROXY_CMD" ;; esac fi @@ -1367,7 +1380,16 @@ case "$TRANSPORT" in if [ "$DAEMON_HAS_AUTO" = true ]; then if [ -n "$CONFIG_TRANSPORT" ]; then TRANSPORT_CLEAR=true; fi else - echo " Note: this pilot-daemon (${TAG:-source}) predates -transport=auto; it keeps its default (udp)." + # Nothing is saved or removed: this release has no auto to go + # back to, and what it runs is what config.json already says. + case "$CONFIG_TRANSPORT" in + compat|udp) + echo " Note: this pilot-daemon (${TAG:-source}) predates -transport=auto; it keeps the" + echo " transport saved in config.json (${CONFIG_TRANSPORT}). Switch with --transport udp or" + echo " --transport compat." ;; + *) + echo " Note: this pilot-daemon (${TAG:-source}) predates -transport=auto; it keeps its default (udp)." ;; + esac fi ;; esac if [ "$CONFIG_TRANSPORT" = "auto" ] && [ "$DAEMON_HAS_AUTO" != true ] && [ -z "$TRANSPORT_TO_SAVE" ]; then @@ -1559,6 +1581,13 @@ fi # There every "start the daemon" instruction below is replaced by the # recipe; elsewhere it is kept, with the condition spelled out. SANDBOX_RECIPE_URL="https://pilotprotocol.network/learn/install-pilot-skills-in-meta-muse" +# The recipe needs Linux (root and `unshare -m` for its SNI router): on any +# other OS the way forward is a release whose pilot-daemon has -proxy. +if [ "$OS" = "linux" ]; then + PROXY_REMEDY="the pilot-sandbox recipe (step 3): ${SANDBOX_RECIPE_URL}" +else + PROXY_REMEDY="re-run this installer once a Pilot release whose pilot-daemon -h lists -proxy is out" +fi PROXY_UNSUPPORTED=false PROXY_ONLY=false if [ -n "$PILOT_PROXY_URL" ] && [ "$DAEMON_HAS_PROXY" != true ]; then @@ -1575,14 +1604,13 @@ if [ -n "$PILOT_PROXY_URL" ] && [ "$DAEMON_HAS_PROXY" != true ]; then echo " WARNING: this host goes out through a proxy ($(redact_proxy "$PILOT_PROXY_URL")), and this" echo " pilot-daemon (${TAG:-source}) cannot use one: \`pilotctl daemon start\` would dial" echo " the registry directly, around the proxy, and never come online." - echo " Do not run it here. Bring the node online with the pilot-sandbox" - echo " recipe (step 3): ${SANDBOX_RECIPE_URL}" + echo " Do not start it here. What brings the node online: ${PROXY_REMEDY}" else echo " WARNING: a proxy is set ($(redact_proxy "$PILOT_PROXY_URL")), but this pilot-daemon" echo " (${TAG:-source}) cannot use one: it dials the registry directly. If the" echo " proxy is this host's only way out (UDP blocked, e.g. an agent sandbox)," - echo " \`pilotctl daemon start\` will not come online with this release: use the" - echo " pilot-sandbox recipe (step 3) instead: ${SANDBOX_RECIPE_URL}" + echo " the daemon will not come online with this release. What does then:" + echo " ${PROXY_REMEDY}" fi fi @@ -1598,18 +1626,25 @@ if [ "$EFFECTIVE_TRANSPORT" = "udp" ] && [ "$DAEMON_HAS_AUTO" != true ] \ UDP_ONLY_HINT=true fi -# start_hint PREFIX COMMAND [NAME] — print how to start the daemon: COMMAND, -# except where this daemon cannot use the proxy (the WARNING above). On a -# proxy-only host (PROXY_ONLY) COMMAND is not printed as something to run: -# only that NAME (default: COMMAND) must not be run there, and the recipe. +# start_hint PREFIX COMMAND [NAME [STOP]] — print how to start the daemon: +# COMMAND, except where this daemon cannot use the proxy (the WARNING above). +# On a proxy-only host (PROXY_ONLY) COMMAND is not printed as something to +# run: only that NAME (default: COMMAND) must not be run there, and +# PROXY_REMEDY. STOP (restart hints) is printed first there: the recipe starts +# a daemon but never stops one, and two daemons must not share an identity. start_hint() { if [ "$PROXY_ONLY" = true ]; then echo "${1}Do not run \`${3:-$2}\` on this host (see the WARNING above)." - echo "${1}Use the pilot-sandbox recipe (step 3): ${SANDBOX_RECIPE_URL}" + if [ -n "${4:-}" ]; then + echo "${1}Stop the running daemon first: ${4}" + echo "${1}then bring it back with ${PROXY_REMEDY}" + else + echo "${1}What brings the node online: ${PROXY_REMEDY}" + fi elif [ "$PROXY_UNSUPPORTED" = true ]; then echo "${1}${2}" - echo "${1}(if the proxy is this host's only way out, use the pilot-sandbox recipe" - echo "${1} instead, see the WARNING above)" + echo "${1}(if the proxy is this host's only way out, it will not come online with" + echo "${1} this release: see the WARNING above)" else echo "${1}${2}" fi @@ -2035,7 +2070,12 @@ UPLIST case " $RESTART_LAUNCHD " in *" network.pilotprotocol.pilot-daemon "*) ;; *) - echo " Start daemon: launchctl load -w $PLIST" + if [ "$PROXY_UNSUPPORTED" = true ]; then + echo " Start daemon:" + start_hint " " "launchctl load -w $PLIST" "launchctl load -w $PLIST" + else + echo " Start daemon: launchctl load -w $PLIST" + fi echo " Stop daemon: launchctl unload $PLIST" ;; esac @@ -2210,7 +2250,7 @@ if [ "$UPDATING" = true ]; then if [ -z "$RESTART_SYSTEMD" ] && [ -z "$RESTART_LAUNCHD" ]; then echo "No managed service was running. If you run the daemon yourself," echo "restart it to pick up the new version:" - start_hint " " "pilotctl daemon stop && pilotctl daemon start" "pilotctl daemon start" + start_hint " " "pilotctl daemon stop && pilotctl daemon start" "pilotctl daemon start" "pilotctl daemon stop" echo "" fi exit 0 @@ -2263,13 +2303,13 @@ echo " export PATH=\"${BIN_DIR}:\$PATH\" # only needed in THIS shell, befo # (see PROXY_UNSUPPORTED / PROXY_ONLY): a proxy-only host gets the recipe. if [ "$PROXY_ONLY" = true ]; then echo " # Do NOT run \`pilotctl daemon start\` on this host: this pilot-daemon (${TAG:-source})" - echo " # cannot use the proxy and would dial the registry directly. Bring the node" - echo " # online with the pilot-sandbox recipe (step 3), then check it here:" - echo " # ${SANDBOX_RECIPE_URL}" + echo " # cannot use the proxy and would dial the registry directly. What brings the" + echo " # node online (then check it here):" + echo " # ${PROXY_REMEDY}" elif [ "$PROXY_UNSUPPORTED" = true ]; then echo " # This pilot-daemon (${TAG:-source}) cannot use the proxy (see the WARNING above). If the" - echo " # proxy is this host's only way out, skip the next line and use the pilot-sandbox" - echo " # recipe (step 3) instead: ${SANDBOX_RECIPE_URL}" + echo " # proxy is this host's only way out, skip the next line; what works then:" + echo " # ${PROXY_REMEDY}" echo " pilotctl daemon start --hostname my-agent # blocks until registered; email already saved" else echo " pilotctl daemon start --hostname my-agent # blocks until registered; email already saved" diff --git a/internal/proxyconf/command.go b/internal/proxyconf/command.go index d0a2004b..ce6dfcb9 100644 --- a/internal/proxyconf/command.go +++ b/internal/proxyconf/command.go @@ -34,6 +34,9 @@ func CommandSource(command string, env []string) func(ctx context.Context) (stri env = append(make([]string, 0, len(env)), env...) } return func(ctx context.Context) (string, error) { + // #nosec G204 -- running the operator's proxy command (-proxy-cmd, + // $PILOT_PROXY_CMD, config.json proxy_cmd) with sh -c is this function's + // purpose; it is set by whoever starts the daemon, never by a peer. cmd := exec.CommandContext(ctx, "sh", "-c", command) if env != nil { cmd.Env = env diff --git a/internal/proxyconf/proxyconf.go b/internal/proxyconf/proxyconf.go index 71958155..6c5593ca 100644 --- a/internal/proxyconf/proxyconf.go +++ b/internal/proxyconf/proxyconf.go @@ -391,15 +391,21 @@ func UnreadableConnectReply(err error) bool { return strings.Contains(msg, "read CONNECT response: ") && strings.Contains(msg, "(response text withheld)") } +// rotatingCredentialsHint ends CredentialHint. It covers both a daemon +// without a proxy command and one whose command is already set (and then +// printed the credentials the proxy refused), so it never sends an +// operator who set -proxy-cmd off to set it again. +const rotatingCredentialsHint = "if the proxy rotates its credentials, the daemon needs -proxy-cmd ($PILOT_PROXY_CMD, config.json proxy_cmd), a command that prints the current proxy URL; when one is set, check what it prints from a fresh shell" + // CredentialHint explains a proxy's rejection of the credentials — a 407, // or an answer that could not be parsed — for a log line or error; "" for // any other error. func CredentialHint(err error) string { switch { case AuthRejected(err): - return "the proxy rejected its credentials (407), also after re-reading them: check HTTPS_PROXY / -proxy; if the proxy rotates its credentials, set -proxy-cmd ($PILOT_PROXY_CMD, config.json proxy_cmd) to a command that prints the current proxy URL" + return "the proxy rejected its credentials (407), also after re-reading them: check HTTPS_PROXY / -proxy; " + rotatingCredentialsHint case UnreadableConnectReply(err): - return "the proxy's answer to CONNECT could not be parsed, which is how some egress proxies (Meta Muse's) reject wrong or expired credentials: check the credentials in HTTPS_PROXY / -proxy; if the proxy rotates them, set -proxy-cmd ($PILOT_PROXY_CMD, config.json proxy_cmd) to a command that prints the current proxy URL" + return "the proxy's answer to CONNECT could not be parsed, which is how some egress proxies (Meta Muse's) reject wrong or expired credentials: check the credentials in HTTPS_PROXY / -proxy; " + rotatingCredentialsHint } return "" } diff --git a/internal/proxyconf/relay.go b/internal/proxyconf/relay.go index 37415d8d..caedb098 100644 --- a/internal/proxyconf/relay.go +++ b/internal/proxyconf/relay.go @@ -112,7 +112,7 @@ func StartRelay(r *netproxy.Resolver, proxyTLS *tls.Config) (*Relay, error) { func startRelayOn(ln net.Listener, r *netproxy.Resolver, proxyTLS *tls.Config) (*Relay, error) { raw := make([]byte, 24) if _, err := rand.Read(raw); err != nil { - ln.Close() + _ = ln.Close() return nil, fmt.Errorf("proxy relay: token: %w", err) } token := hex.EncodeToString(raw) @@ -184,7 +184,7 @@ func (rl *Relay) Close() error { rl.closed = true err := rl.ln.Close() for c := range rl.conns { - c.Close() + _ = c.Close() } rl.mu.Unlock() rl.cancel() @@ -212,7 +212,7 @@ func (rl *Relay) untrack(c net.Conn) { rl.mu.Lock() delete(rl.conns, c) rl.mu.Unlock() - c.Close() + _ = c.Close() } func (rl *Relay) serve() { @@ -235,7 +235,7 @@ func (rl *Relay) serve() { } backoff = 5 * time.Millisecond if !rl.track(c, true) { - c.Close() + _ = c.Close() return } go func() { @@ -295,7 +295,7 @@ func (rl *Relay) handle(c net.Conn) { return } if !rl.track(up, false) { - up.Close() + _ = up.Close() return } defer rl.untrack(up) @@ -421,5 +421,5 @@ func closeWrite(c net.Conn) { return } } - c.Close() + _ = c.Close() } From 30c034bbde6ea5dd9e697ce6d49ad168520e4169 Mon Sep 17 00:00:00 2001 From: Teodor Calin Date: Thu, 24 Sep 2026 12:22:46 +0300 Subject: [PATCH 17/19] proxyconf: name the rotation hint so gosec G101 does not read it as a credential Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/proxyconf/proxyconf.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/proxyconf/proxyconf.go b/internal/proxyconf/proxyconf.go index 6c5593ca..27aba91d 100644 --- a/internal/proxyconf/proxyconf.go +++ b/internal/proxyconf/proxyconf.go @@ -391,11 +391,11 @@ func UnreadableConnectReply(err error) bool { return strings.Contains(msg, "read CONNECT response: ") && strings.Contains(msg, "(response text withheld)") } -// rotatingCredentialsHint ends CredentialHint. It covers both a daemon +// rotatingProxyHint ends CredentialHint. It covers both a daemon // without a proxy command and one whose command is already set (and then // printed the credentials the proxy refused), so it never sends an // operator who set -proxy-cmd off to set it again. -const rotatingCredentialsHint = "if the proxy rotates its credentials, the daemon needs -proxy-cmd ($PILOT_PROXY_CMD, config.json proxy_cmd), a command that prints the current proxy URL; when one is set, check what it prints from a fresh shell" +const rotatingProxyHint = "if the proxy rotates its credentials, the daemon needs -proxy-cmd ($PILOT_PROXY_CMD, config.json proxy_cmd), a command that prints the current proxy URL; when one is set, check what it prints from a fresh shell" // CredentialHint explains a proxy's rejection of the credentials — a 407, // or an answer that could not be parsed — for a log line or error; "" for @@ -403,9 +403,9 @@ const rotatingCredentialsHint = "if the proxy rotates its credentials, the daemo func CredentialHint(err error) string { switch { case AuthRejected(err): - return "the proxy rejected its credentials (407), also after re-reading them: check HTTPS_PROXY / -proxy; " + rotatingCredentialsHint + return "the proxy rejected its credentials (407), also after re-reading them: check HTTPS_PROXY / -proxy; " + rotatingProxyHint case UnreadableConnectReply(err): - return "the proxy's answer to CONNECT could not be parsed, which is how some egress proxies (Meta Muse's) reject wrong or expired credentials: check the credentials in HTTPS_PROXY / -proxy; " + rotatingCredentialsHint + return "the proxy's answer to CONNECT could not be parsed, which is how some egress proxies (Meta Muse's) reject wrong or expired credentials: check the credentials in HTTPS_PROXY / -proxy; " + rotatingProxyHint } return "" } From b01d0c45c4092d28ece6114b7d95517768bfe204 Mon Sep 17 00:00:00 2001 From: Teodor Calin Date: Thu, 24 Sep 2026 13:02:46 +0300 Subject: [PATCH 18/19] test(proxyconf): fixed order in the relay garbled-rejection test TestConfigureTransportViaRelayHandlesGarbledRejection ranged over a map of the two transports and afterwards restored the proxy to "p-clone", the password the clone iteration rotates to. Go randomizes map order, so half the runs restored a password the relay no longer held and the 403 check saw a garbled 407 instead (architecture-gates race run on 30c034bb). Iterate in a fixed order. -race -count=5 passes. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/proxyconf/relay_test.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/proxyconf/relay_test.go b/internal/proxyconf/relay_test.go index 0476cad2..bfbd0cf3 100644 --- a/internal/proxyconf/relay_test.go +++ b/internal/proxyconf/relay_test.go @@ -380,7 +380,13 @@ func TestConfigureTransportViaRelayHandlesGarbledRejection(t *testing.T) { tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} // #nosec G402 -- test server ConfigureTransport(tr, r, relay.URL()) clone := tr.Clone() // a plugin that cloned DefaultTransport - for name, rt := range map[string]*http.Transport{"configured": tr, "clone": clone} { + // A fixed order: the credentials restored below are the last ones + // rotated in here (map order would make that random). + for _, c := range []struct { + name string + rt *http.Transport + }{{"configured", tr}, {"clone", clone}} { + name, rt := c.name, c.rt client := &http.Client{Transport: rt, Timeout: 20 * time.Second} get := func() error { req, _ := http.NewRequest(http.MethodGet, "https://plugin.pilot.invalid/x", nil) From 5dd85bf821277e47f713287ccbaa3d4d6d913e4c Mon Sep 17 00:00:00 2001 From: Teodor Calin Date: Thu, 24 Sep 2026 13:25:46 +0300 Subject: [PATCH 19/19] deps: skillinject v0.2.5 (gated Meta Muse injection target) Co-Authored-By: Claude Opus 5.5 (1M context) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2c557dad..9e9fb614 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/pilot-protocol/policy v0.2.3 github.com/pilot-protocol/rendezvous v0.2.8 github.com/pilot-protocol/runtime v0.3.2 - github.com/pilot-protocol/skillinject v0.2.4 + github.com/pilot-protocol/skillinject v0.2.5 github.com/pilot-protocol/trustedagents v0.2.6 github.com/pilot-protocol/updater v0.2.5 github.com/pilot-protocol/webhook v0.2.0 diff --git a/go.sum b/go.sum index 8972a52b..a5c40796 100644 --- a/go.sum +++ b/go.sum @@ -255,8 +255,8 @@ github.com/pilot-protocol/rendezvous v0.2.8 h1:GRJyNplqNrfhPuCUgOAaY5Dg5En4G2aO+ github.com/pilot-protocol/rendezvous v0.2.8/go.mod h1:sv0TuqAosOCe3f3EIuZ9lmrNaHJTqyy5k/UGFqHskB4= github.com/pilot-protocol/runtime v0.3.2 h1:21lgUfYNvpls0Vd3V2v9lfs13G7mT8pcT3D2QzcMueE= github.com/pilot-protocol/runtime v0.3.2/go.mod h1:CXEmjKF/HozhIxn9QZxO13Lxdnkok3XKEkYS/jFjQKs= -github.com/pilot-protocol/skillinject v0.2.4 h1:2pJgTHwzha1p9Sdbjo3Q9N/9GlvrPCD1MLpCf7t7uv4= -github.com/pilot-protocol/skillinject v0.2.4/go.mod h1:gw4XxJS94YlHwbHt9t77cNMy+7Bc5JMgA89iIQdx3yg= +github.com/pilot-protocol/skillinject v0.2.5 h1:ZLYjhBbEj/yB9Y4lEMxq7CiBS3FKoTVZiUY6EQqovpI= +github.com/pilot-protocol/skillinject v0.2.5/go.mod h1:xzb3Bobbac9RwWp1svSIYvyBSEVP9HYSmPc59ntN89s= github.com/pilot-protocol/trustedagents v0.2.6 h1:dFKr6V+lJr947v3iUyR7EflUTusGfKkItAmsqJfWc8g= github.com/pilot-protocol/trustedagents v0.2.6/go.mod h1:JAk89O4bg9NeXLnMJh4gUsCQ1R5BIHTQHjrGwNXaqZM= github.com/pilot-protocol/updater v0.2.5 h1:klfIV0cUUOQZzcnb0deO9ohkNXveLFzyEbw5X1BbvRY=