diff --git a/internal/vm/nethttp_bind.go b/internal/vm/nethttp_bind.go index 459e30cd..9c1c7975 100644 --- a/internal/vm/nethttp_bind.go +++ b/internal/vm/nethttp_bind.go @@ -13,6 +13,7 @@ import ( "io" "net" "net/url" + "os" "strconv" "strings" "time" @@ -186,20 +187,28 @@ func (vm *VM) registerNetHTTPInstanceMethods(http *RClass) { } // --- proxy accessors -------------------------------------------------------- + // Each reflects the *effective* proxy: an explicitly-configured one, or — when + // the instance was built with :ENV (MRI's default) — the environment proxy + // resolved against the instance's current scheme (@use_ssl) and target host. http.define("proxy?", func(_ *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { - return object.Bool(!object.IsNil(getIvar(self, "@proxy_address"))) + addr, _, _, _ := nethttpEffectiveProxy(self) + return object.Bool(!object.IsNil(addr)) }) http.define("proxy_address", func(_ *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { - return getIvar(self, "@proxy_address") + addr, _, _, _ := nethttpEffectiveProxy(self) + return addr }) http.define("proxy_port", func(_ *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { - return getIvar(self, "@proxy_port") + _, port, _, _ := nethttpEffectiveProxy(self) + return port }) http.define("proxy_user", func(_ *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { - return getIvar(self, "@proxy_user") + _, _, user, _ := nethttpEffectiveProxy(self) + return user }) http.define("proxy_pass", func(_ *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { - return getIvar(self, "@proxy_pass") + _, _, _, pass := nethttpEffectiveProxy(self) + return pass }) // Instance verb helpers: get/head/delete/options take (path[, header]); the @@ -384,13 +393,13 @@ func (vm *VM) nethttpExecInstance(inst object.Value, method, path string, body [ writeTO: nethttpDuration(getIvar(inst, "@write_timeout")), } cfg.dialHost, cfg.dialPort = host, portStr - if pa := getIvar(inst, "@proxy_address"); !object.IsNil(pa) { + if pa, pp, pu, ppw := nethttpEffectiveProxy(inst); !object.IsNil(pa) { cfg.proxied = true cfg.dialHost = strArg(pa) - cfg.dialPort = portString(getIvar(inst, "@proxy_port")) + cfg.dialPort = portString(pp) cfg.connectTunnel = scheme == "https" - if u := getIvar(inst, "@proxy_user"); !object.IsNil(u) { - cfg.proxyAuth = basicAuthHeader(strArg(u), strArg(getIvar(inst, "@proxy_pass"))) + if !object.IsNil(pu) { + cfg.proxyAuth = basicAuthHeader(strArg(pu), strArg(ppw)) } } if getIvar(inst, "@started").Truthy() { @@ -924,10 +933,15 @@ func (vm *VM) nethttpResponseClass(netMod *RClass, class, category string) *RCla // nethttpNewInstance builds a Net::HTTP instance from // (address[, port[, p_addr[, p_port[, p_user[, p_pass]]]]]), defaulting the port -// to 80, use_ssl to false and each timeout to 60s (MRI's defaults). A proxy is -// configured only when p_addr is an explicit host String; a nil or a bare -// :ENV symbol (MRI's default, which would consult http_proxy) leaves the -// instance proxy-less — environment-proxy resolution is a flagged follow-up. +// to 80, use_ssl to false and each timeout to 60s (MRI's defaults). The p_addr +// argument selects the proxy policy, mirroring MRI: +// - absent, or the :ENV symbol (MRI's default) → resolve the proxy from the +// environment at request time (http_proxy/https_proxy, honoring no_proxy); +// - nil (or false) → no proxy, even if the environment sets one; +// - an explicit host String → that fixed proxy (with p_port/p_user/p_pass). +// +// Environment resolution is deferred to nethttpEffectiveProxy because it depends +// on the instance's scheme (@use_ssl, settable after .new) and target host. func (vm *VM) nethttpNewInstance(http *RClass, args []object.Value) object.Value { o := &RObject{class: http, ivars: map[string]object.Value{}} host := "" @@ -946,11 +960,14 @@ func (vm *VM) nethttpNewInstance(http *RClass, args []object.Value) object.Value o.ivars["@open_timeout"] = object.IntValue(60) o.ivars["@read_timeout"] = object.IntValue(60) o.ivars["@write_timeout"] = object.IntValue(60) - // Proxy (p_addr, p_port, p_user, p_pass): only an explicit host String enables it. + // Proxy (p_addr, p_port, p_user, p_pass). An explicit host String pins a fixed + // proxy; the :ENV symbol (or an omitted p_addr) selects environment resolution + // via @proxy_from_env; nil/false disables the proxy outright. o.ivars["@proxy_address"] = object.NilV o.ivars["@proxy_port"] = object.NilV o.ivars["@proxy_user"] = object.NilV o.ivars["@proxy_pass"] = object.NilV + o.ivars["@proxy_from_env"] = object.Bool(len(args) <= 2 || nethttpIsENV(args[2])) if len(args) > 2 { if ps, ok := args[2].(*object.String); ok && ps.Str() != "" { o.ivars["@proxy_address"] = args[2] @@ -970,6 +987,157 @@ func (vm *VM) nethttpNewInstance(http *RClass, args []object.Value) object.Value return o } +// nethttpIsENV reports whether a p_addr argument is MRI's :ENV symbol, which +// selects environment-based proxy resolution. +func nethttpIsENV(v object.Value) bool { + s, ok := v.(object.Symbol) + return ok && string(s) == "ENV" +} + +// nethttpEffectiveProxy resolves the proxy that applies to a Net::HTTP instance +// right now, returning (address, port, user, pass) as Ruby values (each NilV when +// absent). An explicitly-configured proxy (@proxy_address set) wins unchanged; +// otherwise, when the instance opted into :ENV (@proxy_from_env), the environment +// is consulted against the instance's current scheme and target host. A direct +// (no-proxy) result yields four NilV values. +func nethttpEffectiveProxy(inst object.Value) (addr, port, user, pass object.Value) { + if pa := getIvar(inst, "@proxy_address"); !object.IsNil(pa) { + return pa, getIvar(inst, "@proxy_port"), getIvar(inst, "@proxy_user"), getIvar(inst, "@proxy_pass") + } + if getIvar(inst, "@proxy_from_env").Truthy() { + host := strArg(getIvar(inst, "@address")) + reqPort := intArg(getIvar(inst, "@port")) + https := getIvar(inst, "@use_ssl").Truthy() + if a, p, u, pw, ok := nethttpResolveEnvProxy(host, reqPort, https); ok { + user, pass = object.NilV, object.NilV + if u != "" { + user = object.NewString(u) + } + if pw != "" { + pass = object.NewString(pw) + } + return object.NewString(a), object.IntValue(p), user, pass + } + } + return object.NilV, object.NilV, object.NilV, object.NilV +} + +// nethttpResolveEnvProxy resolves the environment proxy for a request to +// host:reqPort over http (https=false) or https (https=true), mirroring MRI's +// URI::Generic#find_proxy. It reads https_proxy/HTTPS_PROXY for TLS requests and +// http_proxy/HTTP_PROXY otherwise, honors no_proxy/NO_PROXY, and returns the proxy +// endpoint (address/port) plus any userinfo credentials. ok is false when no proxy +// applies (a direct connection). +func nethttpResolveEnvProxy(host string, reqPort int64, https bool) (addr string, port int64, user, pass string, ok bool) { + raw := nethttpProxyEnvValue(https) + if raw == "" { + return "", 0, "", "", false + } + if nethttpNoProxyBypass(host, reqPort, nethttpEnvAny("no_proxy", "NO_PROXY")) { + return "", 0, "", "", false + } + u, err := url.Parse(raw) + if err != nil || u.Host == "" { + // A bare "host:port" (no scheme) is a common form MRI's URI.parse would + // reject as relative; retry it as an http:// URL before giving up. + u, err = url.Parse("http://" + raw) + if err != nil || u.Host == "" { + return "", 0, "", "", false + } + } + addr = u.Hostname() + port = 80 + if u.Scheme == "https" { + port = 443 + } + if ps := u.Port(); ps != "" { + if n, e := strconv.ParseInt(ps, 10, 64); e == nil { + port = n + } + } + if u.User != nil { + user = u.User.Username() + pass, _ = u.User.Password() + } + return addr, port, user, pass, true +} + +// nethttpProxyEnvValue returns the proxy URL string from the environment for the +// request scheme. For https it prefers https_proxy then HTTPS_PROXY. For http it +// normally prefers http_proxy then HTTP_PROXY, but when REQUEST_METHOD is set (a +// CGI context) it deliberately ignores the uppercase HTTP_PROXY — which in CGI is +// attacker-controlled via the Proxy request header — and honors only the lowercase +// http_proxy, matching MRI's find_proxy CGI-safety rule. +func nethttpProxyEnvValue(https bool) string { + if https { + return nethttpEnvAny("https_proxy", "HTTPS_PROXY") + } + if os.Getenv("REQUEST_METHOD") != "" { + return os.Getenv("http_proxy") + } + return nethttpEnvAny("http_proxy", "HTTP_PROXY") +} + +// nethttpEnvAny returns the first non-empty value among the named environment +// variables, in order. +func nethttpEnvAny(names ...string) string { + for _, n := range names { + if v := os.Getenv(n); v != "" { + return v + } + } + return "" +} + +// nethttpNoProxyBypass reports whether host:reqPort matches the no_proxy list and +// should therefore connect directly. Each comma/space-separated entry is a host, +// a domain suffix (leading '.', or a bare name that also matches its subdomains), +// or a CIDR/IP (matched when host is an IP literal), optionally suffixed with +// ":port" to restrict it to that port. This mirrors MRI's URI::Generic.use_proxy? +// (inverted: use_proxy? false ⇒ bypass true). +func nethttpNoProxyBypass(host string, reqPort int64, noProxy string) bool { + host = strings.ToLower(strings.TrimSpace(host)) + if host == "" || noProxy == "" { + return false + } + for _, ent := range strings.FieldsFunc(noProxy, func(r rune) bool { + return r == ',' || r == ' ' || r == '\t' || r == '\n' || r == '\r' + }) { + ehost := ent + // An optional trailing ":port" restricts the entry to that request port. + // (A CIDR mask like "/24" or an IPv6 literal has no bare numeric tail here.) + if i := strings.LastIndex(ent, ":"); i >= 0 { + if n, e := strconv.ParseInt(ent[i+1:], 10, 64); e == nil { + if n != reqPort { + continue + } + ehost = ent[:i] + } + } + el := strings.ToLower(ehost) + if strings.HasPrefix(el, ".") { + if strings.HasSuffix(host, el) { + return true + } + } else if el != "" { + if host == el || strings.HasSuffix("."+host, "."+el) { + return true + } + } + // A CIDR or IP entry matches when the request host is an IP literal. + if ip := net.ParseIP(host); ip != nil { + if _, cidr, e := net.ParseCIDR(ehost); e == nil { + if cidr.Contains(ip) { + return true + } + } else if eip := net.ParseIP(ehost); eip != nil && eip.Equal(ip) { + return true + } + } + } + return false +} + // nethttpGetURI resolves the argument forms of Net::HTTP.get / get_response: a // single URI (String or URI object), or (host, path[, port]). func (vm *VM) nethttpGetURI(args []object.Value) *url.URL { diff --git a/internal/vm/nethttp_bind_test.go b/internal/vm/nethttp_bind_test.go index a46a7fcd..7a597bdd 100644 --- a/internal/vm/nethttp_bind_test.go +++ b/internal/vm/nethttp_bind_test.go @@ -943,9 +943,12 @@ func TestNetHTTPTransportUnits(t *testing.T) { } // TestNetHTTPProxyNilArms covers nethttpNewInstance's proxy-argument arms not -// exercised elsewhere: a nil / :ENV / empty p_addr leaves the instance proxy-less, -// and proxy_user / proxy_pass default to nil. +// exercised elsewhere: with no proxy in the environment a nil / :ENV / omitted / +// empty p_addr leaves the instance proxy-less, and proxy_user / proxy_pass default +// to nil. The proxy environment is cleared so the :ENV arms resolve to direct +// regardless of the host's own settings. func TestNetHTTPProxyNilArms(t *testing.T) { + nethttpClearProxyEnv(t) got := runSrc(t, `require "net/http" puts Net::HTTP.new("h", 80).proxy? puts Net::HTTP.new("h", 80, nil).proxy? @@ -1098,3 +1101,261 @@ func (c *scriptConn) RemoteAddr() net.Addr { return nil } func (c *scriptConn) SetDeadline(_ time.Time) error { return nil } func (c *scriptConn) SetReadDeadline(_ time.Time) error { return nil } func (c *scriptConn) SetWriteDeadline(_ time.Time) error { return nil } + +// nethttpClearProxyEnv blanks every proxy-related environment variable for the +// duration of a test so that :ENV resolution is driven only by what the test sets +// (t.Setenv restores the prior values on cleanup). +func nethttpClearProxyEnv(t *testing.T) { + t.Helper() + for _, k := range []string{ + "http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY", + "no_proxy", "NO_PROXY", "REQUEST_METHOD", + } { + t.Setenv(k, "") + } +} + +// TestNetHTTPEnvProxyRouting is the end-to-end :ENV proof: with http_proxy set to +// an in-test forwarding proxy, Net::HTTP.new(host, port) (p_addr defaulting to +// :ENV) routes through it — proxy? / proxy_address / proxy_port reflect the +// resolved endpoint and the proxy sees the request. no_proxy on the target host, +// and an explicit nil p_addr, both force a direct connection even with the +// environment set. +func TestNetHTTPEnvProxyRouting(t *testing.T) { + nethttpClearProxyEnv(t) + + backend := httptest.NewServer(nethttpTestMux()) + defer backend.Close() + + var proxyHits int64 + var sawAuth string + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt64(&proxyHits, 1) + sawAuth = r.Header.Get("Proxy-Authorization") + resp, err := http.Get(r.RequestURI) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + w.Write(body) + })) + defer proxy.Close() + + bu, _ := url.Parse(backend.URL) + pu, _ := url.Parse(proxy.URL) + + // http_proxy carries userinfo so the resolved credentials become a + // Proxy-Authorization header on the routed request. + t.Setenv("http_proxy", "http://usr:pwd@"+pu.Host) + + got := runSrc(t, fmt.Sprintf(`require "net/http" +h = Net::HTTP.new(%q, %s) +puts h.proxy? +puts h.proxy_address +puts h.proxy_port +puts h.proxy_user +puts h.proxy_pass +puts h.get("/echo").body`, bu.Hostname(), bu.Port())) + want := "true\n" + pu.Hostname() + "\n" + pu.Port() + "\nusr\npwd\nGET|/echo||" + if got != want { + t.Fatalf("env proxy GET = %q, want %q", got, want) + } + if n := atomic.LoadInt64(&proxyHits); n < 1 { + t.Fatalf("env proxy was not hit (%d)", n) + } + if wantAuth := "Basic " + base64Std("usr:pwd"); sawAuth != wantAuth { + t.Fatalf("Proxy-Authorization = %q, want %q", sawAuth, wantAuth) + } + + // no_proxy on the target host bypasses the proxy: a direct connection reaches + // the backend and the proxy stays untouched. + atomic.StoreInt64(&proxyHits, 0) + t.Setenv("no_proxy", bu.Hostname()) + got = runSrc(t, fmt.Sprintf(`require "net/http" +h = Net::HTTP.new(%q, %s) +puts h.proxy? +puts h.get("/echo").body`, bu.Hostname(), bu.Port())) + if want = "false\nGET|/echo||"; got != want { + t.Fatalf("no_proxy bypass GET = %q, want %q", got, want) + } + if n := atomic.LoadInt64(&proxyHits); n != 0 { + t.Fatalf("no_proxy: proxy was hit (%d)", n) + } + + // An explicit nil p_addr opts out of environment resolution entirely, so even + // with http_proxy set (and no_proxy cleared) the request goes direct. + atomic.StoreInt64(&proxyHits, 0) + t.Setenv("no_proxy", "") + got = runSrc(t, fmt.Sprintf(`require "net/http" +h = Net::HTTP.new(%q, %s, nil) +puts h.proxy? +puts h.get("/echo").body`, bu.Hostname(), bu.Port())) + if want = "false\nGET|/echo||"; got != want { + t.Fatalf("nil p_addr direct GET = %q, want %q", got, want) + } + if n := atomic.LoadInt64(&proxyHits); n != 0 { + t.Fatalf("nil p_addr: proxy was hit (%d)", n) + } +} + +// TestNetHTTPEnvProxyHTTPS proves the https_proxy arm end-to-end: with use_ssl set +// the instance resolves https_proxy and reaches the TLS backend through a +// CONNECT-tunnelling proxy. +func TestNetHTTPEnvProxyHTTPS(t *testing.T) { + nethttpClearProxyEnv(t) + + backend := httptest.NewTLSServer(nethttpTestMux()) + defer backend.Close() + + var connects int64 + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodConnect { + http.Error(w, "expected CONNECT", http.StatusMethodNotAllowed) + return + } + atomic.AddInt64(&connects, 1) + dst, err := net.Dial("tcp", r.Host) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + hj, ok := w.(http.Hijacker) + if !ok { + http.Error(w, "no hijack", http.StatusInternalServerError) + dst.Close() + return + } + conn, _, err := hj.Hijack() + if err != nil { + dst.Close() + return + } + conn.Write([]byte("HTTP/1.1 200 Connection Established\r\n\r\n")) + go func() { io.Copy(dst, conn); dst.Close() }() + io.Copy(conn, dst) + conn.Close() + })) + defer proxy.Close() + + bu, _ := url.Parse(backend.URL) + pu, _ := url.Parse(proxy.URL) + // Uppercase HTTPS_PROXY exercises the fallback lookup order. + t.Setenv("HTTPS_PROXY", "http://"+pu.Host) + + got := runSrc(t, fmt.Sprintf(`require "net/http" +h = Net::HTTP.new(%q, %s) +h.use_ssl = true +h.verify_mode = OpenSSL::SSL::VERIFY_NONE +puts h.proxy? +puts h.proxy_address +puts h.get("/").body`, bu.Hostname(), bu.Port())) + want := "true\n" + pu.Hostname() + "\nhello net/http" + if got != want { + t.Fatalf("env https proxy GET = %q, want %q", got, want) + } + if n := atomic.LoadInt64(&connects); n < 1 { + t.Fatalf("CONNECT tunnel was not used (%d)", n) + } +} + +// TestNetHTTPEnvProxyCGISafety covers MRI's CGI-safety rule: when REQUEST_METHOD is +// present, the uppercase HTTP_PROXY (attacker-controllable via the Proxy header in +// CGI) is ignored, while the lowercase http_proxy is still honored. +func TestNetHTTPEnvProxyCGISafety(t *testing.T) { + nethttpClearProxyEnv(t) + t.Setenv("REQUEST_METHOD", "GET") + + // Uppercase HTTP_PROXY alone under CGI ⇒ ignored ⇒ direct. + t.Setenv("HTTP_PROXY", "http://blackhole.invalid:8080") + if got := runSrc(t, `require "net/http" +puts Net::HTTP.new("h", 80).proxy?`); got != "false" { + t.Fatalf("CGI HTTP_PROXY (upper) proxy? = %q, want false", got) + } + + // Lowercase http_proxy under CGI ⇒ honored. + t.Setenv("http_proxy", "http://proxy.internal:3128") + got := runSrc(t, `require "net/http" +h = Net::HTTP.new("h", 80) +puts h.proxy? +puts h.proxy_address +puts h.proxy_port`) + if want := "true\nproxy.internal\n3128"; got != want { + t.Fatalf("CGI http_proxy (lower) = %q, want %q", got, want) + } +} + +// TestNetHTTPEnvProxyResolveUnits exercises the resolver and no_proxy matcher arms +// directly, including the branches the end-to-end tests do not naturally reach +// (scheme-less proxy strings, malformed values, https default port, CIDR / bare +// domain / port-qualified no_proxy entries). +func TestNetHTTPEnvProxyResolveUnits(t *testing.T) { + if !nethttpIsENV(object.Symbol("ENV")) || nethttpIsENV(object.NewString("ENV")) { + t.Fatal("nethttpIsENV must match only the :ENV symbol") + } + + // No proxy in the environment ⇒ direct. + nethttpClearProxyEnv(t) + if _, _, _, _, ok := nethttpResolveEnvProxy("h", 80, false); ok { + t.Fatal("empty env should resolve direct") + } + + // A scheme-less "host:port" is retried as an http:// URL. + t.Setenv("http_proxy", "proxy.example:8080") + if a, p, _, _, ok := nethttpResolveEnvProxy("h", 80, false); !ok || a != "proxy.example" || p != 8080 { + t.Fatalf("scheme-less proxy = (%q,%d,%v), want proxy.example,8080,true", a, p, ok) + } + + // A value malformed even after the http:// retry ⇒ direct. + t.Setenv("http_proxy", "%zz") + if _, _, _, _, ok := nethttpResolveEnvProxy("h", 80, false); ok { + t.Fatal("malformed proxy should resolve direct") + } + + // The https branch defaults the port to 443 when the URL omits it. + t.Setenv("https_proxy", "https://secure.proxy") + if a, p, _, _, ok := nethttpResolveEnvProxy("h", 443, true); !ok || a != "secure.proxy" || p != 443 { + t.Fatalf("https default-port proxy = (%q,%d,%v), want secure.proxy,443,true", a, p, ok) + } + + // no_proxy bypass arms. + if !nethttpNoProxyBypass("api.example.com", 80, ".example.com") { + t.Fatal("leading-dot suffix should bypass") + } + if !nethttpNoProxyBypass("api.example.com", 80, "example.com") { + t.Fatal("bare domain should bypass its subdomain") + } + if !nethttpNoProxyBypass("host", 8080, "host:8080") { + t.Fatal("port-qualified entry on matching port should bypass") + } + if nethttpNoProxyBypass("host", 80, "host:8080") { + t.Fatal("port-qualified entry on other port should not bypass") + } + if !nethttpNoProxyBypass("10.1.2.3", 80, "10.0.0.0/8") { + t.Fatal("CIDR should bypass a contained IP host") + } + if !nethttpNoProxyBypass("10.1.2.3", 80, "10.1.2.3") { + t.Fatal("exact IP entry should bypass") + } + // Differently-spelled but equal IPs match via net.IP equality, not the string + // compare (::0.0.0.1 and ::1 are the same address). + if !nethttpNoProxyBypass("::1", 80, "::0.0.0.1") { + t.Fatal("equal-but-differently-spelled IP entry should bypass") + } + if nethttpNoProxyBypass("other.net", 80, "example.com, .foo.com") { + t.Fatal("non-matching host should not bypass") + } + if nethttpNoProxyBypass("10.1.2.3", 80, "192.168.0.0/16") { + t.Fatal("IP host outside the CIDR should not bypass") + } + if nethttpNoProxyBypass("10.1.2.3", 80, "10.9.9.9") { + t.Fatal("non-matching IP entry should not bypass") + } + if nethttpNoProxyBypass("10.1.2.3", 80, ":80") { + t.Fatal("a port-only entry (empty host) should not bypass") + } + if nethttpNoProxyBypass("", 80, "example.com") || nethttpNoProxyBypass("h", 80, "") { + t.Fatal("empty host or empty list should not bypass") + } +}