diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cd6cecc..95f65e9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -70,7 +70,7 @@ jobs: container=$(ruby -ryaml -e 'puts YAML.load_file("recipes/targets.yml").fetch("targets").fetch(ENV.fetch("TARGET")).fetch("container")') echo "container=$container" >> "$GITHUB_OUTPUT" - - name: Build portable Ruby in manylinux2014 + - name: Build portable Ruby in manylinux env: VERSION: ${{ inputs.version }} TARGET: ${{ matrix.target }} diff --git a/CLAUDE.md b/CLAUDE.md index 954918b..9b356e1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,7 +20,7 @@ Build a Ruby version locally: bin/package 3.4.9 --target x86_64_linux --no-yjit --output rubies ``` -Linux targets must be built inside their manylinux2014 container; `bin/package-linux` does that from any Docker host, including macOS. +Linux targets must be built inside their pinned manylinux_2_28 container; `bin/package-linux` does that from any Docker host, including macOS. Builds need a baseruby of Ruby 3.0.0 or newer; set `JDX_RUBY_BASERUBY` when the shell default is older. Ruby 3.2.x builds require `JDX_RUBY_BASERUBY` to point to an existing Ruby executable with the same version. @@ -33,14 +33,14 @@ YJIT builds require `rustup` or `rustc` in `PATH`. Set `JDX_RUBY_RUSTUP_HOME` to - `recipes/rubies.yml`: Ruby source URL, SHA256, series, and prerelease version metadata. - `recipes/dependencies.yml`: portable dependency source URL and SHA256 metadata. - `recipes/series.yml`: per-series behavior such as libedit, bundled gems, and baseruby requirements. -- `recipes/targets.yml`: release targets, artifact platform names, and pinned manylinux2014 containers. +- `recipes/targets.yml`: release targets, artifact platform names, and pinned manylinux_2_28 containers. ### Commands - `bin/package`: Builds portable dependencies, builds Ruby, runs runtime/linkage/ABI checks, and writes release tarballs. - `bin/validate-recipes`: Validates YAML shape, required fields, duplicate versions, URL/SHA256 formats, and target matrix completeness. - `bin/update-ruby-recipe`: Adds or updates a Ruby entry in `recipes/rubies.yml`; used by autobump. -- `bin/package-linux`: Runs `bin/package` for a Linux target inside its pinned manylinux2014 container, mirroring the build workflow, for local Linux builds. +- `bin/package-linux`: Runs `bin/package` for a Linux target inside its pinned manylinux container, mirroring the build workflow, for local Linux builds. - `bin/recipe-info`: Prints a version's effective series settings (`legacy`, `yjit`, ...) for workflows. - `bin/prune-revisions`: Deletes a version's oldest revision releases and tags, keeping the newest `KEEP_REVISIONS` (2); run by the release workflow after publishing. - `bin/build-fingerprint`: Prints a digest of everything in the repo that affects a version's build; the release workflow skips rebuilding when it matches the newest revision release's. @@ -51,7 +51,7 @@ YJIT builds require `rustup` or `rustc` in `PATH`. Set `JDX_RUBY_RUSTUP_HOME` to ### Key Build Details -- Linux builds use pinned manylinux2014/glibc 2.17 containers for both YJIT and no-YJIT artifacts. +- Linux builds use pinned manylinux_2_28 containers (AlmaLinux 8, glibc 2.28, GCC 14) for both YJIT and no-YJIT artifacts. glibc 2.28 is the newest that still runs on Ubuntu 20.04 (2.31), the oldest release Hatchbox provisions. - `pkgconf` is built as a bootstrap tool so the build has no host package-manager dependency. - OpenSSL, libyaml, libffi, libxcrypt, zlib, ncurses, and libedit are source-built into an isolated prefix as needed. - Bundled `msgpack` and `bootsnap` gems are staged during the Ruby build. diff --git a/README.md b/README.md index 542c811..f75e4e0 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,14 @@ maintains them to install Ruby on its customers' servers without compiling it th Every tarball is self-contained: OpenSSL, libyaml, libffi, zlib and libxcrypt (and libedit and ncurses where a series uses them) are linked in statically, so the only shared libraries -a build needs are glibc's. They are built against glibc 2.17, which means any Ubuntu LTS -from 20.04 (Focal) on, Debian 11 on, and other distributions of that vintage or newer, on -x86_64 and arm64. Headers, static libraries and pkg-config files for the bundled -dependencies ship in the tarball, so native gems compile after it has been moved. +a build needs are glibc's. They are built against glibc 2.28 (AlmaLinux 8), which means any +Ubuntu LTS from 20.04 (Focal) on, Debian 10 on, RHEL 8 on, and other distributions of that +vintage or newer, on x86_64 and arm64. At run time they use the system's glibc; the build +glibc only limits which of its functions Ruby can use. 2.28 is as new as it can go while +still running on Ubuntu 20.04, and it gives Ruby 2.7 and later `File.birthtime` (through +`statx`) and `copy_file_range` for `IO.copy_stream`, which glibc 2.17 builds lacked. +Headers, static libraries and pkg-config files for the bundled dependencies ship in the +tarball, so native gems compile after it has been moved. ## How do I use these rubies @@ -191,14 +195,14 @@ bin/package 3.4.9 --target x86_64_linux --no-yjit --output rubies ``` `bin/package-linux VERSION TARGET [--yjit|--no-yjit]` runs the same thing inside the pinned -manylinux2014 container for a Linux target, the way CI does, so a Linux tarball can be built +manylinux container for a Linux target, the way CI does, so a Linux tarball can be built and tested locally without a Linux machine. -Linux release builds are expected to run in the pinned manylinux2014 containers from `recipes/targets.yml`. Builds need a baseruby of Ruby 3.0.0 or newer; set `JDX_RUBY_BASERUBY` when your shell default is older. Ruby 3.2 needs a baseruby of exactly the version being built: the build makes one first, or uses `JDX_RUBY_BASERUBY` when it points at one. YJIT builds use rustup/rustc from `PATH`, with optional `JDX_RUBY_RUSTUP_HOME`. +Linux release builds are expected to run in the pinned manylinux_2_28 containers (GCC 14) from `recipes/targets.yml`. Builds need a baseruby of Ruby 3.0.0 or newer; set `JDX_RUBY_BASERUBY` when your shell default is older. Ruby 3.2 needs a baseruby of exactly the version being built: the build makes one first, or uses `JDX_RUBY_BASERUBY` when it points at one. YJIT builds use rustup/rustc from `PATH`, with optional `JDX_RUBY_RUSTUP_HOME`. Every build ends by testing the packaged tree from a different directory: the standard library and its extensions load, a native gem compiles and loads, nothing links to a shared -library outside glibc or needs a glibc newer than 2.17, no OpenSSL symbol is exported +library outside glibc or needs a glibc newer than 2.28, no OpenSSL symbol is exported (see [below](#alongside-the-systems-openssl)), and nothing a native gem is built from still names the build tree (see [Native gems](#native-gems)). Pull requests only build Ruby 3.4.1 (all four artifacts), so build a change to an older series locally with `bin/package-linux` diff --git a/bin/package-linux b/bin/package-linux index ab78f71..045d3f0 100755 --- a/bin/package-linux +++ b/bin/package-linux @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Run bin/package for a Linux target inside its pinned manylinux2014 container, the way +# Run bin/package for a Linux target inside its pinned manylinux container, the way # .github/workflows/build.yml does, so a Linux tarball can be built and tested locally. # # bin/package-linux 3.4.9 x86_64_linux --no-yjit @@ -30,7 +30,10 @@ esac # has released 3.4.7. bootstrap_version=3.4.7 bootstrap_repo=${BOOTSTRAP_REPO:-hatchboxio/precompiled-ruby} -image="precompiled-ruby-build:${target}-${bootstrap_version}" +# Keyed by the container too, so a new pin in recipes/targets.yml gets a new image and a +# new build volume instead of reusing ones made with the old toolchain. +container_id=$(printf %s "$container" | cksum | cut -d" " -f1) +image="precompiled-ruby-build:${target}-${bootstrap_version}-${container_id}" if ! docker image inspect "$image" >/dev/null 2>&1; then echo "Building $image from $container..." @@ -55,10 +58,10 @@ mkdir -p rubies .cache # filesystem — libffi's generated Makefile came out unreadable on one — and CI builds on # a native one. The volume keeps a failed build's tree around for inspection: # -# docker run --rm -it -v precompiled-ruby-build-$target:/work/.build $image bash +# docker run --rm -it -v precompiled-ruby-build-$target-$container_id:/work/.build $image bash docker run --rm --platform "$platform" \ -v "$PWD:/src:ro" -v "$PWD/.cache:/work/.cache" -v "$PWD/rubies:/work/rubies" \ - -v "precompiled-ruby-build-${target}:/work/.build" \ + -v "precompiled-ruby-build-${target}-${container_id}:/work/.build" \ -e JDX_RUBY_RUSTUP_HOME="/work/.build/rustup-${target}" -e JDX_RUBY_JOBS \ "$image" /bin/bash -lc " set -euo pipefail diff --git a/libexec/package.rb b/libexec/package.rb index 2c0273b..4b0da55 100755 --- a/libexec/package.rb +++ b/libexec/package.rb @@ -127,7 +127,7 @@ def validate_host! glibc = `getconf GNU_LIBC_VERSION 2>/dev/null`.split.last if glibc && glibc != @target_recipe["max_glibc"] raise PackageError, "expected glibc #{@target_recipe["max_glibc"]}, found #{glibc}; " \ - "Linux targets must be built inside the manylinux2014 container" + "Linux targets must be built inside the pinned manylinux container" end elsif !host_os.include?("darwin") raise PackageError, "macos target must be built on macOS" @@ -386,6 +386,7 @@ def build_ruby! remove_build_rpaths end copy_native_gem_dependencies + relocate_pkgconfig_files bundle_certificates end @@ -803,6 +804,8 @@ def patch_rbconfig content.gsub!(%r{ ?-B#{Regexp.escape(@build_root)}[^ "']*}, "") content.gsub!(%r{ ?-Wl,-rpath-link=#{Regexp.escape(@build_root)}[^ "']*}, "") content.gsub!(%r{ ?-Wl,(?:-rpath,|-R)#{Regexp.escape(@build_root)}[^ "']*}, "") + # Where Ruby's own build found the YJIT static library; nothing after the build uses it. + content.gsub!(/(CONFIG\["RUST_LIB"\] = )"[^"]*#{Regexp.escape(@build_root)}[^"]*"/, '\\1""') content.gsub!(/(CONFIG\["CC"\] = )"[^"]*gcc(?:-\d+)?"/, '\\1"cc"') content.gsub!(/(CONFIG\["LDSHARED"\] = )"[^"]*gcc(?:-\d+)?/, '\\1"cc') content.gsub!(/(CONFIG\["CXX"\] = )"[^"]*g\+\+(?:-\d+)?"/, '\\1"c++"') @@ -821,11 +824,16 @@ def patch_rbconfig # strscan 3.1.8 on Ruby 2.4-2.7 with "undefined symbol: rb_deprecate_constant". Merging # the archive into one object and making its hidden symbols local leaves exactly the # exported API visible, as a shared libruby would. + # + # As one object it is linked whole into every probe, so its debug info goes too: 900 MB + # of linker memory per have_func on 3.4, enough to get gem installs OOM-killed on small + # servers (and a killed link reads as "function missing"). Nothing debugs through this + # archive; the ruby executable keeps its own debug info. Stripped, a probe takes ~50 MB. def localize_static_libruby Dir.glob(File.join(@install_prefix, "lib", "libruby*static.a")).each do |archive| merged = File.join(@build_root, "libruby-merged.o") run "ld", "-r", "--whole-archive", archive, "-o", merged - run "objcopy", "--localize-hidden", merged + run "objcopy", "--localize-hidden", "--strip-debug", merged FileUtils.rm_f(archive) run "ar", "rcs", archive, merged FileUtils.rm_f(merged) @@ -931,6 +939,19 @@ module RbConfig RUBY end + # pkg-config files from the dependencies' own installs, and Ruby's ruby-X.pc, name the + # directories they were installed to. Everything they describe is now in this prefix. + def relocate_pkgconfig_files + relocated = "${pcfiledir}/../.." + Dir.glob(File.join(@install_prefix, "lib", "pkgconfig", "*.pc")).each do |pc| + next if File.symlink?(pc) + + content = File.read(pc) + updated = content.gsub(%r{#{Regexp.escape(@deps_root)}/[^/\s]+}, relocated).gsub(@install_prefix, relocated) + File.write(pc, updated) unless updated == content + end + end + def copy_native_gem_dependencies deps = [dep_prefix("libyaml"), dep_prefix("openssl")] deps += [dep_prefix("libffi"), dep_prefix("zlib")] @@ -1130,6 +1151,12 @@ def check_no_build_paths!(root, ruby, env) stale = [@deps_root, @install_prefix] raise PackageError, "RbConfig still names the build tree:\n#{config}" if stale.any? { |dir| config.include?(dir) } + pkgconfig = Dir.glob(File.join(root, "lib", "pkgconfig", "*.pc")).select do |pc| + body = File.read(pc) + stale.any? { |dir| body.include?(dir) } + end + raise PackageError, "pkg-config files name the build tree:\n #{pkgconfig.join("\n ")}" unless pkgconfig.empty? + binaries = Dir.glob(File.join(root, "**", "*.so")) + Dir.glob(File.join(root, "{bin,libexec}", "ruby")) offenders = binaries.select do |path| rpath = capture("patchelf", "--print-rpath", path, allow_failure: true) @@ -1142,6 +1169,9 @@ def check_no_build_paths!(root, ruby, env) hidden = capture("readelf", "-sW", archive).lines.count { |line| line.match?(/ (GLOBAL|WEAK)\s+HIDDEN /) } raise PackageError, "#{archive} still has #{hidden} linkable hidden symbols" unless hidden.zero? + + debug = capture("readelf", "-SW", archive).lines.count { |line| line.include?(".debug_") } + raise PackageError, "#{archive} still has #{debug} debug sections" unless debug.zero? end def check_abi!(root) @@ -1206,8 +1236,8 @@ def download(name, recipe) FileUtils.rm_f(path) # Alternate between the primary URL and the mirror, in rounds with a growing pause: # curl's own --retry only covers HTTP errors, not a host that is down (the - # manylinux2014 curl is too old for --retry-all-errors), and a single upstream - # outage should not fail a build. + # manylinux image's curl, 7.61, is too old for --retry-all-errors), and a single + # upstream outage should not fail a build. urls = [recipe["url"], recipe["mirror"]].compact attempts = DOWNLOAD_RETRY_DELAYS.flat_map { |delay| urls.map { |url| [url, delay] } } attempts.each_with_index do |(url, delay), index| diff --git a/recipes/series.yml b/recipes/series.yml index c6ef0b6..346d35a 100644 --- a/recipes/series.yml +++ b/recipes/series.yml @@ -40,6 +40,10 @@ series: # libedit; no bundled msgpack/bootsnap; a version-appropriate native gem as the test; # init_only_exports so their openssl and digest extensions can't collide with a system # OpenSSL 3 that the pg gem loads into the same process. + # Up to 2.2 their sources call undeclared functions and use K&R declarations, which GCC + # 14 rejects (and which break configure's own checks for OpenSSL functions), so those + # errors are downgraded to warnings. Only flags that older compilers also accept: CFLAGS + # stays in rbconfig and reaches native gems built on servers with GCC 9 or older. "1.8": openssl: openssl1.0 baseruby: none @@ -47,7 +51,7 @@ series: use_libedit: true install_doc: false install_target: install-nodoc # 1.8 has no --disable-install-doc - cflags: "-O3 -fno-strict-overflow" + cflags: "-O3 -fno-strict-overflow -Wno-error=implicit-function-declaration -Wno-error=implicit-int -Wno-error=int-conversion -Wno-error=incompatible-pointer-types" patches: [lex_c99, init_only_exports] freshen_config_guess: true extra_out_ext: [tk, tcltklib, gdbm, dbm] # removed from the tree: 1.8 has no --with-out-ext @@ -63,7 +67,7 @@ series: baseruby: none use_libedit: true install_doc: false - cflags: "-O3 -fno-strict-overflow" + cflags: "-O3 -fno-strict-overflow -Wno-error=implicit-function-declaration -Wno-error=implicit-int -Wno-error=int-conversion -Wno-error=incompatible-pointer-types" make_jobs: 1 # parallel make races in ext/ on 1.9.3 freshen_config_guess: true extra_out_ext: [tk, tcl, gdbm, dbm] @@ -78,7 +82,7 @@ series: baseruby: none use_libedit: true install_doc: false - cflags: "-O3 -fno-strict-overflow" + cflags: "-O3 -fno-strict-overflow -Wno-error=implicit-function-declaration -Wno-error=implicit-int -Wno-error=int-conversion -Wno-error=incompatible-pointer-types" freshen_config_guess: true # 2.0's config.guess predates aarch64 extra_out_ext: [tk, tcl, gdbm, dbm] bundler: ["1.17.3"] @@ -92,7 +96,7 @@ series: baseruby: none use_libedit: true install_doc: false - cflags: "-O3 -fno-strict-overflow" + cflags: "-O3 -fno-strict-overflow -Wno-error=implicit-function-declaration -Wno-error=implicit-int -Wno-error=int-conversion -Wno-error=incompatible-pointer-types" extra_out_ext: [tk, tcl, gdbm, dbm] bundler: ["1.17.3"] bundled_gems: ~ @@ -105,7 +109,7 @@ series: baseruby: none use_libedit: true install_doc: false - cflags: "-O3 -fno-strict-overflow" + cflags: "-O3 -fno-strict-overflow -Wno-error=implicit-function-declaration -Wno-error=implicit-int -Wno-error=int-conversion -Wno-error=incompatible-pointer-types" extra_out_ext: [tk, tcl, gdbm, dbm] bundler: ["1.17.3"] bundled_gems: ~ diff --git a/recipes/targets.yml b/recipes/targets.yml index 3b74d0d..11ee6fb 100644 --- a/recipes/targets.yml +++ b/recipes/targets.yml @@ -4,10 +4,10 @@ targets: x86_64_linux: os: linux artifact_platform: x86_64_linux - container: quay.io/pypa/manylinux2014_x86_64:2026.06.25-1@sha256:e19f84d04229d95eff831b6d4d9a44b9ba8d25a76ee2fce0eb9d6f0785e62f02 - max_glibc: "2.17" + container: quay.io/pypa/manylinux_2_28_x86_64:2026.09.29-1@sha256:407f771c51a2c3e83ebe5a7970b4289ead3a6db21d9b9c089168775cad11d328 + max_glibc: "2.28" arm64_linux: os: linux artifact_platform: arm64_linux - container: quay.io/pypa/manylinux2014_aarch64:2026.06.25-1@sha256:9c8e0985470699c10aade65c740240c5cf83e4eb96bbfc5d8fe5f91bb26e4dcf - max_glibc: "2.17" + container: quay.io/pypa/manylinux_2_28_aarch64:2026.09.29-1@sha256:c22ffd129ac99a8a42d1f2c2f4e88a9089288dd9ee987a7092da1c7dc48f27a9 + max_glibc: "2.28"