From 9a7acaec09519bb8aaa895afb4cecdeb997de1a1 Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Sun, 14 Aug 2022 20:01:52 +0800 Subject: [PATCH 1/4] build: Improve testing infra for repo tests. --- .github/workflows/ci.yml | 11 +- .github/workflows/release.yml | 1 + .github/workflows/setup-bazel.sh | 14 +- .gitignore | 2 + .tool-versions | 1 - BUILD | 1 + README.md | 2 +- WORKSPACE | 4 +- .../scip-ruby/CONTRIBUTING.md | 82 ++++++++-- docs/scip-ruby/DESIGN.md | 77 +++++++++ gems/scip-ruby/BUILD | 8 +- gems/scip-ruby/bin/scip-ruby | 38 ++++- gems/scip-ruby/build.sh | 28 ++-- gems/scip-ruby/build_gems.bzl | 41 ++++- gems/scip-ruby/install_ruby.sh | 40 +++++ test/scip/repos/BUILD | 9 +- .../{brew-3.5.7.patch => brew-3.5.9.patch} | 0 test/scip/repos/bundle_cache.sh | 58 +++++++ test/scip/repos/index_gem.sh | 122 ++++++++++++++ test/scip/repos/index_oss_repo.sh | 89 ----------- test/scip/repos/scip_repos_test.bzl | 149 +++++++++++++----- .../shopify-api-ruby-v11.1.0-darwin.patch | 24 +++ .../shopify-api-ruby-v11.1.0-linux.patch | 24 +++ third_party/externals.bzl | 9 ++ third_party/test_gem.BUILD | 5 + third_party/test_gem_data.bzl | 41 +++++ xbrew | 7 + 27 files changed, 708 insertions(+), 179 deletions(-) delete mode 100644 .tool-versions rename scip-ruby-CONTRIBUTING.md => docs/scip-ruby/CONTRIBUTING.md (77%) create mode 100644 docs/scip-ruby/DESIGN.md create mode 100755 gems/scip-ruby/install_ruby.sh rename test/scip/repos/{brew-3.5.7.patch => brew-3.5.9.patch} (100%) create mode 100755 test/scip/repos/bundle_cache.sh create mode 100755 test/scip/repos/index_gem.sh delete mode 100755 test/scip/repos/index_oss_repo.sh create mode 100644 test/scip/repos/shopify-api-ruby-v11.1.0-darwin.patch create mode 100644 test/scip/repos/shopify-api-ruby-v11.1.0-linux.patch create mode 100644 third_party/test_gem.BUILD create mode 100644 third_party/test_gem_data.bzl create mode 100755 xbrew diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb982f083f..e73f1c00d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,8 +13,6 @@ jobs: steps: - uses: actions/checkout@v3 - uses: ruby/setup-ruby@v1 - with: - ruby-version: '2.7' # Keep in sync with release.yml - name: Manually evict cache entry if applicable run: ACCESS_TOKEN='${{ secrets.GITHUB_TOKEN }}' python3 .github/workflows/evict.py @@ -46,4 +44,11 @@ jobs: - name: "🏋️‍♂️ Run snapshot tests" run: rm log && ./bazel test //test/scip --config=dbg - name: "🏋️‍♂️ Run repo tests" - run: ./bazel test //test/scip/repos --test_env GITHUB_ACTIONS=1 --test_env GEM_PATH="$(dirname "$(which gem)")" --config=dbg + run: | + sudo apt-get install --reinstall build-essential + ls "$(dirname "$(which gcc)")" + echo "$PATH" + ls /usr/bin /usr/local/bin /bin + # Technically, this is for Ruby 2.7.0 but it's probably fine. + sudo apt-get install ruby2.7-dev + ./bazel test //test/scip/repos --test_env GITHUB_ACTIONS=1 --test_env GEM_PATH="$(dirname "$(which gem)")" --config=dbg diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a4eb3eff3a..12a1ab8b38 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,6 +35,7 @@ jobs: steps: - uses: actions/checkout@v3 # Keep in sync with ci.yml + - uses: actions/setup-ruby@v1 - name: Manually evict cache entry if applicable if: ${{ runner.os == 'Linux' }} run: ACCESS_TOKEN='${{ secrets.GITHUB_TOKEN }}' python3 .github/workflows/evict.py diff --git a/.github/workflows/setup-bazel.sh b/.github/workflows/setup-bazel.sh index 49d58c9a3b..7ca3a7d9e4 100755 --- a/.github/workflows/setup-bazel.sh +++ b/.github/workflows/setup-bazel.sh @@ -4,10 +4,22 @@ set -euo pipefail CACHE_ROOT="$HOME/bazelcache" mkdir -p "$CACHE_ROOT/output-base" "$CACHE_ROOT/build" "$CACHE_ROOT/repos" + +os="$(uname -s)" +if [ "$os" == "Linux" ]; then + SETUP_RUBY_TOOLS_DIR="/opt/hostedtoolcache/Ruby" +elif [ "$os" == "Darwin" ]; then + SETUP_RUBY_TOOLS_DIR="/Users/runner/hostedtoolcache/Ruby" +else + echo "Check https://github.com/ruby/setup-ruby#using-self-hosted-runners and note setup-ruby's cache directory here" + exit 1 +fi + { - echo 'common --curses=no --color=yes' + echo "common --curses=no --color=yes " echo "startup --output_base=$CACHE_ROOT/output-base" echo "build --disk_cache=$CACHE_ROOT/build --repository_cache=$CACHE_ROOT/repos" + echo "build --define SCIP_RUBY_CACHE_RUBY_DIR=$SETUP_RUBY_TOOLS_DIR --define SCIP_RUBY_RBENV_EXE=RUNNING_IN_CI_RBENV_NOT_NEEDED" if [ -n "${VERSION:-}" ]; then echo "build --//gems/scip-ruby:version=$VERSION" fi diff --git a/.gitignore b/.gitignore index bb2fa7b97e..2039344e1d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ /bazel-* /.bazelrc.local +/.cache_ruby /compile_commands.json +/.homebrew-x86_64 /.gdb_history /tags diff --git a/.tool-versions b/.tool-versions deleted file mode 100644 index 9eb38ed717..0000000000 --- a/.tool-versions +++ /dev/null @@ -1 +0,0 @@ -ruby 2.7.2 diff --git a/BUILD b/BUILD index e69de29bb2..50acbb8fcb 100644 --- a/BUILD +++ b/BUILD @@ -0,0 +1 @@ +exports_files([".ruby-version"]) diff --git a/README.md b/README.md index 9e502972b1..829f36b5e2 100644 --- a/README.md +++ b/README.md @@ -94,4 +94,4 @@ using the [Sourcegraph CLI](https://github.com/sourcegraph/src-cli). ## Contributing -See the [Contributing docs](./scip-ruby-CONTRIBUTING.md). +See the [Contributing docs](./docs/scip-ruby/CONTRIBUTING.md). diff --git a/WORKSPACE b/WORKSPACE index 5b9f7740c4..3b8789276d 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,9 +1,11 @@ workspace(name = "com_stripe_ruby_typer") -load("//third_party:externals.bzl", "register_sorbet_dependencies") +load("//third_party:externals.bzl", "register_scip_ruby_dependencies", "register_sorbet_dependencies") register_sorbet_dependencies() +register_scip_ruby_dependencies() + load("@com_grail_bazel_compdb//:deps.bzl", "bazel_compdb_deps") bazel_compdb_deps() diff --git a/scip-ruby-CONTRIBUTING.md b/docs/scip-ruby/CONTRIBUTING.md similarity index 77% rename from scip-ruby-CONTRIBUTING.md rename to docs/scip-ruby/CONTRIBUTING.md index 3adc381ad2..be34bf31c0 100644 --- a/scip-ruby-CONTRIBUTING.md +++ b/docs/scip-ruby/CONTRIBUTING.md @@ -1,5 +1,13 @@ # Working on scip-ruby +This document covers the day-to-day aspects of `scip-ruby`. +For questions on why certain things are the way they are, +see the [Design Decisions doc][] doc. + +[Design Decisions doc]: DESIGN.md + +- [Install dependencies](#install-dependencies) +- [Configuring Ruby (optional)](#configuring-ruby-optional) - [Building](#building) - [IDE integration](#ide-integration) - [Writing a new snapshot test](#writing-a-new-snapshot-test) @@ -10,11 +18,58 @@ - [Cutting a release](#cutting-a-release) - Troubleshooting - [Known build issues][] - - [Known Ruby installation issues][] - [Known RubyGems related issues](#known-rubygems-related-issues) + - [Known testing issues](#known-testing-issues) [Known build issues]: #known-build-issues -[Known Ruby installation issues]: #known-ruby-installation-issues + +## Install dependencies + +1. C++ toolchain (gcc or clang): Used for bootstrapping. +2. [rbenv][]: (optional) This is used for [Configuring Ruby](#configuring-ruby-optional). + +[rbenv]: https://github.com/rbenv/rbenv#installation + +## Configuring Ruby (optional) + +If you're going to be running the repository tests or building the +scip-ruby gem locally, follow these steps before building stuff. + +### macOS pre-requisites + +arm64 macOS builds aren't supported yet, +so it is simpler to have a consistent set of tools +where Homebrew is running under Rosetta too. + +``` +mkdir ~/.homebrew-x86_64 +curl -L https://github.com/Homebrew/brew/tarball/master | tar xz --strip 1 -C ~/.homebrew-x86_64 +./xbrew --version # check that it works +``` + +**WARNING:** Do not put this `brew` on your PATH! +That may lead to confusing errors when working on other projects. + +### rbenv setup + +#### Linux + +Install [rbenv][] as per the official instructions. Then run: + +```bash +echo "build --define SCIP_RUBY_CACHE_RUBY_DIR='$PWD/.cache_ruby' --define SCIP_RUBY_RBENV_EXE='$(which rbenv)'" >> .bazelrc.local +``` + +#### macOS + +```bash +./xbrew install rbenv +echo "build --define SCIP_RUBY_CACHE_RUBY_DIR='$PWD/.cache_ruby' --define SCIP_RUBY_RBENV_EXE='$(./xbrew where rbenv)'" >> .bazelrc.local +``` + +If you're wondering why we have this separate kind of caching, +instead of having everything happen through the Magic of Bazel (TM), +see the [Design Decisions doc][]. ## Building @@ -92,7 +147,7 @@ First, clone the repo using Sorbet locally and check if you can index it. Typically, the commands will be something like: -``` +```bash BUNDLE_WITH=sorbet bundle install # Replace srb binary with scip-ruby binary @@ -101,7 +156,7 @@ bundle exec srb --index-file index.scip --gem-metadata "name@version" ``` In case there are any type errors, create a patch and save it: -``` +```bash git diff > /path/to/test/scip/repos/name-version.patch ``` @@ -200,15 +255,6 @@ See the [release workflow](/.github/workflows/release.yml) for details. ^~~~~~~~~~~~~~~~~~~~~~~ ``` -### Known Ruby installation issues - -1. On macOS, Ruby 2.7.x may fail to install due to a combination of `-Werror` - and a warning in OpenSSL. As a workaround, you can use the following asdf invocation: - ``` - OPENSSL_CFLAGS=-Wno-error=implicit-function-declaration asdf install ruby 2.7.2 - ``` - This works with 2.7.2, but not with 2.7.0. - ### Known RubyGems related issues 1. If you're trying to run `bundle install` for a gem @@ -227,3 +273,13 @@ See the [release workflow](/.github/workflows/release.yml) for details. Network Preferences > Advanced > TCP/IP > Configure IPv6: set to Link-local only. You need to Apply the settings after changing that field. + +### Known testing issues + +I can't quite figure out why, but with some of the custom rules, +the executable script being modified doesn't trigger a rebuild. +If you're seeing stale results, try something like: + +```bash +rm -rf .cache_ruby/ && cd bazel-bin && find . -name '*.tgz' -type f | xargs rm -f && cd - +``` \ No newline at end of file diff --git a/docs/scip-ruby/DESIGN.md b/docs/scip-ruby/DESIGN.md new file mode 100644 index 0000000000..0a6473c817 --- /dev/null +++ b/docs/scip-ruby/DESIGN.md @@ -0,0 +1,77 @@ +# Design Decisions + +## Repo tests + + + +The repo tests have a couple of unusual things: +#. Tests are broken up into 3 stages, + instead of being a simple script that does clone + + apply patch + invoke dev tool (`scip-ruby`). +#. There is "manual" caching going on for the intermediate results + of these steps. The cache cannot be used in parallel; repo tests run serially. + +I'll cover each of these one-by-one. + +Some constraints to keep in mind while reading the sections below: +#. We want to run tests using as close to "real-world" tools (such as Bundler) + as possible instead of trying to mimic what they do. +#. We want to cache intermediate results as much as possible, + as many steps in Ruby installation + Gem installation are slow. +#. Tests should hopefully run reasonably quickly, especially if you only + modify the indexer (most common situation). +#. Tests should be isolated from each other. We don't one test to "install" + `scip-ruby` and have that be visible to another test. + +Before going into details about the caching, first let's cover +an outline of the 3 stages. +### 3 stages of repo tests + +#. Create a standalone ruby installation: This is cached + and the installation is blown away. +#. Test prep: This step involves installing the dependencies of the gem, + including any custom bundler version it may require. + The pristine toolchain from the first step is extracted + and we run `bundle cache`. + After this is done, the _entire_ Ruby toolchain is cached again + (because it will be have been modified by `bundle cache`). + The modified source tree is also cached (I tried `--frozen` earlier but + ran into some error with it that I didn't have time to look into). +#. Index: This step reuses the two caches from the previous step, + applies a patch, copies the `scip-ruby` gem into the `vendor/cache` + directory, runs `bundle install --local` (so nothing should be + fetched from the internet). Now, everything is in a state which mostly + mimics what a user would have. Then we invoke `scip-ruby` normally. + +Intermediate results use tarballs for caching directory trees instead of copying +directories because it seemed much simpler to deal with a tarball in Bazel. + +Why do we go through all this trouble to cache things "manually"? +### Manual caching + +Here are some obstacles the caching system needs to overcome. + +* Ruby installation is [path-dependent](https://github.com/ruby/setup-ruby#using-self-hosted-runners). +* Ruby installation is relatively slow as it builds everything from source: + it takes a few minutes even on a Macbook Pro. + Installing it per-test would be too time-consuming. + +These two factors together imply that we should cache a Ruby installation +inside a stable directory, not a temporary test directory. + +Additionally, we only want one Ruby toolchain cache; to ensure test isolation, +we run them in serial on the same directory (a GIL for the test suite). + +* We want to ensure reproducibility, and not mess with the global environment. + +To achieve this, we delete the toolchain directory at the start of a test, +and restore it from the cache in the 'Test prep' step. + +### Bonus: Bundler weirdness + +If you run `bundle install --local` for a test gem X, +after copying an updated `scip-ruby` gem (same version, new checksum) +into X's `vendor/cache` directory, +you would naively expect `bundle` to re-install the updated `scip-ruby` gem. +(install = copy/setup stuff in toolchain-adjacent directories) +But `bundle` doesn't actually do that. 🙈 \ No newline at end of file diff --git a/gems/scip-ruby/BUILD b/gems/scip-ruby/BUILD index 44bec2149a..7110108446 100644 --- a/gems/scip-ruby/BUILD +++ b/gems/scip-ruby/BUILD @@ -1,10 +1,15 @@ -load(":build_gems.bzl", "build_gems", "version") +load(":build_gems.bzl", "build_gems", "setup_standalone_ruby", "version") version( name = "version", build_setting_default = "0.0.0", ) +setup_standalone_ruby( + name = "standalone-ruby", + visibility = ["//visibility:public"], +) + build_gems( name = "scip-ruby", srcs = [ @@ -20,4 +25,5 @@ build_gems( "//tools/config:darwin": "darwin", }), scip_ruby_target = "//main:scip-ruby", + visibility = ["//visibility:public"], ) diff --git a/gems/scip-ruby/bin/scip-ruby b/gems/scip-ruby/bin/scip-ruby index a6a7f55e96..e05c9566a9 100755 --- a/gems/scip-ruby/bin/scip-ruby +++ b/gems/scip-ruby/bin/scip-ruby @@ -20,15 +20,24 @@ if [ -n "${DEBUG:-}" ]; then set -x fi -if [ -z "${BUNDLE_GEMFILE:-}" ]; then - echo "Expected to be invoke through bundle exec scip-ruby" +die() { + echo "Expected to be invoked as 'bundle exec scip-ruby'" echo "Did you invoke this binary directly?" exit 1 +} + +if [ -z "${BUNDLE_GEMFILE:-}" ]; then + echo "Missing BUNDLE_GEMFILE environment variable" + die +elif [ -z "${BUNDLE_BIN_PATH:-}" ]; then + echo "Missing BUNDLE_BIN_PATH environment variable" + die fi -if grep -q -e "scip-ruby-debug" "$BUNDLE_GEMFILE"; then +GEMFILE_CONTENTS="$(< "$BUNDLE_GEMFILE")" +if [ "$GEMFILE_CONTENTS" == *"scip-ruby-debug"* ]; then SCIP_RUBY="scip-ruby-debug" -elif grep -q -e "scip-ruby" "$BUNDLE_GEMFILE"; then +elif [ "$GEMFILE_CONTENTS" == *"scip-ruby"* ]; then SCIP_RUBY="scip-ruby" elif grep -q -e "scip-ruby-debug" "$(dirname "$BUNDLE_GEMFILE")"/*.gemspec; then SCIP_RUBY="scip-ruby-debug" @@ -36,7 +45,26 @@ else SCIP_RUBY="scip-ruby" fi -GEM_INSTALL_PATH="$(bundle info "$SCIP_RUBY" | grep 'Path: ' | sed -E 's/^\s+Path: //')" +# Calling the bundle in BUNDLE_BIN_PATH tries to mess with system directories. +if [[ "$BUNDLE_BIN_PATH" =~ (.*)/lib/ruby/gems/.* ]]; then + BUNDLE_BIN_PATH="${BASH_REMATCH[1]}/bin/bundle" + file "$BUNDLE_BIN_PATH" > /dev/null || (echo "scip-ruby: warning: bundle not found at $BUNDLE_BIN_PATH, this may lead to errors") +else + echo "scip-ruby: warning: unexpected BUNDLE_BIN_PATH = $BUNDLE_BIN_PATH" + echo "scip-ruby: warning: this may lead to further errors" +fi + +# Careful about trying to "optimize" the regex here. +# I tried to use -E but that doesn't match with BSD sed. +GEM_INSTALL_PATH="$("$BUNDLE_BIN_PATH" info "$SCIP_RUBY" | grep 'Path:' | sed -e 's/\s*Path://')" +# From https://stackoverflow.com/a/3352015/2682729 +trim() { + local var="$*" + var="${var#"${var%%[![:space:]]*}"}" + var="${var%"${var##*[![:space:]]}"}" + printf '%s' "$var" +} +GEM_INSTALL_PATH="$(trim "$GEM_INSTALL_PATH")" chmod +x "$GEM_INSTALL_PATH/native/scip-ruby" exec "$GEM_INSTALL_PATH/native/scip-ruby" "$@" diff --git a/gems/scip-ruby/build.sh b/gems/scip-ruby/build.sh index 27eb74dc52..e7938f4573 100755 --- a/gems/scip-ruby/build.sh +++ b/gems/scip-ruby/build.sh @@ -1,27 +1,17 @@ #!/usr/bin/env bash -# Based on build-static-release.sh +# See also: NOTE[repo-test-structure] set -eu -exit_fail() { - echo 'This is likely a bug in the bazel code running the build' - exit 1 -} - -if [ -z "${VERSION:-}" ]; then - echo 'Missing value for VERSION environment variable' - exit_fail -elif [ -z "${NAME:-}" ]; then - echo 'Missing value for NAME environment variable' - exit_fail -elif [ -z "${SCIP_RUBY_BINARY:-}" ]; then - echo 'Missing value for SCIP_RUBY_BINARY environment variable' - exit_fail -elif [ -z "${OUT_DIR:-}" ]; then - echo 'Missing value for OUT_DIR environment variable' - exit_fail -fi +ENV_VARS=("VERSION" "NAME" "SCIP_RUBY_BINARY" "OUT_DIR") +for ENV_VAR in "${ENV_VARS[@]}"; do + if eval "[ -z \"$(printf '${%s:-}' $ENV_VAR)\" ]"; then + echo "Missing definition for $ENV_VAR environment variable" + echo 'This is likely a bug in the bazel code running the build' + exit 1 + fi +done cp -R gems/scip-ruby out mkdir -p out/native diff --git a/gems/scip-ruby/build_gems.bzl b/gems/scip-ruby/build_gems.bzl index 3b2e5b9b4f..1fef82469a 100644 --- a/gems/scip-ruby/build_gems.bzl +++ b/gems/scip-ruby/build_gems.bzl @@ -31,7 +31,7 @@ def _build_gems(ctx): outputs = output_files, inputs = inputs, mnemonic = "BuildGems", - executable = ctx.attr._build_script.files.to_list()[0], + executable = ctx.file._build_script, env = { "NAME": name, "DARWIN_VERSIONS": " ".join([str(dv) for dv in darwin_versions]), @@ -40,13 +40,48 @@ def _build_gems(ctx): "OUT_DIR": output_files[0].dirname, }, ) - return [DefaultInfo(files = depset(output_files))] + runfiles = ctx.runfiles(files = output_files) + return [DefaultInfo(files = depset(output_files), runfiles = runfiles)] + +def _setup_standalone_ruby(ctx): + ruby_archive = ctx.actions.declare_file("cache/ruby.tgz") + inputs = [ctx.file._ruby_version] + outputs = [ruby_archive] + ctx.actions.run( + outputs = outputs, + inputs = inputs, + mnemonic = "StandaloneRuby", + executable = ctx.file._install_script, + env = { + # Ideally, we would also pass in a C compiler here, but 🤷🏽 + "SCIP_RUBY_RBENV_EXE": ctx.var["SCIP_RUBY_RBENV_EXE"], + "SCIP_RUBY_CACHE_RUBY_DIR": ctx.var["SCIP_RUBY_CACHE_RUBY_DIR"], + "RUBY_VERSION_FILE": ctx.file._ruby_version.path, + "OUT_TGZ_PATH": ruby_archive.path, + }, + ) + runfiles = ctx.runfiles(files = outputs) + return [DefaultInfo(files = depset(outputs), runfiles = runfiles)] + +setup_standalone_ruby = rule( + implementation = _setup_standalone_ruby, + attrs = { + "_ruby_version": attr.label(default = "//:.ruby-version", allow_single_file = True), + "_install_script": attr.label(default = "install_ruby.sh", allow_single_file = True), + }, + doc = """ + Creates a standalone ruby installation using rbenv that is only for test use, + without interfering with any system Ruby. We are not using bazelruby/rules_ruby here + because it seems largely oriented towards *building* Ruby code, whereas what we + want to do is install dependencies in a way that mimics common usage (through Bundler). + """, +) build_gems = rule( implementation = _build_gems, attrs = { "_version": attr.label(default = ":version"), - "_build_script": attr.label(default = "build.sh", allow_files = True), + "_build_script": attr.label(default = "build.sh", allow_single_file = True), "srcs": attr.label_list(allow_files = True), "scip_ruby_target": attr.label(), "gem_name": attr.string(), diff --git a/gems/scip-ruby/install_ruby.sh b/gems/scip-ruby/install_ruby.sh new file mode 100755 index 0000000000..6dcecfbd08 --- /dev/null +++ b/gems/scip-ruby/install_ruby.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Remove after https://github.com/rbenv/ruby-build/issues/2024 is fixed. +NUM_CPUS="$(getconf _NPROCESSORS_ONLN)" +ENV_VARS=("SCIP_RUBY_CACHE_RUBY_DIR" "SCIP_RUBY_RBENV_EXE" "RUBY_VERSION_FILE" "OUT_TGZ_PATH" "NUM_CPUS") +for ENV_VAR in "${ENV_VARS[@]}"; do + if eval "[ -z \"$(printf '${%s:-}' $ENV_VAR)\" ]"; then + echo "Missing definition for $ENV_VAR environment variable" + exit 1 + fi +done +export MAKE_OPTS="-j ${NUM_CPUS:-4}" + +mkdir -p "$SCIP_RUBY_CACHE_RUBY_DIR" + +# Make ruby-build available to rbenv. +export PATH="$(dirname "$SCIP_RUBY_RBENV_EXE"):$PATH" + +# Debugging tip: If there is a failure in this step, replace it with: +# +# CC="$SCIP_RUBY_CACHE_RUBY_DIR/cc_wrapper" "$SCIP_RUBY_RBENV_EXE" install --keep --force +# +# In cc_wrapper, add the following: +# +# #!/usr/bin/env bash +# echo "cc $*" >> "$(dirname "${BASH_SOURCE[0]}")/cc.log" +# exec cc "$@" +# +# This will persist the build directory and record invocations which +# can be replayed for faster triage. + +if [[ "$SCIP_RUBY_RBENV_EXE" != "RUNNING_IN_CI_RBENV_NOT_NEEDED" ]]; then + RBENV_ROOT="$SCIP_RUBY_CACHE_RUBY_DIR" "$SCIP_RUBY_RBENV_EXE" install --skip-existing +fi + +mkdir -p "$(dirname "$OUT_TGZ_PATH")" +rm -f "$OUT_TGZ_PATH" +tar -czf "$OUT_TGZ_PATH" -C "$SCIP_RUBY_CACHE_RUBY_DIR" . \ No newline at end of file diff --git a/test/scip/repos/BUILD b/test/scip/repos/BUILD index 4ac53416f3..adcc4792d9 100644 --- a/test/scip/repos/BUILD +++ b/test/scip/repos/BUILD @@ -1,3 +1,8 @@ -load(":scip_repos_test.bzl", "scip_repos_test_suite") +load(":scip_repos_test.bzl", "register_scip_repo_test_caches", "scip_repos_test_suite") -scip_repos_test_suite(patch_paths = glob(["*.patch"])) +register_scip_repo_test_caches() + +scip_repos_test_suite( + name = "repos", + patch_paths = glob(["*.patch"]), +) diff --git a/test/scip/repos/brew-3.5.7.patch b/test/scip/repos/brew-3.5.9.patch similarity index 100% rename from test/scip/repos/brew-3.5.7.patch rename to test/scip/repos/brew-3.5.9.patch diff --git a/test/scip/repos/bundle_cache.sh b/test/scip/repos/bundle_cache.sh new file mode 100755 index 0000000000..85eb163a07 --- /dev/null +++ b/test/scip/repos/bundle_cache.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ENV_VARS=("SCIP_RUBY_CACHE_RUBY_DIR" "RUBY_VERSION_FILE" "PRISTINE_TOOLCHAIN_TGZ_PATH" "TEST_GEM_ZIP" "TEST_GEM_ZIP_PREFIX" "TEST_GEM_SUBDIR" "OUT_MODIFIED_TOOLCHAIN_TGZ_PATH" "OUT_GEM_WITH_VENDOR_TGZ_PATH") +for ENV_VAR in "${ENV_VARS[@]}"; do + if eval "[ -z \"$(printf '${%s:-}' $ENV_VAR)\" ]"; then + echo "Missing definition for $ENV_VAR environment variable" + exit 1 + fi +done + +SCIP_RUBY_SPECIFIC_RUBY_ROOT="$SCIP_RUBY_CACHE_RUBY_DIR" +if [ -d "$SCIP_RUBY_SPECIFIC_RUBY_ROOT/versions" ]; then + # rbenv creates an extra versions subdirectory, which doesn't apply in CI. + # This is to avoid finding the 'gem' from shims/ + SCIP_RUBY_SPECIFIC_RUBY_ROOT="$SCIP_RUBY_SPECIFIC_RUBY_ROOT/versions" +fi +SCIP_RUBY_SPECIFIC_RUBY_ROOT="$SCIP_RUBY_SPECIFIC_RUBY_ROOT/$(< "$RUBY_VERSION_FILE")" + +rm -rf "$SCIP_RUBY_CACHE_RUBY_DIR" +mkdir -p "$SCIP_RUBY_CACHE_RUBY_DIR" +tar -xzf "$PRISTINE_TOOLCHAIN_TGZ_PATH" -C "$SCIP_RUBY_CACHE_RUBY_DIR" + +GEM_EXE="$(find "$SCIP_RUBY_SPECIFIC_RUBY_ROOT" -name 'gem' -type f)" +BUNDLE_EXE="$(dirname "$GEM_EXE")/bundle" + +rm -rf "$TEST_GEM_ZIP_PREFIX" +unzip -q "$TEST_GEM_ZIP" + +# Is there a more reliable way to get the version? 😬 +_BUNDLER_VERSION="$(tail -n 1 < $TEST_GEM_ZIP_PREFIX/$TEST_GEM_SUBDIR/Gemfile.lock)" +_BUNDLER_VERSION="${_BUNDLER_VERSION// /}" +echo "BUNDLER_VERSION = $_BUNDLER_VERSION" +"$GEM_EXE" install "bundler:$_BUNDLER_VERSION" + +pushd "$TEST_GEM_ZIP_PREFIX/$TEST_GEM_SUBDIR" +git init -b main +git add . +GIT_AUTHOR_DATE=1993-05-16T15:04:05Z GIT_COMMITTER_DATE=1993-05-16T15:04:05Z \ + git -c user.name='iu' -c user.email='i@u' commit -a -m '(^_^)' --quiet +set +e + +if ! "$BUNDLE_EXE" cache --quiet 2> >(tee stderr.log >&2); then + err="$(< stderr.log)" + while IFS= read -r line; do + if [[ "$line" == *"lib/ruby/gems"*"mkmf.log" ]]; then + cat "$line" + fi + done <<< "$err" + exit 1 +fi +set -e +git restore . +popd + +tar -czf "$OUT_MODIFIED_TOOLCHAIN_TGZ_PATH" -C "$SCIP_RUBY_CACHE_RUBY_DIR" . +tar -czf "$OUT_GEM_WITH_VENDOR_TGZ_PATH" -C "$TEST_GEM_ZIP_PREFIX" . diff --git a/test/scip/repos/index_gem.sh b/test/scip/repos/index_gem.sh new file mode 100755 index 0000000000..8ddddd7c89 --- /dev/null +++ b/test/scip/repos/index_gem.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# See NOTE[repo-test-structure] for more context. + +TEST_DIR="$PWD" +ENV_VARS=("SCIP_RUBY_CACHE_RUBY_DIR" "RUBY_VERSION_FILE" "SCIP_RUBY_GEMS" "CACHED_TGZS" "REPO_NAME" "PATCH_PATH") # "PREP_CMD" "RUN_CMD" ) +for ENV_VAR in "${ENV_VARS[@]}"; do + if eval "[ -z \"$(printf '${%s:-}' $ENV_VAR)\" ]"; then + echo "Missing definition for $ENV_VAR environment variable" + exit 1 + fi +done +TEST_NAME="$REPO_NAME" + +# TODO: Librarify this, the env var check, and the GITHUB_ACTIONS var and use it consistently. +get_date() { + date '+%s' +} +PREV_TIME="$(get_date)" +PREV_CMD="" +print_time_elapsed() { + CUR_TIME="$(get_date)" + local DELTA=$(( CUR_TIME - PREV_TIME )) + if [ "$DELTA" -ne 0 ]; then + echo "---------------------------" + echo "Ran in ${DELTA}s: $PREV_CMD" + echo "---------------------------" + fi + PREV_CMD="$BASH_COMMAND" + PREV_TIME="$CUR_TIME" +} +trap print_time_elapsed DEBUG + +if [ -n "${GITHUB_ACTIONS:-}" ]; then + echo "::group::$TEST_NAME" + trap "echo ::end-group::$TEST_NAME" EXIT +fi + +CACHED_TGZS_ARRAY=($CACHED_TGZS) +TGZ_CACHE_DIR="$(dirname ${CACHED_TGZS_ARRAY[0]})" + +rm -rf "$SCIP_RUBY_CACHE_RUBY_DIR" +mkdir -p "$SCIP_RUBY_CACHE_RUBY_DIR" +tar -xzf "$TGZ_CACHE_DIR/ruby-post-bundle-cache.tgz" -C "$SCIP_RUBY_CACHE_RUBY_DIR" + +rm -rf repo +mkdir repo +tar -xzf "$TGZ_CACHE_DIR/gem-post-vendor.tgz" -C repo + +SCIP_RUBY_GEMS_ARRAY=($SCIP_RUBY_GEMS) +for SCIP_RUBY_GEM in "${SCIP_RUBY_GEMS_ARRAY[@]}"; do + # NOTE: Keep in sync with scip_repos_test.bzl + if [[ "$SCIP_RUBY_GEM" != *"darwin"* || "$SCIP_RUBY_GEM" == *"20"* ]]; then + cp "$SCIP_RUBY_GEM" repo/vendor/cache/ + fi +done + +SCIP_RUBY_SPECIFIC_RUBY_ROOT="$SCIP_RUBY_CACHE_RUBY_DIR" +if [ -d "$SCIP_RUBY_SPECIFIC_RUBY_ROOT/versions" ]; then + # rbenv creates an extra versions subdirectory, which doesn't apply in CI. + # This is to avoid finding the 'gem' from shims/ + SCIP_RUBY_SPECIFIC_RUBY_ROOT="$SCIP_RUBY_SPECIFIC_RUBY_ROOT/versions" +fi +SCIP_RUBY_SPECIFIC_RUBY_ROOT="$SCIP_RUBY_SPECIFIC_RUBY_ROOT/$(< "$RUBY_VERSION_FILE")" + +# Little indirect because apparently there may be more bundle binaries inside. +GEM_EXE="$(find "$SCIP_RUBY_SPECIFIC_RUBY_ROOT" -name 'gem' -type f)" +BUNDLE_EXE="$(dirname "$GEM_EXE")/bundle" + +pushd repo +git diff --exit-code # No changes should've been made before applying the patch +git apply "../$PATCH_PATH" + +"$BUNDLE_EXE" exec 'echo $PATH' +"$BUNDLE_EXE" install -j "$(getconf _NPROCESSORS_ONLN)" --local + +set +e +_INFO="$("$BUNDLE_EXE" info scip-ruby-debug)" +if [ $? != 0 ]; then + set -e + _INFO="$("$BUNDLE_EXE" info scip-ruby)" +fi +set -e + +printf 'bundle info output:\n%s\n' "$_INFO" + +# Ideally, we'd write something like: +# +# "$BUNDLE_EXE" exec scip-ruby --index-file index.scip --gem-metadata "$REPO_NAME@99.99.99" \\ +# +# However, there is a discrepancy in paths between bundle install/info +# vs bundle exec (see https://github.com/rubygems/rubygems/issues/5838). +# Work around that by relying on 'bundle info` as the source of truth. +GEM_INSTALL_PATH="$(echo "$_INFO" | grep 'Path:' | sed -e 's/\s*Path: //')" +# From https://stackoverflow.com/a/3352015/2682729 +trim() { + local var="$*" + var="${var#"${var%%[![:space:]]*}"}" + var="${var%"${var##*[![:space:]]}"}" + printf '%s' "$var" +} +GEM_INSTALL_PATH="$(trim "$GEM_INSTALL_PATH")" + +echo "GEM_INSTALL_PATH = $GEM_INSTALL_PATH" +echo "Calling shim" + +{ +echo "Shim @ $GEM_INSTALL_PATH/bin/scip-ruby:" +echo "-----------------------------------" +cat "$GEM_INSTALL_PATH/bin/scip-ruby" +echo "-----------------------------------" +echo "This is for debugging/reproduction; I was seeing stale shims earlier, not sure why" +} >&2 + + +# Call the shim, not the native binary, to test that the shim works. +DEBUG=1 "$BUNDLE_EXE" exec "$GEM_INSTALL_PATH/bin/scip-ruby" --index-file index.scip --gem-metadata "$REPO_NAME@99.99.99" +file index.scip + +popd \ No newline at end of file diff --git a/test/scip/repos/index_oss_repo.sh b/test/scip/repos/index_oss_repo.sh deleted file mode 100755 index ad2902665b..0000000000 --- a/test/scip/repos/index_oss_repo.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env bash - -# Some tests populate caches based on $HOME. To avoid polluting -# the outside env, such as on a dev machine, use the current -# directory, as this script is invoked inside a temporary -# test directory by Bazel. -export HOME="$PWD" - -# GEM_PATH should be provided from outside: -# -# ./bazel test --test_env GEM_PATH="$(dirname "$(which gem)")" -# -# Avoid passing --test_env PATH="$PATH" to reduce risk of -# dependencies on random stuff in a dev environment. -if [ -z "$GEM_PATH" ]; then - echo '$GEM_PATH environment variable was not set.' - exit 1 -fi -export PATH="$GEM_PATH:$PATH" - -get_date() { - date '+%s' -} -PREV_TIME="$(get_date)" -PREV_CMD="" -print_time_elapsed() { - CUR_TIME="$(get_date)" - local DELTA=$(( CUR_TIME - PREV_TIME )) - if [ "$DELTA" -ne 0 ]; then - echo "Ran in ${DELTA}s: $PREV_CMD\n" - fi - PREV_CMD="$BASH_COMMAND" - PREV_TIME="$CUR_TIME" -} -trap print_time_elapsed DEBUG - -if [ "$#" -lt 6 ] || [ "$#" -gt 7 ]; then - echo "Expected 5-6 arguments: got $#" - echo "1. Test name" - echo "2. Clone URL" - echo "3. git tag to use" - echo "4. git SHA for double-checking" - echo "5. Prep command to run before applying the patch" - echo "6. Command to run for type-checking" - echo "7. (optional) absolute path to patch to apply" - exit 1 -fi - -set -eu - -TEST_NAME="$1" -CLONE_URL="$2" -GIT_TAG="$3" -GIT_SHA="$4" -PREP_CMD="$5" -RUN_CMD="$6" -PATCH_ABSPATH="" -TEST_DIR="$PWD" -if [ "$#" -eq 7 ]; then - PATCH_ABSPATH="$TEST_DIR/$7" -fi - -if [ -n "${GITHUB_ACTIONS:-}" ]; then - echo "::group::$TEST_NAME" - trap "echo ::end-group::$TEST_NAME" EXIT -fi - -rm -rf repo -mkdir repo -pushd repo - -git -c advice.detachedHead=false clone "$CLONE_URL" . --branch "$GIT_TAG" --depth=1 -if [ "$(git rev-parse HEAD)" != "$GIT_SHA" ]; then - echo "Expected SHA: $GIT_SHA" - echo "Obtained SHA: $(git rev-parse HEAD)" - exit 1 -fi - -eval "$PREP_CMD" - -if [ -n "$PATCH_ABSPATH" ]; then - git apply "$PATCH_ABSPATH" - git diff -U0 -fi - -eval "$RUN_CMD" - -popd -rm -rf repo diff --git a/test/scip/repos/scip_repos_test.bzl b/test/scip/repos/scip_repos_test.bzl index ab21fe30db..b1ecea32b6 100644 --- a/test/scip/repos/scip_repos_test.bzl +++ b/test/scip/repos/scip_repos_test.bzl @@ -1,54 +1,123 @@ -_data = [ - { - "name": "brew", - "clone_url": "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/Homebrew/brew.git", - "tag": "3.5.7", - "git_sha": "ce8ef89ec0cfe875c48c3a83a843af9074c05930", - "prep_cmd": ("pushd Library/Homebrew" + - " && gem install bundler:1.17.3" + - " && popd" + - " && ./bin/brew typecheck"), - "run_cmd": ("pushd Library/Homebrew" + - " && find . -name srb -type f -exec cp $${TEST_DIR}/$(location //main:scip-ruby) {} \\;" + - " && popd" + - " && ./bin/brew typecheck"), +load("//third_party:test_gem_data.bzl", "gem_build_info") + +# For more context, see NOTE[repo-test-structure] + +def _scip_repo_test_cache_rule(ctx): + target_zip_list = ctx.attr.target.files.to_list() + inputs = target_zip_list + [ctx.file._standalone_ruby_tgz] + gem_name = ctx.label.name + toolchain_cache = ctx.actions.declare_file("cache/{}/ruby-post-bundle-cache.tgz".format(gem_name)) + gem_with_vendor_cache = ctx.actions.declare_file("cache/{}/gem-post-vendor.tgz".format(gem_name)) + outputs = [toolchain_cache, gem_with_vendor_cache] + if len(target_zip_list) != 1: + fail("Expected exactly 1 zip file in filegroup but found {} for {}".format( + len(target_zip_list), + ctx.attr.target, + )) + ctx.actions.run( + outputs = outputs, + inputs = inputs, + mnemonic = "BundleCache", + executable = ctx.file._bundle_cache_script, + env = { + # Ideally, we would pass in a C compiler here too, but 🤷🏽 + "SCIP_RUBY_CACHE_RUBY_DIR": ctx.var["SCIP_RUBY_CACHE_RUBY_DIR"], + "RUBY_VERSION_FILE": ctx.file._ruby_version.path, + "PRISTINE_TOOLCHAIN_TGZ_PATH": ctx.file._standalone_ruby_tgz.path, + "TEST_GEM_ZIP": target_zip_list[0].path, + "TEST_GEM_ZIP_PREFIX": ctx.attr.strip_prefix, + "TEST_GEM_SUBDIR": ctx.attr.subdir, + "OUT_MODIFIED_TOOLCHAIN_TGZ_PATH": toolchain_cache.path, + "OUT_GEM_WITH_VENDOR_TGZ_PATH": gem_with_vendor_cache.path, + }, + ) + return [DefaultInfo(files = depset(outputs), runfiles = ctx.runfiles(files = outputs))] + +scip_repo_test_cache_rule = rule( + implementation = _scip_repo_test_cache_rule, + attrs = { + "_standalone_ruby_tgz": attr.label(default = "//gems/scip-ruby:standalone-ruby", allow_single_file = True), + "_bundle_cache_script": attr.label(default = ":bundle_cache.sh", allow_single_file = True), + "_ruby_version": attr.label(default = "//:.ruby-version", allow_single_file = True), + "target": attr.label(mandatory = True), + "strip_prefix": attr.string(mandatory = True), + "subdir": attr.string(mandatory = True), }, -] + doc = "Intermediate cache archives for gems after running 'bundle cache'.", +) -def scip_repos_test_suite(patch_paths): - patch_paths = {p: None for p in patch_paths} +def register_scip_repo_test_caches(): + for gem_data in gem_build_info: + gem_name = gem_data["repo_name"] + scip_repo_test_cache_rule( + name = "prep_{}".format(gem_name), + target = "@{}_zip//file".format(gem_name), + strip_prefix = gem_data["strip_prefix"], + subdir = gem_data["gem_subdir"], + ) + +def scip_repos_test_suite(name, patch_paths): + available_patches = {p: None for p in patch_paths} test_names = [] - for testdata in _data: - test_name = testdata["name"] - patch_path = "{}-{}.patch".format(test_name, testdata["tag"]) - patch_dep = [] - patch_arg = [] - if patch_path in patch_paths: - patch_dep.append(patch_path) - patch_arg.append("$(location {})".format(patch_path)) - patch_paths.pop(patch_path, None) + consumed = {} + for gem_data in gem_build_info: + gem_name = gem_data["repo_name"] + patch_basename = "{}-{}".format(gem_name, gem_data["ref"]) + extra_deps = {} + gem_dep = {} + patch_arg = {} + for os in ["//tools/config:linux", "//tools/config:darwin"]: + candidates = [patch_basename + ".patch"] + if "linux" in os: + candidates.append(patch_basename + "-linux.patch") + gem_dep[os] = "//gems/scip-ruby/scip-ruby-debug-0.0.0-x86_64-linux.gem" + else: + candidates.append(patch_basename + "-darwin.patch") + + # NOTE: Keep in sync with index_gem.sh. + gem_dep[os] = "//gems/scip-ruby/scip-ruby-debug-0.0.0-universal-darwin-20.gem" + extra_deps[os] = [] + for patch_path in candidates: + if patch_path in available_patches: + extra_deps[os].append(patch_path) + patch_arg[os] = "$(location {})".format(patch_path) + consumed[patch_path] = None + if "linux" in patch_path or "darwin" in patch_path: + available_patches.pop(patch_path, None) + prep_target = ":prep_{}".format(gem_name) + data = [prep_target, "//gems/scip-ruby", "//:.ruby-version"] + + test_name = gem_name native.sh_test( name = test_name, size = "medium", # Should finish in 5 minutes - srcs = ["index_oss_repo.sh"], - deps = [], - data = ["//main:scip-ruby"] + patch_dep, - args = [ - testdata["name"], - testdata["clone_url"], - testdata["tag"], - testdata["git_sha"], - "'{}'".format(testdata["prep_cmd"]), - "'{}'".format(testdata["run_cmd"]), - ] + patch_arg, + srcs = ["index_gem.sh"], + data = data + select(extra_deps), + # Tests run serially since they share the same cache dir. + # For more context, see NOTE[repo-test-structure]. + tags = ["exclusive"], + env = select( + {os: { + "SCIP_RUBY_CACHE_RUBY_DIR": "$(SCIP_RUBY_CACHE_RUBY_DIR)", + "RUBY_VERSION_FILE": "$(location //:.ruby-version)", + "SCIP_RUBY_GEMS": "$(locations //gems/scip-ruby)", + "CACHED_TGZS": "$(locations {})".format(prep_target), + "REPO_NAME": gem_data["repo_name"], + "PATCH_PATH": pa, + } for os, pa in patch_arg.items()}, + ), ) test_names.append(test_name) - if len(patch_paths) != 0: - fail("Patches {} were not used by any tests".format(patch_paths)) + for p in consumed: + available_patches.pop(p, None) + + # if len(available_patches) != 0: + # fail("Patches {} were not used by any tests".format(available_patches)) + # FIXME(varun): Get the brew test working and turn this on. native.test_suite( - name = "repos", + name = name, tests = test_names, ) diff --git a/test/scip/repos/shopify-api-ruby-v11.1.0-darwin.patch b/test/scip/repos/shopify-api-ruby-v11.1.0-darwin.patch new file mode 100644 index 0000000000..464aa36961 --- /dev/null +++ b/test/scip/repos/shopify-api-ruby-v11.1.0-darwin.patch @@ -0,0 +1,24 @@ +diff --git a/Gemfile.lock b/Gemfile.lock +index 9fef6a6..4b720c0 100644 +--- a/Gemfile.lock ++++ b/Gemfile.lock +@@ -81,2 +81,3 @@ GEM + ruby-progressbar (1.11.0) ++ scip-ruby-debug (0.0.0-universal-darwin-21) + securerandom (0.2.0) +@@ -125,2 +126,3 @@ PLATFORMS + arm64-darwin-21 ++ ruby + x86_64-linux +@@ -136,2 +138,3 @@ DEPENDENCIES + rubocop-sorbet ++ scip-ruby-debug + shopify_api! +diff --git a/shopify_api.gemspec b/shopify_api.gemspec +index da77b24..93fba5c 100644 +--- a/shopify_api.gemspec ++++ b/shopify_api.gemspec +@@ -51,2 +51,3 @@ Gem::Specification.new do |s| + s.add_development_dependency("tapioca") ++ s.add_development_dependency("scip-ruby-debug") + end diff --git a/test/scip/repos/shopify-api-ruby-v11.1.0-linux.patch b/test/scip/repos/shopify-api-ruby-v11.1.0-linux.patch new file mode 100644 index 0000000000..84ee512afd --- /dev/null +++ b/test/scip/repos/shopify-api-ruby-v11.1.0-linux.patch @@ -0,0 +1,24 @@ +diff --git a/Gemfile.lock b/Gemfile.lock +index 9fef6a6..54db244 100644 +--- a/Gemfile.lock ++++ b/Gemfile.lock +@@ -81,2 +81,3 @@ GEM + ruby-progressbar (1.11.0) ++ scip-ruby-debug (0.0.0-x86_64-linux) + securerandom (0.2.0) +@@ -125,2 +126,3 @@ PLATFORMS + arm64-darwin-21 ++ ruby + x86_64-linux +@@ -136,2 +138,3 @@ DEPENDENCIES + rubocop-sorbet ++ scip-ruby-debug + shopify_api! +diff --git a/shopify_api.gemspec b/shopify_api.gemspec +index da77b24..93fba5c 100644 +--- a/shopify_api.gemspec ++++ b/shopify_api.gemspec +@@ -51,2 +51,3 @@ Gem::Specification.new do |s| + s.add_development_dependency("tapioca") ++ s.add_development_dependency("scip-ruby-debug") + end diff --git a/third_party/externals.bzl b/third_party/externals.bzl index f019985c44..2e9fef70e3 100644 --- a/third_party/externals.bzl +++ b/third_party/externals.bzl @@ -1,5 +1,6 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file") load("//third_party:ruby_externals.bzl", "register_ruby_dependencies") +load("//third_party:test_gem_data.bzl", "gem_build_info") # We define our externals here instead of directly in WORKSPACE def register_sorbet_dependencies(): @@ -341,6 +342,14 @@ def register_sorbet_dependencies(): register_ruby_dependencies() +def register_scip_ruby_dependencies(): + for data in gem_build_info: + http_file( + name = data["repo_name"] + "_zip", + urls = [data["archive_url"]], + sha256 = data["archive_sha256"], + ) + def _github_public_urls(path): """ Produce a url list that works both with github, and stripe's internal artifact cache. diff --git a/third_party/test_gem.BUILD b/third_party/test_gem.BUILD new file mode 100644 index 0000000000..7d69da58e8 --- /dev/null +++ b/third_party/test_gem.BUILD @@ -0,0 +1,5 @@ +filegroup( + name = "all", + srcs = glob(["**/*"]), + visibility = ["//visibility:public"], +) \ No newline at end of file diff --git a/third_party/test_gem_data.bzl b/third_party/test_gem_data.bzl new file mode 100644 index 0000000000..750505434b --- /dev/null +++ b/third_party/test_gem_data.bzl @@ -0,0 +1,41 @@ +gem_build_info = [ + # { + # "repo_name": "brew", + # # "kind": "self_sandboxing", + # "archive_url": "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/Homebrew/brew/archive/refs/tags/3.5.9.zip", + # "archive_sha256": "28a66005cc6f0f22a9a112ce84cedb833efe6926072dec2c68623195c008852f", + # "ref": "3.5.9", + # "strip_prefix": "brew-3.5.9", + # "gem_subdir": "Library/Homebrew", + # # "prep_cmd": """ + # # pushd Library/Homebrew \\ + # # && gem install bundler:1.17.3 \\ + # # && popd \\ + # # && ./bin/brew typecheck + # # """, + # # "run_cmd": """ + # # pushd Library/Homebrew \\ + # # && find . -name srb -type f -exec cp $${TEST_DIR}/$(location //main:scip-ruby) {} \\; \\ + # # && popd \\ + # # && ./bin/brew typecheck + # # """, + # }, + { + "repo_name": "shopify-api-ruby", + # "kind": "needs_sandboxing", + "archive_url": "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/Shopify/shopify-api-ruby/archive/refs/tags/v11.1.0.zip", + "archive_sha256": "95a209308d6479491ca282af66254bd6c4e1c5b6edb8e0409e2fe29424a51157", + "ref": "v11.1.0", + "strip_prefix": "shopify-api-ruby-11.1.0", + # "prep_cmd": """ + # bundle cache --all \\ + # && for p in $(locations //gems/scip-ruby); do cp $${TEST_DIR}/$$p vendor/cache/; done \\ + # """, + # "run_cmd": """ + # bundle install --local \\ + # && bundle exec scip-ruby --index-file index.scip --gem-metadata 'shopify-api-ruby@11.1.0' \\ + # && file index.scip + # """, + "gem_subdir": ".", + }, +] diff --git a/xbrew b/xbrew new file mode 100755 index 0000000000..83ba9aa0c8 --- /dev/null +++ b/xbrew @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +if [ "$1" == "where" ]; then + set -e + echo "$($HOME/.homebrew-x86_64/bin/brew --prefix)/bin/$2" + exit 0 +fi +exec arch -x86_64 "$HOME/.homebrew-x86_64/bin/brew" "$@" From f8b5954a4e8bb490a95ecc4957a87a84cec5cc11 Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 15 Aug 2022 10:30:11 +0800 Subject: [PATCH 2/4] more docs + vm_setup + update scripts a bit --- docs/scip-ruby/CONTRIBUTING.md | 46 ++++++++++++++++++++++++++++++--- docs/scip-ruby/vm_setup.sh | 39 ++++++++++++++++++++++++++++ gems/scip-ruby/install_ruby.sh | 7 +++-- test/scip/repos/bundle_cache.sh | 4 ++- 4 files changed, 88 insertions(+), 8 deletions(-) create mode 100644 docs/scip-ruby/vm_setup.sh diff --git a/docs/scip-ruby/CONTRIBUTING.md b/docs/scip-ruby/CONTRIBUTING.md index be34bf31c0..db0514f7e1 100644 --- a/docs/scip-ruby/CONTRIBUTING.md +++ b/docs/scip-ruby/CONTRIBUTING.md @@ -14,6 +14,9 @@ see the [Design Decisions doc][] doc. - [Writing a new repo test](#writing-a-new-snapshot-test) - [Debugging with print statements](#debugging-with-print-statements) - [Debugging with LLDB](#debugging-with-lldb) +- [Debugging build issues](#debugging-build-issues) + - [Debugging Bazel](#debugging-bazel) + - [Debugging on Linux](#debugging-on-linux) - [Creating PRs](#creating-prs) - [Cutting a release](#cutting-a-release) - Troubleshooting @@ -54,7 +57,28 @@ That may lead to confusing errors when working on other projects. #### Linux -Install [rbenv][] as per the official instructions. Then run: +Install [rbenv][] as per the official instructions. For `ruby-build`, +do NOT install it as a plugin. + +
+Why not install `ruby-build` as a plugin? + +The `RBENV_ROOT` variable does double-duty; +it serves as the location to install the tools, and it is also used to +locate plugins (like `ruby-build` -- if you install it as a plugin). +However, in our case, since we are using a different tool path, +`rbenv` will be unable to use the `install` plugin. + +To mimic the configuration on macOS, I recommend +installing `ruby-build` next to the `rbenv` binary. +
+ +``` +git clone https://github.com/rbenv/ruby-build.git --depth=1 +# Assuming you installed rbenv through git as is recommended in the README. +PREFIX="$HOME/.rbenv" ./ruby-build/install.sh +rm -rf ruby-build +``` ```bash echo "build --define SCIP_RUBY_CACHE_RUBY_DIR='$PWD/.cache_ruby' --define SCIP_RUBY_RBENV_EXE='$(which rbenv)'" >> .bazelrc.local @@ -205,6 +229,22 @@ popd unset TEST_DIR ``` +## Debugging build issues + +### Debugging Bazel + +See Keith Smiley's blog post [Debugging bazel actions](https://www.smileykeith.com/2022/03/02/debugging-bazel-actions/). ([archive link](https://web.archive.org/web/20220711000725/https://www.smileykeith.com/2022/03/02/debugging-bazel-actions/)) + +### Debugging on Linux + +Debugging a build issue in GitHub Actions can get emotionally draining quickly. + +If you work at Sourcegraph, set up a GCP VM. +You may find the companion [VM setup script](./vm_setup.sh) helpful. +It is not tested in CI, so you may need to tweak it a bit. + +Don't forget to delete the instance after you're done investigating! + ## Creating PRs PRs created through the GitHub UI default to being made @@ -281,5 +321,5 @@ the executable script being modified doesn't trigger a rebuild. If you're seeing stale results, try something like: ```bash -rm -rf .cache_ruby/ && cd bazel-bin && find . -name '*.tgz' -type f | xargs rm -f && cd - -``` \ No newline at end of file +rm -rf .cache_ruby/ && (cd bazel-bin && find . -name '*.tgz' -type f -delete) +``` diff --git a/docs/scip-ruby/vm_setup.sh b/docs/scip-ruby/vm_setup.sh new file mode 100644 index 0000000000..83894fadee --- /dev/null +++ b/docs/scip-ruby/vm_setup.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# Approximate setup script for a Google Cloud VM. +# +# I recommend using a 8 core or 16 core CPU-optimized instance +# with Ubuntu 20.04 (there is a build issue with 22.04 for +# inscrutable reasons). 25 GB disk is more than sufficient + +sudo apt update +sudo apt autoremove +sudo apt upgrade +sudo apt install wget unzip ripgrep fd-find gcc libncurses5-dev libncursesw5-dev xxd libtinfo5 make zlib1g-dev autoconf bison build-essential libssl-dev libyaml-dev libreadline6-dev libffi-dev libgdbm6 libgdbm-dev libdb-dev uuid-dev + +rm -rf ~/.rbenv +git clone https://github.com/rbenv/rbenv.git ~/.rbenv --depth=1 +(cd ~/.rbenv && src/configure && make -C src) +echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bashrc +echo 'eval "$(rbenv init - bash)"' >> ~/.bashrc +source ~/.bashrc + +git clone https://github.com/rbenv/ruby-build.git --depth=1 +PREFIX="$HOME/.rbenv" ./ruby-build/install.sh +rm -rf ruby-build + +curl -fsSL https://github.com/rbenv/rbenv-installer/raw/main/bin/rbenv-doctor | bash + +git clone https://github.com/sourcegraph/scip-ruby.git +cd scip-ruby + +export JOB_NAME=test +export CACHE_DIR="$HOME/bae" +mkdir -p "$CACHE_DIR/output-bases/${JOB_NAME}" "$CACHE_DIR/build" "$CACHE_DIR/repos" +{ + echo 'common --curses=no --color=yes' + echo "startup --output_base=$CACHE_DIR/output-bases/${JOB_NAME}" + echo "build --disk_cache=$CACHE_DIR/build --repository_cache=$CACHE_DIR/repos" + echo "test --disk_cache=$CACHE_DIR/build --repository_cache=$CACHE_DIR/repos" + echo "build --define SCIP_RUBY_CACHE_RUBY_DIR='$PWD/.cache_ruby' --define SCIP_RUBY_RBENV_EXE='$(which rbenv)'" +} > .bazelrc.local diff --git a/gems/scip-ruby/install_ruby.sh b/gems/scip-ruby/install_ruby.sh index 6dcecfbd08..0c23cfec85 100755 --- a/gems/scip-ruby/install_ruby.sh +++ b/gems/scip-ruby/install_ruby.sh @@ -15,9 +15,6 @@ export MAKE_OPTS="-j ${NUM_CPUS:-4}" mkdir -p "$SCIP_RUBY_CACHE_RUBY_DIR" -# Make ruby-build available to rbenv. -export PATH="$(dirname "$SCIP_RUBY_RBENV_EXE"):$PATH" - # Debugging tip: If there is a failure in this step, replace it with: # # CC="$SCIP_RUBY_CACHE_RUBY_DIR/cc_wrapper" "$SCIP_RUBY_RBENV_EXE" install --keep --force @@ -32,9 +29,11 @@ export PATH="$(dirname "$SCIP_RUBY_RBENV_EXE"):$PATH" # can be replayed for faster triage. if [[ "$SCIP_RUBY_RBENV_EXE" != "RUNNING_IN_CI_RBENV_NOT_NEEDED" ]]; then + # Make ruby-build available to rbenv. + export PATH="$(dirname "$SCIP_RUBY_RBENV_EXE"):$PATH" RBENV_ROOT="$SCIP_RUBY_CACHE_RUBY_DIR" "$SCIP_RUBY_RBENV_EXE" install --skip-existing fi mkdir -p "$(dirname "$OUT_TGZ_PATH")" rm -f "$OUT_TGZ_PATH" -tar -czf "$OUT_TGZ_PATH" -C "$SCIP_RUBY_CACHE_RUBY_DIR" . \ No newline at end of file +tar -czf "$OUT_TGZ_PATH" -C "$SCIP_RUBY_CACHE_RUBY_DIR" . diff --git a/test/scip/repos/bundle_cache.sh b/test/scip/repos/bundle_cache.sh index 85eb163a07..fce6c1b265 100755 --- a/test/scip/repos/bundle_cache.sh +++ b/test/scip/repos/bundle_cache.sh @@ -35,7 +35,7 @@ echo "BUNDLER_VERSION = $_BUNDLER_VERSION" "$GEM_EXE" install "bundler:$_BUNDLER_VERSION" pushd "$TEST_GEM_ZIP_PREFIX/$TEST_GEM_SUBDIR" -git init -b main +git init -q git add . GIT_AUTHOR_DATE=1993-05-16T15:04:05Z GIT_COMMITTER_DATE=1993-05-16T15:04:05Z \ git -c user.name='iu' -c user.email='i@u' commit -a -m '(^_^)' --quiet @@ -45,7 +45,9 @@ if ! "$BUNDLE_EXE" cache --quiet 2> >(tee stderr.log >&2); then err="$(< stderr.log)" while IFS= read -r line; do if [[ "$line" == *"lib/ruby/gems"*"mkmf.log" ]]; then + echo "----------- Printing likely bad mkmf.log -----------" cat "$line" + echo "----------------------------------------------------" fi done <<< "$err" exit 1 From 7c137b96d598e68a00be3a24b3d3db9afedbfc3c Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 15 Aug 2022 12:56:02 +0800 Subject: [PATCH 3/4] Work around gcc weirdness in Sorbet's config. --- docs/scip-ruby/CONTRIBUTING.md | 29 +++++++++++++++++++++++++++++ test/scip/repos/bundle_cache.sh | 8 +++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/docs/scip-ruby/CONTRIBUTING.md b/docs/scip-ruby/CONTRIBUTING.md index db0514f7e1..2835cb2da7 100644 --- a/docs/scip-ruby/CONTRIBUTING.md +++ b/docs/scip-ruby/CONTRIBUTING.md @@ -235,6 +235,35 @@ unset TEST_DIR See Keith Smiley's blog post [Debugging bazel actions](https://www.smileykeith.com/2022/03/02/debugging-bazel-actions/). ([archive link](https://web.archive.org/web/20220711000725/https://www.smileykeith.com/2022/03/02/debugging-bazel-actions/)) +One KEY thing to keep in mind is that some problems only manifest +inside the sandbox but not outside because Bazel sandbox changes +the binaries available at certain paths to use non-system tools. + +For example, `uname -m` on an M1 Mac would normally return `arm64` +but inside an x86_64 Bazel environment (currently the default) +it will return `x86_64`. + +For compiler issues inside a sandbox, +it helpful to test a small code snippet first. + +```bash +{ + echo '--- Try compiling some simple stuff ---' + { + echo '#include ' + echo '' + echo 'int main() {' + echo ' printf("Hello %s\n", "World!");' + echo ' return 0;' + echo '}' + } > tmp.c + gcc tmp.c + ./a.out + rm tmp.c ./a.out + echo '--------------------------------------' +} >&2 +``` + ### Debugging on Linux Debugging a build issue in GitHub Actions can get emotionally draining quickly. diff --git a/test/scip/repos/bundle_cache.sh b/test/scip/repos/bundle_cache.sh index fce6c1b265..2fcf60ce57 100755 --- a/test/scip/repos/bundle_cache.sh +++ b/test/scip/repos/bundle_cache.sh @@ -41,7 +41,13 @@ GIT_AUTHOR_DATE=1993-05-16T15:04:05Z GIT_COMMITTER_DATE=1993-05-16T15:04:05Z \ git -c user.name='iu' -c user.email='i@u' commit -a -m '(^_^)' --quiet set +e -if ! "$BUNDLE_EXE" cache --quiet 2> >(tee stderr.log >&2); then +# For reasons that are unclear to me, the 'env -' usage works around +# a gcc problem which manifests as 'cc1' not being executable when invoked +# as 'gcc blah' and 'ld' not being found when invoked as '/usr/bin/gcc'. +# +# This seems specific to some Sorbet configuration; I have not been +# able to reproduce this weirdness with a minimal configuration outside. +if ! env - PATH="$PATH" PWD="$PWD" "$BUNDLE_EXE" cache --quiet 2> >(tee stderr.log >&2); then err="$(< stderr.log)" while IFS= read -r line; do if [[ "$line" == *"lib/ruby/gems"*"mkmf.log" ]]; then From 3ccc25896f8ef5ca218ce19c4f399a4445e3c54e Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 15 Aug 2022 14:19:38 +0800 Subject: [PATCH 4/4] build: Make gem build use standalone ruby. --- gems/scip-ruby/BUILD | 1 - gems/scip-ruby/build_gems.bzl | 95 +++++++++++-------- .../{build.sh => build_scip_ruby_gems.sh} | 21 +++- ...all_ruby.sh => install_standalone_ruby.sh} | 0 4 files changed, 74 insertions(+), 43 deletions(-) rename gems/scip-ruby/{build.sh => build_scip_ruby_gems.sh} (54%) rename gems/scip-ruby/{install_ruby.sh => install_standalone_ruby.sh} (100%) diff --git a/gems/scip-ruby/BUILD b/gems/scip-ruby/BUILD index 7110108446..af2b12c708 100644 --- a/gems/scip-ruby/BUILD +++ b/gems/scip-ruby/BUILD @@ -24,6 +24,5 @@ build_gems( "//tools/config:linux": "linux", "//tools/config:darwin": "darwin", }), - scip_ruby_target = "//main:scip-ruby", visibility = ["//visibility:public"], ) diff --git a/gems/scip-ruby/build_gems.bzl b/gems/scip-ruby/build_gems.bzl index 1fef82469a..44e9204a7e 100644 --- a/gems/scip-ruby/build_gems.bzl +++ b/gems/scip-ruby/build_gems.bzl @@ -1,3 +1,37 @@ +def _setup_standalone_ruby(ctx): + ruby_archive = ctx.actions.declare_file("cache/ruby.tgz") + inputs = [ctx.file._ruby_version] + outputs = [ruby_archive] + ctx.actions.run( + outputs = outputs, + inputs = inputs, + mnemonic = "StandaloneRuby", + executable = ctx.file._install_script, + env = { + # Ideally, we would also pass in a C compiler here, but 🤷🏽 + "SCIP_RUBY_RBENV_EXE": ctx.var["SCIP_RUBY_RBENV_EXE"], + "SCIP_RUBY_CACHE_RUBY_DIR": ctx.var["SCIP_RUBY_CACHE_RUBY_DIR"], + "RUBY_VERSION_FILE": ctx.file._ruby_version.path, + "OUT_TGZ_PATH": ruby_archive.path, + }, + ) + runfiles = ctx.runfiles(files = outputs) + return [DefaultInfo(files = depset(outputs), runfiles = runfiles)] + +setup_standalone_ruby = rule( + implementation = _setup_standalone_ruby, + attrs = { + "_ruby_version": attr.label(default = "//:.ruby-version", allow_single_file = True), + "_install_script": attr.label(default = "install_standalone_ruby.sh", allow_single_file = True), + }, + doc = """ + Creates a standalone ruby installation using rbenv that is only for test use, + without interfering with any system Ruby. We are not using bazelruby/rules_ruby here + because it seems largely oriented towards *building* Ruby code, whereas what we + want to do is install dependencies in a way that mimics common usage (through Bundler). + """, +) + VersionProvider = provider(fields = ["version"]) def _version(ctx): @@ -24,66 +58,49 @@ def _build_gems(ctx): output_files = [ctx.actions.declare_file(out) for out in outs] inputs = ctx.attr._build_script.files.to_list() - for src in (ctx.attr.srcs + [ctx.attr.scip_ruby_target]): + for src in ctx.attr.srcs: inputs += src.files.to_list() + inputs.append(ctx.file._scip_ruby_binary) + inputs.append(ctx.file._ruby_version) + + # Upside of depending on our Ruby installation: + # - Reduce risk of funky issues caused due to having two Ruby toolchains. + # Downside of depending on our Ruby installation: + # - The build is more serialized: + # setup standalone ruby -> build gems -> run tests + # Instead of build gems running in parallel with the ruby setup + # (due to an environmental 'gem' command) + # We could potentially revisit this... + inputs.append(ctx.file._standalone_ruby_tgz) ctx.actions.run( outputs = output_files, inputs = inputs, - mnemonic = "BuildGems", + mnemonic = "BuildSCIPRubyGems", executable = ctx.file._build_script, env = { + "SCIP_RUBY_CACHE_RUBY_DIR": ctx.var["SCIP_RUBY_CACHE_RUBY_DIR"], + "RUBY_VERSION_FILE": ctx.file._ruby_version.path, + "PRISTINE_TOOLCHAIN_TGZ_PATH": ctx.file._standalone_ruby_tgz.path, "NAME": name, "DARWIN_VERSIONS": " ".join([str(dv) for dv in darwin_versions]), "VERSION": version, - "SCIP_RUBY_BINARY": ctx.attr.scip_ruby_target.files.to_list()[0].path, + "SCIP_RUBY_BINARY": ctx.file._scip_ruby_binary.path, "OUT_DIR": output_files[0].dirname, }, ) runfiles = ctx.runfiles(files = output_files) return [DefaultInfo(files = depset(output_files), runfiles = runfiles)] -def _setup_standalone_ruby(ctx): - ruby_archive = ctx.actions.declare_file("cache/ruby.tgz") - inputs = [ctx.file._ruby_version] - outputs = [ruby_archive] - ctx.actions.run( - outputs = outputs, - inputs = inputs, - mnemonic = "StandaloneRuby", - executable = ctx.file._install_script, - env = { - # Ideally, we would also pass in a C compiler here, but 🤷🏽 - "SCIP_RUBY_RBENV_EXE": ctx.var["SCIP_RUBY_RBENV_EXE"], - "SCIP_RUBY_CACHE_RUBY_DIR": ctx.var["SCIP_RUBY_CACHE_RUBY_DIR"], - "RUBY_VERSION_FILE": ctx.file._ruby_version.path, - "OUT_TGZ_PATH": ruby_archive.path, - }, - ) - runfiles = ctx.runfiles(files = outputs) - return [DefaultInfo(files = depset(outputs), runfiles = runfiles)] - -setup_standalone_ruby = rule( - implementation = _setup_standalone_ruby, - attrs = { - "_ruby_version": attr.label(default = "//:.ruby-version", allow_single_file = True), - "_install_script": attr.label(default = "install_ruby.sh", allow_single_file = True), - }, - doc = """ - Creates a standalone ruby installation using rbenv that is only for test use, - without interfering with any system Ruby. We are not using bazelruby/rules_ruby here - because it seems largely oriented towards *building* Ruby code, whereas what we - want to do is install dependencies in a way that mimics common usage (through Bundler). - """, -) - build_gems = rule( implementation = _build_gems, attrs = { "_version": attr.label(default = ":version"), - "_build_script": attr.label(default = "build.sh", allow_single_file = True), + "_build_script": attr.label(default = "build_scip_ruby_gems.sh", allow_single_file = True), "srcs": attr.label_list(allow_files = True), - "scip_ruby_target": attr.label(), + "_scip_ruby_binary": attr.label(default = "//main:scip-ruby", allow_single_file = True), + "_ruby_version": attr.label(default = "//:.ruby-version", allow_single_file = True), + "_standalone_ruby_tgz": attr.label(default = "//gems/scip-ruby:standalone-ruby", allow_single_file = True), "gem_name": attr.string(), "gem_target_os": attr.string(), }, diff --git a/gems/scip-ruby/build.sh b/gems/scip-ruby/build_scip_ruby_gems.sh similarity index 54% rename from gems/scip-ruby/build.sh rename to gems/scip-ruby/build_scip_ruby_gems.sh index e7938f4573..5aab1d3246 100755 --- a/gems/scip-ruby/build.sh +++ b/gems/scip-ruby/build_scip_ruby_gems.sh @@ -4,7 +4,7 @@ set -eu -ENV_VARS=("VERSION" "NAME" "SCIP_RUBY_BINARY" "OUT_DIR") +ENV_VARS=("PRISTINE_TOOLCHAIN_TGZ_PATH" "SCIP_RUBY_CACHE_RUBY_DIR" "RUBY_VERSION_FILE" "VERSION" "NAME" "SCIP_RUBY_BINARY" "OUT_DIR") for ENV_VAR in "${ENV_VARS[@]}"; do if eval "[ -z \"$(printf '${%s:-}' $ENV_VAR)\" ]"; then echo "Missing definition for $ENV_VAR environment variable" @@ -24,6 +24,21 @@ cleanup() { } trap cleanup EXIT +rm -rf "$SCIP_RUBY_CACHE_RUBY_DIR" +mkdir -p "$SCIP_RUBY_CACHE_RUBY_DIR" +tar -xzf "$PRISTINE_TOOLCHAIN_TGZ_PATH" -C "$SCIP_RUBY_CACHE_RUBY_DIR" + +SCIP_RUBY_SPECIFIC_RUBY_ROOT="$SCIP_RUBY_CACHE_RUBY_DIR" +if [ -d "$SCIP_RUBY_SPECIFIC_RUBY_ROOT/versions" ]; then + # rbenv creates an extra versions subdirectory, which doesn't apply in CI. + # This is to avoid finding the 'gem' from shims/ + SCIP_RUBY_SPECIFIC_RUBY_ROOT="$SCIP_RUBY_SPECIFIC_RUBY_ROOT/versions" +fi +SCIP_RUBY_SPECIFIC_RUBY_ROOT="$SCIP_RUBY_SPECIFIC_RUBY_ROOT/$(< "$RUBY_VERSION_FILE")" + +GEM_EXE="$(find "$SCIP_RUBY_SPECIFIC_RUBY_ROOT" -name 'gem' -type f)" +file "$GEM_EXE" + pushd out cat scip-ruby.template.gemspec \ @@ -36,11 +51,11 @@ if [ "$(uname -s)" == "Darwin" ]; then DARWIN_VERSIONS=($DARWIN_VERSIONS) for i in "${DARWIN_VERSIONS[@]}"; do sed -i.bak "s/Gem::Platform::CURRENT/'universal-darwin-$i'/" "$GEMSPEC" - gem build "$GEMSPEC" + "$GEM_EXE" build "$GEMSPEC" mv "$GEMSPEC.bak" "$GEMSPEC" done else - gem build "$GEMSPEC" + "$GEM_EXE" build "$GEMSPEC" fi popd diff --git a/gems/scip-ruby/install_ruby.sh b/gems/scip-ruby/install_standalone_ruby.sh similarity index 100% rename from gems/scip-ruby/install_ruby.sh rename to gems/scip-ruby/install_standalone_ruby.sh