diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index cb982f083f..e73f1c00d0 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -13,8 +13,6 @@ jobs:
steps:
- uses: actions/checkout@v3
- uses: ruby/setup-ruby@v1
- with:
- ruby-version: '2.7'
# Keep in sync with release.yml
- name: Manually evict cache entry if applicable
run: ACCESS_TOKEN='${{ secrets.GITHUB_TOKEN }}' python3 .github/workflows/evict.py
@@ -46,4 +44,11 @@ jobs:
- name: "🏋️♂️ Run snapshot tests"
run: rm log && ./bazel test //test/scip --config=dbg
- name: "🏋️♂️ Run repo tests"
- run: ./bazel test //test/scip/repos --test_env GITHUB_ACTIONS=1 --test_env GEM_PATH="$(dirname "$(which gem)")" --config=dbg
+ run: |
+ sudo apt-get install --reinstall build-essential
+ ls "$(dirname "$(which gcc)")"
+ echo "$PATH"
+ ls /usr/bin /usr/local/bin /bin
+ # Technically, this is for Ruby 2.7.0 but it's probably fine.
+ sudo apt-get install ruby2.7-dev
+ ./bazel test //test/scip/repos --test_env GITHUB_ACTIONS=1 --test_env GEM_PATH="$(dirname "$(which gem)")" --config=dbg
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index a4eb3eff3a..12a1ab8b38 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -35,6 +35,7 @@ jobs:
steps:
- uses: actions/checkout@v3
# Keep in sync with ci.yml
+ - uses: actions/setup-ruby@v1
- name: Manually evict cache entry if applicable
if: ${{ runner.os == 'Linux' }}
run: ACCESS_TOKEN='${{ secrets.GITHUB_TOKEN }}' python3 .github/workflows/evict.py
diff --git a/.github/workflows/setup-bazel.sh b/.github/workflows/setup-bazel.sh
index 49d58c9a3b..7ca3a7d9e4 100755
--- a/.github/workflows/setup-bazel.sh
+++ b/.github/workflows/setup-bazel.sh
@@ -4,10 +4,22 @@ set -euo pipefail
CACHE_ROOT="$HOME/bazelcache"
mkdir -p "$CACHE_ROOT/output-base" "$CACHE_ROOT/build" "$CACHE_ROOT/repos"
+
+os="$(uname -s)"
+if [ "$os" == "Linux" ]; then
+ SETUP_RUBY_TOOLS_DIR="/opt/hostedtoolcache/Ruby"
+elif [ "$os" == "Darwin" ]; then
+ SETUP_RUBY_TOOLS_DIR="/Users/runner/hostedtoolcache/Ruby"
+else
+ echo "Check https://github.com/ruby/setup-ruby#using-self-hosted-runners and note setup-ruby's cache directory here"
+ exit 1
+fi
+
{
- echo 'common --curses=no --color=yes'
+ echo "common --curses=no --color=yes "
echo "startup --output_base=$CACHE_ROOT/output-base"
echo "build --disk_cache=$CACHE_ROOT/build --repository_cache=$CACHE_ROOT/repos"
+ echo "build --define SCIP_RUBY_CACHE_RUBY_DIR=$SETUP_RUBY_TOOLS_DIR --define SCIP_RUBY_RBENV_EXE=RUNNING_IN_CI_RBENV_NOT_NEEDED"
if [ -n "${VERSION:-}" ]; then
echo "build --//gems/scip-ruby:version=$VERSION"
fi
diff --git a/.gitignore b/.gitignore
index bb2fa7b97e..2039344e1d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,8 @@
/bazel-*
/.bazelrc.local
+/.cache_ruby
/compile_commands.json
+/.homebrew-x86_64
/.gdb_history
/tags
diff --git a/.tool-versions b/.tool-versions
deleted file mode 100644
index 9eb38ed717..0000000000
--- a/.tool-versions
+++ /dev/null
@@ -1 +0,0 @@
-ruby 2.7.2
diff --git a/BUILD b/BUILD
index e69de29bb2..50acbb8fcb 100644
--- a/BUILD
+++ b/BUILD
@@ -0,0 +1 @@
+exports_files([".ruby-version"])
diff --git a/README.md b/README.md
index 9e502972b1..829f36b5e2 100644
--- a/README.md
+++ b/README.md
@@ -94,4 +94,4 @@ using the [Sourcegraph CLI](https://github.com/sourcegraph/src-cli).
## Contributing
-See the [Contributing docs](./scip-ruby-CONTRIBUTING.md).
+See the [Contributing docs](./docs/scip-ruby/CONTRIBUTING.md).
diff --git a/WORKSPACE b/WORKSPACE
index 5b9f7740c4..3b8789276d 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -1,9 +1,11 @@
workspace(name = "com_stripe_ruby_typer")
-load("//third_party:externals.bzl", "register_sorbet_dependencies")
+load("//third_party:externals.bzl", "register_scip_ruby_dependencies", "register_sorbet_dependencies")
register_sorbet_dependencies()
+register_scip_ruby_dependencies()
+
load("@com_grail_bazel_compdb//:deps.bzl", "bazel_compdb_deps")
bazel_compdb_deps()
diff --git a/scip-ruby-CONTRIBUTING.md b/docs/scip-ruby/CONTRIBUTING.md
similarity index 62%
rename from scip-ruby-CONTRIBUTING.md
rename to docs/scip-ruby/CONTRIBUTING.md
index 3adc381ad2..2835cb2da7 100644
--- a/scip-ruby-CONTRIBUTING.md
+++ b/docs/scip-ruby/CONTRIBUTING.md
@@ -1,20 +1,99 @@
# Working on scip-ruby
+This document covers the day-to-day aspects of `scip-ruby`.
+For questions on why certain things are the way they are,
+see the [Design Decisions doc][] doc.
+
+[Design Decisions doc]: DESIGN.md
+
+- [Install dependencies](#install-dependencies)
+- [Configuring Ruby (optional)](#configuring-ruby-optional)
- [Building](#building)
- [IDE integration](#ide-integration)
- [Writing a new snapshot test](#writing-a-new-snapshot-test)
- [Writing a new repo test](#writing-a-new-snapshot-test)
- [Debugging with print statements](#debugging-with-print-statements)
- [Debugging with LLDB](#debugging-with-lldb)
+- [Debugging build issues](#debugging-build-issues)
+ - [Debugging Bazel](#debugging-bazel)
+ - [Debugging on Linux](#debugging-on-linux)
- [Creating PRs](#creating-prs)
- [Cutting a release](#cutting-a-release)
- Troubleshooting
- [Known build issues][]
- - [Known Ruby installation issues][]
- [Known RubyGems related issues](#known-rubygems-related-issues)
+ - [Known testing issues](#known-testing-issues)
[Known build issues]: #known-build-issues
-[Known Ruby installation issues]: #known-ruby-installation-issues
+
+## Install dependencies
+
+1. C++ toolchain (gcc or clang): Used for bootstrapping.
+2. [rbenv][]: (optional) This is used for [Configuring Ruby](#configuring-ruby-optional).
+
+[rbenv]: https://github.com/rbenv/rbenv#installation
+
+## Configuring Ruby (optional)
+
+If you're going to be running the repository tests or building the
+scip-ruby gem locally, follow these steps before building stuff.
+
+### macOS pre-requisites
+
+arm64 macOS builds aren't supported yet,
+so it is simpler to have a consistent set of tools
+where Homebrew is running under Rosetta too.
+
+```
+mkdir ~/.homebrew-x86_64
+curl -L https://github.com/Homebrew/brew/tarball/master | tar xz --strip 1 -C ~/.homebrew-x86_64
+./xbrew --version # check that it works
+```
+
+**WARNING:** Do not put this `brew` on your PATH!
+That may lead to confusing errors when working on other projects.
+
+### rbenv setup
+
+#### Linux
+
+Install [rbenv][] as per the official instructions. For `ruby-build`,
+do NOT install it as a plugin.
+
+
+Why not install `ruby-build` as a plugin?
+
+The `RBENV_ROOT` variable does double-duty;
+it serves as the location to install the tools, and it is also used to
+locate plugins (like `ruby-build` -- if you install it as a plugin).
+However, in our case, since we are using a different tool path,
+`rbenv` will be unable to use the `install` plugin.
+
+To mimic the configuration on macOS, I recommend
+installing `ruby-build` next to the `rbenv` binary.
+
+
+```
+git clone https://github.com/rbenv/ruby-build.git --depth=1
+# Assuming you installed rbenv through git as is recommended in the README.
+PREFIX="$HOME/.rbenv" ./ruby-build/install.sh
+rm -rf ruby-build
+```
+
+```bash
+echo "build --define SCIP_RUBY_CACHE_RUBY_DIR='$PWD/.cache_ruby' --define SCIP_RUBY_RBENV_EXE='$(which rbenv)'" >> .bazelrc.local
+```
+
+#### macOS
+
+```bash
+./xbrew install rbenv
+echo "build --define SCIP_RUBY_CACHE_RUBY_DIR='$PWD/.cache_ruby' --define SCIP_RUBY_RBENV_EXE='$(./xbrew where rbenv)'" >> .bazelrc.local
+```
+
+If you're wondering why we have this separate kind of caching,
+instead of having everything happen through the Magic of Bazel (TM),
+see the [Design Decisions doc][].
## Building
@@ -92,7 +171,7 @@ First, clone the repo using Sorbet locally
and check if you can index it.
Typically, the commands will be something like:
-```
+```bash
BUNDLE_WITH=sorbet bundle install
# Replace srb binary with scip-ruby binary
@@ -101,7 +180,7 @@ bundle exec srb --index-file index.scip --gem-metadata "name@version"
```
In case there are any type errors, create a patch and save it:
-```
+```bash
git diff > /path/to/test/scip/repos/name-version.patch
```
@@ -150,6 +229,51 @@ popd
unset TEST_DIR
```
+## Debugging build issues
+
+### Debugging Bazel
+
+See Keith Smiley's blog post [Debugging bazel actions](https://www.smileykeith.com/2022/03/02/debugging-bazel-actions/). ([archive link](https://web.archive.org/web/20220711000725/https://www.smileykeith.com/2022/03/02/debugging-bazel-actions/))
+
+One KEY thing to keep in mind is that some problems only manifest
+inside the sandbox but not outside because Bazel sandbox changes
+the binaries available at certain paths to use non-system tools.
+
+For example, `uname -m` on an M1 Mac would normally return `arm64`
+but inside an x86_64 Bazel environment (currently the default)
+it will return `x86_64`.
+
+For compiler issues inside a sandbox,
+it helpful to test a small code snippet first.
+
+```bash
+{
+ echo '--- Try compiling some simple stuff ---'
+ {
+ echo '#include '
+ echo ''
+ echo 'int main() {'
+ echo ' printf("Hello %s\n", "World!");'
+ echo ' return 0;'
+ echo '}'
+ } > tmp.c
+ gcc tmp.c
+ ./a.out
+ rm tmp.c ./a.out
+ echo '--------------------------------------'
+} >&2
+```
+
+### Debugging on Linux
+
+Debugging a build issue in GitHub Actions can get emotionally draining quickly.
+
+If you work at Sourcegraph, set up a GCP VM.
+You may find the companion [VM setup script](./vm_setup.sh) helpful.
+It is not tested in CI, so you may need to tweak it a bit.
+
+Don't forget to delete the instance after you're done investigating!
+
## Creating PRs
PRs created through the GitHub UI default to being made
@@ -200,15 +324,6 @@ See the [release workflow](/.github/workflows/release.yml) for details.
^~~~~~~~~~~~~~~~~~~~~~~
```
-### Known Ruby installation issues
-
-1. On macOS, Ruby 2.7.x may fail to install due to a combination of `-Werror`
- and a warning in OpenSSL. As a workaround, you can use the following asdf invocation:
- ```
- OPENSSL_CFLAGS=-Wno-error=implicit-function-declaration asdf install ruby 2.7.2
- ```
- This works with 2.7.2, but not with 2.7.0.
-
### Known RubyGems related issues
1. If you're trying to run `bundle install` for a gem
@@ -227,3 +342,13 @@ See the [release workflow](/.github/workflows/release.yml) for details.
Network Preferences > Advanced > TCP/IP >
Configure IPv6: set to Link-local only.
You need to Apply the settings after changing that field.
+
+### Known testing issues
+
+I can't quite figure out why, but with some of the custom rules,
+the executable script being modified doesn't trigger a rebuild.
+If you're seeing stale results, try something like:
+
+```bash
+rm -rf .cache_ruby/ && (cd bazel-bin && find . -name '*.tgz' -type f -delete)
+```
diff --git a/docs/scip-ruby/DESIGN.md b/docs/scip-ruby/DESIGN.md
new file mode 100644
index 0000000000..0a6473c817
--- /dev/null
+++ b/docs/scip-ruby/DESIGN.md
@@ -0,0 +1,77 @@
+# Design Decisions
+
+## Repo tests
+
+
+
+The repo tests have a couple of unusual things:
+#. Tests are broken up into 3 stages,
+ instead of being a simple script that does clone
+ + apply patch + invoke dev tool (`scip-ruby`).
+#. There is "manual" caching going on for the intermediate results
+ of these steps. The cache cannot be used in parallel; repo tests run serially.
+
+I'll cover each of these one-by-one.
+
+Some constraints to keep in mind while reading the sections below:
+#. We want to run tests using as close to "real-world" tools (such as Bundler)
+ as possible instead of trying to mimic what they do.
+#. We want to cache intermediate results as much as possible,
+ as many steps in Ruby installation + Gem installation are slow.
+#. Tests should hopefully run reasonably quickly, especially if you only
+ modify the indexer (most common situation).
+#. Tests should be isolated from each other. We don't one test to "install"
+ `scip-ruby` and have that be visible to another test.
+
+Before going into details about the caching, first let's cover
+an outline of the 3 stages.
+### 3 stages of repo tests
+
+#. Create a standalone ruby installation: This is cached
+ and the installation is blown away.
+#. Test prep: This step involves installing the dependencies of the gem,
+ including any custom bundler version it may require.
+ The pristine toolchain from the first step is extracted
+ and we run `bundle cache`.
+ After this is done, the _entire_ Ruby toolchain is cached again
+ (because it will be have been modified by `bundle cache`).
+ The modified source tree is also cached (I tried `--frozen` earlier but
+ ran into some error with it that I didn't have time to look into).
+#. Index: This step reuses the two caches from the previous step,
+ applies a patch, copies the `scip-ruby` gem into the `vendor/cache`
+ directory, runs `bundle install --local` (so nothing should be
+ fetched from the internet). Now, everything is in a state which mostly
+ mimics what a user would have. Then we invoke `scip-ruby` normally.
+
+Intermediate results use tarballs for caching directory trees instead of copying
+directories because it seemed much simpler to deal with a tarball in Bazel.
+
+Why do we go through all this trouble to cache things "manually"?
+### Manual caching
+
+Here are some obstacles the caching system needs to overcome.
+
+* Ruby installation is [path-dependent](https://github.com/ruby/setup-ruby#using-self-hosted-runners).
+* Ruby installation is relatively slow as it builds everything from source:
+ it takes a few minutes even on a Macbook Pro.
+ Installing it per-test would be too time-consuming.
+
+These two factors together imply that we should cache a Ruby installation
+inside a stable directory, not a temporary test directory.
+
+Additionally, we only want one Ruby toolchain cache; to ensure test isolation,
+we run them in serial on the same directory (a GIL for the test suite).
+
+* We want to ensure reproducibility, and not mess with the global environment.
+
+To achieve this, we delete the toolchain directory at the start of a test,
+and restore it from the cache in the 'Test prep' step.
+
+### Bonus: Bundler weirdness
+
+If you run `bundle install --local` for a test gem X,
+after copying an updated `scip-ruby` gem (same version, new checksum)
+into X's `vendor/cache` directory,
+you would naively expect `bundle` to re-install the updated `scip-ruby` gem.
+(install = copy/setup stuff in toolchain-adjacent directories)
+But `bundle` doesn't actually do that. 🙈
\ No newline at end of file
diff --git a/docs/scip-ruby/vm_setup.sh b/docs/scip-ruby/vm_setup.sh
new file mode 100644
index 0000000000..83894fadee
--- /dev/null
+++ b/docs/scip-ruby/vm_setup.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# Approximate setup script for a Google Cloud VM.
+#
+# I recommend using a 8 core or 16 core CPU-optimized instance
+# with Ubuntu 20.04 (there is a build issue with 22.04 for
+# inscrutable reasons). 25 GB disk is more than sufficient
+
+sudo apt update
+sudo apt autoremove
+sudo apt upgrade
+sudo apt install wget unzip ripgrep fd-find gcc libncurses5-dev libncursesw5-dev xxd libtinfo5 make zlib1g-dev autoconf bison build-essential libssl-dev libyaml-dev libreadline6-dev libffi-dev libgdbm6 libgdbm-dev libdb-dev uuid-dev
+
+rm -rf ~/.rbenv
+git clone https://github.com/rbenv/rbenv.git ~/.rbenv --depth=1
+(cd ~/.rbenv && src/configure && make -C src)
+echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bashrc
+echo 'eval "$(rbenv init - bash)"' >> ~/.bashrc
+source ~/.bashrc
+
+git clone https://github.com/rbenv/ruby-build.git --depth=1
+PREFIX="$HOME/.rbenv" ./ruby-build/install.sh
+rm -rf ruby-build
+
+curl -fsSL https://github.com/rbenv/rbenv-installer/raw/main/bin/rbenv-doctor | bash
+
+git clone https://github.com/sourcegraph/scip-ruby.git
+cd scip-ruby
+
+export JOB_NAME=test
+export CACHE_DIR="$HOME/bae"
+mkdir -p "$CACHE_DIR/output-bases/${JOB_NAME}" "$CACHE_DIR/build" "$CACHE_DIR/repos"
+{
+ echo 'common --curses=no --color=yes'
+ echo "startup --output_base=$CACHE_DIR/output-bases/${JOB_NAME}"
+ echo "build --disk_cache=$CACHE_DIR/build --repository_cache=$CACHE_DIR/repos"
+ echo "test --disk_cache=$CACHE_DIR/build --repository_cache=$CACHE_DIR/repos"
+ echo "build --define SCIP_RUBY_CACHE_RUBY_DIR='$PWD/.cache_ruby' --define SCIP_RUBY_RBENV_EXE='$(which rbenv)'"
+} > .bazelrc.local
diff --git a/gems/scip-ruby/BUILD b/gems/scip-ruby/BUILD
index 44bec2149a..af2b12c708 100644
--- a/gems/scip-ruby/BUILD
+++ b/gems/scip-ruby/BUILD
@@ -1,10 +1,15 @@
-load(":build_gems.bzl", "build_gems", "version")
+load(":build_gems.bzl", "build_gems", "setup_standalone_ruby", "version")
version(
name = "version",
build_setting_default = "0.0.0",
)
+setup_standalone_ruby(
+ name = "standalone-ruby",
+ visibility = ["//visibility:public"],
+)
+
build_gems(
name = "scip-ruby",
srcs = [
@@ -19,5 +24,5 @@ build_gems(
"//tools/config:linux": "linux",
"//tools/config:darwin": "darwin",
}),
- scip_ruby_target = "//main:scip-ruby",
+ visibility = ["//visibility:public"],
)
diff --git a/gems/scip-ruby/bin/scip-ruby b/gems/scip-ruby/bin/scip-ruby
index a6a7f55e96..e05c9566a9 100755
--- a/gems/scip-ruby/bin/scip-ruby
+++ b/gems/scip-ruby/bin/scip-ruby
@@ -20,15 +20,24 @@ if [ -n "${DEBUG:-}" ]; then
set -x
fi
-if [ -z "${BUNDLE_GEMFILE:-}" ]; then
- echo "Expected to be invoke through bundle exec scip-ruby"
+die() {
+ echo "Expected to be invoked as 'bundle exec scip-ruby'"
echo "Did you invoke this binary directly?"
exit 1
+}
+
+if [ -z "${BUNDLE_GEMFILE:-}" ]; then
+ echo "Missing BUNDLE_GEMFILE environment variable"
+ die
+elif [ -z "${BUNDLE_BIN_PATH:-}" ]; then
+ echo "Missing BUNDLE_BIN_PATH environment variable"
+ die
fi
-if grep -q -e "scip-ruby-debug" "$BUNDLE_GEMFILE"; then
+GEMFILE_CONTENTS="$(< "$BUNDLE_GEMFILE")"
+if [ "$GEMFILE_CONTENTS" == *"scip-ruby-debug"* ]; then
SCIP_RUBY="scip-ruby-debug"
-elif grep -q -e "scip-ruby" "$BUNDLE_GEMFILE"; then
+elif [ "$GEMFILE_CONTENTS" == *"scip-ruby"* ]; then
SCIP_RUBY="scip-ruby"
elif grep -q -e "scip-ruby-debug" "$(dirname "$BUNDLE_GEMFILE")"/*.gemspec; then
SCIP_RUBY="scip-ruby-debug"
@@ -36,7 +45,26 @@ else
SCIP_RUBY="scip-ruby"
fi
-GEM_INSTALL_PATH="$(bundle info "$SCIP_RUBY" | grep 'Path: ' | sed -E 's/^\s+Path: //')"
+# Calling the bundle in BUNDLE_BIN_PATH tries to mess with system directories.
+if [[ "$BUNDLE_BIN_PATH" =~ (.*)/lib/ruby/gems/.* ]]; then
+ BUNDLE_BIN_PATH="${BASH_REMATCH[1]}/bin/bundle"
+ file "$BUNDLE_BIN_PATH" > /dev/null || (echo "scip-ruby: warning: bundle not found at $BUNDLE_BIN_PATH, this may lead to errors")
+else
+ echo "scip-ruby: warning: unexpected BUNDLE_BIN_PATH = $BUNDLE_BIN_PATH"
+ echo "scip-ruby: warning: this may lead to further errors"
+fi
+
+# Careful about trying to "optimize" the regex here.
+# I tried to use -E but that doesn't match with BSD sed.
+GEM_INSTALL_PATH="$("$BUNDLE_BIN_PATH" info "$SCIP_RUBY" | grep 'Path:' | sed -e 's/\s*Path://')"
+# From https://stackoverflow.com/a/3352015/2682729
+trim() {
+ local var="$*"
+ var="${var#"${var%%[![:space:]]*}"}"
+ var="${var%"${var##*[![:space:]]}"}"
+ printf '%s' "$var"
+}
+GEM_INSTALL_PATH="$(trim "$GEM_INSTALL_PATH")"
chmod +x "$GEM_INSTALL_PATH/native/scip-ruby"
exec "$GEM_INSTALL_PATH/native/scip-ruby" "$@"
diff --git a/gems/scip-ruby/build.sh b/gems/scip-ruby/build.sh
deleted file mode 100755
index 27eb74dc52..0000000000
--- a/gems/scip-ruby/build.sh
+++ /dev/null
@@ -1,58 +0,0 @@
-#!/usr/bin/env bash
-
-# Based on build-static-release.sh
-
-set -eu
-
-exit_fail() {
- echo 'This is likely a bug in the bazel code running the build'
- exit 1
-}
-
-if [ -z "${VERSION:-}" ]; then
- echo 'Missing value for VERSION environment variable'
- exit_fail
-elif [ -z "${NAME:-}" ]; then
- echo 'Missing value for NAME environment variable'
- exit_fail
-elif [ -z "${SCIP_RUBY_BINARY:-}" ]; then
- echo 'Missing value for SCIP_RUBY_BINARY environment variable'
- exit_fail
-elif [ -z "${OUT_DIR:-}" ]; then
- echo 'Missing value for OUT_DIR environment variable'
- exit_fail
-fi
-
-cp -R gems/scip-ruby out
-mkdir -p out/native
-cp "$SCIP_RUBY_BINARY" out/native/scip-ruby
-
-GEMSPEC="$NAME.gemspec"
-
-cleanup() {
- rm -rf out
-}
-trap cleanup EXIT
-
-pushd out
-
-cat scip-ruby.template.gemspec \
- | sed -e "s/VERSION_PLACEHOLDER/$VERSION/" -e "s/NAME_PLACEHOLDER/$NAME/" \
- > "$GEMSPEC"
-
-if [ "$(uname -s)" == "Darwin" ]; then
- # Darwin 20 ~ macOS 11 (Big Sur) was released in mid-2020.
- # We can publish older releases if someone asks for them.
- DARWIN_VERSIONS=($DARWIN_VERSIONS)
- for i in "${DARWIN_VERSIONS[@]}"; do
- sed -i.bak "s/Gem::Platform::CURRENT/'universal-darwin-$i'/" "$GEMSPEC"
- gem build "$GEMSPEC"
- mv "$GEMSPEC.bak" "$GEMSPEC"
- done
-else
- gem build "$GEMSPEC"
-fi
-
-popd
-
-mv out/*.gem "$OUT_DIR/"
diff --git a/gems/scip-ruby/build_gems.bzl b/gems/scip-ruby/build_gems.bzl
index 3b2e5b9b4f..44e9204a7e 100644
--- a/gems/scip-ruby/build_gems.bzl
+++ b/gems/scip-ruby/build_gems.bzl
@@ -1,3 +1,37 @@
+def _setup_standalone_ruby(ctx):
+ ruby_archive = ctx.actions.declare_file("cache/ruby.tgz")
+ inputs = [ctx.file._ruby_version]
+ outputs = [ruby_archive]
+ ctx.actions.run(
+ outputs = outputs,
+ inputs = inputs,
+ mnemonic = "StandaloneRuby",
+ executable = ctx.file._install_script,
+ env = {
+ # Ideally, we would also pass in a C compiler here, but 🤷🏽
+ "SCIP_RUBY_RBENV_EXE": ctx.var["SCIP_RUBY_RBENV_EXE"],
+ "SCIP_RUBY_CACHE_RUBY_DIR": ctx.var["SCIP_RUBY_CACHE_RUBY_DIR"],
+ "RUBY_VERSION_FILE": ctx.file._ruby_version.path,
+ "OUT_TGZ_PATH": ruby_archive.path,
+ },
+ )
+ runfiles = ctx.runfiles(files = outputs)
+ return [DefaultInfo(files = depset(outputs), runfiles = runfiles)]
+
+setup_standalone_ruby = rule(
+ implementation = _setup_standalone_ruby,
+ attrs = {
+ "_ruby_version": attr.label(default = "//:.ruby-version", allow_single_file = True),
+ "_install_script": attr.label(default = "install_standalone_ruby.sh", allow_single_file = True),
+ },
+ doc = """
+ Creates a standalone ruby installation using rbenv that is only for test use,
+ without interfering with any system Ruby. We are not using bazelruby/rules_ruby here
+ because it seems largely oriented towards *building* Ruby code, whereas what we
+ want to do is install dependencies in a way that mimics common usage (through Bundler).
+ """,
+)
+
VersionProvider = provider(fields = ["version"])
def _version(ctx):
@@ -24,31 +58,49 @@ def _build_gems(ctx):
output_files = [ctx.actions.declare_file(out) for out in outs]
inputs = ctx.attr._build_script.files.to_list()
- for src in (ctx.attr.srcs + [ctx.attr.scip_ruby_target]):
+ for src in ctx.attr.srcs:
inputs += src.files.to_list()
+ inputs.append(ctx.file._scip_ruby_binary)
+ inputs.append(ctx.file._ruby_version)
+
+ # Upside of depending on our Ruby installation:
+ # - Reduce risk of funky issues caused due to having two Ruby toolchains.
+ # Downside of depending on our Ruby installation:
+ # - The build is more serialized:
+ # setup standalone ruby -> build gems -> run tests
+ # Instead of build gems running in parallel with the ruby setup
+ # (due to an environmental 'gem' command)
+ # We could potentially revisit this...
+ inputs.append(ctx.file._standalone_ruby_tgz)
ctx.actions.run(
outputs = output_files,
inputs = inputs,
- mnemonic = "BuildGems",
- executable = ctx.attr._build_script.files.to_list()[0],
+ mnemonic = "BuildSCIPRubyGems",
+ executable = ctx.file._build_script,
env = {
+ "SCIP_RUBY_CACHE_RUBY_DIR": ctx.var["SCIP_RUBY_CACHE_RUBY_DIR"],
+ "RUBY_VERSION_FILE": ctx.file._ruby_version.path,
+ "PRISTINE_TOOLCHAIN_TGZ_PATH": ctx.file._standalone_ruby_tgz.path,
"NAME": name,
"DARWIN_VERSIONS": " ".join([str(dv) for dv in darwin_versions]),
"VERSION": version,
- "SCIP_RUBY_BINARY": ctx.attr.scip_ruby_target.files.to_list()[0].path,
+ "SCIP_RUBY_BINARY": ctx.file._scip_ruby_binary.path,
"OUT_DIR": output_files[0].dirname,
},
)
- return [DefaultInfo(files = depset(output_files))]
+ runfiles = ctx.runfiles(files = output_files)
+ return [DefaultInfo(files = depset(output_files), runfiles = runfiles)]
build_gems = rule(
implementation = _build_gems,
attrs = {
"_version": attr.label(default = ":version"),
- "_build_script": attr.label(default = "build.sh", allow_files = True),
+ "_build_script": attr.label(default = "build_scip_ruby_gems.sh", allow_single_file = True),
"srcs": attr.label_list(allow_files = True),
- "scip_ruby_target": attr.label(),
+ "_scip_ruby_binary": attr.label(default = "//main:scip-ruby", allow_single_file = True),
+ "_ruby_version": attr.label(default = "//:.ruby-version", allow_single_file = True),
+ "_standalone_ruby_tgz": attr.label(default = "//gems/scip-ruby:standalone-ruby", allow_single_file = True),
"gem_name": attr.string(),
"gem_target_os": attr.string(),
},
diff --git a/gems/scip-ruby/build_scip_ruby_gems.sh b/gems/scip-ruby/build_scip_ruby_gems.sh
new file mode 100755
index 0000000000..5aab1d3246
--- /dev/null
+++ b/gems/scip-ruby/build_scip_ruby_gems.sh
@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+
+# See also: NOTE[repo-test-structure]
+
+set -eu
+
+ENV_VARS=("PRISTINE_TOOLCHAIN_TGZ_PATH" "SCIP_RUBY_CACHE_RUBY_DIR" "RUBY_VERSION_FILE" "VERSION" "NAME" "SCIP_RUBY_BINARY" "OUT_DIR")
+for ENV_VAR in "${ENV_VARS[@]}"; do
+ if eval "[ -z \"$(printf '${%s:-}' $ENV_VAR)\" ]"; then
+ echo "Missing definition for $ENV_VAR environment variable"
+ echo 'This is likely a bug in the bazel code running the build'
+ exit 1
+ fi
+done
+
+cp -R gems/scip-ruby out
+mkdir -p out/native
+cp "$SCIP_RUBY_BINARY" out/native/scip-ruby
+
+GEMSPEC="$NAME.gemspec"
+
+cleanup() {
+ rm -rf out
+}
+trap cleanup EXIT
+
+rm -rf "$SCIP_RUBY_CACHE_RUBY_DIR"
+mkdir -p "$SCIP_RUBY_CACHE_RUBY_DIR"
+tar -xzf "$PRISTINE_TOOLCHAIN_TGZ_PATH" -C "$SCIP_RUBY_CACHE_RUBY_DIR"
+
+SCIP_RUBY_SPECIFIC_RUBY_ROOT="$SCIP_RUBY_CACHE_RUBY_DIR"
+if [ -d "$SCIP_RUBY_SPECIFIC_RUBY_ROOT/versions" ]; then
+ # rbenv creates an extra versions subdirectory, which doesn't apply in CI.
+ # This is to avoid finding the 'gem' from shims/
+ SCIP_RUBY_SPECIFIC_RUBY_ROOT="$SCIP_RUBY_SPECIFIC_RUBY_ROOT/versions"
+fi
+SCIP_RUBY_SPECIFIC_RUBY_ROOT="$SCIP_RUBY_SPECIFIC_RUBY_ROOT/$(< "$RUBY_VERSION_FILE")"
+
+GEM_EXE="$(find "$SCIP_RUBY_SPECIFIC_RUBY_ROOT" -name 'gem' -type f)"
+file "$GEM_EXE"
+
+pushd out
+
+cat scip-ruby.template.gemspec \
+ | sed -e "s/VERSION_PLACEHOLDER/$VERSION/" -e "s/NAME_PLACEHOLDER/$NAME/" \
+ > "$GEMSPEC"
+
+if [ "$(uname -s)" == "Darwin" ]; then
+ # Darwin 20 ~ macOS 11 (Big Sur) was released in mid-2020.
+ # We can publish older releases if someone asks for them.
+ DARWIN_VERSIONS=($DARWIN_VERSIONS)
+ for i in "${DARWIN_VERSIONS[@]}"; do
+ sed -i.bak "s/Gem::Platform::CURRENT/'universal-darwin-$i'/" "$GEMSPEC"
+ "$GEM_EXE" build "$GEMSPEC"
+ mv "$GEMSPEC.bak" "$GEMSPEC"
+ done
+else
+ "$GEM_EXE" build "$GEMSPEC"
+fi
+
+popd
+
+mv out/*.gem "$OUT_DIR/"
diff --git a/gems/scip-ruby/install_standalone_ruby.sh b/gems/scip-ruby/install_standalone_ruby.sh
new file mode 100755
index 0000000000..0c23cfec85
--- /dev/null
+++ b/gems/scip-ruby/install_standalone_ruby.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Remove after https://github.com/rbenv/ruby-build/issues/2024 is fixed.
+NUM_CPUS="$(getconf _NPROCESSORS_ONLN)"
+ENV_VARS=("SCIP_RUBY_CACHE_RUBY_DIR" "SCIP_RUBY_RBENV_EXE" "RUBY_VERSION_FILE" "OUT_TGZ_PATH" "NUM_CPUS")
+for ENV_VAR in "${ENV_VARS[@]}"; do
+ if eval "[ -z \"$(printf '${%s:-}' $ENV_VAR)\" ]"; then
+ echo "Missing definition for $ENV_VAR environment variable"
+ exit 1
+ fi
+done
+export MAKE_OPTS="-j ${NUM_CPUS:-4}"
+
+mkdir -p "$SCIP_RUBY_CACHE_RUBY_DIR"
+
+# Debugging tip: If there is a failure in this step, replace it with:
+#
+# CC="$SCIP_RUBY_CACHE_RUBY_DIR/cc_wrapper" "$SCIP_RUBY_RBENV_EXE" install --keep --force
+#
+# In cc_wrapper, add the following:
+#
+# #!/usr/bin/env bash
+# echo "cc $*" >> "$(dirname "${BASH_SOURCE[0]}")/cc.log"
+# exec cc "$@"
+#
+# This will persist the build directory and record invocations which
+# can be replayed for faster triage.
+
+if [[ "$SCIP_RUBY_RBENV_EXE" != "RUNNING_IN_CI_RBENV_NOT_NEEDED" ]]; then
+ # Make ruby-build available to rbenv.
+ export PATH="$(dirname "$SCIP_RUBY_RBENV_EXE"):$PATH"
+ RBENV_ROOT="$SCIP_RUBY_CACHE_RUBY_DIR" "$SCIP_RUBY_RBENV_EXE" install --skip-existing
+fi
+
+mkdir -p "$(dirname "$OUT_TGZ_PATH")"
+rm -f "$OUT_TGZ_PATH"
+tar -czf "$OUT_TGZ_PATH" -C "$SCIP_RUBY_CACHE_RUBY_DIR" .
diff --git a/test/scip/repos/BUILD b/test/scip/repos/BUILD
index 4ac53416f3..adcc4792d9 100644
--- a/test/scip/repos/BUILD
+++ b/test/scip/repos/BUILD
@@ -1,3 +1,8 @@
-load(":scip_repos_test.bzl", "scip_repos_test_suite")
+load(":scip_repos_test.bzl", "register_scip_repo_test_caches", "scip_repos_test_suite")
-scip_repos_test_suite(patch_paths = glob(["*.patch"]))
+register_scip_repo_test_caches()
+
+scip_repos_test_suite(
+ name = "repos",
+ patch_paths = glob(["*.patch"]),
+)
diff --git a/test/scip/repos/brew-3.5.7.patch b/test/scip/repos/brew-3.5.9.patch
similarity index 100%
rename from test/scip/repos/brew-3.5.7.patch
rename to test/scip/repos/brew-3.5.9.patch
diff --git a/test/scip/repos/bundle_cache.sh b/test/scip/repos/bundle_cache.sh
new file mode 100755
index 0000000000..2fcf60ce57
--- /dev/null
+++ b/test/scip/repos/bundle_cache.sh
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+ENV_VARS=("SCIP_RUBY_CACHE_RUBY_DIR" "RUBY_VERSION_FILE" "PRISTINE_TOOLCHAIN_TGZ_PATH" "TEST_GEM_ZIP" "TEST_GEM_ZIP_PREFIX" "TEST_GEM_SUBDIR" "OUT_MODIFIED_TOOLCHAIN_TGZ_PATH" "OUT_GEM_WITH_VENDOR_TGZ_PATH")
+for ENV_VAR in "${ENV_VARS[@]}"; do
+ if eval "[ -z \"$(printf '${%s:-}' $ENV_VAR)\" ]"; then
+ echo "Missing definition for $ENV_VAR environment variable"
+ exit 1
+ fi
+done
+
+SCIP_RUBY_SPECIFIC_RUBY_ROOT="$SCIP_RUBY_CACHE_RUBY_DIR"
+if [ -d "$SCIP_RUBY_SPECIFIC_RUBY_ROOT/versions" ]; then
+ # rbenv creates an extra versions subdirectory, which doesn't apply in CI.
+ # This is to avoid finding the 'gem' from shims/
+ SCIP_RUBY_SPECIFIC_RUBY_ROOT="$SCIP_RUBY_SPECIFIC_RUBY_ROOT/versions"
+fi
+SCIP_RUBY_SPECIFIC_RUBY_ROOT="$SCIP_RUBY_SPECIFIC_RUBY_ROOT/$(< "$RUBY_VERSION_FILE")"
+
+rm -rf "$SCIP_RUBY_CACHE_RUBY_DIR"
+mkdir -p "$SCIP_RUBY_CACHE_RUBY_DIR"
+tar -xzf "$PRISTINE_TOOLCHAIN_TGZ_PATH" -C "$SCIP_RUBY_CACHE_RUBY_DIR"
+
+GEM_EXE="$(find "$SCIP_RUBY_SPECIFIC_RUBY_ROOT" -name 'gem' -type f)"
+BUNDLE_EXE="$(dirname "$GEM_EXE")/bundle"
+
+rm -rf "$TEST_GEM_ZIP_PREFIX"
+unzip -q "$TEST_GEM_ZIP"
+
+# Is there a more reliable way to get the version? 😬
+_BUNDLER_VERSION="$(tail -n 1 < $TEST_GEM_ZIP_PREFIX/$TEST_GEM_SUBDIR/Gemfile.lock)"
+_BUNDLER_VERSION="${_BUNDLER_VERSION// /}"
+echo "BUNDLER_VERSION = $_BUNDLER_VERSION"
+"$GEM_EXE" install "bundler:$_BUNDLER_VERSION"
+
+pushd "$TEST_GEM_ZIP_PREFIX/$TEST_GEM_SUBDIR"
+git init -q
+git add .
+GIT_AUTHOR_DATE=1993-05-16T15:04:05Z GIT_COMMITTER_DATE=1993-05-16T15:04:05Z \
+ git -c user.name='iu' -c user.email='i@u' commit -a -m '(^_^)' --quiet
+set +e
+
+# For reasons that are unclear to me, the 'env -' usage works around
+# a gcc problem which manifests as 'cc1' not being executable when invoked
+# as 'gcc blah' and 'ld' not being found when invoked as '/usr/bin/gcc'.
+#
+# This seems specific to some Sorbet configuration; I have not been
+# able to reproduce this weirdness with a minimal configuration outside.
+if ! env - PATH="$PATH" PWD="$PWD" "$BUNDLE_EXE" cache --quiet 2> >(tee stderr.log >&2); then
+ err="$(< stderr.log)"
+ while IFS= read -r line; do
+ if [[ "$line" == *"lib/ruby/gems"*"mkmf.log" ]]; then
+ echo "----------- Printing likely bad mkmf.log -----------"
+ cat "$line"
+ echo "----------------------------------------------------"
+ fi
+ done <<< "$err"
+ exit 1
+fi
+set -e
+git restore .
+popd
+
+tar -czf "$OUT_MODIFIED_TOOLCHAIN_TGZ_PATH" -C "$SCIP_RUBY_CACHE_RUBY_DIR" .
+tar -czf "$OUT_GEM_WITH_VENDOR_TGZ_PATH" -C "$TEST_GEM_ZIP_PREFIX" .
diff --git a/test/scip/repos/index_gem.sh b/test/scip/repos/index_gem.sh
new file mode 100755
index 0000000000..8ddddd7c89
--- /dev/null
+++ b/test/scip/repos/index_gem.sh
@@ -0,0 +1,122 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# See NOTE[repo-test-structure] for more context.
+
+TEST_DIR="$PWD"
+ENV_VARS=("SCIP_RUBY_CACHE_RUBY_DIR" "RUBY_VERSION_FILE" "SCIP_RUBY_GEMS" "CACHED_TGZS" "REPO_NAME" "PATCH_PATH") # "PREP_CMD" "RUN_CMD" )
+for ENV_VAR in "${ENV_VARS[@]}"; do
+ if eval "[ -z \"$(printf '${%s:-}' $ENV_VAR)\" ]"; then
+ echo "Missing definition for $ENV_VAR environment variable"
+ exit 1
+ fi
+done
+TEST_NAME="$REPO_NAME"
+
+# TODO: Librarify this, the env var check, and the GITHUB_ACTIONS var and use it consistently.
+get_date() {
+ date '+%s'
+}
+PREV_TIME="$(get_date)"
+PREV_CMD=""
+print_time_elapsed() {
+ CUR_TIME="$(get_date)"
+ local DELTA=$(( CUR_TIME - PREV_TIME ))
+ if [ "$DELTA" -ne 0 ]; then
+ echo "---------------------------"
+ echo "Ran in ${DELTA}s: $PREV_CMD"
+ echo "---------------------------"
+ fi
+ PREV_CMD="$BASH_COMMAND"
+ PREV_TIME="$CUR_TIME"
+}
+trap print_time_elapsed DEBUG
+
+if [ -n "${GITHUB_ACTIONS:-}" ]; then
+ echo "::group::$TEST_NAME"
+ trap "echo ::end-group::$TEST_NAME" EXIT
+fi
+
+CACHED_TGZS_ARRAY=($CACHED_TGZS)
+TGZ_CACHE_DIR="$(dirname ${CACHED_TGZS_ARRAY[0]})"
+
+rm -rf "$SCIP_RUBY_CACHE_RUBY_DIR"
+mkdir -p "$SCIP_RUBY_CACHE_RUBY_DIR"
+tar -xzf "$TGZ_CACHE_DIR/ruby-post-bundle-cache.tgz" -C "$SCIP_RUBY_CACHE_RUBY_DIR"
+
+rm -rf repo
+mkdir repo
+tar -xzf "$TGZ_CACHE_DIR/gem-post-vendor.tgz" -C repo
+
+SCIP_RUBY_GEMS_ARRAY=($SCIP_RUBY_GEMS)
+for SCIP_RUBY_GEM in "${SCIP_RUBY_GEMS_ARRAY[@]}"; do
+ # NOTE: Keep in sync with scip_repos_test.bzl
+ if [[ "$SCIP_RUBY_GEM" != *"darwin"* || "$SCIP_RUBY_GEM" == *"20"* ]]; then
+ cp "$SCIP_RUBY_GEM" repo/vendor/cache/
+ fi
+done
+
+SCIP_RUBY_SPECIFIC_RUBY_ROOT="$SCIP_RUBY_CACHE_RUBY_DIR"
+if [ -d "$SCIP_RUBY_SPECIFIC_RUBY_ROOT/versions" ]; then
+ # rbenv creates an extra versions subdirectory, which doesn't apply in CI.
+ # This is to avoid finding the 'gem' from shims/
+ SCIP_RUBY_SPECIFIC_RUBY_ROOT="$SCIP_RUBY_SPECIFIC_RUBY_ROOT/versions"
+fi
+SCIP_RUBY_SPECIFIC_RUBY_ROOT="$SCIP_RUBY_SPECIFIC_RUBY_ROOT/$(< "$RUBY_VERSION_FILE")"
+
+# Little indirect because apparently there may be more bundle binaries inside.
+GEM_EXE="$(find "$SCIP_RUBY_SPECIFIC_RUBY_ROOT" -name 'gem' -type f)"
+BUNDLE_EXE="$(dirname "$GEM_EXE")/bundle"
+
+pushd repo
+git diff --exit-code # No changes should've been made before applying the patch
+git apply "../$PATCH_PATH"
+
+"$BUNDLE_EXE" exec 'echo $PATH'
+"$BUNDLE_EXE" install -j "$(getconf _NPROCESSORS_ONLN)" --local
+
+set +e
+_INFO="$("$BUNDLE_EXE" info scip-ruby-debug)"
+if [ $? != 0 ]; then
+ set -e
+ _INFO="$("$BUNDLE_EXE" info scip-ruby)"
+fi
+set -e
+
+printf 'bundle info output:\n%s\n' "$_INFO"
+
+# Ideally, we'd write something like:
+#
+# "$BUNDLE_EXE" exec scip-ruby --index-file index.scip --gem-metadata "$REPO_NAME@99.99.99" \\
+#
+# However, there is a discrepancy in paths between bundle install/info
+# vs bundle exec (see https://github.com/rubygems/rubygems/issues/5838).
+# Work around that by relying on 'bundle info` as the source of truth.
+GEM_INSTALL_PATH="$(echo "$_INFO" | grep 'Path:' | sed -e 's/\s*Path: //')"
+# From https://stackoverflow.com/a/3352015/2682729
+trim() {
+ local var="$*"
+ var="${var#"${var%%[![:space:]]*}"}"
+ var="${var%"${var##*[![:space:]]}"}"
+ printf '%s' "$var"
+}
+GEM_INSTALL_PATH="$(trim "$GEM_INSTALL_PATH")"
+
+echo "GEM_INSTALL_PATH = $GEM_INSTALL_PATH"
+echo "Calling shim"
+
+{
+echo "Shim @ $GEM_INSTALL_PATH/bin/scip-ruby:"
+echo "-----------------------------------"
+cat "$GEM_INSTALL_PATH/bin/scip-ruby"
+echo "-----------------------------------"
+echo "This is for debugging/reproduction; I was seeing stale shims earlier, not sure why"
+} >&2
+
+
+# Call the shim, not the native binary, to test that the shim works.
+DEBUG=1 "$BUNDLE_EXE" exec "$GEM_INSTALL_PATH/bin/scip-ruby" --index-file index.scip --gem-metadata "$REPO_NAME@99.99.99"
+file index.scip
+
+popd
\ No newline at end of file
diff --git a/test/scip/repos/index_oss_repo.sh b/test/scip/repos/index_oss_repo.sh
deleted file mode 100755
index ad2902665b..0000000000
--- a/test/scip/repos/index_oss_repo.sh
+++ /dev/null
@@ -1,89 +0,0 @@
-#!/usr/bin/env bash
-
-# Some tests populate caches based on $HOME. To avoid polluting
-# the outside env, such as on a dev machine, use the current
-# directory, as this script is invoked inside a temporary
-# test directory by Bazel.
-export HOME="$PWD"
-
-# GEM_PATH should be provided from outside:
-#
-# ./bazel test --test_env GEM_PATH="$(dirname "$(which gem)")"
-#
-# Avoid passing --test_env PATH="$PATH" to reduce risk of
-# dependencies on random stuff in a dev environment.
-if [ -z "$GEM_PATH" ]; then
- echo '$GEM_PATH environment variable was not set.'
- exit 1
-fi
-export PATH="$GEM_PATH:$PATH"
-
-get_date() {
- date '+%s'
-}
-PREV_TIME="$(get_date)"
-PREV_CMD=""
-print_time_elapsed() {
- CUR_TIME="$(get_date)"
- local DELTA=$(( CUR_TIME - PREV_TIME ))
- if [ "$DELTA" -ne 0 ]; then
- echo "Ran in ${DELTA}s: $PREV_CMD\n"
- fi
- PREV_CMD="$BASH_COMMAND"
- PREV_TIME="$CUR_TIME"
-}
-trap print_time_elapsed DEBUG
-
-if [ "$#" -lt 6 ] || [ "$#" -gt 7 ]; then
- echo "Expected 5-6 arguments: got $#"
- echo "1. Test name"
- echo "2. Clone URL"
- echo "3. git tag to use"
- echo "4. git SHA for double-checking"
- echo "5. Prep command to run before applying the patch"
- echo "6. Command to run for type-checking"
- echo "7. (optional) absolute path to patch to apply"
- exit 1
-fi
-
-set -eu
-
-TEST_NAME="$1"
-CLONE_URL="$2"
-GIT_TAG="$3"
-GIT_SHA="$4"
-PREP_CMD="$5"
-RUN_CMD="$6"
-PATCH_ABSPATH=""
-TEST_DIR="$PWD"
-if [ "$#" -eq 7 ]; then
- PATCH_ABSPATH="$TEST_DIR/$7"
-fi
-
-if [ -n "${GITHUB_ACTIONS:-}" ]; then
- echo "::group::$TEST_NAME"
- trap "echo ::end-group::$TEST_NAME" EXIT
-fi
-
-rm -rf repo
-mkdir repo
-pushd repo
-
-git -c advice.detachedHead=false clone "$CLONE_URL" . --branch "$GIT_TAG" --depth=1
-if [ "$(git rev-parse HEAD)" != "$GIT_SHA" ]; then
- echo "Expected SHA: $GIT_SHA"
- echo "Obtained SHA: $(git rev-parse HEAD)"
- exit 1
-fi
-
-eval "$PREP_CMD"
-
-if [ -n "$PATCH_ABSPATH" ]; then
- git apply "$PATCH_ABSPATH"
- git diff -U0
-fi
-
-eval "$RUN_CMD"
-
-popd
-rm -rf repo
diff --git a/test/scip/repos/scip_repos_test.bzl b/test/scip/repos/scip_repos_test.bzl
index ab21fe30db..b1ecea32b6 100644
--- a/test/scip/repos/scip_repos_test.bzl
+++ b/test/scip/repos/scip_repos_test.bzl
@@ -1,54 +1,123 @@
-_data = [
- {
- "name": "brew",
- "clone_url": "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/Homebrew/brew.git",
- "tag": "3.5.7",
- "git_sha": "ce8ef89ec0cfe875c48c3a83a843af9074c05930",
- "prep_cmd": ("pushd Library/Homebrew" +
- " && gem install bundler:1.17.3" +
- " && popd" +
- " && ./bin/brew typecheck"),
- "run_cmd": ("pushd Library/Homebrew" +
- " && find . -name srb -type f -exec cp $${TEST_DIR}/$(location //main:scip-ruby) {} \\;" +
- " && popd" +
- " && ./bin/brew typecheck"),
+load("//third_party:test_gem_data.bzl", "gem_build_info")
+
+# For more context, see NOTE[repo-test-structure]
+
+def _scip_repo_test_cache_rule(ctx):
+ target_zip_list = ctx.attr.target.files.to_list()
+ inputs = target_zip_list + [ctx.file._standalone_ruby_tgz]
+ gem_name = ctx.label.name
+ toolchain_cache = ctx.actions.declare_file("cache/{}/ruby-post-bundle-cache.tgz".format(gem_name))
+ gem_with_vendor_cache = ctx.actions.declare_file("cache/{}/gem-post-vendor.tgz".format(gem_name))
+ outputs = [toolchain_cache, gem_with_vendor_cache]
+ if len(target_zip_list) != 1:
+ fail("Expected exactly 1 zip file in filegroup but found {} for {}".format(
+ len(target_zip_list),
+ ctx.attr.target,
+ ))
+ ctx.actions.run(
+ outputs = outputs,
+ inputs = inputs,
+ mnemonic = "BundleCache",
+ executable = ctx.file._bundle_cache_script,
+ env = {
+ # Ideally, we would pass in a C compiler here too, but 🤷🏽
+ "SCIP_RUBY_CACHE_RUBY_DIR": ctx.var["SCIP_RUBY_CACHE_RUBY_DIR"],
+ "RUBY_VERSION_FILE": ctx.file._ruby_version.path,
+ "PRISTINE_TOOLCHAIN_TGZ_PATH": ctx.file._standalone_ruby_tgz.path,
+ "TEST_GEM_ZIP": target_zip_list[0].path,
+ "TEST_GEM_ZIP_PREFIX": ctx.attr.strip_prefix,
+ "TEST_GEM_SUBDIR": ctx.attr.subdir,
+ "OUT_MODIFIED_TOOLCHAIN_TGZ_PATH": toolchain_cache.path,
+ "OUT_GEM_WITH_VENDOR_TGZ_PATH": gem_with_vendor_cache.path,
+ },
+ )
+ return [DefaultInfo(files = depset(outputs), runfiles = ctx.runfiles(files = outputs))]
+
+scip_repo_test_cache_rule = rule(
+ implementation = _scip_repo_test_cache_rule,
+ attrs = {
+ "_standalone_ruby_tgz": attr.label(default = "//gems/scip-ruby:standalone-ruby", allow_single_file = True),
+ "_bundle_cache_script": attr.label(default = ":bundle_cache.sh", allow_single_file = True),
+ "_ruby_version": attr.label(default = "//:.ruby-version", allow_single_file = True),
+ "target": attr.label(mandatory = True),
+ "strip_prefix": attr.string(mandatory = True),
+ "subdir": attr.string(mandatory = True),
},
-]
+ doc = "Intermediate cache archives for gems after running 'bundle cache'.",
+)
-def scip_repos_test_suite(patch_paths):
- patch_paths = {p: None for p in patch_paths}
+def register_scip_repo_test_caches():
+ for gem_data in gem_build_info:
+ gem_name = gem_data["repo_name"]
+ scip_repo_test_cache_rule(
+ name = "prep_{}".format(gem_name),
+ target = "@{}_zip//file".format(gem_name),
+ strip_prefix = gem_data["strip_prefix"],
+ subdir = gem_data["gem_subdir"],
+ )
+
+def scip_repos_test_suite(name, patch_paths):
+ available_patches = {p: None for p in patch_paths}
test_names = []
- for testdata in _data:
- test_name = testdata["name"]
- patch_path = "{}-{}.patch".format(test_name, testdata["tag"])
- patch_dep = []
- patch_arg = []
- if patch_path in patch_paths:
- patch_dep.append(patch_path)
- patch_arg.append("$(location {})".format(patch_path))
- patch_paths.pop(patch_path, None)
+ consumed = {}
+ for gem_data in gem_build_info:
+ gem_name = gem_data["repo_name"]
+ patch_basename = "{}-{}".format(gem_name, gem_data["ref"])
+ extra_deps = {}
+ gem_dep = {}
+ patch_arg = {}
+ for os in ["//tools/config:linux", "//tools/config:darwin"]:
+ candidates = [patch_basename + ".patch"]
+ if "linux" in os:
+ candidates.append(patch_basename + "-linux.patch")
+ gem_dep[os] = "//gems/scip-ruby/scip-ruby-debug-0.0.0-x86_64-linux.gem"
+ else:
+ candidates.append(patch_basename + "-darwin.patch")
+
+ # NOTE: Keep in sync with index_gem.sh.
+ gem_dep[os] = "//gems/scip-ruby/scip-ruby-debug-0.0.0-universal-darwin-20.gem"
+ extra_deps[os] = []
+ for patch_path in candidates:
+ if patch_path in available_patches:
+ extra_deps[os].append(patch_path)
+ patch_arg[os] = "$(location {})".format(patch_path)
+ consumed[patch_path] = None
+ if "linux" in patch_path or "darwin" in patch_path:
+ available_patches.pop(patch_path, None)
+ prep_target = ":prep_{}".format(gem_name)
+ data = [prep_target, "//gems/scip-ruby", "//:.ruby-version"]
+
+ test_name = gem_name
native.sh_test(
name = test_name,
size = "medium", # Should finish in 5 minutes
- srcs = ["index_oss_repo.sh"],
- deps = [],
- data = ["//main:scip-ruby"] + patch_dep,
- args = [
- testdata["name"],
- testdata["clone_url"],
- testdata["tag"],
- testdata["git_sha"],
- "'{}'".format(testdata["prep_cmd"]),
- "'{}'".format(testdata["run_cmd"]),
- ] + patch_arg,
+ srcs = ["index_gem.sh"],
+ data = data + select(extra_deps),
+ # Tests run serially since they share the same cache dir.
+ # For more context, see NOTE[repo-test-structure].
+ tags = ["exclusive"],
+ env = select(
+ {os: {
+ "SCIP_RUBY_CACHE_RUBY_DIR": "$(SCIP_RUBY_CACHE_RUBY_DIR)",
+ "RUBY_VERSION_FILE": "$(location //:.ruby-version)",
+ "SCIP_RUBY_GEMS": "$(locations //gems/scip-ruby)",
+ "CACHED_TGZS": "$(locations {})".format(prep_target),
+ "REPO_NAME": gem_data["repo_name"],
+ "PATCH_PATH": pa,
+ } for os, pa in patch_arg.items()},
+ ),
)
test_names.append(test_name)
- if len(patch_paths) != 0:
- fail("Patches {} were not used by any tests".format(patch_paths))
+ for p in consumed:
+ available_patches.pop(p, None)
+
+ # if len(available_patches) != 0:
+ # fail("Patches {} were not used by any tests".format(available_patches))
+ # FIXME(varun): Get the brew test working and turn this on.
native.test_suite(
- name = "repos",
+ name = name,
tests = test_names,
)
diff --git a/test/scip/repos/shopify-api-ruby-v11.1.0-darwin.patch b/test/scip/repos/shopify-api-ruby-v11.1.0-darwin.patch
new file mode 100644
index 0000000000..464aa36961
--- /dev/null
+++ b/test/scip/repos/shopify-api-ruby-v11.1.0-darwin.patch
@@ -0,0 +1,24 @@
+diff --git a/Gemfile.lock b/Gemfile.lock
+index 9fef6a6..4b720c0 100644
+--- a/Gemfile.lock
++++ b/Gemfile.lock
+@@ -81,2 +81,3 @@ GEM
+ ruby-progressbar (1.11.0)
++ scip-ruby-debug (0.0.0-universal-darwin-21)
+ securerandom (0.2.0)
+@@ -125,2 +126,3 @@ PLATFORMS
+ arm64-darwin-21
++ ruby
+ x86_64-linux
+@@ -136,2 +138,3 @@ DEPENDENCIES
+ rubocop-sorbet
++ scip-ruby-debug
+ shopify_api!
+diff --git a/shopify_api.gemspec b/shopify_api.gemspec
+index da77b24..93fba5c 100644
+--- a/shopify_api.gemspec
++++ b/shopify_api.gemspec
+@@ -51,2 +51,3 @@ Gem::Specification.new do |s|
+ s.add_development_dependency("tapioca")
++ s.add_development_dependency("scip-ruby-debug")
+ end
diff --git a/test/scip/repos/shopify-api-ruby-v11.1.0-linux.patch b/test/scip/repos/shopify-api-ruby-v11.1.0-linux.patch
new file mode 100644
index 0000000000..84ee512afd
--- /dev/null
+++ b/test/scip/repos/shopify-api-ruby-v11.1.0-linux.patch
@@ -0,0 +1,24 @@
+diff --git a/Gemfile.lock b/Gemfile.lock
+index 9fef6a6..54db244 100644
+--- a/Gemfile.lock
++++ b/Gemfile.lock
+@@ -81,2 +81,3 @@ GEM
+ ruby-progressbar (1.11.0)
++ scip-ruby-debug (0.0.0-x86_64-linux)
+ securerandom (0.2.0)
+@@ -125,2 +126,3 @@ PLATFORMS
+ arm64-darwin-21
++ ruby
+ x86_64-linux
+@@ -136,2 +138,3 @@ DEPENDENCIES
+ rubocop-sorbet
++ scip-ruby-debug
+ shopify_api!
+diff --git a/shopify_api.gemspec b/shopify_api.gemspec
+index da77b24..93fba5c 100644
+--- a/shopify_api.gemspec
++++ b/shopify_api.gemspec
+@@ -51,2 +51,3 @@ Gem::Specification.new do |s|
+ s.add_development_dependency("tapioca")
++ s.add_development_dependency("scip-ruby-debug")
+ end
diff --git a/third_party/externals.bzl b/third_party/externals.bzl
index f019985c44..2e9fef70e3 100644
--- a/third_party/externals.bzl
+++ b/third_party/externals.bzl
@@ -1,5 +1,6 @@
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file")
load("//third_party:ruby_externals.bzl", "register_ruby_dependencies")
+load("//third_party:test_gem_data.bzl", "gem_build_info")
# We define our externals here instead of directly in WORKSPACE
def register_sorbet_dependencies():
@@ -341,6 +342,14 @@ def register_sorbet_dependencies():
register_ruby_dependencies()
+def register_scip_ruby_dependencies():
+ for data in gem_build_info:
+ http_file(
+ name = data["repo_name"] + "_zip",
+ urls = [data["archive_url"]],
+ sha256 = data["archive_sha256"],
+ )
+
def _github_public_urls(path):
"""
Produce a url list that works both with github, and stripe's internal artifact cache.
diff --git a/third_party/test_gem.BUILD b/third_party/test_gem.BUILD
new file mode 100644
index 0000000000..7d69da58e8
--- /dev/null
+++ b/third_party/test_gem.BUILD
@@ -0,0 +1,5 @@
+filegroup(
+ name = "all",
+ srcs = glob(["**/*"]),
+ visibility = ["//visibility:public"],
+)
\ No newline at end of file
diff --git a/third_party/test_gem_data.bzl b/third_party/test_gem_data.bzl
new file mode 100644
index 0000000000..750505434b
--- /dev/null
+++ b/third_party/test_gem_data.bzl
@@ -0,0 +1,41 @@
+gem_build_info = [
+ # {
+ # "repo_name": "brew",
+ # # "kind": "self_sandboxing",
+ # "archive_url": "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/Homebrew/brew/archive/refs/tags/3.5.9.zip",
+ # "archive_sha256": "28a66005cc6f0f22a9a112ce84cedb833efe6926072dec2c68623195c008852f",
+ # "ref": "3.5.9",
+ # "strip_prefix": "brew-3.5.9",
+ # "gem_subdir": "Library/Homebrew",
+ # # "prep_cmd": """
+ # # pushd Library/Homebrew \\
+ # # && gem install bundler:1.17.3 \\
+ # # && popd \\
+ # # && ./bin/brew typecheck
+ # # """,
+ # # "run_cmd": """
+ # # pushd Library/Homebrew \\
+ # # && find . -name srb -type f -exec cp $${TEST_DIR}/$(location //main:scip-ruby) {} \\; \\
+ # # && popd \\
+ # # && ./bin/brew typecheck
+ # # """,
+ # },
+ {
+ "repo_name": "shopify-api-ruby",
+ # "kind": "needs_sandboxing",
+ "archive_url": "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/Shopify/shopify-api-ruby/archive/refs/tags/v11.1.0.zip",
+ "archive_sha256": "95a209308d6479491ca282af66254bd6c4e1c5b6edb8e0409e2fe29424a51157",
+ "ref": "v11.1.0",
+ "strip_prefix": "shopify-api-ruby-11.1.0",
+ # "prep_cmd": """
+ # bundle cache --all \\
+ # && for p in $(locations //gems/scip-ruby); do cp $${TEST_DIR}/$$p vendor/cache/; done \\
+ # """,
+ # "run_cmd": """
+ # bundle install --local \\
+ # && bundle exec scip-ruby --index-file index.scip --gem-metadata 'shopify-api-ruby@11.1.0' \\
+ # && file index.scip
+ # """,
+ "gem_subdir": ".",
+ },
+]
diff --git a/xbrew b/xbrew
new file mode 100755
index 0000000000..83ba9aa0c8
--- /dev/null
+++ b/xbrew
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+if [ "$1" == "where" ]; then
+ set -e
+ echo "$($HOME/.homebrew-x86_64/bin/brew --prefix)/bin/$2"
+ exit 0
+fi
+exec arch -x86_64 "$HOME/.homebrew-x86_64/bin/brew" "$@"