chore: Merge upstream master into scip-ruby/master

.bazelrc

@@ -76,8 +76,8 @@ test:coverage --spawn_strategy=local
 build --define=versioned=false
 build:versioned --workspace_status_command=tools/buildstamp/get_workspace_status --stamp --define=versioned=true
 
-build:dbg-darwin --config=dbg --platforms=@//tools/config:darwin_x86_64
-build:dbg-linux --config=dbg --platforms=@//tools/config:linux_x86_64
+build:dbg-darwin --config=dbg --platforms=@//tools/platforms:darwin_x86_64
+build:dbg-linux --config=dbg --platforms=@//tools/platforms:linux_x86_64
 
 ##
 ## Configurations used for releases
@@ -94,9 +94,19 @@ build:release-debug-common --config=forcedebug
 build:release-debug-common --config=debugsymbols
 build:release-debug-common --config=skipslowenforce
 
+# This mode has been added to allow tracking the symbols
+# which are to blame for usages of untyped code in a codebase.
+# This allows identifying what symbols would be the highest impact
+# to type.
+#
+# Earlier, we allowed for such statistic collection only in DEBUG mode
+# However, that was far too slow for our purposes. Thus, this mode allows
+# you to build in release mode and track these statistics.
+build:untyped-blame --copt=-DTRACK_UNTYPED_BLAME_MODE
+
 # harden: mark relocation sections read-only
 build:release-linux --linkopt=-Wl,-z,relro,-z,now
-build:release-linux --config=lto-linux --config=release-common --platforms=@//tools/config:linux_x86_64
+build:release-linux --config=lto-linux --config=release-common
 
 # This is to turn on vector instructions where available.
 # We used to do this unconditionally, but Rosetta 2 doesn't translate all vector instructions well.
@@ -109,7 +119,7 @@ build:release-linux --config=lto-linux --config=release-common --platforms=@//to
 build:release-linux           --copt=-march=sandybridge
 build:release-sanitized-linux --copt=-march=sandybridge
 
-build:release-mac --config=release-common --platforms=@//tools/config:darwin_x86_64
+build:release-mac --config=release-common --platforms=@//tools/platforms:darwin_x86_64
 
 build:release-debug-linux --config=release-linux
 build:release-debug-linux --config=release-debug-common
@@ -170,6 +180,9 @@ build:lto --config=static-libs
 
 ## flags that substantially increase Clang&LLVMs ability to devirtualize calls
 build:lto-linux --linkopt=-Wl,--icf=all
+
+# Improves linking time on arm64 hosts
+build:lto-linux --linkopt=-Wl,--thinlto-jobs=all
 build:lto-linux --config=lto
 
 # By default, we make static builds, but we can also use dynamic linking if asked to.

.github/workflows/lint.yml

@@ -12,14 +12,20 @@ jobs:
     runs-on: 'ubuntu-20.04'
     steps:
       - uses: actions/checkout@v3
+      - name: Format Bazel files
+        run: |
+          ./tools/scripts/format_build_files.sh
+          if ! git diff --quiet; then
+            git diff
+            echo ""
+            echo "-----------------------------------------------------"
+            echo "Re-run ./tools/scripts/format_build_files.sh and push"
+            echo "-----------------------------------------------------"
+            exit 1
+          fi
       - uses: actions/setup-go@v3
         with:
           go-version: '>=1.19'
-      - name: Format Bazel files
-        run: |
-          set -e
-          GOBIN="$PWD" go install github.com/bazelbuild/buildtools/buildifier@latest
-          git ls-files -- '**.bzl' '**/BUILD' WORKSPACE | xargs ./buildifier -mode=diff
       - name: Lint workflow files
         run: |
           go install github.com/rhysd/actionlint/cmd/actionlint@latest

.ruby-version

@@ -1 +1 @@
-2.7.2
+2.7.7

.sorbet-buildkite/build-sorbet-runtime.sh

@@ -7,7 +7,7 @@ pushd gems/sorbet-runtime
 echo "--- setup :ruby:"
 eval "$(rbenv init -)"
 
-runtime_versions=(2.7.2 3.1.2)
+runtime_versions=(2.7.7 3.1.2)
 
 for runtime_version in "${runtime_versions[@]}"; do
   rbenv install --skip-existing "$runtime_version"
@@ -23,7 +23,7 @@ for runtime_version in "${runtime_versions[@]}"; do
 
   failed=
 
-  if [ "$runtime_version" = "2.7.2" ]; then
+  if [ "$runtime_version" = "2.7.7" ]; then
     # Our Rubocop version doesn't understand Ruby 3.1 as a valid Ruby version
     echo "+++ rubocop ($runtime_version)"
     if ! rbenv exec bundle exec rake rubocop; then
@@ -36,6 +36,17 @@ for runtime_version in "${runtime_versions[@]}"; do
     failed=1
   fi
 
+  pushd test/wholesome
+
+  rbenv exec bundle config set path 'vendor/bundle'
+  rbenv exec bundle install
+
+  if ! rbenv exec bundle exec rake test; then
+    failed=1
+  fi
+
+  popd
+
   if [ "$failed" != "" ]; then
     exit 1
   fi

.sorbet-buildkite/build-static-release.sh

@@ -5,20 +5,24 @@ set -euo pipefail
 export JOB_NAME=build-static-release
 source .buildkite/tools/setup-bazel.sh
 
-unameOut="$(uname -s)"
-case "${unameOut}" in
-    Linux*)     platform="linux";;
-    Darwin*)    platform="mac";;
-    *)          exit 1
+kernel_name="$(uname -s | tr 'A-Z' 'a-z')"
+processor_name="$(uname -m)"
+
+platform="${kernel_name}-${processor_name}"
+case "$platform" in
+  linux-x86_64|linux-aarch64)
+    CONFIG_OPTS="--config=release-linux"
+    ;;
+  darwin-x86_64|darwin-arm64)
+    CONFIG_OPTS="--config=release-mac"
+    command -v autoconf >/dev/null 2>&1 || brew install autoconf
+    ;;
+  *)
+    echo >&2 "Building on $platform is not implemented"
+    exit 1
+    ;;
 esac
 
-if [[ "linux" == "$platform" ]]; then
-  CONFIG_OPTS="--config=release-linux"
-elif [[ "mac" == "$platform" ]]; then
-  CONFIG_OPTS="--config=release-mac"
-  command -v autoconf >/dev/null 2>&1 || brew install autoconf
-fi
-
 echo will run with $CONFIG_OPTS
 
 ./bazel build //main:sorbet --strip=always $CONFIG_OPTS
@@ -32,7 +36,7 @@ pushd gems/sorbet-static
 git_commit_count=$(git rev-list --count HEAD)
 release_version="0.5.${git_commit_count}"
 sed -i.bak "s/0\\.0\\.0/${release_version}/" sorbet-static.gemspec
-if [[ "mac" == "$platform" ]]; then
+if [[ "darwin" == "$kernel_name" ]]; then
     # Our binary should work on almost all OSes. The oldest v8 publishes is -14
     # so I'm going with that for now.
     for i in {14..22}; do
@@ -70,11 +74,11 @@ rbenv exec gem uninstall --all --executables --ignore-dependencies minitest moch
 rbenv exec gem uninstall --all --executables --ignore-dependencies sorbet sorbet-static
 trap 'rbenv exec gem uninstall --all --executables --ignore-dependencies sorbet sorbet-static' EXIT
 
-if [[ "mac" == "$platform" ]]; then
+if [[ "darwin" == "$kernel_name" ]]; then
   gem_platform="$(ruby -e "(platform = Gem::Platform.local).cpu = 'universal'; puts(platform.to_s)")"
   rbenv exec gem install ../../gems/sorbet-static/sorbet-static-*-"$gem_platform".gem
 else
-  rbenv exec gem install ../../gems/sorbet-static/sorbet-static-*-x86_64-linux.gem
+  rbenv exec gem install ../../gems/sorbet-static/sorbet-static-*-"$processor_name"-linux.gem
 fi
 rbenv exec gem install sorbet-*.gem
 
@@ -104,7 +108,7 @@ rm -rf _out_
 mkdir -p _out_/gems
 
 mv gems/sorbet-static/sorbet-static-*.gem _out_/gems/
-if [[ "linux" == "$platform" ]]; then
+if [[ "$kernel_name" == "linux" ]]; then
   mv gems/sorbet/sorbet*.gem _out_/gems/
 fi
 

.sorbet-buildkite/publish-ruby-gems.sh

@@ -26,46 +26,52 @@ source .buildkite/tools/with_backoff.sh
 
 rbenv install --skip-existing
 
+publish_sorbet_static_gem() {
+  gem_archive=$1
+  platform=$2
+
+  if gem list --remote rubygems.org --exact 'sorbet-static' | grep -q "${release_version}[^,]*${platform}"; then
+    echo "$gem_archive already published."
+    return
+  fi
+
+  # This is last so the exit code is used as the status code for with_backoff
+  gem push --verbose "$gem_archive"
+}
+
 # Push the sorbet-static gems first, in case they fail. We don't want to end
 # up in a weird state where 'sorbet' requires a pinned version of
 # sorbet-static, but the sorbet-static gem push failed.
 #
 # (By failure here, we mean that RubyGems.org 502'd for some reason.)
 for gem_archive in "_out_/gems/sorbet-static-$release_version"-*.gem; do
-  echo "Attempting to publish $gem_archive"
-  if [[ "$gem_archive" =~ _out_/gems/sorbet-static-([^-]*)-([^.]*).gem ]]; then
-    platform="${BASH_REMATCH[2]}"
-    if ! gem list --remote rubygems.org --exact 'sorbet-static' | grep -q "${release_version}[^,]*${platform}"; then
-      with_backoff gem push --verbose "$gem_archive"
-    else
-      echo "$gem_archive already published."
-    fi
-  else
+  if ! [[ "$gem_archive" =~ _out_/gems/sorbet-static-([^-]*)-([^.]*).gem ]]; then
     echo "Regex match failed. This should never happen."
     exit 1
   fi
+
+  platform="${BASH_REMATCH[2]}"
+
+  echo "Attempting to publish $gem_archive"
+  with_backoff publish_sorbet_static_gem "$gem_archive" "$platform"
 done
 
-gem_archive="_out_/gems/sorbet-runtime-$release_version.gem"
-echo "Attempting to publish $gem_archive"
-if ! gem list --remote rubygems.org --exact 'sorbet-runtime' | grep -q "$release_version"; then
-  with_backoff gem push --verbose "$gem_archive"
-else
-  echo "$gem_archive already published."
-fi
+# Sometimes the 'gem push' times out, but after the connection dies, the server
+# decides to finish publishing the gem. So we have to interleave 'gem list' and
+# 'gem push' calls--it's not enough to just check whether it exists once.
+publish_gem() {
+  gem_name=$1
+  gem_archive="_out_/gems/$gem_name-$release_version.gem"
 
-gem_archive="_out_/gems/sorbet-$release_version.gem"
-echo "Attempting to publish $gem_archive"
-if ! gem list --remote rubygems.org --exact 'sorbet' | grep -q "$release_version"; then
-  with_backoff gem push --verbose "$gem_archive"
-else
-  echo "$gem_archive already published."
-fi
+  if gem list --remote rubygems.org --exact "$gem_name" | grep -q "$release_version"; then
+    echo "$gem_archive already published."
+    return
+  fi
 
-gem_archive="_out_/gems/sorbet-static-and-runtime-$release_version.gem"
-echo "Attempting to publish $gem_archive"
-if ! gem list --remote rubygems.org --exact 'sorbet-static-and-runtime' | grep -q "$release_version"; then
-  with_backoff gem push --verbose "$gem_archive"
-else
-  echo "$gem_archive already published."
-fi
+  # This is last so the exit code is used as the status code for with_backoff
+  gem push --verbose "$gem_archive"
+}
+
+with_backoff publish_gem "sorbet-runtime"
+with_backoff publish_gem "sorbet"
+with_backoff publish_gem "sorbet-static-and-runtime"

.sorbet-buildkite/test-compiler.sh

@@ -10,6 +10,7 @@ case "${unameOut}" in
 esac
 
 if [[ "linux" == "$platform" ]]; then
+    curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
     apt-get update
     apt-get install -yy libncurses5-dev libncursesw5-dev xxd
 elif [[ "mac" == "$platform" ]]; then
@@ -38,6 +39,10 @@ mkdir -p _out_
 test_args=(
   "//test:compiler"
   "//test/cli/compiler"
+  # These are two tests that depend on sorbet_ruby, and it's annoying to have
+  # to build sorbet_ruby on the static sanitized job (delays test start time)
+  "//test:end_to_end_rbi_test"
+  "//test:single_package_runner"
   "-c"
   "opt"
   "--config=forcedebug"

.sorbet-buildkite/test-vscode-extension.sh

@@ -4,6 +4,7 @@ set -euo pipefail
 
 # TODO: We probably want to move this into https://github.com/sorbet/sorbet-build-image eventually
 
+curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
 apt-get update
 apt-get install -y libxshmfence-dev libnss3-dev libatk1.0-0 libatk-bridge2.0-0 \
  libdrm2 xvfb libgdk-pixbuf2.0-0 libgtk-3-0 libgbm1	libasound2

.sorbet-buildkite/tools/with_backoff.sh

@@ -6,7 +6,7 @@ echo "--- Loading with_backoff helper"
 
 # https://stackoverflow.com/a/8351489
 with_backoff() {
-  local attempts=5
+  local attempts=8
   local timeout=1 # doubles each failure
 
   local attempt=0

.vscode/settings.json

@@ -1,4 +1,6 @@
 {
+  "files.insertFinalNewline": true,
+  "editor.renderFinalNewline": "off",
   "editor.insertSpaces": true,
   "editor.rulers": [120],
   // Clangd 9.0 has a new semantic highlighting feature, but it's buggy and causes

README.md

@@ -50,6 +50,7 @@ Otherwise, add this line to your `Gemfile`:
 ```ruby
 gem 'scip-ruby', require: false, :group => :development
 ```
+
 After either of those steps, run `bundle install`
 to download and install fetch `scip-ruby`.
 

WORKSPACE

@@ -61,6 +61,8 @@ BAZEL_VERSION = "5.2.0"
 
 BAZEL_INSTALLER_VERSION_LINUX_X86_64_SHA = "7d9ef51beab5726c55725fb36675c6fed0518576d3ba51fb4067580ddf7627c4"
 
+BAZEL_INSTALLER_VERSION_LINUX_ARM64_SHA = "ae50cb7d64aebee986287134ff8ca0335651a0c1685348b3216f3fdfa20ff7e7"
+
 BAZEL_INSTALLER_VERSION_DARWIN_X86_64_SHA = "645e7c335efc3207905e98f0c56a598b7cb0282d54d9470e80f38fb698064fb3"
 
 BAZEL_INSTALLER_VERSION_DARWIN_ARM64_SHA = "bc018ee7980cdf1c3f0099ec1568847a1756a3c00f1f9440bca44c26ceb3d90f"

ast/Helpers.cc

@@ -28,7 +28,9 @@ bool definesBehavior(const ExpressionPtr &expr) {
         },
 
         [&](const ast::Assign &asgn) {
-            if (ast::isa_tree<ast::ConstantLit>(asgn.lhs)) {
+            // this check can fire before the namer converts lhs constants in assignments from UnresolvedConstantLit ->
+            // ConstantLit, so we have to allow for both types.
+            if (ast::isa_tree<ast::ConstantLit>(asgn.lhs) || ast::isa_tree<ast::UnresolvedConstantLit>(asgn.lhs)) {
                 result = false;
             } else {
                 result = true;

ast/Helpers.h

@@ -375,6 +375,11 @@ class MK {
                                                std::move(type));
     }
 
+    static ExpressionPtr AssumeType(core::LocOffsets loc, ExpressionPtr value, ExpressionPtr type) {
+        return ast::make_expression<ast::Cast>(loc, core::Types::todo(), std::move(value), core::Names::assumeType(),
+                                               std::move(type));
+    }
+
     static ExpressionPtr ClassOf(core::LocOffsets loc, ExpressionPtr value) {
         return Send1(loc, T(loc), core::Names::classOf(), loc, std::move(value));
     }
@@ -426,11 +431,6 @@ class MK {
         return Constant(loc, core::Symbols::Magic());
     }
 
-    static ExpressionPtr SelfNew(core::LocOffsets loc, core::LocOffsets funLoc, int numPosArgs,
-                                 ast::Send::ARGS_store args, Send::Flags flags = {}) {
-        return Send(loc, Magic(loc), core::Names::selfNew(), funLoc, numPosArgs, std::move(args), flags);
-    }
-
     static ExpressionPtr DefineTopClassOrModule(core::LocOffsets loc, core::ClassOrModuleRef klass) {
         Send::Flags flags;
         flags.isRewriterSynthesized = true;
@@ -452,6 +452,14 @@ class MK {
         return ret;
     }
 
+    static ExpressionPtr RaiseTypedUnimplemented(core::LocOffsets loc) {
+        auto kernel = Constant(loc, core::Symbols::Kernel());
+        auto msg = String(loc, core::Names::rewriterRaiseUnimplemented());
+        auto ret = Send1(loc, std::move(kernel), core::Names::raise(), loc, std::move(msg));
+        cast_tree<ast::Send>(ret)->flags.isRewriterSynthesized = true;
+        return ret;
+    }
+
     static bool isRootScope(const ast::ExpressionPtr &scope) {
         if (ast::isa_tree<ast::EmptyTree>(scope)) {
             return true;
@@ -469,11 +477,7 @@ class MK {
     }
 
     static bool isSelfNew(ast::Send *send) {
-        if (send->fun != core::Names::selfNew()) {
-            return false;
-        }
-
-        return isMagicClass(send->recv);
+        return send->fun == core::Names::new_() && send->recv.isSelfReference();
     }
 
     static core::NameRef arg2Name(const ExpressionPtr &arg) {