From 0e5c748d7eac17476e347c581091539e49841f51 Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 07:45:20 +0800 Subject: [PATCH 01/30] ci: Add Buildkite testing action to CI. --- .buildkite/all-succeeded.sh | 1 + .buildkite/linters.sh | 1 + .buildkite/pipeline.sh | 30 +++++++++++++++ .buildkite/pipeline.yaml | 24 ++++++++++++ .buildkite/test-indexer.sh | 74 +++++++++++++++++++++++++++++++++++++ .buildkite/tools | 1 + 6 files changed, 131 insertions(+) create mode 120000 .buildkite/all-succeeded.sh create mode 120000 .buildkite/linters.sh create mode 100755 .buildkite/pipeline.sh create mode 100644 .buildkite/pipeline.yaml create mode 100755 .buildkite/test-indexer.sh create mode 120000 .buildkite/tools diff --git a/.buildkite/all-succeeded.sh b/.buildkite/all-succeeded.sh new file mode 120000 index 0000000000..f65a1ca21e --- /dev/null +++ b/.buildkite/all-succeeded.sh @@ -0,0 +1 @@ +../.sorbet-buildkite/all-succeeded.sh \ No newline at end of file diff --git a/.buildkite/linters.sh b/.buildkite/linters.sh new file mode 120000 index 0000000000..1f5a099076 --- /dev/null +++ b/.buildkite/linters.sh @@ -0,0 +1 @@ +../.sorbet-buildkite/linters.sh \ No newline at end of file diff --git a/.buildkite/pipeline.sh b/.buildkite/pipeline.sh new file mode 100755 index 0000000000..482a5a7289 --- /dev/null +++ b/.buildkite/pipeline.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# Largely based off .sorbet-buildkite/pipeline.sh + +set -euo pipefail + +whitelisted=0 + +echo "BUILDKITE_PULL_REQUEST=$BUILDKITE_PULL_REQUEST" +echo "BUILDKITE_PULL_REQUEST_REPO=$BUILDKITE_PULL_REQUEST_REPO" + +if [[ "$BUILDKITE_PULL_REQUEST" == "false" ]]; then + # whitelist commits that are triggered in branch builds of github.com/sourcegraph/scip-ruby + echo "Automatically running build for non-PR commit pushed to sourcegraph/scip-ruby" + whitelisted=1 +fi + +if [[ "$BUILDKITE_PULL_REQUEST_REPO" == "git://github.com/sourcegraph/scip-ruby.git" ]] || + [[ "$BUILDKITE_PULL_REQUEST_REPO" == "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/sourcegraph/scip-ruby.git" ]]; then + # whitelist folks with write access to github.com/sorbet/sorbet + echo "Automatically running build for non-fork PR created against sourcegraph/scip-ruby" + whitelisted=1 +fi + +if [[ "${whitelisted}" -ne 1 ]] ; then + (echo -e "steps:\\n - block: \":key: Needs contributor approval!\"\\n - wait: ~\\n"; + grep -v "steps:" .buildkite/pipeline.yaml ) | buildkite-agent pipeline upload +else + buildkite-agent pipeline upload +fi diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml new file mode 100644 index 0000000000..938b5c06d4 --- /dev/null +++ b/.buildkite/pipeline.yaml @@ -0,0 +1,24 @@ +# Loosely based off .sorbet-buildkite/pipeline.yaml + +steps: + +# Basic Phase - Run super fast things and things that commonly catch bugs + - label: ":linux: linters.sh" + command: .buildkite/linters.sh + <<: &stateless + agents: + os: linux + queue: stateless + + - label: ":linux: test-indexer.sh" + command: .buildkite/test-indexer.sh + artifact_paths: _out_/profile.json + <<: *stateless + + - wait: ~ + +# Success Phase - Allow the PR to be merged + - label: "All tests and builds succeeded" + command: .buildkite/all-succeeded.sh + + - wait: ~ diff --git a/.buildkite/test-indexer.sh b/.buildkite/test-indexer.sh new file mode 100755 index 0000000000..873ff64841 --- /dev/null +++ b/.buildkite/test-indexer.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash + +# Largely based off test-compiler.sh from the Sorbet buildkite config. + +set -euo pipefail + +unameOut="$(uname -s)" +case "${unameOut}" in + Linux*) platform="linux";; + Darwin*) platform="mac";; + *) exit 1 +esac + +if [[ "linux" == "$platform" ]]; then + apt-get update + apt-get install -yy libncurses5-dev libncursesw5-dev xxd +elif [[ "mac" == "$platform" ]]; then + if ! [ -x "$(command -v wget)" ]; then + brew install wget + fi +fi + +export JOB_NAME=test +source .buildkite/tools/setup-bazel.sh + +err=0 + +echo "+++ running tests" + +mkdir -p _out_ + +# `-c opt` is required, otherwise the tests are too slow +# forcedebug is really the ~only thing in `--config=dbg` we care about. +# must come after `-c opt` because `-c opt` will define NDEBUG on its own +test_args=( + "//test/scip" + "//test/scip/long" + "-c" + "opt" + "--config=forcedebug" + "--spawn_strategy=local" +) + +./bazel test \ + --experimental_generate_json_trace_profile \ + --profile=_out_/profile.json \ + --test_summary=terse \ + --test_output=errors \ + "${test_args[@]}" || err=$? + +if [ "$err" -ne 0 ]; then + echo "--- annotating build result" + failing_tests="$(mktemp)" + + echo 'Run this command to run failing tests locally:' >> "$failing_tests" + echo >> "$failing_tests" + echo '```bash' >> "$failing_tests" + echo "./bazel test \\" >> "$failing_tests" + + # Take the lines that start with target labels. + # Lines look like "//foo FAILED in 10s" + { ./bazel test --test_summary=terse "${test_args[@]}" || true ; } | \ + grep '^//' | \ + sed -e 's/ .*/ \\/' | \ + sed -e 's/^/ /' >> "$failing_tests" + + # Put this last as an easy way to not have a `\` on the last line. + echo ' -c opt --config=forcedebug' >> "$failing_tests" + echo '```' >> "$failing_tests" + + buildkite-agent annotate --context "test-indexer.sh" --style error --append < "$failing_tests" + + exit "$err" +fi diff --git a/.buildkite/tools b/.buildkite/tools new file mode 120000 index 0000000000..76620aa7ff --- /dev/null +++ b/.buildkite/tools @@ -0,0 +1 @@ +../.sorbet-buildkite/tools/ \ No newline at end of file From b57f47513e8504e27e72ac644e37252f4e09b916 Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 08:17:55 +0800 Subject: [PATCH 02/30] ci: Add Buildkite hooks like Sorbet. --- .buildkite/hooks/gh-status | 64 +++++++++++++++++++++++++++++++++++ .buildkite/hooks/post-command | 12 +++++++ .buildkite/hooks/pre-command | 8 +++++ 3 files changed, 84 insertions(+) create mode 100755 .buildkite/hooks/gh-status create mode 100755 .buildkite/hooks/post-command create mode 100755 .buildkite/hooks/pre-command diff --git a/.buildkite/hooks/gh-status b/.buildkite/hooks/gh-status new file mode 100755 index 0000000000..652f1e5390 --- /dev/null +++ b/.buildkite/hooks/gh-status @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +set -e +[ -z "$DEBUG" ] || set -x; + +usage() { + echo "$0 " >&2; +} + +if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then + usage + cat >&2 < +Generate this access token at https://github.com/settings/tokens and make sure +it has access to the \`"repo"\` scope. +EOS + exit 1; +fi + +[ -n "$5" ] || (usage; exit 1); + +REPO="$1" +shift + +COMMIT="$1" +shift + +STATE="$1" +shift + +TARGET_URL="$1" +shift + +CONTEXT="$1" +shift + +BODY="" +[ -t 0 ] || BODY=$(cat); + +payload=$( + jq --null-input \ + --arg state "$STATE" \ + --arg target_url "$TARGET_URL" \ + --arg description "$DESCRIPTION" \ + --arg context "$CONTEXT" \ + '{ state: $state, target_url: $target_url, description: $description, context: $context}' +) + +response=$( + curl --fail \ + --netrc \ + --silent \ + --location \ + --data "$payload" \ + "https://api.github.com/repos/${REPO}/statuses/${COMMIT}" +) diff --git a/.buildkite/hooks/post-command b/.buildkite/hooks/post-command new file mode 100755 index 0000000000..9f02713c69 --- /dev/null +++ b/.buildkite/hooks/post-command @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -euo pipefail +if [ "HEAD" == "${BUILDKITE_COMMIT}" ]; then + echo "not using commit marks as this is HEAD build" +else + if [ "$BUILDKITE_COMMAND_EXIT_STATUS" -ne 0 ]; then + .buildkite/hooks/gh-status sourcegraph/scip-ruby "${BUILDKITE_COMMIT}" failure "${BUILDKITE_BUILD_URL}#${BUILDKITE_JOB_ID}" "${BUILDKITE_LABEL}" + else + .buildkite/hooks/gh-status sourcegraph/scip-ruby "${BUILDKITE_COMMIT}" success "${BUILDKITE_BUILD_URL}#${BUILDKITE_JOB_ID}" "${BUILDKITE_LABEL}" + fi +fi diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command new file mode 100755 index 0000000000..5f32e1dfc2 --- /dev/null +++ b/.buildkite/hooks/pre-command @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail +if [ "HEAD" == "${BUILDKITE_COMMIT}" ]; then + echo "not using commit marks as this is HEAD build" +else + .buildkite/hooks/gh-status sourcegraph/scip-ruby "${BUILDKITE_COMMIT}" pending "${BUILDKITE_BUILD_URL}#${BUILDKITE_JOB_ID}" "${BUILDKITE_LABEL}" +fi From 9bed1ef7f47467bb10213c0ef4523d99b19c382f Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 08:27:18 +0800 Subject: [PATCH 03/30] ci: Remove os filter for agent queue. --- .buildkite/pipeline.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index 938b5c06d4..6ae9399251 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -7,7 +7,6 @@ steps: command: .buildkite/linters.sh <<: &stateless agents: - os: linux queue: stateless - label: ":linux: test-indexer.sh" From 0f29cd6617298fc31dddad90dcf76ecb7d154714 Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 08:41:28 +0800 Subject: [PATCH 04/30] ci: Temporarily make hooks unused. --- .buildkite/{hooks => unused-hooks}/gh-status | 0 .buildkite/{hooks => unused-hooks}/post-command | 0 .buildkite/{hooks => unused-hooks}/pre-command | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename .buildkite/{hooks => unused-hooks}/gh-status (100%) rename .buildkite/{hooks => unused-hooks}/post-command (100%) rename .buildkite/{hooks => unused-hooks}/pre-command (100%) diff --git a/.buildkite/hooks/gh-status b/.buildkite/unused-hooks/gh-status similarity index 100% rename from .buildkite/hooks/gh-status rename to .buildkite/unused-hooks/gh-status diff --git a/.buildkite/hooks/post-command b/.buildkite/unused-hooks/post-command similarity index 100% rename from .buildkite/hooks/post-command rename to .buildkite/unused-hooks/post-command diff --git a/.buildkite/hooks/pre-command b/.buildkite/unused-hooks/pre-command similarity index 100% rename from .buildkite/hooks/pre-command rename to .buildkite/unused-hooks/pre-command From 6929dbb8e6bc24c2298aa4b52bb110a7db9fed30 Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 08:43:10 +0800 Subject: [PATCH 05/30] ci: Fix test target for Buildkite. --- .buildkite/test-indexer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/test-indexer.sh b/.buildkite/test-indexer.sh index 873ff64841..5b22fd1f9f 100755 --- a/.buildkite/test-indexer.sh +++ b/.buildkite/test-indexer.sh @@ -34,7 +34,7 @@ mkdir -p _out_ # must come after `-c opt` because `-c opt` will define NDEBUG on its own test_args=( "//test/scip" - "//test/scip/long" + "//test/scip/repos" "-c" "opt" "--config=forcedebug" From 6dee8f65615ce2256506c7c60b23c30eb7cef11a Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 08:47:57 +0800 Subject: [PATCH 06/30] cleanup: Format Bazel files. --- main/BUILD | 2 +- scip_indexer/BUILD | 8 +-- test/BUILD | 16 +++--- test/scip/BUILD | 8 ++- test/scip/repos/scip_repos_test.bzl | 76 ++++++++++++++--------------- test/scip/scip_test.bzl | 6 +-- 6 files changed, 63 insertions(+), 53 deletions(-) diff --git a/main/BUILD b/main/BUILD index 4555e9e2ca..6d1a98b192 100644 --- a/main/BUILD +++ b/main/BUILD @@ -60,8 +60,8 @@ cc_binary( visibility = ["//visibility:public"], deps = [ "realmain", - "//scip_indexer:scip_indexer", "//payload", + "//scip_indexer", ], ) diff --git a/scip_indexer/BUILD b/scip_indexer/BUILD index a7f094f65a..9fd9429011 100644 --- a/scip_indexer/BUILD +++ b/scip_indexer/BUILD @@ -4,14 +4,14 @@ cc_library( visibility = ["//visibility:public"], deps = [ "//main/pipeline/semantic_extension:interface", - ] + ], ) cc_library( name = "scip_utils", srcs = [ - "SCIPUtils.h", "SCIPUtils.cc", + "SCIPUtils.h", ], linkstatic = select({ "@com_stripe_ruby_typer//tools/config:linkshared": 0, @@ -45,10 +45,10 @@ cc_library( "//main/lsp", "//proto", "//sorbet_version", - "@com_google_absl//absl/synchronization", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/synchronization", "@cxxopts", "@spdlog", ], -) \ No newline at end of file +) diff --git a/test/BUILD b/test/BUILD index 34a9460192..1469d84be4 100644 --- a/test/BUILD +++ b/test/BUILD @@ -61,14 +61,18 @@ cc_binary( name = "scip_test_runner", testonly = 1, srcs = [ - "scip_test_runner.cc" + "scip_test_runner.cc", ], linkstatic = select({ "//tools/config:linkshared": 0, "//conditions:default": 1, }), - visibility = ["//tools:__pkg__", "//test/scip:__pkg__"], - deps = [ # TODO(varun): Revisit dependencies after rewriting test runner. + visibility = [ + "//test/scip:__pkg__", + "//tools:__pkg__", + ], + deps = [ + # TODO(varun): Revisit dependencies after rewriting test runner. "//ast/desugar", "//ast/treemap", "//cfg/builder", @@ -78,20 +82,20 @@ cc_binary( "//definition_validator", "//infer", "//local_vars", - "//main/autogen", # TODO(varun): why is this needed? + "//main/autogen", # TODO(varun): why is this needed? "//namer", "//payload", "//proto", "//resolver", "//rewriter", "//scip_indexer:interface", - "//scip_indexer:scip_indexer", + "//scip_indexer", "//test/helpers", "@com_google_absl//absl/strings", "@cxxopts", "@doctest", "@doctest//:doctest_custom_main", - ] + ], ) cc_binary( diff --git a/test/scip/BUILD b/test/scip/BUILD index de4082bcaa..d4cae216e3 100644 --- a/test/scip/BUILD +++ b/test/scip/BUILD @@ -1,3 +1,9 @@ load(":scip_test.bzl", "scip_test_suite") -scip_test_suite(paths = glob(["testdata/*"], exclude_directories = 1), multifile_paths = glob(["testdata/multifile/*/*"])) +scip_test_suite( + multifile_paths = glob(["testdata/multifile/*/*"]), + paths = glob( + ["testdata/*"], + exclude_directories = 1, + ), +) diff --git a/test/scip/repos/scip_repos_test.bzl b/test/scip/repos/scip_repos_test.bzl index ee5133d962..c005aae952 100644 --- a/test/scip/repos/scip_repos_test.bzl +++ b/test/scip/repos/scip_repos_test.bzl @@ -1,54 +1,54 @@ -_data = [ +_data = [ { "name": "brew", "clone_url": "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/Homebrew/brew.git", "tag": "3.5.7", "git_sha": "ce8ef89ec0cfe875c48c3a83a843af9074c05930", - "prep_cmd": ("pushd Library/Homebrew" - + " && gem install bundler:1.17.3" - + " && popd" - + " && ./bin/brew typecheck"), - "run_cmd": ("pushd Library/Homebrew" - + " && $$(find . -name srb -type f) | while IFS= read -r line ; do cp $${TEST_DIR}/$(location //main:scip-ruby) $$line ; done" - + " && popd" - + " && ./bin/brew typecheck") - } + "prep_cmd": ("pushd Library/Homebrew" + + " && gem install bundler:1.17.3" + + " && popd" + + " && ./bin/brew typecheck"), + "run_cmd": ("pushd Library/Homebrew" + + " && $$(find . -name srb -type f) | while IFS= read -r line ; do cp $${TEST_DIR}/$(location //main:scip-ruby) $$line ; done" + + " && popd" + + " && ./bin/brew typecheck"), + }, ] def scip_repos_test_suite(patch_paths): - patch_paths = { p: None for p in patch_paths } + patch_paths = {p: None for p in patch_paths} test_names = [] for testdata in _data: - test_name = testdata["name"] - patch_path = "{}-{}.patch".format(test_name, testdata["tag"]) - patch_dep = [] - patch_arg = [] - if patch_path in patch_paths: - patch_dep.append(patch_path) - patch_arg.append("$(location {})".format(patch_path)) - patch_paths.pop(patch_path, None) + test_name = testdata["name"] + patch_path = "{}-{}.patch".format(test_name, testdata["tag"]) + patch_dep = [] + patch_arg = [] + if patch_path in patch_paths: + patch_dep.append(patch_path) + patch_arg.append("$(location {})".format(patch_path)) + patch_paths.pop(patch_path, None) - native.sh_test( - name = test_name, - size = "medium", # Should finish in 5 minutes - srcs = ["index_oss_repo.sh"], - deps = [], - data = ["//main:scip-ruby"] + patch_dep, - args = [ - testdata["name"], - testdata["clone_url"], - testdata["tag"], - testdata["git_sha"], - "'{}'".format(testdata["prep_cmd"]), - "'{}'".format(testdata["run_cmd"]), - ] + patch_arg, - ) - test_names.append(test_name) + native.sh_test( + name = test_name, + size = "medium", # Should finish in 5 minutes + srcs = ["index_oss_repo.sh"], + deps = [], + data = ["//main:scip-ruby"] + patch_dep, + args = [ + testdata["name"], + testdata["clone_url"], + testdata["tag"], + testdata["git_sha"], + "'{}'".format(testdata["prep_cmd"]), + "'{}'".format(testdata["run_cmd"]), + ] + patch_arg, + ) + test_names.append(test_name) if len(patch_paths) != 0: - fail("Patches {} were not used by any tests".format(patch_paths)) + fail("Patches {} were not used by any tests".format(patch_paths)) native.test_suite( - name = "repos", - tests = test_names, + name = "repos", + tests = test_names, ) diff --git a/test/scip/scip_test.bzl b/test/scip/scip_test.bzl index 1abe736eef..4a98f9d7ea 100644 --- a/test/scip/scip_test.bzl +++ b/test/scip/scip_test.bzl @@ -42,7 +42,7 @@ def scip_test_suite(paths, multifile_paths): ) def scip_test(path): - if not path.endswith(".rb") or path.endswith(".snapshot.rb"): + if not path.endswith(".rb") or path.endswith(".snapshot.rb"): return None test_name = basename(path)[:-3] snapshot_path = path[:-3] + ".snapshot.rb" @@ -57,7 +57,7 @@ def scip_multifile_test(dir, filepaths): path_without_ext, ext = split_extension(filepath) if (ext == "rb" or ext == "rbi") and not path_without_ext.endswith(".snapshot"): data.append(filepath) - if not filepath.endswith("scip-ruby-args.rb"): # Special file for reading Gem-level args. + if not filepath.endswith("scip-ruby-args.rb"): # Special file for reading Gem-level args. data.append(path_without_ext + ".snapshot." + ext) if not dir.startswith("testdata/multifile/"): fail("Expected directory to be under multifile/") @@ -85,4 +85,4 @@ def _make_test(test_name, args, data): tags = ["manual"], size = "small", ) - return (test_name, update_test_name) \ No newline at end of file + return (test_name, update_test_name) From 4cf02f74b617aa0cab79f9768384049c8a4984c6 Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 08:52:18 +0800 Subject: [PATCH 07/30] cleanup: Auto-format C++ code. --- cfg/builder/builder_walk.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cfg/builder/builder_walk.cc b/cfg/builder/builder_walk.cc index 3e0b5887cf..01f268b801 100644 --- a/cfg/builder/builder_walk.cc +++ b/cfg/builder/builder_walk.cc @@ -103,7 +103,7 @@ pair unresolvedIdent2Local(CFGContext cctx, const ast::Unresolve } return {fnd->second.variable, hasError && isAssign}; } else { - return {global2Local(cctx, sym, id.loc), false}; + return {global2Local(cctx, sym, id.loc), false}; } } From bbf7b14b4a4b2a2f404bd01558752c5fd28cee46 Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 09:23:46 +0800 Subject: [PATCH 08/30] ci: Specify Ruby version using ASDF. --- .tool-versions | 1 + 1 file changed, 1 insertion(+) create mode 100644 .tool-versions diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 0000000000..44d859d8d6 --- /dev/null +++ b/.tool-versions @@ -0,0 +1 @@ +ruby 2.7.0 From 076d0b171b50354c32b72a45717ec695983ddaa6 Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 09:24:24 +0800 Subject: [PATCH 09/30] ci: Break up steps in test-indexer.sh --- .buildkite/pipeline.yaml | 3 ++- .buildkite/test-indexer.sh | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index 6ae9399251..e993b7f654 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -11,7 +11,8 @@ steps: - label: ":linux: test-indexer.sh" command: .buildkite/test-indexer.sh - artifact_paths: _out_/profile.json + artifact_paths: + - "_out_/profile*.json" <<: *stateless - wait: ~ diff --git a/.buildkite/test-indexer.sh b/.buildkite/test-indexer.sh index 5b22fd1f9f..17170fba20 100755 --- a/.buildkite/test-indexer.sh +++ b/.buildkite/test-indexer.sh @@ -32,18 +32,49 @@ mkdir -p _out_ # `-c opt` is required, otherwise the tests are too slow # forcedebug is really the ~only thing in `--config=dbg` we care about. # must come after `-c opt` because `-c opt` will define NDEBUG on its own +build_args=( + "//main:scip-ruby" + "//test:scip_test_runner" + "-c" + "opt" + "--config=forcedebug" + "--spawn_strategy=local" +) + +./bazel build \ + --experimental_generate_json_trace_profile \ + --profile=_out_/profile_build.json \ + "${build_args[@]}" || err=$? + test_args=( "//test/scip" + "-c" + "opt" + "--config=forcedebug" + "--spawn_strategy=local" +) + +./bazel test \ + --experimental_generate_json_trace_profile \ + --profile=_out_/profile_snapshot_tests.json \ + --test_summary=terse \ + --test_output=errors \ + "${test_args[@]}" || err=$? + +test_args=( "//test/scip/repos" "-c" "opt" + "--test_env GITHUB_ACTIONS=1" + "--test_env PATH=${PATH}" + "--test_env HOME=${HOME}" "--config=forcedebug" "--spawn_strategy=local" ) ./bazel test \ --experimental_generate_json_trace_profile \ - --profile=_out_/profile.json \ + --profile=_out_/profile_repo_tests.json \ --test_summary=terse \ --test_output=errors \ "${test_args[@]}" || err=$? From 876d1e9ae9f20c7ce6a505c93e7eeb03ae499fa4 Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 09:49:59 +0800 Subject: [PATCH 10/30] ci: Disable compdb generation linter. --- .buildkite/linters.sh | 89 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 88 insertions(+), 1 deletion(-) mode change 120000 => 100755 .buildkite/linters.sh diff --git a/.buildkite/linters.sh b/.buildkite/linters.sh deleted file mode 120000 index 1f5a099076..0000000000 --- a/.buildkite/linters.sh +++ /dev/null @@ -1 +0,0 @@ -../.sorbet-buildkite/linters.sh \ No newline at end of file diff --git a/.buildkite/linters.sh b/.buildkite/linters.sh new file mode 100755 index 0000000000..096918cab6 --- /dev/null +++ b/.buildkite/linters.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +set -euo pipefail +export JOB_NAME=linters + +# shellcheck source=SCRIPTDIR/tools/setup-bazel.sh +source .buildkite/tools/setup-bazel.sh + +set -x +globalErr=0 + +echo "~~~ Checking build files" +if ! ./tools/scripts/format_build_files.sh -t &> buildifier; then + globalErr=1 + echo "^^^ +++" + buildkite-agent annotate --context tools/scripts/format_build_files.sh --style error --append < buildifier +fi + +echo "~~~ Checking c++ formatting" +if ! ./tools/scripts/format_cxx.sh -t &> format_cxx; then + globalErr=1 + echo "^^^ +++" + buildkite-agent annotate --context tools/scripts/format_cxx.sh --style error --append < format_cxx +fi + +echo "~~~ Checking that the compilation db builds" +if ! ./tools/scripts/build_compilation_db.sh &> compdb; then + globalErr=1 + echo "^^^ +++" + buildkite-agent annotate --context tools/scripts/build_compilation_db.sh --style error --append < compdb +fi + +# Skip this because bazel query is failing due to third-party stuff +# echo "~~~ Checking compilation db targets" +# if ! ./tools/scripts/generate_compdb_targets.sh -t &> compdb-targets; then +# globalErr=1 +# echo "^^^ +++" +# buildkite-agent annotate --context tools/scripts/generate_compdb_targets.sh --style error --append < compdb-targets +# fi + +echo "~~~ Linting uses of \`using namespace std\`" +if ! ./tools/scripts/check_using_namespace_std.sh &> std_check; then + globalErr=1 + echo "^^^ +++" + buildkite-agent annotate --context tools/scripts/check_using_namespace_std.sh --style error --append < std_check +fi + +echo "~~~ Checking ErrorClass error code numbers" +if ! ./tools/scripts/check_error_classes.sh &> error_class_check; then + globalErr=1 + echo "^^^ +++" + buildkite-agent annotate --context tools/scripts/check_error_classes.sh --style error --append < error_class_check +fi + +echo "~~~ Running shellcheck" +if ! ./tools/scripts/lint_sh.sh -t &> lint_sh; then + globalErr=1 + echo "^^^ +++" + buildkite-agent annotate --context tools/scripts/lint_sh.sh --style error --append < lint_sh +fi + +echo "~~~ Checking markdown formatting" +if ! ./tools/scripts/format_website.sh -t &> format_website; then + globalErr=1 + echo "^^^ +++" + buildkite-agent annotate --context tools/scripts/format_website.sh --style error --append < format_website +fi + +echo "~~~ Checking the vscode extension" +pushd vscode_extension +yarn install +if ! yarn lint --output-file=yarn_lint; then + globalErr=1 + echo "^^^ +++" + buildkite-agent annotate --context 'yarn lint' --style error --append < Date: Mon, 8 Aug 2022 09:50:15 +0800 Subject: [PATCH 11/30] ci: Fix --test_env flag passed to bazel. --- .buildkite/test-indexer.sh | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/.buildkite/test-indexer.sh b/.buildkite/test-indexer.sh index 17170fba20..66f1aa65f4 100755 --- a/.buildkite/test-indexer.sh +++ b/.buildkite/test-indexer.sh @@ -35,8 +35,7 @@ mkdir -p _out_ build_args=( "//main:scip-ruby" "//test:scip_test_runner" - "-c" - "opt" + "-c" "opt" "--config=forcedebug" "--spawn_strategy=local" ) @@ -48,8 +47,7 @@ build_args=( test_args=( "//test/scip" - "-c" - "opt" + "-c" "opt" "--config=forcedebug" "--spawn_strategy=local" ) @@ -63,11 +61,10 @@ test_args=( test_args=( "//test/scip/repos" - "-c" - "opt" - "--test_env GITHUB_ACTIONS=1" - "--test_env PATH=${PATH}" - "--test_env HOME=${HOME}" + "-c" "opt" + "--test_env" "GITHUB_ACTIONS=1" + "--test_env" "PATH=${PATH}" + "--test_env" "HOME=${HOME}" "--config=forcedebug" "--spawn_strategy=local" ) From 04e59858c914b7fb000d51a0c18bc918d40c187c Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 10:01:32 +0800 Subject: [PATCH 12/30] ci: Remove lint check for VS Code extension. --- .buildkite/linters.sh | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.buildkite/linters.sh b/.buildkite/linters.sh index 096918cab6..fae77eb5ad 100755 --- a/.buildkite/linters.sh +++ b/.buildkite/linters.sh @@ -66,22 +66,6 @@ if ! ./tools/scripts/format_website.sh -t &> format_website; then buildkite-agent annotate --context tools/scripts/format_website.sh --style error --append < format_website fi -echo "~~~ Checking the vscode extension" -pushd vscode_extension -yarn install -if ! yarn lint --output-file=yarn_lint; then - globalErr=1 - echo "^^^ +++" - buildkite-agent annotate --context 'yarn lint' --style error --append < Date: Mon, 8 Aug 2022 10:16:52 +0800 Subject: [PATCH 13/30] ci: Update test-indexer script timing, installing Ruby. --- .buildkite/test-indexer.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.buildkite/test-indexer.sh b/.buildkite/test-indexer.sh index 66f1aa65f4..f1b46c540b 100755 --- a/.buildkite/test-indexer.sh +++ b/.buildkite/test-indexer.sh @@ -2,6 +2,22 @@ # Largely based off test-compiler.sh from the Sorbet buildkite config. +get_date() { + date '+%s' +} +PREV_TIME="$(get_date)" +PREV_CMD="" +print_time_elapsed() { + CUR_TIME="$(get_date)" + local DELTA=$(( CUR_TIME - PREV_TIME )) + if [ "$DELTA" -ne 0 ]; then + echo "Ran in ${DELTA}s: $PREV_CMD\n" + fi + PREV_CMD="$BASH_COMMAND" + PREV_TIME="$CUR_TIME" +} +trap print_time_elapsed DEBUG + set -euo pipefail unameOut="$(uname -s)" @@ -62,13 +78,14 @@ test_args=( test_args=( "//test/scip/repos" "-c" "opt" - "--test_env" "GITHUB_ACTIONS=1" "--test_env" "PATH=${PATH}" "--test_env" "HOME=${HOME}" "--config=forcedebug" "--spawn_strategy=local" ) +OPENSSL_CFLAGS=-Wno-error=implicit-function-declaration asdf install ruby + ./bazel test \ --experimental_generate_json_trace_profile \ --profile=_out_/profile_repo_tests.json \ From 6df6301275768cf10a022f57fea3cc2cb86c2547 Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 10:28:28 +0800 Subject: [PATCH 14/30] ci: Work around asdf-ruby bug. --- .buildkite/test-indexer.sh | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.buildkite/test-indexer.sh b/.buildkite/test-indexer.sh index f1b46c540b..94cad2aa9c 100755 --- a/.buildkite/test-indexer.sh +++ b/.buildkite/test-indexer.sh @@ -41,7 +41,7 @@ source .buildkite/tools/setup-bazel.sh err=0 -echo "+++ running tests" +echo "--- Building" mkdir -p _out_ @@ -61,6 +61,8 @@ build_args=( --profile=_out_/profile_build.json \ "${build_args[@]}" || err=$? +echo "+++ Running snapshot tests" + test_args=( "//test/scip" "-c" "opt" @@ -75,6 +77,15 @@ test_args=( --test_output=errors \ "${test_args[@]}" || err=$? +echo "+++ Installing Ruby" + +# From https://github.com/asdf-vm/asdf-ruby/issues/125#issuecomment-958941354 +rm -rf .asdf/shims +OPENSSL_CFLAGS=-Wno-error=implicit-function-declaration asdf install ruby +asdf reshim ruby 2.7.0 + +echo "+++ Running repo tests" + test_args=( "//test/scip/repos" "-c" "opt" @@ -84,8 +95,6 @@ test_args=( "--spawn_strategy=local" ) -OPENSSL_CFLAGS=-Wno-error=implicit-function-declaration asdf install ruby - ./bazel test \ --experimental_generate_json_trace_profile \ --profile=_out_/profile_repo_tests.json \ From 7d56b6a94770dc8f7ed9efb085a0e12bec2c7aeb Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 12:24:58 +0800 Subject: [PATCH 15/30] ci: Add Buildkite caching. --- .buildkite/pipeline.yaml | 43 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index e993b7f654..394f52ef7a 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -1,19 +1,58 @@ # Loosely based off .sorbet-buildkite/pipeline.yaml -steps: +s3-settings: &s3-settings + backend: s3 + s3: + bucket: sourcegraph_buildkite_cache + endpoint: https://storage.googleapis.com + profile: buildkite + region: us-central1 + tarball: {} + +bazel-full-cache: &bazel-full-cache + id: bazel-full + key: "bazel-full" + restore-keys: + - "bazel-full" + paths: + - /usr/local/var/bazelcache + <<: *s3-settings + +bazel-repo-cache: &bazel-repo-cache + id: bazel-repos + key: "bazel-repos-{{ checksum WORKSPACE }}" + restore-keys: + - "bazel-repos-{{ checksum WORKSPACE }}" + paths: + - /usr/local/var/bazelcache/repos + <<: *s3-settings -# Basic Phase - Run super fast things and things that commonly catch bugs +bazel-build-cache: &bazel-build-cache + id: bazel-build + key: "bazel-build" + restore-keys: + - "bazel-build" + paths: + - /usr/local/var/bazelcache/build + <<: *s3-settings + +steps: - label: ":linux: linters.sh" command: .buildkite/linters.sh <<: &stateless agents: queue: stateless + plugins: + - https://github.com/sourcegraph/cache-buildkite-plugin.git#master: *bazel-full-cache - label: ":linux: test-indexer.sh" command: .buildkite/test-indexer.sh artifact_paths: - "_out_/profile*.json" <<: *stateless + plugins: + - https://github.com/sourcegraph/cache-buildkite-plugin.git#master: *bazel-repo-cache + - https://github.com/sourcegraph/cache-buildkite-plugin.git#master: *bazel-build-cache - wait: ~ From 7790eb2f6f813910981a057a612a1609af33f054 Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 12:25:21 +0800 Subject: [PATCH 16/30] ci: Hide Ruby installation in Buildkite. --- .buildkite/test-indexer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/test-indexer.sh b/.buildkite/test-indexer.sh index 94cad2aa9c..96b98980cd 100755 --- a/.buildkite/test-indexer.sh +++ b/.buildkite/test-indexer.sh @@ -77,7 +77,7 @@ test_args=( --test_output=errors \ "${test_args[@]}" || err=$? -echo "+++ Installing Ruby" +echo "--- Installing Ruby" # From https://github.com/asdf-vm/asdf-ruby/issues/125#issuecomment-958941354 rm -rf .asdf/shims From 017e45ae914ab6d0418f048edd43c4a93de1c167 Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 12:25:37 +0800 Subject: [PATCH 17/30] test: Make git clone of repo quiet. --- test/scip/repos/index_oss_repo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/scip/repos/index_oss_repo.sh b/test/scip/repos/index_oss_repo.sh index f488ba0fca..6c2e382701 100755 --- a/test/scip/repos/index_oss_repo.sh +++ b/test/scip/repos/index_oss_repo.sh @@ -51,7 +51,7 @@ rm -rf repo mkdir repo pushd repo -git clone "$CLONE_URL" . --branch "$GIT_TAG" --depth=1 +git clone -q "$CLONE_URL" . --branch "$GIT_TAG" --depth=1 if [ "$(git rev-parse HEAD)" -ne "$GIT_SHA" ]; then echo "Expected SHA: $GIT_SHA" echo "Obtained SHA: $(git rev-parse HEAD)" From b12aa1da5811fda87eb664eb0f074980b0a1d42e Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 12:27:09 +0800 Subject: [PATCH 18/30] ci: Reshim Ruby without explicit version. --- .buildkite/test-indexer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/test-indexer.sh b/.buildkite/test-indexer.sh index 96b98980cd..afb3ae608d 100755 --- a/.buildkite/test-indexer.sh +++ b/.buildkite/test-indexer.sh @@ -82,7 +82,7 @@ echo "--- Installing Ruby" # From https://github.com/asdf-vm/asdf-ruby/issues/125#issuecomment-958941354 rm -rf .asdf/shims OPENSSL_CFLAGS=-Wno-error=implicit-function-declaration asdf install ruby -asdf reshim ruby 2.7.0 +asdf reshim ruby echo "+++ Running repo tests" From f4ec4ef568f31266dbff34913b967d336ae3c790 Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 13:11:18 +0800 Subject: [PATCH 19/30] ci: Fix incorrect usage of cache keys. --- .buildkite/pipeline.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index 394f52ef7a..34168976cc 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -23,15 +23,17 @@ bazel-repo-cache: &bazel-repo-cache key: "bazel-repos-{{ checksum WORKSPACE }}" restore-keys: - "bazel-repos-{{ checksum WORKSPACE }}" + - "bazel-repos-" paths: - /usr/local/var/bazelcache/repos <<: *s3-settings bazel-build-cache: &bazel-build-cache id: bazel-build - key: "bazel-build" + key: "bazel-build-{{ env.BUILDKITE_COMMIT }}" restore-keys: - - "bazel-build" + - "bazel-build-{{ env.BUILDKITE_COMMIT }}" + - "bazel-build-" paths: - /usr/local/var/bazelcache/build <<: *s3-settings From 642bedce8dc8428e0b6434f5c0a26ca8e70ee148 Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 13:19:56 +0800 Subject: [PATCH 20/30] ci: Add pre-command hook to install aws CLI. --- .buildkite/hooks/pre-command | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100755 .buildkite/hooks/pre-command diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command new file mode 100755 index 0000000000..658af36824 --- /dev/null +++ b/.buildkite/hooks/pre-command @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -e + +if echo "$BUILDKITE_PLUGINS" | grep "cache-buildkite-plugin"; then + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip awscliv2.zip + sudo ./aws/install +fi From 104b9c6d81834cbccd116cf28c08ea81b33e178b Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 19:02:29 +0800 Subject: [PATCH 21/30] ci: Try passing AWS env vars + fix agent config. --- .buildkite/pipeline.yaml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index 34168976cc..0072a1db8d 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -38,12 +38,17 @@ bazel-build-cache: &bazel-build-cache - /usr/local/var/bazelcache/build <<: *s3-settings +aws-envvars: &aws-envvars + AWS_CONFIG_FILE: /buildkite/.aws/config + AWS_SHARED_CREDENTIALS_FILE: /buildkite/.aws/credentials + steps: - label: ":linux: linters.sh" command: .buildkite/linters.sh - <<: &stateless - agents: - queue: stateless + agents: + queue: stateless + env: + <<: *aws-envvars plugins: - https://github.com/sourcegraph/cache-buildkite-plugin.git#master: *bazel-full-cache @@ -51,7 +56,10 @@ steps: command: .buildkite/test-indexer.sh artifact_paths: - "_out_/profile*.json" - <<: *stateless + agents: + queue: stateless + env: + <<: *aws-envvars plugins: - https://github.com/sourcegraph/cache-buildkite-plugin.git#master: *bazel-repo-cache - https://github.com/sourcegraph/cache-buildkite-plugin.git#master: *bazel-build-cache From e34e4157c8c5925e5710740bedfcfceab19dcd6d Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 19:50:52 +0800 Subject: [PATCH 22/30] ci: Create AWS config files before trying to use them. --- .buildkite/hooks/pre-command | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 658af36824..14e9816fe1 100755 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -6,4 +6,14 @@ if echo "$BUILDKITE_PLUGINS" | grep "cache-buildkite-plugin"; then curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" unzip awscliv2.zip sudo ./aws/install + + # From https://sourcegraph.sourcegraph.com/github.com/sourcegraph/sourcegraph/-/blob/enterprise/cmd/server/pre-build.sh?L40-47 + AWS_CONFIG_DIR_PATH="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/buildkite/.aws" + mkdir -p "$AWS_CONFIG_DIR_PATH" + AWS_CONFIG_FILE="$AWS_CONFIG_DIR_PATH/config" + export AWS_CONFIG_FILE + AWS_SHARED_CREDENTIALS_FILE="/buildkite/.aws/credentials" + export AWS_SHARED_CREDENTIALS_FILE + aws configure set aws_access_key_id "$BUILDKITE_HMAC_KEY" --profile buildkite + aws configure set aws_secret_access_key "$BUILDKITE_HMAC_SECRET" --profile buildkite fi From 3c64ab8f7186887ad2caf34a3a025bb673b31dfd Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 20:21:19 +0800 Subject: [PATCH 23/30] ci: Turn off prettier check as it is failing inexplicably. --- .buildkite/linters.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.buildkite/linters.sh b/.buildkite/linters.sh index fae77eb5ad..345095a185 100755 --- a/.buildkite/linters.sh +++ b/.buildkite/linters.sh @@ -59,13 +59,6 @@ if ! ./tools/scripts/lint_sh.sh -t &> lint_sh; then buildkite-agent annotate --context tools/scripts/lint_sh.sh --style error --append < lint_sh fi -echo "~~~ Checking markdown formatting" -if ! ./tools/scripts/format_website.sh -t &> format_website; then - globalErr=1 - echo "^^^ +++" - buildkite-agent annotate --context tools/scripts/format_website.sh --style error --append < format_website -fi - echo "~~~" if [ "$globalErr" -ne 0 ]; then exit $globalErr From 816d60622e6e1463344f5d017292aaab5aea1c7e Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 20:47:12 +0800 Subject: [PATCH 24/30] ci: Minor cleanups for repo test script. --- test/scip/repos/index_oss_repo.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/scip/repos/index_oss_repo.sh b/test/scip/repos/index_oss_repo.sh index 6c2e382701..2af179ae82 100755 --- a/test/scip/repos/index_oss_repo.sh +++ b/test/scip/repos/index_oss_repo.sh @@ -52,7 +52,7 @@ mkdir repo pushd repo git clone -q "$CLONE_URL" . --branch "$GIT_TAG" --depth=1 -if [ "$(git rev-parse HEAD)" -ne "$GIT_SHA" ]; then +if [ "$(git rev-parse HEAD)" != "$GIT_SHA" ]; then echo "Expected SHA: $GIT_SHA" echo "Obtained SHA: $(git rev-parse HEAD)" exit 1 @@ -60,7 +60,7 @@ fi eval "$PREP_CMD" -if [ ! -z "$PATCH_ABSPATH" ]; then +if [ -n "$PATCH_ABSPATH" ]; then git apply "$PATCH_ABSPATH" git diff -U0 fi From e724b9991a27e62d085afc4b17b659246e229956 Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 20:49:11 +0800 Subject: [PATCH 25/30] ci: Set global ruby version with asdf. --- .buildkite/test-indexer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/test-indexer.sh b/.buildkite/test-indexer.sh index afb3ae608d..1a5377e48c 100755 --- a/.buildkite/test-indexer.sh +++ b/.buildkite/test-indexer.sh @@ -82,7 +82,7 @@ echo "--- Installing Ruby" # From https://github.com/asdf-vm/asdf-ruby/issues/125#issuecomment-958941354 rm -rf .asdf/shims OPENSSL_CFLAGS=-Wno-error=implicit-function-declaration asdf install ruby -asdf reshim ruby +asdf global ruby 2.7.0 echo "+++ Running repo tests" From 737171776d25a5f038839e2cf632d88e7c74c31c Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 21:04:11 +0800 Subject: [PATCH 26/30] ci: Unpack AWS CLI in temporary directory. --- .buildkite/hooks/pre-command | 37 ++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 14e9816fe1..f4297f7a8e 100755 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -3,17 +3,30 @@ set -e if echo "$BUILDKITE_PLUGINS" | grep "cache-buildkite-plugin"; then - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - unzip awscliv2.zip - sudo ./aws/install + if ! which aws; then + echo "pwd: $PWD" + ls - # From https://sourcegraph.sourcegraph.com/github.com/sourcegraph/sourcegraph/-/blob/enterprise/cmd/server/pre-build.sh?L40-47 - AWS_CONFIG_DIR_PATH="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/buildkite/.aws" - mkdir -p "$AWS_CONFIG_DIR_PATH" - AWS_CONFIG_FILE="$AWS_CONFIG_DIR_PATH/config" - export AWS_CONFIG_FILE - AWS_SHARED_CREDENTIALS_FILE="/buildkite/.aws/credentials" - export AWS_SHARED_CREDENTIALS_FILE - aws configure set aws_access_key_id "$BUILDKITE_HMAC_KEY" --profile buildkite - aws configure set aws_secret_access_key "$BUILDKITE_HMAC_SECRET" --profile buildkite + AWS_TEMPDIR="$(mktemp -d -t '-awscli')" + pushd AWS_TEMPDIR + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip awscliv2.zip + sudo ./aws/install + ls + popd + rm -rf "$AWS_TEMPDIR" + + echo "pwd: $PWD" + ls + + # From https://sourcegraph.sourcegraph.com/github.com/sourcegraph/sourcegraph/-/blob/enterprise/cmd/server/pre-build.sh?L40-47 + AWS_CONFIG_DIR_PATH="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/buildkite/.aws" + mkdir -p "$AWS_CONFIG_DIR_PATH" + AWS_CONFIG_FILE="$AWS_CONFIG_DIR_PATH/config" + export AWS_CONFIG_FILE + AWS_SHARED_CREDENTIALS_FILE="/buildkite/.aws/credentials" + export AWS_SHARED_CREDENTIALS_FILE + aws configure set aws_access_key_id "$BUILDKITE_HMAC_KEY" --profile buildkite + aws configure set aws_secret_access_key "$BUILDKITE_HMAC_SECRET" --profile buildkite + fi fi From ac66f850b8ce988ff440db25fdcea45dc5eb84d8 Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 21:23:31 +0800 Subject: [PATCH 27/30] ci: Work around buggy argument parsing in mktemp --- .buildkite/hooks/pre-command | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index f4297f7a8e..bda94291af 100755 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -7,7 +7,7 @@ if echo "$BUILDKITE_PLUGINS" | grep "cache-buildkite-plugin"; then echo "pwd: $PWD" ls - AWS_TEMPDIR="$(mktemp -d -t '-awscli')" + AWS_TEMPDIR="$(mktemp -d -t "_awscli")" pushd AWS_TEMPDIR curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" unzip awscliv2.zip From 6d9244156bd68dc483de7b3c730592fe3ab95656 Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 21:31:50 +0800 Subject: [PATCH 28/30] ci: Create template with Xs for Linux mktemp. --- .buildkite/hooks/pre-command | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index bda94291af..736a321a27 100755 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -7,7 +7,7 @@ if echo "$BUILDKITE_PLUGINS" | grep "cache-buildkite-plugin"; then echo "pwd: $PWD" ls - AWS_TEMPDIR="$(mktemp -d -t "_awscli")" + AWS_TEMPDIR="$(mktemp -d -t "awscli-XXXXXXXX")" pushd AWS_TEMPDIR curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" unzip awscliv2.zip From d357a7f1a5ffc2535b8507c02239a5012dd19a1f Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 21:40:29 +0800 Subject: [PATCH 29/30] ci: I apologize for writing Bash, I apologize for writing Bash, I apologize for writing Bash --- .buildkite/hooks/pre-command | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 736a321a27..81fc90367d 100755 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -8,7 +8,7 @@ if echo "$BUILDKITE_PLUGINS" | grep "cache-buildkite-plugin"; then ls AWS_TEMPDIR="$(mktemp -d -t "awscli-XXXXXXXX")" - pushd AWS_TEMPDIR + pushd "$AWS_TEMPDIR" curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" unzip awscliv2.zip sudo ./aws/install From bb5ed5e3fc0ecc4b987a5df417efde5712584038 Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 8 Aug 2022 22:04:33 +0800 Subject: [PATCH 30/30] ci: Remove lint step from buildkite. We'll add it to GitHub Actions later. --- .buildkite/pipeline.yaml | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index 0072a1db8d..aa11e32748 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -9,15 +9,6 @@ s3-settings: &s3-settings region: us-central1 tarball: {} -bazel-full-cache: &bazel-full-cache - id: bazel-full - key: "bazel-full" - restore-keys: - - "bazel-full" - paths: - - /usr/local/var/bazelcache - <<: *s3-settings - bazel-repo-cache: &bazel-repo-cache id: bazel-repos key: "bazel-repos-{{ checksum WORKSPACE }}" @@ -38,20 +29,7 @@ bazel-build-cache: &bazel-build-cache - /usr/local/var/bazelcache/build <<: *s3-settings -aws-envvars: &aws-envvars - AWS_CONFIG_FILE: /buildkite/.aws/config - AWS_SHARED_CREDENTIALS_FILE: /buildkite/.aws/credentials - steps: - - label: ":linux: linters.sh" - command: .buildkite/linters.sh - agents: - queue: stateless - env: - <<: *aws-envvars - plugins: - - https://github.com/sourcegraph/cache-buildkite-plugin.git#master: *bazel-full-cache - - label: ":linux: test-indexer.sh" command: .buildkite/test-indexer.sh artifact_paths: @@ -59,7 +37,8 @@ steps: agents: queue: stateless env: - <<: *aws-envvars + AWS_CONFIG_FILE: /buildkite/.aws/config + AWS_SHARED_CREDENTIALS_FILE: /buildkite/.aws/credentials plugins: - https://github.com/sourcegraph/cache-buildkite-plugin.git#master: *bazel-repo-cache - https://github.com/sourcegraph/cache-buildkite-plugin.git#master: *bazel-build-cache