From 65b340bb9acad1305fc1eab94bdd2498a65ac579 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Thu, 26 Jan 2023 16:46:32 +0500 Subject: [PATCH 01/58] Add GCP image workflow --- .github/workflows/manual_gcp_image_stgn.yml | 39 +++++++++++++ runner/ami/packer/gcp-image.json | 64 +++++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 .github/workflows/manual_gcp_image_stgn.yml create mode 100644 runner/ami/packer/gcp-image.json diff --git a/.github/workflows/manual_gcp_image_stgn.yml b/.github/workflows/manual_gcp_image_stgn.yml new file mode 100644 index 0000000000..9cbf054622 --- /dev/null +++ b/.github/workflows/manual_gcp_image_stgn.yml @@ -0,0 +1,39 @@ +name: Manual GCP image build for STGN + +on: + workflow_dispatch: + push: + +jobs: + gcp-image-build: + defaults: + run: + working-directory: runner + strategy: + matrix: + variants: [ gcp-image ] + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + steps: + - uses: actions/checkout@v3 + - name: 'Authenticate to Google Cloud' + uses: 'google-github-actions/auth@v1' + with: + workload_identity_provider: 'projects/531508670106/locations/global/workloadIdentityPools/github-identity-pool/providers/github-id-provider' + service_account: 'github-actions@dstack.iam.gserviceaccount.com' + create_credentials_file: true + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v1' + - name: Download packer + run: | + wget https://releases.hashicorp.com/packer/1.8.0/packer_1.8.0_linux_amd64.zip + unzip packer_1.8.0_linux_amd64.zip + chmod +x packer + cp -R ami/packer/* . + - name: Run packer + run: | + VERSION=${{ github.run_number }} + ./packer build -var build_id=$VERSION ${{ matrix.variants }}.json + gcloud compute images add-iam-policy-binding stgn-dstack-$VERSION --member='allAuthenticatedUsers' --role='roles/compute.imageUser' diff --git a/runner/ami/packer/gcp-image.json b/runner/ami/packer/gcp-image.json new file mode 100644 index 0000000000..b4fafe903b --- /dev/null +++ b/runner/ami/packer/gcp-image.json @@ -0,0 +1,64 @@ +{ + "variables": { + "build_id": "{{timestamp}}", + "build_prefix": "stgn-", + "docker_version": "20.10.17", + "dstack_stage": "STGN" + }, + "builders": [ + { + "type": "googlecompute", + "project_id": "dstack", + "source_image": "ubuntu-1804-bionic-v20220616", + "image_name": "{{user `build_prefix`}}dstack-{{user `build_id`}}", + "instance_name": "{{user `build_prefix`}}dstack-{{user `build_id`}}", + "ssh_username": "ubuntu", + "zone": "us-central1-a" + } + ], + "provisioners": [ + { + "type": "shell", + "inline": ["cloud-init status --long --wait"] + }, + { + "type": "shell", + "scripts": [ + "provisioners/kernel/apt-upgrade.sh", + "provisioners/kernel/apt-daily.sh", + "provisioners/kernel/apt-packages.sh", + "provisioners/kernel/kernel-tuning.sh" + ] + }, + { + "type": "shell", + "environment_vars": ["DSTACK_STAGE={{user `dstack_stage`}}"], + "script": "provisioners/get-dstack-runner.sh" + }, + { + "type": "file", + "source": "provisioners/install-docker.sh", + "destination": "/tmp/install-docker.sh" + }, + { + "type": "file", + "source": "provisioners/run-docker", + "destination": "/tmp/run-docker" + }, + { + "type": "shell", + "inline": [ + "cd /tmp", + "chmod +x install-docker.sh", + "./install-docker.sh --version {{user `docker_version`}}"] + }, + { + "type": "shell", + "script": "provisioners/docker-image-without-cuda.sh" + }, + { + "type": "shell", + "inline": ["/usr/local/bin/dstack-runner --version"] + } + ] +} From 8d16dc80d725846a882fc011dc0c6b227f44d628 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Mon, 30 Jan 2023 17:15:04 +0500 Subject: [PATCH 02/58] Move runners.py to core --- cli/dstack/backend/__init__.py | 2 +- cli/dstack/backend/aws/local.py | 2 +- cli/dstack/backend/aws/runners.py | 2 +- cli/dstack/{ => core}/runners.py | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename cli/dstack/{ => core}/runners.py (100%) diff --git a/cli/dstack/backend/__init__.py b/cli/dstack/backend/__init__.py index 87ada3448a..b3671df817 100644 --- a/cli/dstack/backend/__init__.py +++ b/cli/dstack/backend/__init__.py @@ -10,7 +10,7 @@ from dstack.core.log_event import LogEvent from dstack.core.tag import TagHead from dstack.core.run import RunHead -from dstack.runners import Runner +from dstack.core.runners import Runner class BackendType(Enum): diff --git a/cli/dstack/backend/aws/local.py b/cli/dstack/backend/aws/local.py index 7a5add394c..a6d407cb79 100644 --- a/cli/dstack/backend/aws/local.py +++ b/cli/dstack/backend/aws/local.py @@ -15,7 +15,7 @@ from dstack import version from dstack.core.config import BackendConfig -from dstack.runners import Resources, Gpu +from dstack.core.runners import Resources, Gpu def _arch() -> str: diff --git a/cli/dstack/backend/aws/runners.py b/cli/dstack/backend/aws/runners.py index 64a586c4ae..08b7044a29 100644 --- a/cli/dstack/backend/aws/runners.py +++ b/cli/dstack/backend/aws/runners.py @@ -14,7 +14,7 @@ from dstack.core.request import RequestStatus, RequestHead from dstack.core.job import Job, JobStatus, Requirements from dstack.core.repo import RepoAddress, _repo_address_path -from dstack.runners import Resources, Runner, Gpu +from dstack.core.runners import Resources, Runner, Gpu CREATE_INSTANCE_RETRY_RATE_SECS = 3 diff --git a/cli/dstack/runners.py b/cli/dstack/core/runners.py similarity index 100% rename from cli/dstack/runners.py rename to cli/dstack/core/runners.py From 9ec5c8e1a9689db077f56d32872ed7649f7f23fc Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Mon, 30 Jan 2023 17:16:53 +0500 Subject: [PATCH 03/58] Fix cli commands import --- cli/dstack/cli/handlers.py | 1 + cli/dstack/core/instance.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cli/dstack/cli/handlers.py b/cli/dstack/cli/handlers.py index 08598e7127..7e4c5f25ac 100644 --- a/cli/dstack/cli/handlers.py +++ b/cli/dstack/cli/handlers.py @@ -5,6 +5,7 @@ def cli_initialize(parser): package = dstack.cli.commands for importer, modname, ispkg in pkgutil.iter_modules(package.__path__): + modname = f"dstack.cli.commands.{modname}" importer.find_module(modname).load_module(modname) commands = [cls(parser=parser) for cls in dstack.cli.commands.BasicCommand.__subclasses__()] # pylint: disable=E1101 diff --git a/cli/dstack/core/instance.py b/cli/dstack/core/instance.py index fb94ffb1f3..95850e9a77 100644 --- a/cli/dstack/core/instance.py +++ b/cli/dstack/core/instance.py @@ -1,4 +1,4 @@ -from dstack.runners import Resources +from dstack.core.runners import Resources class InstanceType: @@ -7,4 +7,4 @@ def __init__(self, instance_name: str, resources: Resources): self.resources = resources def __str__(self) -> str: - return f'InstanceType(instance_name="{self.instance_name}", resources={self.resources})'.__str__() + return f'InstanceType(instance_name="{self.instance_name}", resources={self.resources})' From 1820fde1a884935dbc91aefa9e16ce0a3a3c54be Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Mon, 30 Jan 2023 18:00:30 +0500 Subject: [PATCH 04/58] Turn off gcp_image workflow on push --- .github/workflows/manual_gcp_image_stgn.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/manual_gcp_image_stgn.yml b/.github/workflows/manual_gcp_image_stgn.yml index 9cbf054622..5226092c76 100644 --- a/.github/workflows/manual_gcp_image_stgn.yml +++ b/.github/workflows/manual_gcp_image_stgn.yml @@ -1,8 +1,6 @@ name: Manual GCP image build for STGN -on: - workflow_dispatch: - push: +on: [workflow_dispatch] jobs: gcp-image-build: @@ -36,4 +34,7 @@ jobs: run: | VERSION=${{ github.run_number }} ./packer build -var build_id=$VERSION ${{ matrix.variants }}.json + - name: Publish image + run: | + VERSION=${{ github.run_number }} gcloud compute images add-iam-policy-binding stgn-dstack-$VERSION --member='allAuthenticatedUsers' --role='roles/compute.imageUser' From dc691e288672f19cc74e71a2c772494160262cd1 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Mon, 30 Jan 2023 18:01:35 +0500 Subject: [PATCH 05/58] GCP WIP --- .gitignore | 2 + cli/dstack/api/backend.py | 29 ++-- cli/dstack/backend/__init__.py | 2 +- cli/dstack/backend/gcp/__init__.py | 132 +++++++++++++++++ cli/dstack/backend/gcp/config.py | 17 +++ cli/dstack/backend/gcp/jobs.py | 63 ++++++++ cli/dstack/backend/gcp/runners.py | 225 +++++++++++++++++++++++++++++ cli/dstack/backend/gcp/runs.py | 41 ++++++ cli/dstack/backend/gcp/storage.py | 23 +++ cli/dstack/backend/gcp/utils.py | 51 +++++++ 10 files changed, 570 insertions(+), 15 deletions(-) create mode 100644 cli/dstack/backend/gcp/__init__.py create mode 100644 cli/dstack/backend/gcp/config.py create mode 100644 cli/dstack/backend/gcp/jobs.py create mode 100644 cli/dstack/backend/gcp/runners.py create mode 100644 cli/dstack/backend/gcp/runs.py create mode 100644 cli/dstack/backend/gcp/storage.py create mode 100644 cli/dstack/backend/gcp/utils.py diff --git a/.gitignore b/.gitignore index 7d20462677..3550bb858e 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ /cli/dstack.egg-info/ /runner/runner + +/my/ diff --git a/cli/dstack/api/backend.py b/cli/dstack/api/backend.py index 28866e5631..8e75ec3dc4 100644 --- a/cli/dstack/api/backend.py +++ b/cli/dstack/api/backend.py @@ -2,30 +2,31 @@ from dstack.backend import Backend from dstack.core.error import BackendError from dstack.backend.aws import AwsBackend +from dstack.backend.gcp import GCPBackend from dstack.backend.local import LocalBackend + DEFAULT_REMOTE = "aws" DEFAULT = "local" +backends_classes = [AwsBackend, GCPBackend, LocalBackend] + + +def get_all_backends(): + return [backend_cls() for backend_cls in backends_classes] + + def list_backends() -> List[Backend]: - all_backends = [cls() for cls in Backend.__subclasses__()] # pylint: disable=E1101 - return [current_backend for current_backend in all_backends if current_backend.loaded()] + return [backend for backend in get_all_backends() if backend.loaded()] def dict_backends() -> Dict[str, Backend]: - all_backends = [cls() for cls in Backend.__subclasses__()] # pylint: disable=E1101 - d = {} - for current_backend in all_backends: - if current_backend.loaded(): - d[current_backend.name] = current_backend - return d + return {backend.name: backend for backend in list_backends()} def get_backend_by_name(name: str) -> Optional[Backend]: - all_backends = [cls() for cls in Backend.__subclasses__()] # pylint: disable=E1101 - for current_backend in all_backends: - if current_backend.loaded(): - if current_backend.NAME == name: - return current_backend - raise BackendError(f"Not a backend named {name}") \ No newline at end of file + backend = dict_backends().get(name) + if backend is None: + raise BackendError(f"Backend {name} not found") + return backend diff --git a/cli/dstack/backend/__init__.py b/cli/dstack/backend/__init__.py index b3671df817..d99b9f8501 100644 --- a/cli/dstack/backend/__init__.py +++ b/cli/dstack/backend/__init__.py @@ -27,7 +27,7 @@ class Backend(object): @property def name(self) -> str: - return self.NAME + "*" if self.type() == BackendType.REMOTE else "" + return self.NAME # + "*" if self.type() == BackendType.REMOTE else "" def configure(self): pass diff --git a/cli/dstack/backend/gcp/__init__.py b/cli/dstack/backend/gcp/__init__.py new file mode 100644 index 0000000000..f5a6284bad --- /dev/null +++ b/cli/dstack/backend/gcp/__init__.py @@ -0,0 +1,132 @@ +from typing import Optional, List, Generator +import uuid + +from dstack.backend.gcp.config import GCPConfig +from dstack.backend import * +from dstack.core.repo import RepoProtocol +from dstack.core.repo import RepoData, RepoAddress +from dstack.backend.gcp import jobs, runs, storage + + +class GCPBackend(Backend): + NAME = "gcp" + + def __init__(self): + self.config = GCPConfig() + self._storage_client = storage.get_client(project_id=self.config.project_id) + self.configure() + self._loaded = True + + def configure(self): + self._bucket = storage.get_or_create_bucket(self._storage_client, self.config.bucket_name) + + def create_run(self, repo_address: RepoAddress) -> str: + return runs.create_run(self._bucket) + + def submit_job(self, job: Job, counter: List[int]): + job.runner_id = uuid.uuid4().hex + jobs.create_job(self._bucket, job) + jobs.run_job(self.config, self._bucket, job) + + def get_job(self, repo_address: RepoAddress, job_id: str) -> Optional[Job]: + return jobs.get_job(self._bucket, repo_address, job_id) + + def list_job_heads(self, repo_address: RepoAddress, run_name: Optional[str] = None) -> List[JobHead]: + return jobs.list_job_heads(self._bucket, repo_address, run_name) + + def list_jobs(self, repo_address: RepoAddress, run_name: str) -> List[Job]: + # Doesn't seem to be used + raise NotImplementedError() + + def stop_job(self, repo_address: RepoAddress, job_id: str, abort: bool): + pass + + def delete_job_head(self, repo_address: RepoAddress, job_id: str): + pass + + def list_run_heads(self, repo_address: RepoAddress, run_name: Optional[str] = None, + include_request_heads: bool = True) -> List[RunHead]: + # Doesn't seem to be used + raise NotImplementedError() + + def get_run_heads(self, repo_address: RepoAddress, job_heads: List[JobHead], + include_request_heads: bool = True) -> List[RunHead]: + return runs.get_run_heads(self._bucket, repo_address, job_heads, include_request_heads) + + def poll_logs(self, repo_address: RepoAddress, job_heads: List[JobHead], start_time: int, + attached: bool) -> Generator[LogEvent, None, None]: + pass + + def query_logs(self, repo_address: RepoAddress, run_name: str, start_time: int, end_time: Optional[int], + next_token: Optional[str], job_host_names: Dict[str, Optional[str]], + job_ports: Dict[str, Optional[List[int]]], job_app_specs: Dict[str, Optional[List[AppSpec]]]) \ + -> Tuple[List[LogEvent], Optional[str], Dict[str, Optional[str]], Dict[str, Optional[List[int]]], + Dict[str, Optional[List[AppSpec]]]]: + pass + + def download_run_artifact_files(self, repo_address: RepoAddress, run_name: str, + output_dir: Optional[str]): + pass + + def list_run_artifact_files(self, repo_address: RepoAddress, run_name: str) -> \ + Generator[Tuple[str, str, int], None, None]: + pass + + def list_tag_heads(self, repo_address: RepoAddress) -> List[TagHead]: + pass + + def get_tag_head(self, repo_address: RepoAddress, tag_name: str) -> Optional[TagHead]: + pass + + def add_tag_from_run(self, repo_address: RepoAddress, tag_name: str, run_name: str, + run_jobs: Optional[List[Job]]): + pass + + def add_tag_from_local_dirs(self, repo_data: RepoData, tag_name: str, local_dirs: List[str]): + pass + + def delete_tag_head(self, repo_address: RepoAddress, tag_head: TagHead): + pass + + def list_repo_heads(self) -> List[RepoHead]: + pass + + def update_repo_last_run_at(self, repo_address: RepoAddress, last_run_at: int): + pass + + def increment_repo_tags_count(self, repo_address: RepoAddress): + pass + + def decrement_repo_tags_count(self, repo_address: RepoAddress): + pass + + def delete_repo(self, repo_address: RepoAddress): + pass + + def get_repo_credentials(self, repo_address: RepoAddress) -> Optional[RepoCredentials]: + return RepoCredentials(RepoProtocol.HTTPS, None, None) + + def save_repo_credentials(self, repo_address: RepoAddress, repo_credentials: RepoCredentials): + pass + + def list_run_artifact_files_and_folders(self, repo_address: RepoAddress, job_id: str, + path: str) -> List[Tuple[str, bool]]: + pass + + def list_secret_names(self, repo_address: RepoAddress) -> List[str]: + pass + + def get_secret(self, repo_address: RepoAddress, secret_name: str) -> Optional[Secret]: + pass + + def add_secret(self, repo_address: RepoAddress, secret: Secret): + pass + + def update_secret(self, repo_address: RepoAddress, secret: Secret): + pass + + def delete_secret(self, repo_address: RepoAddress, secret_name: str): + pass + + def type(self) -> BackendType: + return BackendType.REMOTE diff --git a/cli/dstack/backend/gcp/config.py b/cli/dstack/backend/gcp/config.py new file mode 100644 index 0000000000..e3bb6f7467 --- /dev/null +++ b/cli/dstack/backend/gcp/config.py @@ -0,0 +1,17 @@ +from pathlib import Path + +from dstack.core.config import BackendConfig, get_config_path + + +class GCPConfig(BackendConfig): + def __init__(self): + super().__init__() + self.project_id = "dstack" + self.zone = "us-central1-a" + self.bucket_name = "dstack-bucket" + + def load(self, path: Path = get_config_path()): + super().load(path=path) + + def save(self, path: Path = get_config_path()): + pass diff --git a/cli/dstack/backend/gcp/jobs.py b/cli/dstack/backend/gcp/jobs.py new file mode 100644 index 0000000000..bc2267f244 --- /dev/null +++ b/cli/dstack/backend/gcp/jobs.py @@ -0,0 +1,63 @@ +from typing import Optional, List + +from google.cloud.storage import Bucket +import yaml + +from dstack.core.instance import InstanceType +from dstack.backend import Job, JobHead, RepoAddress +from dstack.core.job import JobStatus +from dstack.core.repo import _repo_address_path +from dstack.core.runners import Resources, Runner +from dstack.backend.aws.jobs import serialize_job, _job_head_key, unserialize_job +from dstack.backend.gcp import runners, storage +from dstack.backend.gcp.config import GCPConfig + + +def create_job(bucket: Bucket, job: Job): + counter = 0 + job_id = f"{job.run_name},{job.workflow_name or ''},{counter}" + job.set_id(job_id) + prefix = f"jobs/{_repo_address_path(job.repo_address)}" + key = f"{prefix}/{job_id}.yaml" + storage.put_object(bucket, key, yaml.dump(serialize_job(job))) + storage.put_object(bucket, _job_head_key(job), '') + + +def run_job(gcp_config: GCPConfig, bucket: Bucket, job: Job): + instance_type = _get_instance_type(job) + runner = Runner(job.runner_id, None, instance_type.resources, job) + runners.create_runner(bucket, runner) + runners.launch_runner(gcp_config, bucket, runner) + + +def _get_instance_type(job: Job) -> InstanceType: + return InstanceType( + instance_name="n1-standard-1", + resources=Resources(cpus=1, memory_mib=3750, gpus=[], interruptible=False, local=False), + ) + + +def list_job_heads(bucket: Bucket, repo_address: RepoAddress, run_name: Optional[str] = None) -> List[JobHead]: + jobs_prefix = f"jobs/{_repo_address_path(repo_address)}" + job_head_key_prefix = f"{jobs_prefix}/l;" + job_head_key_run_prefix = job_head_key_prefix + run_name if run_name else job_head_key_prefix + # TODO pagination + blobs = bucket.client.list_blobs(bucket.name, prefix=job_head_key_run_prefix) + job_heads = [] + for blob in blobs: + t = blob.name.split(";") + _, job_id, provider_name, local_repo_user_name, submitted_at, status, artifacts, app_names, tag_name = t + run_name, workflow_name, job_index = tuple(job_id.split(',')) + job_heads.append(JobHead(job_id, repo_address, run_name, workflow_name or None, provider_name, + local_repo_user_name, JobStatus(status), int(submitted_at), + artifacts.split(',') if artifacts else None, tag_name or None, + app_names.split(',') or None)) + return job_heads + + +def get_job(bucket: Bucket, repo_address: RepoAddress, job_id: str) -> Optional[Job]: + prefix = f"jobs/{_repo_address_path(repo_address)}" + key = f"{prefix}/{job_id}.yaml" + obj = storage.read_object(bucket, key) + job = unserialize_job(yaml.load(obj, yaml.FullLoader)) + return job diff --git a/cli/dstack/backend/gcp/runners.py b/cli/dstack/backend/gcp/runners.py new file mode 100644 index 0000000000..f8fa2dd1da --- /dev/null +++ b/cli/dstack/backend/gcp/runners.py @@ -0,0 +1,225 @@ +import re +from typing import Any, List +import warnings + +from google.cloud.storage import Bucket +from google.cloud import compute_v1 +import yaml + +from dstack.backend.gcp import storage, utils as gcp_utils +from dstack.core.runners import Runner +from dstack.backend.aws.runners import _serialize_runner +from dstack.backend.gcp.config import GCPConfig + + +def create_runner(bucket: Bucket, runner: Runner): + runner_filepath = f"runners/{runner.runner_id}.yaml" + storage.put_object(bucket, runner_filepath, yaml.dump(_serialize_runner(runner))) + + +def launch_runner(gcp_config: GCPConfig, bucket: Bucket, runner: Runner): + return + _launch_instance( + project_id=gcp_config.project_id, + zone=gcp_config.zone, + image_name="stgn-dstack-5", + instance_name=f"dstack-{runner.job.run_name}", + ) + + +def _launch_instance(project_id: str, zone: str, image_name: str, instance_name: str) -> compute_v1.Instance: + disk = _disk_from_image( + disk_type=f"zones/{zone}/diskTypes/pd-balanced", + disk_size_gb=20, + boot=True, + source_image=f"projects/dstack/global/images/{image_name}", + auto_delete=False, + ) + instance = _create_instance( + project_id=project_id, + zone=zone, + instance_name=instance_name, + disks=[disk], + ) + return instance + + +def _disk_from_image( + disk_type: str, + disk_size_gb: int, + boot: bool, + source_image: str, + auto_delete: bool = True, +) -> compute_v1.AttachedDisk: + """ + Create an AttachedDisk object to be used in VM instance creation. Uses an image as the + source for the new disk. + + Args: + disk_type: the type of disk you want to create. This value uses the following format: + "zones/{zone}/diskTypes/(pd-standard|pd-ssd|pd-balanced|pd-extreme)". + For example: "zones/us-west3-b/diskTypes/pd-ssd" + disk_size_gb: size of the new disk in gigabytes + boot: boolean flag indicating whether this disk should be used as a boot disk of an instance + source_image: source image to use when creating this disk. You must have read access to this disk. This can be one + of the publicly available images or an image from one of your projects. + This value uses the following format: "projects/{project_name}/global/images/{image_name}" + auto_delete: boolean flag indicating whether this disk should be deleted with the VM that uses it + + Returns: + AttachedDisk object configured to be created using the specified image. + """ + boot_disk = compute_v1.AttachedDisk() + initialize_params = compute_v1.AttachedDiskInitializeParams() + initialize_params.source_image = source_image + initialize_params.disk_size_gb = disk_size_gb + initialize_params.disk_type = disk_type + boot_disk.initialize_params = initialize_params + # Remember to set auto_delete to True if you want the disk to be deleted when you delete + # your VM instance. + boot_disk.auto_delete = auto_delete + boot_disk.boot = boot + return boot_disk + + +def _create_instance( + project_id: str, + zone: str, + instance_name: str, + disks: List[compute_v1.AttachedDisk], + machine_type: str = "n1-standard-1", + network_link: str = "global/networks/default", + subnetwork_link: str = None, + internal_ip: str = None, + external_access: bool = False, + external_ipv4: str = None, + accelerators: List[compute_v1.AcceleratorConfig] = None, + preemptible: bool = False, + spot: bool = False, + instance_termination_action: str = "STOP", + custom_hostname: str = None, + delete_protection: bool = False, +) -> compute_v1.Instance: + """ + Send an instance creation request to the Compute Engine API and wait for it to complete. + + Args: + project_id: project ID or project number of the Cloud project you want to use. + zone: name of the zone to create the instance in. For example: "us-west3-b" + instance_name: name of the new virtual machine (VM) instance. + disks: a list of compute_v1.AttachedDisk objects describing the disks + you want to attach to your new instance. + machine_type: machine type of the VM being created. This value uses the + following format: "zones/{zone}/machineTypes/{type_name}". + For example: "zones/europe-west3-c/machineTypes/f1-micro" + network_link: name of the network you want the new instance to use. + For example: "global/networks/default" represents the network + named "default", which is created automatically for each project. + subnetwork_link: name of the subnetwork you want the new instance to use. + This value uses the following format: + "regions/{region}/subnetworks/{subnetwork_name}" + internal_ip: internal IP address you want to assign to the new instance. + By default, a free address from the pool of available internal IP addresses of + used subnet will be used. + external_access: boolean flag indicating if the instance should have an external IPv4 + address assigned. + external_ipv4: external IPv4 address to be assigned to this instance. If you specify + an external IP address, it must live in the same region as the zone of the instance. + This setting requires `external_access` to be set to True to work. + accelerators: a list of AcceleratorConfig objects describing the accelerators that will + be attached to the new instance. + preemptible: boolean value indicating if the new instance should be preemptible + or not. Preemptible VMs have been deprecated and you should now use Spot VMs. + spot: boolean value indicating if the new instance should be a Spot VM or not. + instance_termination_action: What action should be taken once a Spot VM is terminated. + Possible values: "STOP", "DELETE" + custom_hostname: Custom hostname of the new VM instance. + Custom hostnames must conform to RFC 1035 requirements for valid hostnames. + delete_protection: boolean value indicating if the new virtual machine should be + protected against deletion or not. + Returns: + Instance object. + """ + instance_client = compute_v1.InstancesClient() + + # Use the network interface provided in the network_link argument. + network_interface = compute_v1.NetworkInterface() + network_interface.name = network_link + if subnetwork_link: + network_interface.subnetwork = subnetwork_link + + if internal_ip: + network_interface.network_i_p = internal_ip + + if external_access: + access = compute_v1.AccessConfig() + access.type_ = compute_v1.AccessConfig.Type.ONE_TO_ONE_NAT.name + access.name = "External NAT" + access.network_tier = access.NetworkTier.PREMIUM.name + if external_ipv4: + access.nat_i_p = external_ipv4 + network_interface.access_configs = [access] + + # Collect information into the Instance object. + instance = compute_v1.Instance() + instance.network_interfaces = [network_interface] + instance.name = instance_name + instance.disks = disks + if re.match(r"^zones/[a-z\d\-]+/machineTypes/[a-z\d\-]+$", machine_type): + instance.machine_type = machine_type + else: + instance.machine_type = f"zones/{zone}/machineTypes/{machine_type}" + + if accelerators: + instance.guest_accelerators = accelerators + + if preemptible: + # Set the preemptible setting + warnings.warn( + "Preemptible VMs are being replaced by Spot VMs.", DeprecationWarning + ) + instance.scheduling = compute_v1.Scheduling() + instance.scheduling.preemptible = True + + if spot: + # Set the Spot VM setting + instance.scheduling = compute_v1.Scheduling() + instance.scheduling.provisioning_model = ( + compute_v1.Scheduling.ProvisioningModel.SPOT.name + ) + instance.scheduling.instance_termination_action = instance_termination_action + + if custom_hostname is not None: + # Set the custom hostname for the instance + instance.hostname = custom_hostname + + if delete_protection: + # Set the delete protection bit + instance.deletion_protection = True + + # Prepare the request to insert an instance. + request = compute_v1.InsertInstanceRequest() + request.zone = zone + request.project = project_id + request.instance_resource = instance + + # Wait for the create operation to complete. + print(f"Creating the {instance_name} instance in {zone}...") + + operation = instance_client.insert(request=request) + + gcp_utils.wait_for_extended_operation(operation, "instance creation") + + print(f"Instance {instance_name} created.") + return instance_client.get(project=project_id, zone=zone, instance=instance_name) + + +def main(): + project_id = "dstack" + image_name = "stgn-dstack-5" + zone= "us-central1-a" + _launch_instance(project_id, zone, image_name, image_name) + + +if __name__ == "__main__": + main() diff --git a/cli/dstack/backend/gcp/runs.py b/cli/dstack/backend/gcp/runs.py new file mode 100644 index 0000000000..a4e97cf031 --- /dev/null +++ b/cli/dstack/backend/gcp/runs.py @@ -0,0 +1,41 @@ +from typing import List + +from google.cloud.storage import Bucket + +from dstack import random_name +from dstack.backend import JobHead, RepoAddress, RunHead +from dstack.core.run import RunHead, ArtifactHead +from dstack.core.app import AppHead +from dstack.backend.gcp import storage + + +def create_run(bucket: Bucket) -> str: + run_name_prefix = random_name.next_name() + run_name_count = 0 + storage.put_object(bucket, f"run-names/{run_name_prefix}.yaml", '') + run_name = f"{run_name_prefix}-{run_name_count+1}" + return run_name + + +def get_run_heads( + bucket: Bucket, repo_address: RepoAddress, job_heads: List[JobHead], + include_request_heads: bool = True +) -> List[RunHead]: + # TODO aggregate jobs of the same run + return [_get_run_head(bucket, job_head, include_request_heads) for job_head in job_heads] + + +def _get_run_head(bucket: Bucket, job_head: JobHead, include_request_heads: bool) -> RunHead: + app_heads = list( + map(lambda app_name: AppHead(job_head.job_id, app_name), job_head.app_names)) if job_head.app_names else None + artifact_heads = list(map(lambda artifact_path: ArtifactHead(job_head.job_id, artifact_path), + job_head.artifact_paths)) if job_head.artifact_paths else None + request_heads = None + if include_request_heads and job_head.status.is_unfinished(): + if request_heads is None: + request_heads = [] + # TODO request heads + run_head = RunHead(job_head.repo_address, job_head.run_name, job_head.workflow_name, job_head.provider_name, + job_head.local_repo_user_name, artifact_heads or None, job_head.status, job_head.submitted_at, + job_head.tag_name, app_heads, request_heads) + return run_head diff --git a/cli/dstack/backend/gcp/storage.py b/cli/dstack/backend/gcp/storage.py new file mode 100644 index 0000000000..b81ac2c4fe --- /dev/null +++ b/cli/dstack/backend/gcp/storage.py @@ -0,0 +1,23 @@ +from google.cloud import storage, exceptions + + +def get_client(project_id: str) -> storage.Client: + return storage.Client(project=project_id) + + +def get_or_create_bucket(storage_client: storage.Client, bucket_name: str): + try: + return storage_client.create_bucket(bucket_name) + except exceptions.Conflict: + return storage_client.bucket(bucket_name) + + +def put_object(bucket: storage.Bucket, object_name: str, data: str): + blob = bucket.blob(object_name) + blob.upload_from_string(data) + + +def read_object(bucket: storage.Bucket, object_name: str) -> str: + blob = bucket.get_blob(object_name) + with blob.open() as f: + return f.read() diff --git a/cli/dstack/backend/gcp/utils.py b/cli/dstack/backend/gcp/utils.py new file mode 100644 index 0000000000..eb5e634c56 --- /dev/null +++ b/cli/dstack/backend/gcp/utils.py @@ -0,0 +1,51 @@ +import sys +from typing import Any + +from google.api_core.extended_operation import ExtendedOperation + + +def wait_for_extended_operation( + operation: ExtendedOperation, verbose_name: str = "operation", timeout: int = 300 +) -> Any: + """ + This method will wait for the extended (long-running) operation to + complete. If the operation is successful, it will return its result. + If the operation ends with an error, an exception will be raised. + If there were any warnings during the execution of the operation + they will be printed to sys.stderr. + + Args: + operation: a long-running operation you want to wait on. + verbose_name: (optional) a more verbose name of the operation, + used only during error and warning reporting. + timeout: how long (in seconds) to wait for operation to finish. + If None, wait indefinitely. + + Returns: + Whatever the operation.result() returns. + + Raises: + This method will raise the exception received from `operation.exception()` + or RuntimeError if there is no exception set, but there is an `error_code` + set for the `operation`. + + In case of an operation taking longer than `timeout` seconds to complete, + a `concurrent.futures.TimeoutError` will be raised. + """ + result = operation.result(timeout=timeout) + + if operation.error_code: + print( + f"Error during {verbose_name}: [Code: {operation.error_code}]: {operation.error_message}", + file=sys.stderr, + flush=True, + ) + print(f"Operation ID: {operation.name}", file=sys.stderr, flush=True) + raise operation.exception() or RuntimeError(operation.error_message) + + if operation.warnings: + print(f"Warnings during {verbose_name}:\n", file=sys.stderr, flush=True) + for warning in operation.warnings: + print(f" - {warning.code}: {warning.message}", file=sys.stderr, flush=True) + + return result From 2f960bcaf06a43923ea6a0d75e5d0a81f29ecc59 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 1 Feb 2023 11:02:58 +0500 Subject: [PATCH 06/58] Implement job stopping and deleting --- cli/dstack/api/backend.py | 4 +- cli/dstack/backend/gcp/__init__.py | 4 +- cli/dstack/backend/gcp/jobs.py | 69 +++++++++++++++++++++--------- cli/dstack/backend/gcp/runners.py | 56 +++++++++++++++++++++--- cli/dstack/backend/gcp/storage.py | 14 ++++++ cli/requirements.txt | 4 +- 6 files changed, 121 insertions(+), 30 deletions(-) diff --git a/cli/dstack/api/backend.py b/cli/dstack/api/backend.py index 8e75ec3dc4..1cc0781997 100644 --- a/cli/dstack/api/backend.py +++ b/cli/dstack/api/backend.py @@ -10,7 +10,9 @@ DEFAULT = "local" -backends_classes = [AwsBackend, GCPBackend, LocalBackend] +# backends_classes = [AwsBackend, GCPBackend, LocalBackend] +# for testing +backends_classes = [GCPBackend, LocalBackend] def get_all_backends(): diff --git a/cli/dstack/backend/gcp/__init__.py b/cli/dstack/backend/gcp/__init__.py index f5a6284bad..da04c7d8ea 100644 --- a/cli/dstack/backend/gcp/__init__.py +++ b/cli/dstack/backend/gcp/__init__.py @@ -39,10 +39,10 @@ def list_jobs(self, repo_address: RepoAddress, run_name: str) -> List[Job]: raise NotImplementedError() def stop_job(self, repo_address: RepoAddress, job_id: str, abort: bool): - pass + jobs.stop_job(self.config, self._bucket, repo_address, job_id, abort) def delete_job_head(self, repo_address: RepoAddress, job_id: str): - pass + jobs.delete_job_head(self._bucket, repo_address, job_id) def list_run_heads(self, repo_address: RepoAddress, run_name: Optional[str] = None, include_request_heads: bool = True) -> List[RunHead]: diff --git a/cli/dstack/backend/gcp/jobs.py b/cli/dstack/backend/gcp/jobs.py index bc2267f244..85e2c1c848 100644 --- a/cli/dstack/backend/gcp/jobs.py +++ b/cli/dstack/backend/gcp/jobs.py @@ -23,29 +23,35 @@ def create_job(bucket: Bucket, job: Job): storage.put_object(bucket, _job_head_key(job), '') -def run_job(gcp_config: GCPConfig, bucket: Bucket, job: Job): - instance_type = _get_instance_type(job) - runner = Runner(job.runner_id, None, instance_type.resources, job) - runners.create_runner(bucket, runner) - runners.launch_runner(gcp_config, bucket, runner) +def get_job(bucket: Bucket, repo_address: RepoAddress, job_id: str) -> Optional[Job]: + prefix = f"jobs/{_repo_address_path(repo_address)}" + key = f"{prefix}/{job_id}.yaml" + obj = storage.read_object(bucket, key) + job = unserialize_job(yaml.load(obj, yaml.FullLoader)) + return job -def _get_instance_type(job: Job) -> InstanceType: - return InstanceType( - instance_name="n1-standard-1", - resources=Resources(cpus=1, memory_mib=3750, gpus=[], interruptible=False, local=False), - ) +def update_job(bucket: Bucket, job: Job): + delete_job_head(bucket, job.repo_address, job.job_id) + create_job(bucket, job) + + +def delete_job_head(bucket: Bucket, repo_address: RepoAddress, job_id: str): + jobs_prefix = f"jobs/{_repo_address_path(repo_address)}" + job_head_key_prefix = f"{jobs_prefix}/l;{job_id};" + object_names = storage.list_objects(bucket, prefix=job_head_key_prefix) + for object_name in object_names: + storage.delete_object(bucket, object_name) def list_job_heads(bucket: Bucket, repo_address: RepoAddress, run_name: Optional[str] = None) -> List[JobHead]: jobs_prefix = f"jobs/{_repo_address_path(repo_address)}" job_head_key_prefix = f"{jobs_prefix}/l;" job_head_key_run_prefix = job_head_key_prefix + run_name if run_name else job_head_key_prefix - # TODO pagination - blobs = bucket.client.list_blobs(bucket.name, prefix=job_head_key_run_prefix) + object_names = storage.list_objects(bucket, prefix=job_head_key_run_prefix) job_heads = [] - for blob in blobs: - t = blob.name.split(";") + for object_name in object_names: + t = object_name.split(";") _, job_id, provider_name, local_repo_user_name, submitted_at, status, artifacts, app_names, tag_name = t run_name, workflow_name, job_index = tuple(job_id.split(',')) job_heads.append(JobHead(job_id, repo_address, run_name, workflow_name or None, provider_name, @@ -55,9 +61,32 @@ def list_job_heads(bucket: Bucket, repo_address: RepoAddress, run_name: Optional return job_heads -def get_job(bucket: Bucket, repo_address: RepoAddress, job_id: str) -> Optional[Job]: - prefix = f"jobs/{_repo_address_path(repo_address)}" - key = f"{prefix}/{job_id}.yaml" - obj = storage.read_object(bucket, key) - job = unserialize_job(yaml.load(obj, yaml.FullLoader)) - return job +def run_job(gcp_config: GCPConfig, bucket: Bucket, job: Job): + instance_type = _get_instance_type(job) + runner = Runner(job.runner_id, None, instance_type.resources, job) + runners.create_runner(bucket, runner) + # We save request_id (aka instance name) but we can infer it from the job/runner + runner.request_id = runners.launch_runner(gcp_config, bucket, runner) + job.request_id = runner.request_id + runners.update_runner(bucket, runner) + update_job(bucket, job) + + +def _get_instance_type(job: Job) -> InstanceType: + return InstanceType( + instance_name="n1-standard-1", + resources=Resources(cpus=1, memory_mib=3750, gpus=[], interruptible=False, local=False), + ) + + +def stop_job(gcp_config: GCPConfig, bucket: Bucket, repo_address: RepoAddress, job_id: str, abort: bool): + job = get_job(bucket, repo_address, job_id) + if job.status.is_finished(): + return + # TODO: proper, graceful stopping + # Update runner instead of stop if Stopping + runner = runners.get_runner(bucket, job.runner_id) + runners.stop_runner(gcp_config, runner) + job.status = JobStatus.STOPPED + update_job(bucket, job) + runners.delete_runner(bucket, job.runner_id) diff --git a/cli/dstack/backend/gcp/runners.py b/cli/dstack/backend/gcp/runners.py index f8fa2dd1da..78f33dcee7 100644 --- a/cli/dstack/backend/gcp/runners.py +++ b/cli/dstack/backend/gcp/runners.py @@ -4,11 +4,12 @@ from google.cloud.storage import Bucket from google.cloud import compute_v1 +import google.api_core.exceptions import yaml from dstack.backend.gcp import storage, utils as gcp_utils from dstack.core.runners import Runner -from dstack.backend.aws.runners import _serialize_runner +from dstack.backend.aws.runners import _serialize_runner, _unserialize_runner from dstack.backend.gcp.config import GCPConfig @@ -17,14 +18,35 @@ def create_runner(bucket: Bucket, runner: Runner): storage.put_object(bucket, runner_filepath, yaml.dump(_serialize_runner(runner))) -def launch_runner(gcp_config: GCPConfig, bucket: Bucket, runner: Runner): - return - _launch_instance( +def update_runner(bucket: Bucket, runner: Runner): + # Simply override the existing runner + create_runner(bucket, runner) + + +def get_runner(bucket: Bucket, runner_id: str) -> Runner: + runner_filepath = f"runners/{runner_id}.yaml" + obj = storage.read_object(bucket, runner_filepath) + return _unserialize_runner(yaml.load(obj, yaml.FullLoader)) + + +def delete_runner(bucket: Bucket, runner_id: str): + runner_filepath = f"runners/{runner_id}.yaml" + storage.delete_object(bucket, runner_filepath) + + +def launch_runner(gcp_config: GCPConfig, bucket: Bucket, runner: Runner) -> str: + instance = _launch_instance( project_id=gcp_config.project_id, zone=gcp_config.zone, image_name="stgn-dstack-5", - instance_name=f"dstack-{runner.job.run_name}", + instance_name=_get_instance_name(runner), ) + return instance.name + + +def _get_instance_name(runner: Runner) -> str: + # TODO support multiple jobs per run + return f"dstack-{runner.job.run_name}" def _launch_instance(project_id: str, zone: str, image_name: str, instance_name: str) -> compute_v1.Instance: @@ -214,11 +236,33 @@ def _create_instance( return instance_client.get(project=project_id, zone=zone, instance=instance_name) +def stop_runner(gcp_config: GCPConfig, runner: Runner): + _delete_instance( + instance_name=_get_instance_name(runner), + project_id=gcp_config.project_id, + zone=gcp_config.zone, + ) + + +def _delete_instance(project_id: str, zone: str, instance_name: str): + delete_request = compute_v1.DeleteInstanceRequest( + instance=instance_name, + project=project_id, + zone=zone, + ) + client = compute_v1.InstancesClient() + try: + client.delete(delete_request) + except google.api_core.exceptions.NotFound: + pass + + def main(): project_id = "dstack" image_name = "stgn-dstack-5" zone= "us-central1-a" - _launch_instance(project_id, zone, image_name, image_name) + # _launch_instance(project_id, zone, image_name, image_name) + _delete_instance(project_id, zone, 'stgn-dstack-5') if __name__ == "__main__": diff --git a/cli/dstack/backend/gcp/storage.py b/cli/dstack/backend/gcp/storage.py index b81ac2c4fe..c601291dcf 100644 --- a/cli/dstack/backend/gcp/storage.py +++ b/cli/dstack/backend/gcp/storage.py @@ -1,3 +1,5 @@ +from typing import List + from google.cloud import storage, exceptions @@ -21,3 +23,15 @@ def read_object(bucket: storage.Bucket, object_name: str) -> str: blob = bucket.get_blob(object_name) with blob.open() as f: return f.read() + + +def delete_object(bucket: storage.Bucket, object_name: str): + blob = bucket.blob(object_name) + blob.delete() + + +def list_objects(bucket: storage.Bucket, prefix: str) -> List[str]: + # TODO pagination + blobs = bucket.client.list_blobs(bucket.name, prefix=prefix) + object_names = [blob.name for blob in blobs] + return object_names diff --git a/cli/requirements.txt b/cli/requirements.txt index 4ee5574d91..ba94fa0491 100644 --- a/cli/requirements.txt +++ b/cli/requirements.txt @@ -14,4 +14,6 @@ psutil websocket-client cursor simple-term-menu -jinja2 \ No newline at end of file +jinja2 +google-cloud-storage +google-cloud-compute From a6931d70905b69200190763e225503f66e771c04 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 8 Feb 2023 15:26:09 +0500 Subject: [PATCH 07/58] Update gcp backend after refactoring --- cli/dstack/backend/gcp/__init__.py | 46 ++++++++++++------------------ cli/dstack/backend/gcp/jobs.py | 7 ++--- 2 files changed, 20 insertions(+), 33 deletions(-) diff --git a/cli/dstack/backend/gcp/__init__.py b/cli/dstack/backend/gcp/__init__.py index 16033dcd2b..301c67e951 100644 --- a/cli/dstack/backend/gcp/__init__.py +++ b/cli/dstack/backend/gcp/__init__.py @@ -31,7 +31,7 @@ def __init__(self): @property def name(self) -> str: - "gcp" + return "gcp" def configure(self): self._bucket = storage.get_or_create_bucket(self._storage_client, self.config.bucket_name) @@ -39,40 +39,41 @@ def configure(self): def create_run(self, repo_address: RepoAddress) -> str: return runs.create_run(self._bucket) - def submit_job(self, job: Job, counter: List[int]): - job.runner_id = uuid.uuid4().hex + def create_job(self, job: Job): jobs.create_job(self._bucket, job) - jobs.run_job(self.config, self._bucket, job) + + # def submit_job(self, job: Job, counter: List[int]): + # job.runner_id = uuid.uuid4().hex + # jobs.create_job(self._bucket, job) + # jobs.run_job(self.config, self._bucket, job) def get_job(self, repo_address: RepoAddress, job_id: str) -> Optional[Job]: return jobs.get_job(self._bucket, repo_address, job_id) - def list_job_heads( - self, repo_address: RepoAddress, run_name: Optional[str] = None - ) -> List[JobHead]: - return jobs.list_job_heads(self._bucket, repo_address, run_name) - def list_jobs(self, repo_address: RepoAddress, run_name: str) -> List[Job]: raise NotImplementedError() def run_job(self, job: Job) -> Runner: - raise NotImplementedError() + jobs.run_job(self.config, self._bucket, job) def stop_job(self, repo_address: RepoAddress, job_id: str, abort: bool): jobs.stop_job(self.config, self._bucket, repo_address, job_id, abort) + def list_job_heads( + self, repo_address: RepoAddress, run_name: Optional[str] = None + ) -> List[JobHead]: + return jobs.list_job_heads(self._bucket, repo_address, run_name) + def delete_job_head(self, repo_address: RepoAddress, job_id: str): jobs.delete_job_head(self._bucket, repo_address, job_id) - def store_job(self, job: Job): - raise NotImplementedError() - - def get_run_heads( + def list_run_heads( self, repo_address: RepoAddress, - job_heads: List[JobHead], + run_name: Optional[str] = None, include_request_heads: bool = True, ) -> List[RunHead]: + job_heads = self.list_job_heads(repo_address, run_name) return runs.get_run_heads(self._bucket, repo_address, job_heads, include_request_heads) def poll_logs( @@ -112,23 +113,12 @@ def add_tag_from_local_dirs(self, repo_data: RepoData, tag_name: str, local_dirs def delete_tag_head(self, repo_address: RepoAddress, tag_head: TagHead): pass - def list_repo_heads(self) -> List[RepoHead]: - pass - def update_repo_last_run_at(self, repo_address: RepoAddress, last_run_at: int): pass - def increment_repo_tags_count(self, repo_address: RepoAddress): - pass - - def decrement_repo_tags_count(self, repo_address: RepoAddress): - pass - - def delete_repo(self, repo_address: RepoAddress): - pass - def get_repo_credentials(self, repo_address: RepoAddress) -> Optional[RepoCredentials]: - pass + # TODO + return RepoCredentials(RepoProtocol.HTTPS, None, None) def save_repo_credentials(self, repo_address: RepoAddress, repo_credentials: RepoCredentials): pass diff --git a/cli/dstack/backend/gcp/jobs.py b/cli/dstack/backend/gcp/jobs.py index 9c9681cb26..096f211173 100644 --- a/cli/dstack/backend/gcp/jobs.py +++ b/cli/dstack/backend/gcp/jobs.py @@ -12,17 +12,14 @@ def create_job(bucket: Bucket, job: Job): - counter = 0 - job_id = f"{job.run_name},{job.workflow_name or ''},{counter}" - job.set_id(job_id) prefix = f"jobs/{job.repo_address.path()}" - key = f"{prefix}/{job_id}.yaml" + key = f"{prefix}/{job.job_id}.yaml" storage.put_object(bucket, key, yaml.dump(job.serialize())) storage.put_object(bucket, job.job_head_key(), "") def get_job(bucket: Bucket, repo_address: RepoAddress, job_id: str) -> Optional[Job]: - prefix = f"jobs/{job.repo_address.path()}" + prefix = f"jobs/{repo_address.path()}" key = f"{prefix}/{job_id}.yaml" obj = storage.read_object(bucket, key) job = Job.unserialize(yaml.load(obj, yaml.FullLoader)) From 0ec273eeb7a1c015af9faf8e2e5c947224d481f8 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Mon, 13 Feb 2023 11:58:17 +0500 Subject: [PATCH 08/58] Refactor gcp to use common backend logic --- cli/dstack/backend/gcp/__init__.py | 80 ++++++----- .../backend/gcp/{runners.py => compute.py} | 133 +++++++++++------- cli/dstack/backend/gcp/jobs.py | 112 --------------- cli/dstack/backend/gcp/runs.py | 65 --------- cli/dstack/backend/gcp/storage.py | 59 ++++---- 5 files changed, 165 insertions(+), 284 deletions(-) rename cli/dstack/backend/gcp/{runners.py => compute.py} (76%) delete mode 100644 cli/dstack/backend/gcp/jobs.py delete mode 100644 cli/dstack/backend/gcp/runs.py diff --git a/cli/dstack/backend/gcp/__init__.py b/cli/dstack/backend/gcp/__init__.py index 301c67e951..8156a422a3 100644 --- a/cli/dstack/backend/gcp/__init__.py +++ b/cli/dstack/backend/gcp/__init__.py @@ -1,31 +1,36 @@ -import uuid +import warnings from pathlib import Path from typing import Generator, List, Optional +from google.auth._default import _CLOUD_SDK_CREDENTIALS_WARNING + from dstack.backend.base import RemoteBackend -from dstack.backend.gcp import jobs, runs, storage +from dstack.backend.base import jobs as base_jobs +from dstack.backend.base import repos as base_repos +from dstack.backend.base import runs as base_runs +from dstack.backend.base import secrets as base_secrets +from dstack.backend.base import tags as base_tags +from dstack.backend.gcp.compute import GCPCompute from dstack.backend.gcp.config import GCPConfig +from dstack.backend.gcp.storage import GCPStorage from dstack.core.artifact import Artifact from dstack.core.job import Job, JobHead from dstack.core.log_event import LogEvent -from dstack.core.repo import ( - LocalRepoData, - RepoAddress, - RepoCredentials, - RepoData, - RepoHead, - RepoProtocol, -) +from dstack.core.repo import RepoAddress, RepoCredentials, RepoData, RepoProtocol from dstack.core.run import RunHead -from dstack.core.runners import Runner from dstack.core.secret import Secret from dstack.core.tag import TagHead +warnings.filterwarnings("ignore", message=_CLOUD_SDK_CREDENTIALS_WARNING) + class GCPBackend(RemoteBackend): def __init__(self): self.config = GCPConfig() - self._storage_client = storage.get_client(project_id=self.config.project_id) + self._storage = GCPStorage( + project_id=self.config.project_id, bucket_name=self.config.bucket_name + ) + self._compute = GCPCompute(gcp_config=self.config) self.configure() self._loaded = True @@ -34,38 +39,33 @@ def name(self) -> str: return "gcp" def configure(self): - self._bucket = storage.get_or_create_bucket(self._storage_client, self.config.bucket_name) + self._storage.configure() def create_run(self, repo_address: RepoAddress) -> str: - return runs.create_run(self._bucket) + return base_runs.create_run(self._storage, repo_address, self.type) def create_job(self, job: Job): - jobs.create_job(self._bucket, job) - - # def submit_job(self, job: Job, counter: List[int]): - # job.runner_id = uuid.uuid4().hex - # jobs.create_job(self._bucket, job) - # jobs.run_job(self.config, self._bucket, job) + base_jobs.create_job(self._storage, job) def get_job(self, repo_address: RepoAddress, job_id: str) -> Optional[Job]: - return jobs.get_job(self._bucket, repo_address, job_id) + return base_jobs.get_job(self._storage, repo_address, job_id) def list_jobs(self, repo_address: RepoAddress, run_name: str) -> List[Job]: - raise NotImplementedError() + return base_jobs.list_jobs(self._storage, repo_address, run_name) - def run_job(self, job: Job) -> Runner: - jobs.run_job(self.config, self._bucket, job) + def run_job(self, job: Job): + base_jobs.run_job(self._storage, self._compute, job) def stop_job(self, repo_address: RepoAddress, job_id: str, abort: bool): - jobs.stop_job(self.config, self._bucket, repo_address, job_id, abort) + base_jobs.stop_job(self._storage, self._compute, repo_address, job_id, abort) def list_job_heads( self, repo_address: RepoAddress, run_name: Optional[str] = None ) -> List[JobHead]: - return jobs.list_job_heads(self._bucket, repo_address, run_name) + return base_jobs.list_job_heads(self._storage, repo_address, run_name) def delete_job_head(self, repo_address: RepoAddress, job_id: str): - jobs.delete_job_head(self._bucket, repo_address, job_id) + base_jobs.delete_job_head(self._storage, repo_address, job_id) def list_run_heads( self, @@ -74,7 +74,9 @@ def list_run_heads( include_request_heads: bool = True, ) -> List[RunHead]: job_heads = self.list_job_heads(repo_address, run_name) - return runs.get_run_heads(self._bucket, repo_address, job_heads, include_request_heads) + return base_runs.get_run_heads( + self._storage, self._compute, job_heads, include_request_heads + ) def poll_logs( self, @@ -93,10 +95,10 @@ def list_run_artifact_files( pass def list_tag_heads(self, repo_address: RepoAddress) -> List[TagHead]: - pass + return base_tags.list_tag_heads(self._storage, repo_address) def get_tag_head(self, repo_address: RepoAddress, tag_name: str) -> Optional[TagHead]: - pass + return base_tags.get_tag_head(self._storage, repo_address, tag_name) def add_tag_from_run( self, @@ -105,20 +107,30 @@ def add_tag_from_run( run_name: str, run_jobs: Optional[List[Job]], ): - pass + base_tags.create_tag_from_run( + self._storage, + repo_address, + tag_name, + run_name, + run_jobs, + ) def add_tag_from_local_dirs(self, repo_data: RepoData, tag_name: str, local_dirs: List[str]): pass def delete_tag_head(self, repo_address: RepoAddress, tag_head: TagHead): - pass + base_tags.delete_tag(self._storage, repo_address, tag_head) def update_repo_last_run_at(self, repo_address: RepoAddress, last_run_at: int): - pass + base_repos.update_repo_last_run_at( + self._storage, + repo_address, + last_run_at, + ) def get_repo_credentials(self, repo_address: RepoAddress) -> Optional[RepoCredentials]: # TODO - return RepoCredentials(RepoProtocol.HTTPS, None, None) + return RepoCredentials(protocol=RepoProtocol.HTTPS, private_key=None, oauth_token=None) def save_repo_credentials(self, repo_address: RepoAddress, repo_credentials: RepoCredentials): pass diff --git a/cli/dstack/backend/gcp/runners.py b/cli/dstack/backend/gcp/compute.py similarity index 76% rename from cli/dstack/backend/gcp/runners.py rename to cli/dstack/backend/gcp/compute.py index 7dbbb0645a..6f206cc5f7 100644 --- a/cli/dstack/backend/gcp/runners.py +++ b/cli/dstack/backend/gcp/compute.py @@ -1,56 +1,84 @@ import re import warnings -from typing import Any, List +from typing import Any, List, Optional, Tuple import google.api_core.exceptions -import yaml from google.cloud import compute_v1 -from google.cloud.storage import Bucket -from dstack.backend.gcp import storage +from dstack.backend.base.compute import Compute from dstack.backend.gcp import utils as gcp_utils from dstack.backend.gcp.config import GCPConfig -from dstack.core.runners import Runner - - -def create_runner(bucket: Bucket, runner: Runner): - runner_filepath = f"runners/{runner.runner_id}.yaml" - storage.put_object(bucket, runner_filepath, yaml.dump(runner.serialize())) - - -def update_runner(bucket: Bucket, runner: Runner): - # Simply override the existing runner - create_runner(bucket, runner) - - -def get_runner(bucket: Bucket, runner_id: str) -> Runner: - runner_filepath = f"runners/{runner_id}.yaml" - obj = storage.read_object(bucket, runner_filepath) - return Runner.unserialize(yaml.load(obj, yaml.FullLoader)) - - -def delete_runner(bucket: Bucket, runner_id: str): - runner_filepath = f"runners/{runner_id}.yaml" - storage.delete_object(bucket, runner_filepath) - - -def launch_runner(gcp_config: GCPConfig, bucket: Bucket, runner: Runner) -> str: - instance = _launch_instance( - project_id=gcp_config.project_id, - zone=gcp_config.zone, - image_name="stgn-dstack-5", - instance_name=_get_instance_name(runner), - ) - return instance.name - - -def _get_instance_name(runner: Runner) -> str: +from dstack.core.instance import InstanceType +from dstack.core.job import Job, Requirements +from dstack.core.request import RequestHead, RequestStatus +from dstack.core.runners import Resources, Runner + + +class GCPCompute(Compute): + def __init__( + self, + gcp_config: GCPConfig, + ): + self.gcp_config = gcp_config + + def get_request_head(self, job: Job, request_id: Optional[str]) -> RequestHead: + if request_id is None: + return RequestHead( + job_id=job.job_id, + status=RequestStatus.TERMINATED, + message="request_id is not specified", + ) + instance_status = _get_instance_status( + project_id=self.gcp_config.project_id, + zone=self.gcp_config.zone, + instance_name=request_id, + ) + return RequestHead( + job_id=job.job_id, + status=instance_status, + message=None, + ) + + def get_instance_type(self, job: Job) -> Optional[InstanceType]: + return InstanceType( + instance_name="n1-standard-1", + resources=Resources( + cpus=1, memory_mib=3750, gpus=[], interruptible=False, local=False + ), + ) + + def run_instance(self, job: Job, instance_type: InstanceType) -> str: + instance = _launch_instance( + project_id=self.gcp_config.project_id, + zone=self.gcp_config.zone, + image_name="stgn-dstack-5", + instance_name=_get_instance_name(job), + ) + return instance.name + + def terminate_instance(self, request_id: str): + _terminate_instance( + gcp_config=self.gcp_config, + instance_name=request_id, + ) + + def cancel_spot_request(self, request_id: str): + _terminate_instance( + gcp_config=self.gcp_config, + instance_name=request_id, + ) + + +def _get_instance_name(job: Job) -> str: # TODO support multiple jobs per run - return f"dstack-{runner.job.run_name}" + return f"dstack-{job.run_name}" def _launch_instance( - project_id: str, zone: str, image_name: str, instance_name: str + project_id: str, + zone: str, + image_name: str, + instance_name: str, ) -> compute_v1.Instance: disk = _disk_from_image( disk_type=f"zones/{zone}/diskTypes/pd-balanced", @@ -234,9 +262,22 @@ def _create_instance( return instance_client.get(project=project_id, zone=zone, instance=instance_name) -def stop_runner(gcp_config: GCPConfig, runner: Runner): +def _get_instance_status(project_id: str, zone: str, instance_name: str) -> RequestStatus: + get_instance_request = compute_v1.GetInstanceRequest( + instance=instance_name, + project=project_id, + zone=zone, + ) + client = compute_v1.InstancesClient() + instance = client.get(get_instance_request) + if instance.status in ["PROVISIONING", "STAGING", "RUNNING"]: + return RequestStatus.RUNNING + return RequestStatus.TERMINATED + + +def _terminate_instance(gcp_config: GCPConfig, instance_name: str): _delete_instance( - instance_name=_get_instance_name(runner), + instance_name=instance_name, project_id=gcp_config.project_id, zone=gcp_config.zone, ) @@ -256,11 +297,7 @@ def _delete_instance(project_id: str, zone: str, instance_name: str): def main(): - project_id = "dstack" - image_name = "stgn-dstack-5" - zone = "us-central1-a" - # _launch_instance(project_id, zone, image_name, image_name) - _delete_instance(project_id, zone, "stgn-dstack-5") + pass if __name__ == "__main__": diff --git a/cli/dstack/backend/gcp/jobs.py b/cli/dstack/backend/gcp/jobs.py deleted file mode 100644 index 096f211173..0000000000 --- a/cli/dstack/backend/gcp/jobs.py +++ /dev/null @@ -1,112 +0,0 @@ -from typing import List, Optional - -import yaml -from google.cloud.storage import Bucket - -from dstack.backend.gcp import runners, storage -from dstack.backend.gcp.config import GCPConfig -from dstack.core.instance import InstanceType -from dstack.core.job import Job, JobHead, JobStatus -from dstack.core.repo import RepoAddress -from dstack.core.runners import Resources, Runner - - -def create_job(bucket: Bucket, job: Job): - prefix = f"jobs/{job.repo_address.path()}" - key = f"{prefix}/{job.job_id}.yaml" - storage.put_object(bucket, key, yaml.dump(job.serialize())) - storage.put_object(bucket, job.job_head_key(), "") - - -def get_job(bucket: Bucket, repo_address: RepoAddress, job_id: str) -> Optional[Job]: - prefix = f"jobs/{repo_address.path()}" - key = f"{prefix}/{job_id}.yaml" - obj = storage.read_object(bucket, key) - job = Job.unserialize(yaml.load(obj, yaml.FullLoader)) - return job - - -def update_job(bucket: Bucket, job: Job): - delete_job_head(bucket, job.repo_address, job.job_id) - create_job(bucket, job) - - -def delete_job_head(bucket: Bucket, repo_address: RepoAddress, job_id: str): - jobs_prefix = f"jobs/{repo_address.path()}" - job_head_key_prefix = f"{jobs_prefix}/l;{job_id};" - object_names = storage.list_objects(bucket, prefix=job_head_key_prefix) - for object_name in object_names: - storage.delete_object(bucket, object_name) - - -def list_job_heads( - bucket: Bucket, repo_address: RepoAddress, run_name: Optional[str] = None -) -> List[JobHead]: - jobs_prefix = f"jobs/{repo_address.path()}" - job_head_key_prefix = f"{jobs_prefix}/l;" - job_head_key_run_prefix = job_head_key_prefix + run_name if run_name else job_head_key_prefix - object_names = storage.list_objects(bucket, prefix=job_head_key_run_prefix) - job_heads = [] - for object_name in object_names: - t = object_name.split(";") - ( - _, - job_id, - provider_name, - local_repo_user_name, - submitted_at, - status, - artifacts, - app_names, - tag_name, - ) = t - run_name, workflow_name, job_index = tuple(job_id.split(",")) - job_heads.append( - JobHead( - job_id, - repo_address, - run_name, - workflow_name or None, - provider_name, - local_repo_user_name, - JobStatus(status), - int(submitted_at), - artifacts.split(",") if artifacts else None, - tag_name or None, - app_names.split(",") or None, - ) - ) - return job_heads - - -def run_job(gcp_config: GCPConfig, bucket: Bucket, job: Job): - instance_type = _get_instance_type(job) - runner = Runner(job.runner_id, None, instance_type.resources, job) - runners.create_runner(bucket, runner) - # We save request_id (aka instance name) but we can infer it from the job/runner - runner.request_id = runners.launch_runner(gcp_config, bucket, runner) - job.request_id = runner.request_id - runners.update_runner(bucket, runner) - update_job(bucket, job) - - -def _get_instance_type(job: Job) -> InstanceType: - return InstanceType( - instance_name="n1-standard-1", - resources=Resources(cpus=1, memory_mib=3750, gpus=[], interruptible=False, local=False), - ) - - -def stop_job( - gcp_config: GCPConfig, bucket: Bucket, repo_address: RepoAddress, job_id: str, abort: bool -): - job = get_job(bucket, repo_address, job_id) - if job.status.is_finished(): - return - # TODO: proper, graceful stopping - # Update runner instead of stop if Stopping - runner = runners.get_runner(bucket, job.runner_id) - runners.stop_runner(gcp_config, runner) - job.status = JobStatus.STOPPED - update_job(bucket, job) - runners.delete_runner(bucket, job.runner_id) diff --git a/cli/dstack/backend/gcp/runs.py b/cli/dstack/backend/gcp/runs.py deleted file mode 100644 index 74bdfe379d..0000000000 --- a/cli/dstack/backend/gcp/runs.py +++ /dev/null @@ -1,65 +0,0 @@ -from typing import List - -from google.cloud.storage import Bucket - -from dstack.backend.gcp import storage -from dstack.core.app import AppHead -from dstack.core.artifact import ArtifactHead -from dstack.core.job import JobHead -from dstack.core.repo import RepoAddress -from dstack.core.run import RunHead, generate_remote_run_name_prefix - - -def create_run(bucket: Bucket) -> str: - run_name_prefix = generate_remote_run_name_prefix() - run_name_count = 0 - run_name = f"{run_name_prefix}-{run_name_count+1}" - storage.put_object(bucket, f"run-names/{run_name}.yaml", "") - return run_name - - -def get_run_heads( - bucket: Bucket, - repo_address: RepoAddress, - job_heads: List[JobHead], - include_request_heads: bool = True, -) -> List[RunHead]: - # TODO aggregate jobs of the same run - return [_get_run_head(bucket, job_head, include_request_heads) for job_head in job_heads] - - -def _get_run_head(bucket: Bucket, job_head: JobHead, include_request_heads: bool) -> RunHead: - app_heads = ( - list(map(lambda app_name: AppHead(job_head.job_id, app_name), job_head.app_names)) - if job_head.app_names - else None - ) - artifact_heads = ( - list( - map( - lambda artifact_path: ArtifactHead(job_head.job_id, artifact_path), - job_head.artifact_paths, - ) - ) - if job_head.artifact_paths - else None - ) - request_heads = None - if include_request_heads and job_head.status.is_unfinished(): - if request_heads is None: - request_heads = [] - # TODO request heads - run_head = RunHead( - job_head.repo_address, - job_head.run_name, - job_head.workflow_name, - job_head.provider_name, - job_head.local_repo_user_name, - artifact_heads or None, - job_head.status, - job_head.submitted_at, - job_head.tag_name, - app_heads, - request_heads, - ) - return run_head diff --git a/cli/dstack/backend/gcp/storage.py b/cli/dstack/backend/gcp/storage.py index 2d66d19353..09845517e8 100644 --- a/cli/dstack/backend/gcp/storage.py +++ b/cli/dstack/backend/gcp/storage.py @@ -1,37 +1,46 @@ -from typing import List +from typing import Dict, List, Optional from google.cloud import exceptions, storage +from dstack.backend.base.storage import Storage -def get_client(project_id: str) -> storage.Client: - return storage.Client(project=project_id) +class GCPStorage(Storage): + def __init__(self, project_id: str, bucket_name: str): + self.storage_client = self._get_client(project_id) + self.bucket_name = bucket_name -def get_or_create_bucket(storage_client: storage.Client, bucket_name: str): - try: - return storage_client.create_bucket(bucket_name) - except exceptions.Conflict: - return storage_client.bucket(bucket_name) + def configure(self): + self.bucket = self._get_or_create_bucket(self.bucket_name) + def put_object(self, key: str, content: str, metadata: Optional[Dict] = None): + blob = self.bucket.blob(key) + blob.upload_from_string(content) -def put_object(bucket: storage.Bucket, object_name: str, data: str): - blob = bucket.blob(object_name) - blob.upload_from_string(data) + def get_object(self, key: str) -> Optional[str]: + blob = self.bucket.get_blob(key) + if blob is None: + return None + with blob.open() as f: + return f.read() + def delete_object(self, key: str): + try: + self.bucket.delete_blob(key) + except exceptions.NotFound: + pass -def read_object(bucket: storage.Bucket, object_name: str) -> str: - blob = bucket.get_blob(object_name) - with blob.open() as f: - return f.read() + def list_objects(self, keys_prefix: str) -> List[str]: + # TODO pagination + blobs = self.bucket.client.list_blobs(self.bucket.name, prefix=keys_prefix) + object_names = [blob.name for blob in blobs] + return object_names + def _get_client(self, project_id: str) -> storage.Client: + return storage.Client(project=project_id) -def delete_object(bucket: storage.Bucket, object_name: str): - blob = bucket.blob(object_name) - blob.delete() - - -def list_objects(bucket: storage.Bucket, prefix: str) -> List[str]: - # TODO pagination - blobs = bucket.client.list_blobs(bucket.name, prefix=prefix) - object_names = [blob.name for blob in blobs] - return object_names + def _get_or_create_bucket(self, bucket_name: str): + try: + return self.storage_client.create_bucket(bucket_name) + except exceptions.Conflict: + return self.storage_client.bucket(bucket_name) From dbf55ce99e58fff410d618a74c5003a57bea741c Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Mon, 13 Feb 2023 17:24:48 +0500 Subject: [PATCH 09/58] Implement secrets for gcp --- cli/dstack/backend/gcp/__init__.py | 39 ++++++++--- cli/dstack/backend/gcp/secrets.py | 109 +++++++++++++++++++++++++++++ cli/requirements.txt | 1 + 3 files changed, 141 insertions(+), 8 deletions(-) create mode 100644 cli/dstack/backend/gcp/secrets.py diff --git a/cli/dstack/backend/gcp/__init__.py b/cli/dstack/backend/gcp/__init__.py index 8156a422a3..252f4d8086 100644 --- a/cli/dstack/backend/gcp/__init__.py +++ b/cli/dstack/backend/gcp/__init__.py @@ -12,6 +12,7 @@ from dstack.backend.base import tags as base_tags from dstack.backend.gcp.compute import GCPCompute from dstack.backend.gcp.config import GCPConfig +from dstack.backend.gcp.secrets import GCPSecretsManager from dstack.backend.gcp.storage import GCPStorage from dstack.core.artifact import Artifact from dstack.core.job import Job, JobHead @@ -31,6 +32,10 @@ def __init__(self): project_id=self.config.project_id, bucket_name=self.config.bucket_name ) self._compute = GCPCompute(gcp_config=self.config) + self._secrets_manager = GCPSecretsManager( + project_id=self.config.project_id, + bucket_name=self.config.bucket_name, + ) self.configure() self._loaded = True @@ -129,26 +134,44 @@ def update_repo_last_run_at(self, repo_address: RepoAddress, last_run_at: int): ) def get_repo_credentials(self, repo_address: RepoAddress) -> Optional[RepoCredentials]: - # TODO - return RepoCredentials(protocol=RepoProtocol.HTTPS, private_key=None, oauth_token=None) + return base_repos.get_repo_credentials(self._secrets_manager, repo_address) def save_repo_credentials(self, repo_address: RepoAddress, repo_credentials: RepoCredentials): - pass + base_repos.save_repo_credentials( + self._secrets_manager, + repo_address, + repo_credentials, + ) def list_secret_names(self, repo_address: RepoAddress) -> List[str]: - pass + return base_secrets.list_secret_names(self._storage, repo_address) def get_secret(self, repo_address: RepoAddress, secret_name: str) -> Optional[Secret]: - pass + return base_secrets.get_secret(self._secrets_manager, repo_address, secret_name) def add_secret(self, repo_address: RepoAddress, secret: Secret): - pass + base_secrets.add_secret( + self._storage, + self._secrets_manager, + repo_address, + secret, + ) def update_secret(self, repo_address: RepoAddress, secret: Secret): - pass + base_secrets.update_secret( + self._storage, + self._secrets_manager, + repo_address, + secret, + ) def delete_secret(self, repo_address: RepoAddress, secret_name: str): - pass + base_secrets.delete_secret( + self._storage, + self._secrets_manager, + repo_address, + secret_name, + ) def download_run_artifact_files( self, diff --git a/cli/dstack/backend/gcp/secrets.py b/cli/dstack/backend/gcp/secrets.py new file mode 100644 index 0000000000..24f5a49906 --- /dev/null +++ b/cli/dstack/backend/gcp/secrets.py @@ -0,0 +1,109 @@ +from typing import List, Optional + +from google.api_core import exceptions +from google.cloud import secretmanager + +from dstack.backend.base.secrets import SecretsManager +from dstack.core.repo import RepoAddress +from dstack.core.secret import Secret + + +class GCPSecretsManager(SecretsManager): + def __init__(self, project_id: str, bucket_name: str): + self.project_id = project_id + self.bucket_name = bucket_name + self.secrets_client = secretmanager.SecretManagerServiceClient() + + def get_secret(self, repo_address: RepoAddress, secret_name: str) -> Optional[Secret]: + secret_value = self._get_secret_value( + _get_secret_key(self.bucket_name, repo_address, secret_name) + ) + if secret_value is None: + return None + return Secret(secret_name=secret_name, secret_value=secret_value) + + def add_secret(self, repo_address: RepoAddress, secret: Secret): + secret_key = _get_secret_key(self.bucket_name, repo_address, secret.secret_name) + self._create_secret(secret_key) + self._add_secret_version( + secret_key=secret_key, + secret_value=secret.secret_value, + ) + + def update_secret(self, repo_address: RepoAddress, secret: Secret): + self._add_secret_version( + secret_key=_get_secret_key(self.bucket_name, repo_address, secret.secret_name), + secret_value=secret.secret_value, + ) + + def delete_secret(self, repo_address: RepoAddress, secret_name: str): + secret_resource = _get_secret_resource( + self.project_id, _get_secret_key(self.bucket_name, repo_address, secret_name) + ) + self.secrets_client.delete_secret(request={"name": secret_resource}) + + def get_credentials(self, repo_address: RepoAddress) -> Optional[str]: + return self._get_secret_value(_get_credentials_key(self.bucket_name, repo_address)) + + def add_credentials(self, repo_address: RepoAddress, data: str): + credentails_key = _get_credentials_key(self.bucket_name, repo_address) + self._create_secret(credentails_key) + self._add_secret_version( + secret_key=credentails_key, + secret_value=data, + ) + + def update_credentials(self, repo_address: RepoAddress, data: str): + self._add_secret_version( + secret_key=_get_credentials_key(self.bucket_name, repo_address), + secret_value=data, + ) + + def _get_secret_value(self, secret_key: str) -> Optional[str]: + secret_version_resource = _get_secret_version_resource(self.project_id, secret_key) + try: + response = self.secrets_client.access_secret_version(name=secret_version_resource) + except exceptions.NotFound: + return None + return response.payload.data + + def _create_secret(self, secret_key: str): + try: + self.secrets_client.create_secret( + parent=_get_project_resource(self.project_id), + secret_id=secret_key, + secret={"replication": {"automatic": {}}}, + ) + except exceptions.AlreadyExists: + pass + + def _add_secret_version(self, secret_key: str, secret_value: str): + self.secrets_client.add_secret_version( + parent=_get_secret_resource(self.project_id, secret_key), + payload={"data": secret_value.encode()}, + ) + + +def _get_project_resource(project_id: str) -> str: + return f"projects/{project_id}" + + +def _get_secret_resource(project_id: str, secret_key: str) -> str: + return f"{_get_project_resource(project_id)}/secrets/{secret_key}" + + +def _get_secret_version_resource(project_id: str, secret_key: str) -> str: + secret_resource = _get_secret_resource(project_id, secret_key) + return f"{secret_resource}/versions/latest" + + +def _get_secret_key(bucket_name: str, repo_address: RepoAddress, secret_name: str) -> str: + key = f"dstack-secrets-{bucket_name}-{repo_address.path(delimiter='-')}-{secret_name}" + key = key.replace(".", "-") + return key + + +def _get_credentials_key(bucket_name: str, repo_address: RepoAddress) -> str: + key = f"dstack-credentials-{bucket_name}-{repo_address.path(delimiter='-')}" + key = key.replace(".", "-") + return key diff --git a/cli/requirements.txt b/cli/requirements.txt index 26d761a6cf..6b6d761e90 100644 --- a/cli/requirements.txt +++ b/cli/requirements.txt @@ -23,4 +23,5 @@ jinja2 pygtail google-cloud-storage google-cloud-compute +google-cloud-secret-manager aiosqlite From 0a31282d07be98350003fec24f8ea53909bf3a28 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 15 Feb 2023 12:47:06 +0500 Subject: [PATCH 10/58] Implement artifacts for gcp --- cli/dstack/api/backend.py | 9 ++-- cli/dstack/backend/gcp/__init__.py | 75 +++++++++++++++++++----------- cli/dstack/backend/gcp/storage.py | 44 ++++++++++++++++-- 3 files changed, 95 insertions(+), 33 deletions(-) diff --git a/cli/dstack/api/backend.py b/cli/dstack/api/backend.py index 664c1968dc..f48cf11fce 100644 --- a/cli/dstack/api/backend.py +++ b/cli/dstack/api/backend.py @@ -1,7 +1,7 @@ from typing import Dict, List, Optional from dstack.backend.aws import AwsBackend -from dstack.backend.base import Backend, RemoteBackend +from dstack.backend.base import Backend, BackendType, RemoteBackend from dstack.backend.gcp import GCPBackend from dstack.backend.hub import HubBackend from dstack.backend.local import LocalBackend @@ -41,8 +41,11 @@ def get_backend_by_name(name: str) -> Optional[Backend]: return backend -def get_current_remote_backend() -> RemoteBackend: - return get_backend_by_name(DEFAULT_REMOTE) +def get_current_remote_backend() -> Optional[RemoteBackend]: + for backend in list_backends(): + if backend.type is BackendType.REMOTE: + return backend + return None def get_local_backend() -> LocalBackend: diff --git a/cli/dstack/backend/gcp/__init__.py b/cli/dstack/backend/gcp/__init__.py index 252f4d8086..b492c04af4 100644 --- a/cli/dstack/backend/gcp/__init__.py +++ b/cli/dstack/backend/gcp/__init__.py @@ -4,7 +4,8 @@ from google.auth._default import _CLOUD_SDK_CREDENTIALS_WARNING -from dstack.backend.base import RemoteBackend +from dstack.backend.base import CloudBackend +from dstack.backend.base import artifacts as base_artifacts from dstack.backend.base import jobs as base_jobs from dstack.backend.base import repos as base_repos from dstack.backend.base import runs as base_runs @@ -17,7 +18,7 @@ from dstack.core.artifact import Artifact from dstack.core.job import Job, JobHead from dstack.core.log_event import LogEvent -from dstack.core.repo import RepoAddress, RepoCredentials, RepoData, RepoProtocol +from dstack.core.repo import LocalRepoData, RepoAddress, RepoCredentials from dstack.core.run import RunHead from dstack.core.secret import Secret from dstack.core.tag import TagHead @@ -25,7 +26,7 @@ warnings.filterwarnings("ignore", message=_CLOUD_SDK_CREDENTIALS_WARNING) -class GCPBackend(RemoteBackend): +class GCPBackend(CloudBackend): def __init__(self): self.config = GCPConfig() self._storage = GCPStorage( @@ -92,12 +93,36 @@ def poll_logs( ) -> Generator[LogEvent, None, None]: pass - def list_run_artifact_files( - self, repo_address: RepoAddress, run_name: str - ) -> Generator[Artifact, None, None]: - # TODO: add a flag for non-recursive listing. - # Backends may implement this via list_run_artifact_files_and_folders() - pass + def list_run_artifact_files(self, repo_address: RepoAddress, run_name: str) -> List[Artifact]: + return base_artifacts.list_run_artifact_files(self._storage, repo_address, run_name) + + def download_run_artifact_files( + self, + repo_address: RepoAddress, + run_name: str, + output_dir: Optional[str], + ): + base_artifacts.download_run_artifact_files( + storage=self._storage, + repo_address=repo_address, + run_name=run_name, + output_dir=output_dir, + ) + + def upload_job_artifact_files( + self, + repo_address: RepoAddress, + job_id: str, + artifact_name: str, + local_path: Path, + ): + base_artifacts.upload_job_artifact_files( + storage=self._storage, + repo_address=repo_address, + job_id=job_id, + artifact_name=artifact_name, + local_path=local_path, + ) def list_tag_heads(self, repo_address: RepoAddress) -> List[TagHead]: return base_tags.list_tag_heads(self._storage, repo_address) @@ -120,8 +145,16 @@ def add_tag_from_run( run_jobs, ) - def add_tag_from_local_dirs(self, repo_data: RepoData, tag_name: str, local_dirs: List[str]): - pass + def add_tag_from_local_dirs( + self, repo_data: LocalRepoData, tag_name: str, local_dirs: List[str] + ): + base_tags.create_tag_from_local_dirs( + self._storage, + repo_data, + tag_name, + local_dirs, + self.type, + ) def delete_tag_head(self, repo_address: RepoAddress, tag_head: TagHead): base_tags.delete_tag(self._storage, repo_address, tag_head) @@ -173,20 +206,8 @@ def delete_secret(self, repo_address: RepoAddress, secret_name: str): secret_name, ) - def download_run_artifact_files( - self, - repo_address: RepoAddress, - run_name: str, - output_dir: Optional[str], - output_job_dirs: bool = True, - ): - pass + def get_signed_download_url(self, object_key: str) -> str: + return self._storage.get_signed_download_url(object_key) - def upload_job_artifact_files( - self, - repo_address: RepoAddress, - job_id: str, - artifact_name: str, - local_path: Path, - ): - pass + def get_signed_upload_url(self, object_key: str) -> str: + return self._storage.get_signed_upload_url(object_key) diff --git a/cli/dstack/backend/gcp/storage.py b/cli/dstack/backend/gcp/storage.py index 09845517e8..bcbeacb88e 100644 --- a/cli/dstack/backend/gcp/storage.py +++ b/cli/dstack/backend/gcp/storage.py @@ -1,11 +1,14 @@ -from typing import Dict, List, Optional +import os +from datetime import timedelta +from typing import Callable, Dict, List, Optional from google.cloud import exceptions, storage -from dstack.backend.base.storage import Storage +from dstack.backend.base.storage import SIGNED_URL_EXPIRATION, CloudStorage +from dstack.core.storage import StorageFile -class GCPStorage(Storage): +class GCPStorage(CloudStorage): def __init__(self, project_id: str, bucket_name: str): self.storage_client = self._get_client(project_id) self.bucket_name = bucket_name @@ -36,6 +39,41 @@ def list_objects(self, keys_prefix: str) -> List[str]: object_names = [blob.name for blob in blobs] return object_names + def list_files(self, dirpath: str) -> List[StorageFile]: + prefix = dirpath + blobs = self.bucket.client.list_blobs(self.bucket.name, prefix=prefix) + files = [] + for blob in blobs: + file = StorageFile( + filepath=blob.name.removeprefix(prefix), + filesize_in_bytes=blob.size, + ) + files.append(file) + return files + + def download_file(self, source_path: str, dest_path: str, callback: Callable[[int], None]): + blob = self.bucket.blob(source_path) + blob.download_to_filename(dest_path) + callback(os.path.getsize(dest_path)) + + def upload_file(self, source_path: str, dest_path: str, callback: Callable[[int], None]): + blob = self.bucket.blob(dest_path) + blob.upload_from_filename(source_path) + callback(os.path.getsize(source_path)) + + def get_signed_download_url(self, key: str) -> str: + blob = self.bucket.blob(key) + url = blob.generate_signed_url(expiration=timedelta(seconds=SIGNED_URL_EXPIRATION)) + return url + + def get_signed_upload_url(self, key: str) -> str: + blob = self.bucket.blob(key) + url = blob.generate_signed_url( + expiration=timedelta(seconds=SIGNED_URL_EXPIRATION), + method="PUT", + ) + return url + def _get_client(self, project_id: str) -> storage.Client: return storage.Client(project=project_id) From 274b78a817672cad7d232be16c60907baea8fb90 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Thu, 16 Feb 2023 10:58:12 +0500 Subject: [PATCH 11/58] Implement gcp poll logs --- cli/dstack/backend/base/logs.py | 14 ++++++--- cli/dstack/backend/gcp/__init__.py | 11 ++++++- cli/dstack/backend/gcp/logs.py | 48 ++++++++++++++++++++++++++++++ cli/requirements.txt | 1 + 4 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 cli/dstack/backend/gcp/logs.py diff --git a/cli/dstack/backend/base/logs.py b/cli/dstack/backend/base/logs.py index f969889a2c..bd9eaac5ee 100644 --- a/cli/dstack/backend/base/logs.py +++ b/cli/dstack/backend/base/logs.py @@ -4,6 +4,7 @@ from dstack.backend.base import jobs from dstack.backend.base.storage import Storage +from dstack.core.job import Job from dstack.core.log_event import LogEvent, LogEventSource from dstack.core.repo import RepoAddress @@ -27,10 +28,7 @@ def render_log_message( job_id = message["job_id"] log = message["log"] job = jobs.get_job(storage, repo_address, job_id) - pat = get_logs_host_replace_pattern(job) - sub = get_logs_host_replace_sub(job) - if pat is not None: - log = re.sub(pat, sub, log) + log = replace_logs_host(log, job) return LogEvent( event_id=event["eventId"], timestamp=event["timestamp"], @@ -42,6 +40,14 @@ def render_log_message( ) +def replace_logs_host(log: str, job: Job) -> str: + pat = get_logs_host_replace_pattern(job) + sub = get_logs_host_replace_sub(job) + if pat is not None: + log = re.sub(pat, sub, log) + return log + + def get_logs_host_replace_pattern(job: str) -> str: if not (job.host_name and job.ports and job.app_specs): return None diff --git a/cli/dstack/backend/gcp/__init__.py b/cli/dstack/backend/gcp/__init__.py index b492c04af4..d44f92802b 100644 --- a/cli/dstack/backend/gcp/__init__.py +++ b/cli/dstack/backend/gcp/__init__.py @@ -13,6 +13,7 @@ from dstack.backend.base import tags as base_tags from dstack.backend.gcp.compute import GCPCompute from dstack.backend.gcp.config import GCPConfig +from dstack.backend.gcp.logs import GCPLogging from dstack.backend.gcp.secrets import GCPSecretsManager from dstack.backend.gcp.storage import GCPStorage from dstack.core.artifact import Artifact @@ -37,6 +38,10 @@ def __init__(self): project_id=self.config.project_id, bucket_name=self.config.bucket_name, ) + self._logging = GCPLogging( + project_id=self.config.project_id, + bucket_name=self.config.bucket_name, + ) self.configure() self._loaded = True @@ -91,7 +96,11 @@ def poll_logs( start_time: int, attached: bool, ) -> Generator[LogEvent, None, None]: - pass + yield from self._logging.poll_logs( + storage=self._storage, + repo_address=repo_address, + run_name=job_heads[0].run_name, + ) def list_run_artifact_files(self, repo_address: RepoAddress, run_name: str) -> List[Artifact]: return base_artifacts.list_run_artifact_files(self._storage, repo_address, run_name) diff --git a/cli/dstack/backend/gcp/logs.py b/cli/dstack/backend/gcp/logs.py new file mode 100644 index 0000000000..40ed9b561c --- /dev/null +++ b/cli/dstack/backend/gcp/logs.py @@ -0,0 +1,48 @@ +from typing import Generator + +from google.cloud import logging + +from dstack.backend.base import jobs +from dstack.backend.base.logs import replace_logs_host +from dstack.backend.base.storage import Storage +from dstack.core.log_event import LogEvent, LogEventSource +from dstack.core.repo import RepoAddress + + +class GCPLogging: + def __init__(self, project_id: str, bucket_name: str): + self.project_id = project_id + self.bucket_name = bucket_name + self.logging_client = logging.Client(project=project_id) + + def poll_logs( + self, + storage: Storage, + repo_address: RepoAddress, + run_name: str, + ) -> Generator[LogEvent, None, None]: + log_name = f"dstack-{self.bucket_name}-{run_name}" + logger = self.logging_client.logger(log_name) + log_entries = logger.list_entries() + for log_entry in log_entries: + yield log_entry_to_log_event(storage, repo_address, log_entry) + + +def log_entry_to_log_event( + storage: Storage, + repo_address: RepoAddress, + log_entry: logging.LogEntry, +) -> LogEvent: + job_id = log_entry.payload["job_id"] + log = log_entry.payload["log"] + job = jobs.get_job(storage, repo_address, job_id) + log = replace_logs_host(log, job) + return LogEvent( + event_id=log_entry.insert_id, + timestamp=log_entry.timestamp, + job_id=job_id, + log_message=log, + log_source=LogEventSource.STDOUT + if log_entry.payload["source"] == "stdout" + else LogEventSource.STDERR, + ) diff --git a/cli/requirements.txt b/cli/requirements.txt index 6b6d761e90..1c70eeff87 100644 --- a/cli/requirements.txt +++ b/cli/requirements.txt @@ -24,4 +24,5 @@ pygtail google-cloud-storage google-cloud-compute google-cloud-secret-manager +google-cloud-logging aiosqlite From b70e6cf75b2913a787954f6661ed6fa1dde49c34 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Thu, 16 Feb 2023 14:03:55 +0500 Subject: [PATCH 12/58] Setup basic config --- cli/dstack/backend/gcp/__init__.py | 9 +++++++-- cli/dstack/backend/gcp/config.py | 22 ++++++++++++++++++---- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/cli/dstack/backend/gcp/__init__.py b/cli/dstack/backend/gcp/__init__.py index d44f92802b..a2798d9440 100644 --- a/cli/dstack/backend/gcp/__init__.py +++ b/cli/dstack/backend/gcp/__init__.py @@ -28,8 +28,13 @@ class GCPBackend(CloudBackend): - def __init__(self): - self.config = GCPConfig() + def __init__(self, config: Optional[GCPConfig] = None): + if config is None: + config = GCPConfig.load() + if config is None: + self._loaded = False + return + self.config = config self._storage = GCPStorage( project_id=self.config.project_id, bucket_name=self.config.bucket_name ) diff --git a/cli/dstack/backend/gcp/config.py b/cli/dstack/backend/gcp/config.py index 7a557fd166..0ea4036365 100644 --- a/cli/dstack/backend/gcp/config.py +++ b/cli/dstack/backend/gcp/config.py @@ -1,11 +1,17 @@ from pathlib import Path +from typing import Dict from dstack.core.config import BackendConfig, get_config_path class GCPConfig(BackendConfig): - def __init__(self): - super().__init__() + @property + def name(self): + return "gcp" + + def __init__( + self, + ): self.project_id = "dstack" self.zone = "us-central1-a" self.bucket_name = "dstack-bucket" @@ -13,8 +19,16 @@ def __init__(self): def configure(self): pass - def load(self, path: Path = get_config_path()): - super().load(path=path) + @classmethod + def load(cls, path: Path = get_config_path()): + return cls() def save(self, path: Path = get_config_path()): pass + + def serialize(self) -> Dict: + return { + "project_id": self.project_id, + "zone": self.zone, + "bucket": self.bucket_name, + } From 1ebb76d7a9821447708c3c40deee1a670dde4c41 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 21 Feb 2023 10:27:40 +0500 Subject: [PATCH 13/58] Implement gcp storage for runner --- runner/go.mod | 9 +- runner/go.sum | 747 ++++++++++++++++++++++++- runner/internal/backend/gcp/storage.go | 88 +++ 3 files changed, 830 insertions(+), 14 deletions(-) create mode 100644 runner/internal/backend/gcp/storage.go diff --git a/runner/go.mod b/runner/go.mod index 3186759b79..295e4eb08a 100644 --- a/runner/go.mod +++ b/runner/go.mod @@ -15,7 +15,7 @@ require ( github.com/go-git/go-git/v5 v5.4.2 github.com/google/uuid v1.3.0 github.com/sirupsen/logrus v1.8.1 - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.8.1 github.com/urfave/cli/v2 v2.3.0 go.uber.org/atomic v1.4.0 golang.org/x/crypto v0.6.0 @@ -23,6 +23,7 @@ require ( ) require ( + cloud.google.com/go/storage v1.29.0 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.15 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.21 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.15 // indirect @@ -35,10 +36,8 @@ require ( github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.1 - golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect - google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6 // indirect - google.golang.org/grpc v1.38.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c + google.golang.org/api v0.106.0 + gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.20.3 ) diff --git a/runner/go.sum b/runner/go.sum index 242e1285ff..182db74c53 100644 --- a/runner/go.sum +++ b/runner/go.sum @@ -10,17 +10,383 @@ cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6T cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go v0.100.1/go.mod h1:fs4QogzfH5n2pBXBP9vRiU+eCny7lD2vmFZy79Iuw1U= +cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= +cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= +cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= +cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= +cloud.google.com/go v0.107.0 h1:qkj22L7bgkl6vIeZDlOY2po43Mx/TIa2Wsa7VR+PEww= +cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= +cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= +cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= +cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp+CE8dkrszb4oK9CWyvD4o= +cloud.google.com/go/accesscontextmanager v1.4.0/go.mod h1:/Kjh7BBu/Gh83sv+K60vN9QE5NJcd80sU33vIe2IFPE= +cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= +cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= +cloud.google.com/go/aiplatform v1.27.0/go.mod h1:Bvxqtl40l0WImSb04d0hXFU7gDOiq9jQmorivIiWcKg= +cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= +cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= +cloud.google.com/go/apigateway v1.3.0/go.mod h1:89Z8Bhpmxu6AmUxuVRg/ECRGReEdiP3vQtk4Z1J9rJk= +cloud.google.com/go/apigateway v1.4.0/go.mod h1:pHVY9MKGaH9PQ3pJ4YLzoj6U5FUDeDFBllIz7WmzJoc= +cloud.google.com/go/apigeeconnect v1.3.0/go.mod h1:G/AwXFAKo0gIXkPTVfZDd2qA1TxBXJ3MgMRBQkIi9jc= +cloud.google.com/go/apigeeconnect v1.4.0/go.mod h1:kV4NwOKqjvt2JYR0AoIWo2QGfoRtn/pkS3QlHp0Ni04= +cloud.google.com/go/appengine v1.4.0/go.mod h1:CS2NhuBuDXM9f+qscZ6V86m1MIIqPj3WC/UoEuR1Sno= +cloud.google.com/go/appengine v1.5.0/go.mod h1:TfasSozdkFI0zeoxW3PTBLiNqRmzraodCWatWI9Dmak= +cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4= +cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= +cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ= +cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk= +cloud.google.com/go/artifactregistry v1.8.0/go.mod h1:w3GQXkJX8hiKN0v+at4b0qotwijQbYUqF2GWkZzAhC0= +cloud.google.com/go/artifactregistry v1.9.0/go.mod h1:2K2RqvA2CYvAeARHRkLDhMDJ3OXy26h3XW+3/Jh2uYc= +cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= +cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s= +cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0= +cloud.google.com/go/asset v1.9.0/go.mod h1:83MOE6jEJBMqFKadM9NLRcs80Gdw76qGuHn8m3h8oHQ= +cloud.google.com/go/asset v1.10.0/go.mod h1:pLz7uokL80qKhzKr4xXGvBQXnzHn5evJAEAtZiIb0wY= +cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= +cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= +cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= +cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo= +cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0= +cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= +cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= +cloud.google.com/go/automl v1.7.0/go.mod h1:RL9MYCCsJEOmt0Wf3z9uzG0a7adTT1fe+aObgSpkCt8= +cloud.google.com/go/automl v1.8.0/go.mod h1:xWx7G/aPEe/NP+qzYXktoBSDfjO+vnKMGgsApGJJquM= +cloud.google.com/go/baremetalsolution v0.3.0/go.mod h1:XOrocE+pvK1xFfleEnShBlNAXf+j5blPPxrhjKgnIFc= +cloud.google.com/go/baremetalsolution v0.4.0/go.mod h1:BymplhAadOO/eBa7KewQ0Ppg4A4Wplbn+PsFKRLo0uI= +cloud.google.com/go/batch v0.3.0/go.mod h1:TR18ZoAekj1GuirsUsR1ZTKN3FC/4UDnScjT8NXImFE= +cloud.google.com/go/batch v0.4.0/go.mod h1:WZkHnP43R/QCGQsZ+0JyG4i79ranE2u8xvjq/9+STPE= +cloud.google.com/go/beyondcorp v0.2.0/go.mod h1:TB7Bd+EEtcw9PCPQhCJtJGjk/7TC6ckmnSFS+xwTfm4= +cloud.google.com/go/beyondcorp v0.3.0/go.mod h1:E5U5lcrcXMsCuoDNyGrpyTm/hn7ne941Jz2vmksAxW8= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= +cloud.google.com/go/bigquery v1.43.0/go.mod h1:ZMQcXHsl+xmU1z36G2jNGZmKp9zNY5BUua5wDgmNCfw= +cloud.google.com/go/bigquery v1.44.0/go.mod h1:0Y33VqXTEsbamHJvJHdFmtqHvMIY28aK1+dFsvaChGc= +cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= +cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= +cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI= +cloud.google.com/go/billing v1.7.0/go.mod h1:q457N3Hbj9lYwwRbnlD7vUpyjq6u5U1RAOArInEiD5Y= +cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= +cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= +cloud.google.com/go/binaryauthorization v1.3.0/go.mod h1:lRZbKgjDIIQvzYQS1p99A7/U1JqvqeZg0wiI5tp6tg0= +cloud.google.com/go/binaryauthorization v1.4.0/go.mod h1:tsSPQrBd77VLplV70GUhBf/Zm3FsKmgSqgm4UmiDItk= +cloud.google.com/go/certificatemanager v1.3.0/go.mod h1:n6twGDvcUBFu9uBgt4eYvvf3sQ6My8jADcOVwHmzadg= +cloud.google.com/go/certificatemanager v1.4.0/go.mod h1:vowpercVFyqs8ABSmrdV+GiFf2H/ch3KyudYQEMM590= +cloud.google.com/go/channel v1.8.0/go.mod h1:W5SwCXDJsq/rg3tn3oG0LOxpAo6IMxNa09ngphpSlnk= +cloud.google.com/go/channel v1.9.0/go.mod h1:jcu05W0my9Vx4mt3/rEHpfxc9eKi9XwsdDL8yBMbKUk= +cloud.google.com/go/cloudbuild v1.3.0/go.mod h1:WequR4ULxlqvMsjDEEEFnOG5ZSRSgWOywXYDb1vPE6U= +cloud.google.com/go/cloudbuild v1.4.0/go.mod h1:5Qwa40LHiOXmz3386FrjrYM93rM/hdRr7b53sySrTqA= +cloud.google.com/go/clouddms v1.3.0/go.mod h1:oK6XsCDdW4Ib3jCCBugx+gVjevp2TMXFtgxvPSee3OM= +cloud.google.com/go/clouddms v1.4.0/go.mod h1:Eh7sUGCC+aKry14O1NRljhjyrr0NFC0G2cjwX0cByRk= +cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY= +cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI= +cloud.google.com/go/cloudtasks v1.7.0/go.mod h1:ImsfdYWwlWNJbdgPIIGJWC+gemEGTBK/SunNQQNCAb4= +cloud.google.com/go/cloudtasks v1.8.0/go.mod h1:gQXUIwCSOI4yPVK7DgTVFiiP0ZW/eQkydWzwVMdHxrI= +cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= +cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= +cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= +cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= +cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= +cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= +cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= +cloud.google.com/go/compute v1.12.0/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute v1.13.0/go.mod h1:5aPTS0cUNMIc1CE546K+Th6weJUNQErARyZtRXDJ8GE= +cloud.google.com/go/compute v1.14.0 h1:hfm2+FfxVmnRlh6LpB7cg1ZNU+5edAHmW679JePztk0= +cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= +cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= +cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= +cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= +cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg= +cloud.google.com/go/container v1.7.0/go.mod h1:Dp5AHtmothHGX3DwwIHPgq45Y8KmNsgN3amoYfxVkLo= +cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= +cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= +cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= +cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= +cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= +cloud.google.com/go/datacatalog v1.7.0/go.mod h1:9mEl4AuDYWw81UGc41HonIHH7/sn52H0/tc8f8ZbZIE= +cloud.google.com/go/datacatalog v1.8.0/go.mod h1:KYuoVOv9BM8EYz/4eMFxrr4DUKhGIOXxZoKYF5wdISM= +cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= +cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= +cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KFMQQNSThKo= +cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE= +cloud.google.com/go/dataform v0.5.0/go.mod h1:GFUYRe8IBa2hcomWplodVmUx/iTL0FrsauObOM3Ipr0= +cloud.google.com/go/datafusion v1.4.0/go.mod h1:1Zb6VN+W6ALo85cXnM1IKiPw+yQMKMhB9TsTSRDo/38= +cloud.google.com/go/datafusion v1.5.0/go.mod h1:Kz+l1FGHB0J+4XF2fud96WMmRiq/wj8N9u007vyXZ2w= +cloud.google.com/go/datalabeling v0.5.0/go.mod h1:TGcJ0G2NzcsXSE/97yWjIZO0bXj0KbVlINXMG9ud42I= +cloud.google.com/go/datalabeling v0.6.0/go.mod h1:WqdISuk/+WIGeMkpw/1q7bK/tFEZxsrFJOJdY2bXvTQ= +cloud.google.com/go/dataplex v1.3.0/go.mod h1:hQuRtDg+fCiFgC8j0zV222HvzFQdRd+SVX8gdmFcZzA= +cloud.google.com/go/dataplex v1.4.0/go.mod h1:X51GfLXEMVJ6UN47ESVqvlsRplbLhcsAt0kZCCKsU0A= +cloud.google.com/go/dataproc v1.7.0/go.mod h1:CKAlMjII9H90RXaMpSxQ8EU6dQx6iAYNPcYPOkSbi8s= +cloud.google.com/go/dataproc v1.8.0/go.mod h1:5OW+zNAH0pMpw14JVrPONsxMQYMBqJuzORhIBfBn9uI= +cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo= +cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/datastore v1.10.0/go.mod h1:PC5UzAmDEkAmkfaknstTYbNpgE49HAgW2J1gcgUfmdM= +cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= +cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= +cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g= +cloud.google.com/go/datastream v1.5.0/go.mod h1:6TZMMNPwjUqZHBKPQ1wwXpb0d5VDVPl2/XoS5yi88q4= +cloud.google.com/go/deploy v1.4.0/go.mod h1:5Xghikd4VrmMLNaF6FiRFDlHb59VM59YoDQnOUdsH/c= +cloud.google.com/go/deploy v1.5.0/go.mod h1:ffgdD0B89tToyW/U/D2eL0jN2+IEV/3EMuXHA0l4r+s= +cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= +cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= +cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= +cloud.google.com/go/dialogflow v1.18.0/go.mod h1:trO7Zu5YdyEuR+BhSNOqJezyFQ3aUzz0njv7sMx/iek= +cloud.google.com/go/dialogflow v1.19.0/go.mod h1:JVmlG1TwykZDtxtTXujec4tQ+D8SBFMoosgy+6Gn0s0= +cloud.google.com/go/dlp v1.6.0/go.mod h1:9eyB2xIhpU0sVwUixfBubDoRwP+GjeUoxxeueZmqvmM= +cloud.google.com/go/dlp v1.7.0/go.mod h1:68ak9vCiMBjbasxeVD17hVPxDEck+ExiHavX8kiHG+Q= +cloud.google.com/go/documentai v1.7.0/go.mod h1:lJvftZB5NRiFSX4moiye1SMxHx0Bc3x1+p9e/RfXYiU= +cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX2Q7UD0c5IhU= +cloud.google.com/go/documentai v1.9.0/go.mod h1:FS5485S8R00U10GhgBC0aNGrJxBP8ZVpEeJ7PQDZd6k= +cloud.google.com/go/documentai v1.10.0/go.mod h1:vod47hKQIPeCfN2QS/jULIvQTugbmdc0ZvxxfQY1bg4= +cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= +cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= +cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= +cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w= +cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= +cloud.google.com/go/essentialcontacts v1.3.0/go.mod h1:r+OnHa5jfj90qIfZDO/VztSFqbQan7HV75p8sA+mdGI= +cloud.google.com/go/essentialcontacts v1.4.0/go.mod h1:8tRldvHYsmnBCHdFpvU+GL75oWiBKl80BiqlFh9tp+8= +cloud.google.com/go/eventarc v1.7.0/go.mod h1:6ctpF3zTnaQCxUjHUdcfgcA1A2T309+omHZth7gDfmc= +cloud.google.com/go/eventarc v1.8.0/go.mod h1:imbzxkyAU4ubfsaKYdQg04WS1NvncblHEup4kvF+4gw= +cloud.google.com/go/filestore v1.3.0/go.mod h1:+qbvHGvXU1HaKX2nD0WEPo92TP/8AQuCVEBXNY9z0+w= +cloud.google.com/go/filestore v1.4.0/go.mod h1:PaG5oDfo9r224f8OYXURtAsY+Fbyq/bLYoINEK8XQAI= +cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE= +cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= +cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= +cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY= +cloud.google.com/go/functions v1.9.0/go.mod h1:Y+Dz8yGguzO3PpIjhLTbnqV1CWmgQ5UwtlpzoyquQ08= +cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM= +cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA= +cloud.google.com/go/gaming v1.7.0/go.mod h1:LrB8U7MHdGgFG851iHAfqUdLcKBdQ55hzXy9xBJz0+w= +cloud.google.com/go/gaming v1.8.0/go.mod h1:xAqjS8b7jAVW0KFYeRUxngo9My3f33kFmua++Pi+ggM= +cloud.google.com/go/gkebackup v0.2.0/go.mod h1:XKvv/4LfG829/B8B7xRkk8zRrOEbKtEam6yNfuQNH60= +cloud.google.com/go/gkebackup v0.3.0/go.mod h1:n/E671i1aOQvUxT541aTkCwExO/bTer2HDlj4TsBRAo= +cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o= +cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= +cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0= +cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= +cloud.google.com/go/gkemulticloud v0.3.0/go.mod h1:7orzy7O0S+5kq95e4Hpn7RysVA7dPs8W/GgfUtsPbrA= +cloud.google.com/go/gkemulticloud v0.4.0/go.mod h1:E9gxVBnseLWCk24ch+P9+B2CoDFJZTyIgLKSalC7tuI= +cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= +cloud.google.com/go/gsuiteaddons v1.3.0/go.mod h1:EUNK/J1lZEZO8yPtykKxLXI6JSVN2rg9bN8SXOa0bgM= +cloud.google.com/go/gsuiteaddons v1.4.0/go.mod h1:rZK5I8hht7u7HxFQcFei0+AtfS9uSushomRlg+3ua1o= +cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c= +cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= +cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= +cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc= +cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= +cloud.google.com/go/iam v0.8.0 h1:E2osAkZzxI/+8pZcxVLcDtAQx/u+hZXVryUaYQ5O0Kk= +cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= +cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= +cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= +cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM= +cloud.google.com/go/ids v1.2.0/go.mod h1:5WXvp4n25S0rA/mQWAg1YEEBBq6/s+7ml1RDCW1IrcY= +cloud.google.com/go/iot v1.3.0/go.mod h1:r7RGh2B61+B8oz0AGE+J72AhA0G7tdXItODWsaA2oLs= +cloud.google.com/go/iot v1.4.0/go.mod h1:dIDxPOn0UvNDUMD8Ger7FIaTuvMkj+aGk94RPP0iV+g= +cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= +cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg= +cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0= +cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= +cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= +cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE= +cloud.google.com/go/language v1.8.0/go.mod h1:qYPVHf7SPoNNiCL2Dr0FfEFNil1qi3pQEyygwpgVKB8= +cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= +cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= +cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw= +cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= +cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= +cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= +cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE= +cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM= +cloud.google.com/go/maps v0.1.0/go.mod h1:BQM97WGyfw9FWEmQMpZ5T6cpovXXSd1cGmFma94eubI= +cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= +cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= +cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= +cloud.google.com/go/memcache v1.5.0/go.mod h1:dk3fCK7dVo0cUU2c36jKb4VqKPS22BTkf81Xq617aWM= +cloud.google.com/go/memcache v1.6.0/go.mod h1:XS5xB0eQZdHtTuTF9Hf8eJkKtR3pVRCcvJwtm68T3rA= +cloud.google.com/go/memcache v1.7.0/go.mod h1:ywMKfjWhNtkQTxrWxCkCFkoPjLHPW6A7WOTVI8xy3LY= +cloud.google.com/go/metastore v1.5.0/go.mod h1:2ZNrDcQwghfdtCwJ33nM0+GrBGlVuh8rakL3vdPY3XY= +cloud.google.com/go/metastore v1.6.0/go.mod h1:6cyQTls8CWXzk45G55x57DVQ9gWg7RiH65+YgPsNh9s= +cloud.google.com/go/metastore v1.7.0/go.mod h1:s45D0B4IlsINu87/AsWiEVYbLaIMeUSoxlKKDqBGFS8= +cloud.google.com/go/metastore v1.8.0/go.mod h1:zHiMc4ZUpBiM7twCIFQmJ9JMEkDSyZS9U12uf7wHqSI= +cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk= +cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4= +cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= +cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= +cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM= +cloud.google.com/go/networkconnectivity v1.7.0/go.mod h1:RMuSbkdbPwNMQjB5HBWD5MpTBnNm39iAVpC3TmsExt8= +cloud.google.com/go/networkmanagement v1.4.0/go.mod h1:Q9mdLLRn60AsOrPc8rs8iNV6OHXaGcDdsIQe1ohekq8= +cloud.google.com/go/networkmanagement v1.5.0/go.mod h1:ZnOeZ/evzUdUsnvRt792H0uYEnHQEMaz+REhhzJRcf4= +cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= +cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= +cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY= +cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34= +cloud.google.com/go/notebooks v1.4.0/go.mod h1:4QPMngcwmgb6uw7Po99B2xv5ufVoIQ7nOGDyL4P8AgA= +cloud.google.com/go/notebooks v1.5.0/go.mod h1:q8mwhnP9aR8Hpfnrc5iN5IBhrXUy8S2vuYs+kBJ/gu0= +cloud.google.com/go/optimization v1.1.0/go.mod h1:5po+wfvX5AQlPznyVEZjGJTMr4+CAkJf2XSTQOOl9l4= +cloud.google.com/go/optimization v1.2.0/go.mod h1:Lr7SOHdRDENsh+WXVmQhQTrzdu9ybg0NecjHidBq6xs= +cloud.google.com/go/orchestration v1.3.0/go.mod h1:Sj5tq/JpWiB//X/q3Ngwdl5K7B7Y0KZ7bfv0wL6fqVA= +cloud.google.com/go/orchestration v1.4.0/go.mod h1:6W5NLFWs2TlniBphAViZEVhrXRSMgUGDfW7vrWKvsBk= +cloud.google.com/go/orgpolicy v1.4.0/go.mod h1:xrSLIV4RePWmP9P3tBl8S93lTmlAxjm06NSm2UTmKvE= +cloud.google.com/go/orgpolicy v1.5.0/go.mod h1:hZEc5q3wzwXJaKrsx5+Ewg0u1LxJ51nNFlext7Tanwc= +cloud.google.com/go/osconfig v1.7.0/go.mod h1:oVHeCeZELfJP7XLxcBGTMBvRO+1nQ5tFG9VQTmYS2Fs= +cloud.google.com/go/osconfig v1.8.0/go.mod h1:EQqZLu5w5XA7eKizepumcvWx+m8mJUhEwiPqWiZeEdg= +cloud.google.com/go/osconfig v1.9.0/go.mod h1:Yx+IeIZJ3bdWmzbQU4fxNl8xsZ4amB+dygAwFPlvnNo= +cloud.google.com/go/osconfig v1.10.0/go.mod h1:uMhCzqC5I8zfD9zDEAfvgVhDS8oIjySWh+l4WK6GnWw= +cloud.google.com/go/oslogin v1.4.0/go.mod h1:YdgMXWRaElXz/lDk1Na6Fh5orF7gvmJ0FGLIs9LId4E= +cloud.google.com/go/oslogin v1.5.0/go.mod h1:D260Qj11W2qx/HVF29zBg+0fd6YCSjSqLUkY/qEenQU= +cloud.google.com/go/oslogin v1.6.0/go.mod h1:zOJ1O3+dTU8WPlGEkFSh7qeHPPSoxrcMbbK1Nm2iX70= +cloud.google.com/go/oslogin v1.7.0/go.mod h1:e04SN0xO1UNJ1M5GP0vzVBFicIe4O53FOfcixIqTyXo= +cloud.google.com/go/phishingprotection v0.5.0/go.mod h1:Y3HZknsK9bc9dMi+oE8Bim0lczMU6hrX0UpADuMefr0= +cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7qni15yVUoAxiFxnlSUA= +cloud.google.com/go/policytroubleshooter v1.3.0/go.mod h1:qy0+VwANja+kKrjlQuOzmlvscn4RNsAc0e15GGqfMxg= +cloud.google.com/go/policytroubleshooter v1.4.0/go.mod h1:DZT4BcRw3QoO8ota9xw/LKtPa8lKeCByYeKTIf/vxdE= +cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= +cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI= +cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= +cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg= +cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= +cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= +cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= +cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7dd5NwwNQUErSgEDit1DLNTdo= +cloud.google.com/go/recaptchaenterprise/v2 v2.4.0/go.mod h1:Am3LHfOuBstrLrNCBrlI5sbwx9LBg3te2N6hGvHn2mE= +cloud.google.com/go/recaptchaenterprise/v2 v2.5.0/go.mod h1:O8LzcHXN3rz0j+LBC91jrwI3R+1ZSZEWrfL7XHgNo9U= +cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg= +cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= +cloud.google.com/go/recommender v1.5.0/go.mod h1:jdoeiBIVrJe9gQjwd759ecLJbxCDED4A6p+mqoqDvTg= +cloud.google.com/go/recommender v1.6.0/go.mod h1:+yETpm25mcoiECKh9DEScGzIRyDKpZ0cEhWGo+8bo+c= +cloud.google.com/go/recommender v1.7.0/go.mod h1:XLHs/W+T8olwlGOgfQenXBTbIseGclClff6lhFVe9Bs= +cloud.google.com/go/recommender v1.8.0/go.mod h1:PkjXrTT05BFKwxaUxQmtIlrtj0kph108r02ZZQ5FE70= +cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y= +cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A= +cloud.google.com/go/redis v1.9.0/go.mod h1:HMYQuajvb2D0LvMgZmLDZW8V5aOC/WxstZHiy4g8OiA= +cloud.google.com/go/redis v1.10.0/go.mod h1:ThJf3mMBQtW18JzGgh41/Wld6vnDDc/F/F35UolRZPM= +cloud.google.com/go/resourcemanager v1.3.0/go.mod h1:bAtrTjZQFJkiWTPDb1WBjzvc6/kifjj4QBYuKCCoqKA= +cloud.google.com/go/resourcemanager v1.4.0/go.mod h1:MwxuzkumyTX7/a3n37gmsT3py7LIXwrShilPh3P1tR0= +cloud.google.com/go/resourcesettings v1.3.0/go.mod h1:lzew8VfESA5DQ8gdlHwMrqZs1S9V87v3oCnKCWoOuQU= +cloud.google.com/go/resourcesettings v1.4.0/go.mod h1:ldiH9IJpcrlC3VSuCGvjR5of/ezRrOxFtpJoJo5SmXg= +cloud.google.com/go/retail v1.8.0/go.mod h1:QblKS8waDmNUhghY2TI9O3JLlFk8jybHeV4BF19FrE4= +cloud.google.com/go/retail v1.9.0/go.mod h1:g6jb6mKuCS1QKnH/dpu7isX253absFl6iE92nHwlBUY= +cloud.google.com/go/retail v1.10.0/go.mod h1:2gDk9HsL4HMS4oZwz6daui2/jmKvqShXKQuB2RZ+cCc= +cloud.google.com/go/retail v1.11.0/go.mod h1:MBLk1NaWPmh6iVFSz9MeKG/Psyd7TAgm6y/9L2B4x9Y= +cloud.google.com/go/run v0.2.0/go.mod h1:CNtKsTA1sDcnqqIFR3Pb5Tq0usWxJJvsWOCPldRU3Do= +cloud.google.com/go/run v0.3.0/go.mod h1:TuyY1+taHxTjrD0ZFk2iAR+xyOXEA0ztb7U3UNA0zBo= +cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= +cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= +cloud.google.com/go/scheduler v1.6.0/go.mod h1:SgeKVM7MIwPn3BqtcBntpLyrIJftQISRrYB5ZtT+KOk= +cloud.google.com/go/scheduler v1.7.0/go.mod h1:jyCiBqWW956uBjjPMMuX09n3x37mtyPJegEWKxRsn44= +cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= +cloud.google.com/go/secretmanager v1.8.0/go.mod h1:hnVgi/bN5MYHd3Gt0SPuTPPp5ENina1/LxM+2W9U9J4= +cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8AL6X1MA01dW8ht4= +cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= +cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= +cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= +cloud.google.com/go/security v1.9.0/go.mod h1:6Ta1bO8LXI89nZnmnsZGp9lVoVWXqsVbIq/t9dzI+2Q= +cloud.google.com/go/security v1.10.0/go.mod h1:QtOMZByJVlibUT2h9afNDWRZ1G96gVywH8T5GUSb9IA= +cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU= +cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= +cloud.google.com/go/securitycenter v1.15.0/go.mod h1:PeKJ0t8MoFmmXLXWm41JidyzI3PJjd8sXWaVqg43WWk= +cloud.google.com/go/securitycenter v1.16.0/go.mod h1:Q9GMaLQFUD+5ZTabrbujNWLtSLZIZF7SAR0wWECrjdk= +cloud.google.com/go/servicecontrol v1.4.0/go.mod h1:o0hUSJ1TXJAmi/7fLJAedOovnujSEvjKCAFNXPQ1RaU= +cloud.google.com/go/servicecontrol v1.5.0/go.mod h1:qM0CnXHhyqKVuiZnGKrIurvVImCs8gmqWsDoqe9sU1s= +cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= +cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= +cloud.google.com/go/servicedirectory v1.6.0/go.mod h1:pUlbnWsLH9c13yGkxCmfumWEPjsRs1RlmJ4pqiNjVL4= +cloud.google.com/go/servicedirectory v1.7.0/go.mod h1:5p/U5oyvgYGYejufvxhgwjL8UVXjkuw7q5XcG10wx1U= +cloud.google.com/go/servicemanagement v1.4.0/go.mod h1:d8t8MDbezI7Z2R1O/wu8oTggo3BI2GKYbdG4y/SJTco= +cloud.google.com/go/servicemanagement v1.5.0/go.mod h1:XGaCRe57kfqu4+lRxaFEAuqmjzF0r+gWHjWqKqBvKFo= +cloud.google.com/go/serviceusage v1.3.0/go.mod h1:Hya1cozXM4SeSKTAgGXgj97GlqUvF5JaoXacR1JTP/E= +cloud.google.com/go/serviceusage v1.4.0/go.mod h1:SB4yxXSaYVuUBYUml6qklyONXNLt83U0Rb+CXyhjEeU= +cloud.google.com/go/shell v1.3.0/go.mod h1:VZ9HmRjZBsjLGXusm7K5Q5lzzByZmJHf1d0IWHEN5X4= +cloud.google.com/go/shell v1.4.0/go.mod h1:HDxPzZf3GkDdhExzD/gs8Grqk+dmYcEjGShZgYa9URw= +cloud.google.com/go/spanner v1.41.0/go.mod h1:MLYDBJR/dY4Wt7ZaMIQ7rXOTLjYrmxLE/5ve9vFfWos= +cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= +cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= +cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0= +cloud.google.com/go/speech v1.9.0/go.mod h1:xQ0jTcmnRFFM2RfX/U+rk6FQNUF6DQlydUSyoooSpco= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= +cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= +cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= +cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI= +cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= +cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= +cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= +cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= +cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= +cloud.google.com/go/talent v1.3.0/go.mod h1:CmcxwJ/PKfRgd1pBjQgU6W3YBwiewmUzQYH5HHmSCmM= +cloud.google.com/go/talent v1.4.0/go.mod h1:ezFtAgVuRf8jRsvyE6EwmbTK5LKciD4KVnHuDEFmOOA= +cloud.google.com/go/texttospeech v1.4.0/go.mod h1:FX8HQHA6sEpJ7rCMSfXuzBcysDAuWusNNNvN9FELDd8= +cloud.google.com/go/texttospeech v1.5.0/go.mod h1:oKPLhR4n4ZdQqWKURdwxMy0uiTS1xU161C8W57Wkea4= +cloud.google.com/go/tpu v1.3.0/go.mod h1:aJIManG0o20tfDQlRIej44FcwGGl/cD0oiRyMKG19IQ= +cloud.google.com/go/tpu v1.4.0/go.mod h1:mjZaX8p0VBgllCzF6wcU2ovUXN9TONFLd7iz227X2Xg= +cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28= +cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y= +cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs= +cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg= +cloud.google.com/go/video v1.8.0/go.mod h1:sTzKFc0bUSByE8Yoh8X0mn8bMymItVGPfTuUBUyRgxk= +cloud.google.com/go/video v1.9.0/go.mod h1:0RhNKFRF5v92f8dQt0yhaHrEuH95m068JYOvLZYnJSw= +cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= +cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= +cloud.google.com/go/videointelligence v1.8.0/go.mod h1:dIcCn4gVDdS7yte/w+koiXn5dWVplOZkE+xwG9FgK+M= +cloud.google.com/go/videointelligence v1.9.0/go.mod h1:29lVRMPDYHikk3v8EdPSaL8Ku+eMzDljjuvRs105XoU= +cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0= +cloud.google.com/go/vision/v2 v2.2.0/go.mod h1:uCdV4PpN1S0jyCyq8sIM42v2Y6zOLkZs+4R9LrGYwFo= +cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiCRx0vSpZoUo= +cloud.google.com/go/vision/v2 v2.4.0/go.mod h1:VtI579ll9RpVTrdKdkMzckdnwMyX2JILb+MhPqRbPsY= +cloud.google.com/go/vision/v2 v2.5.0/go.mod h1:MmaezXOOE+IWa+cS7OhRRLK2cNv1ZL98zhqFFZaaH2E= +cloud.google.com/go/vmmigration v1.2.0/go.mod h1:IRf0o7myyWFSmVR1ItrBSFLFD/rJkfDCUTO4vLlJvsE= +cloud.google.com/go/vmmigration v1.3.0/go.mod h1:oGJ6ZgGPQOFdjHuocGcLqX4lc98YQ7Ygq8YQwHh9A7g= +cloud.google.com/go/vmwareengine v0.1.0/go.mod h1:RsdNEf/8UDvKllXhMz5J40XxDrNJNN4sagiox+OI208= +cloud.google.com/go/vpcaccess v1.4.0/go.mod h1:aQHVbTWDYUR1EbTApSVvMq1EnT57ppDmQzZ3imqIk4w= +cloud.google.com/go/vpcaccess v1.5.0/go.mod h1:drmg4HLk9NkZpGfCmZ3Tz0Bwnm2+DKqViEpeEpOq0m8= +cloud.google.com/go/webrisk v1.4.0/go.mod h1:Hn8X6Zr+ziE2aNd8SliSDWpEnSS1u4R9+xXZmFiHmGE= +cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuWDEEsqMTg= +cloud.google.com/go/webrisk v1.6.0/go.mod h1:65sW9V9rOosnc9ZY7A7jsy1zoHS5W9IAXv6dGqhMQMc= +cloud.google.com/go/webrisk v1.7.0/go.mod h1:mVMHgEYH0r337nmt1JyLthzMr6YxwN1aAIEc2fTcq7A= +cloud.google.com/go/websecurityscanner v1.3.0/go.mod h1:uImdKm2wyeXQevQJXeh8Uun/Ym1VqworNDlBXQevGMo= +cloud.google.com/go/websecurityscanner v1.4.0/go.mod h1:ebit/Fp0a+FWu5j4JOmJEV8S8CzdTkAS77oDsiSqYWQ= +cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0= +cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= +cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vfKf5Af+to4M= +cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= @@ -71,6 +437,7 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= @@ -147,6 +514,7 @@ github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8n github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= @@ -162,7 +530,14 @@ github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= @@ -307,7 +682,12 @@ github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -367,12 +747,17 @@ github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -386,19 +771,25 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -406,12 +797,26 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= @@ -421,9 +826,24 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/googleapis/enterprise-certificate-proxy v0.2.1 h1:RY7tHKZcRlk788d5WSo/e83gOyyy742E8GSs771ySpg= +github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= +github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= +github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= +github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= +github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= +github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= +github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -440,6 +860,7 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= @@ -449,6 +870,7 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -621,6 +1043,7 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -661,16 +1084,21 @@ github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -700,8 +1128,11 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1: github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= @@ -715,6 +1146,12 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -760,6 +1197,8 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -768,6 +1207,9 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -787,6 +1229,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -798,14 +1241,37 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.6.0 h1:L4ZwwTvKW9gr0ZMS1yrHD9GZhIuVjOBBnaKH+SPQK0Q= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -813,16 +1279,42 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 h1:nt+Q6cXKz4MosCSpnbMtqiQ8Oz0pxTef2B4Vca2lvfk= +golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -867,11 +1359,18 @@ golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -881,14 +1380,43 @@ golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -902,7 +1430,12 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -910,8 +1443,9 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= +golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -946,16 +1480,42 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -966,12 +1526,58 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= +google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= +google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= +google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= +google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= +google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.77.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= +google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= +google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= +google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g= +google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.93.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.95.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaETEI= +google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.99.0/go.mod h1:1YOf74vkVndF7pG6hIHuINsM7eWwpVTAfNMNiL91A08= +google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= +google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= +google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= +google.golang.org/api v0.106.0 h1:ffmW0faWCwKkpbbtvlY/K/8fUl+JKvNS5CVzRoyfCv8= +google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -992,11 +1598,101 @@ google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6 h1:4Xw2NwItrJOFR5s6PnK98PI6Bgw1LhMP1j/rO5WP0S4= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= +google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220722212130-b98a9ff5e252/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE= +google.golang.org/genproto v0.0.0-20220801145646-83ce21fca29f/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc= +google.golang.org/genproto v0.0.0-20220815135757-37a418bb8959/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220817144833-d7fd3f11b9b1/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220829144015-23454907ede3/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220829175752-36a9c930ecbf/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220913154956-18f8339a66a5/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220914142337-ca0e39ece12f/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220915135415-7fd63a7952de/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220916172020-2692e8806bfa/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220919141832-68c03719ef51/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006/go.mod h1:ht8XFiar2npT/g4vkk7O0WYS1sHOHbdujxbEp7CJWbw= +google.golang.org/genproto v0.0.0-20220926165614-551eb538f295/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= +google.golang.org/genproto v0.0.0-20220926220553-6981cbe3cfce/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= +google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqwhZAwq4wsRUaVG555sVgsNmIjRtO7t/JH29U= +google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221024153911-1573dae28c9c/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= +google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= +google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= +google.golang.org/genproto v0.0.0-20221114212237-e4508ebdbee1/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221117204609-8f9c96812029/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE= +google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1009,10 +1705,36 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= +google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1022,8 +1744,13 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1056,8 +1783,9 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= @@ -1069,6 +1797,7 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= diff --git a/runner/internal/backend/gcp/storage.go b/runner/internal/backend/gcp/storage.go new file mode 100644 index 0000000000..bd82807c9d --- /dev/null +++ b/runner/internal/backend/gcp/storage.go @@ -0,0 +1,88 @@ +package gcp + +import ( + "bytes" + "context" + "io" + + "cloud.google.com/go/storage" + "github.com/dstackai/dstack/runner/internal/gerrors" + "google.golang.org/api/iterator" +) + +type GCPStorage struct { + client *storage.Client + bucket *storage.BucketHandle + project string + bucketName string +} + +func NewGCPStorage(project, bucketName string) (*GCPStorage, error) { + ctx := context.TODO() + client, _ := storage.NewClient(ctx) + bucket := client.Bucket(bucketName) + if bucket == nil { + return nil, gerrors.New("Cannot access bucket") + } + return &GCPStorage{ + client: client, + bucket: bucket, + project: project, + bucketName: bucketName, + }, nil +} + +func (gstorage *GCPStorage) GetFile(ctx context.Context, key string) ([]byte, error) { + obj := gstorage.bucket.Object(key) + reader, err := obj.NewReader(ctx) + if err != nil { + return nil, gerrors.Wrap(err) + } + defer reader.Close() + buffer := new(bytes.Buffer) + io.Copy(buffer, reader) + return buffer.Bytes(), nil +} + +func (gstorage *GCPStorage) PutFile(ctx context.Context, key string, contents []byte) error { + obj := gstorage.bucket.Object(key) + writer := obj.NewWriter(ctx) + reader := bytes.NewReader(contents) + io.Copy(writer, reader) + return writer.Close() +} + +func (gstorage *GCPStorage) ListFile(ctx context.Context, prefix string) ([]string, error) { + query := &storage.Query{Prefix: prefix} + names := make([]string, 0) + it := gstorage.bucket.Objects(ctx, query) + for { + attrs, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return nil, gerrors.Wrap(err) + } + names = append(names, attrs.Name) + } + return names, nil +} + +func (gstorage *GCPStorage) DeleteFile(ctx context.Context, key string) error { + obj := gstorage.bucket.Object(key) + err := obj.Delete(ctx) + return gerrors.Wrap(err) +} + +func (gstorage *GCPStorage) RenameFile(ctx context.Context, oldKey, newKey string) error { + src := gstorage.bucket.Object(oldKey) + dst := gstorage.bucket.Object(newKey) + copier := dst.CopierFrom(src) + _, err := copier.Run(ctx) + if err != nil { + return gerrors.Wrap(err) + } + err = src.Delete(ctx) + return gerrors.Wrap(err) +} From 05192791a3f9f6d1590ab35642dd406c86b54951 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 21 Feb 2023 11:46:56 +0500 Subject: [PATCH 14/58] Implement gcp secrets for runner --- runner/go.mod | 1 + runner/go.sum | 2 + runner/internal/backend/gcp/secret.go | 72 +++++++++++++++++++++++++++ runner/internal/models/backend.go | 18 ++++++- 4 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 runner/internal/backend/gcp/secret.go diff --git a/runner/go.mod b/runner/go.mod index 295e4eb08a..c7ccae5505 100644 --- a/runner/go.mod +++ b/runner/go.mod @@ -23,6 +23,7 @@ require ( ) require ( + cloud.google.com/go/secretmanager v1.10.0 cloud.google.com/go/storage v1.29.0 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.15 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.21 // indirect diff --git a/runner/go.sum b/runner/go.sum index 182db74c53..e397246e9c 100644 --- a/runner/go.sum +++ b/runner/go.sum @@ -311,6 +311,8 @@ cloud.google.com/go/scheduler v1.7.0/go.mod h1:jyCiBqWW956uBjjPMMuX09n3x37mtyPJe cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= cloud.google.com/go/secretmanager v1.8.0/go.mod h1:hnVgi/bN5MYHd3Gt0SPuTPPp5ENina1/LxM+2W9U9J4= cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8AL6X1MA01dW8ht4= +cloud.google.com/go/secretmanager v1.10.0 h1:pu03bha7ukxF8otyPKTFdDz+rr9sE3YauS5PliDXK60= +cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU= cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= diff --git a/runner/internal/backend/gcp/secret.go b/runner/internal/backend/gcp/secret.go new file mode 100644 index 0000000000..37e904cb3b --- /dev/null +++ b/runner/internal/backend/gcp/secret.go @@ -0,0 +1,72 @@ +package gcp + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + secretmanager "cloud.google.com/go/secretmanager/apiv1" + "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb" + "github.com/dstackai/dstack/runner/internal/gerrors" + "github.com/dstackai/dstack/runner/internal/models" +) + +type GCPSecretManager struct { + client *secretmanager.Client + project string + bucket string +} + +func NewGCPSecretManager(project, bucket string) *GCPSecretManager { + ctx := context.TODO() + client, err := secretmanager.NewClient(ctx) + if err != nil { + return nil + } + return &GCPSecretManager{ + client: client, + project: project, + bucket: bucket, + } +} + +func (sm *GCPSecretManager) FetchSecret(ctx context.Context, repoData *models.RepoData, name string) (string, error) { + key := getSecretKey(sm.bucket, repoData, name) + return sm.getSecretValue(ctx, key) +} + +func (sm *GCPSecretManager) FetchCredentials(ctx context.Context, repoData *models.RepoData) (*models.GitCredentials, error) { + key := getCreadentialsKey(sm.bucket, repoData) + creds := new(models.GitCredentials) + data, err := sm.getSecretValue(ctx, key) + if err != nil { + return nil, gerrors.Wrap(err) + } + json.Unmarshal([]byte(data), creds) + return creds, nil +} + +func (sm *GCPSecretManager) getSecretValue(ctx context.Context, key string) (string, error) { + resource := fmt.Sprintf("projects/%s/secrets/%s/versions/latest", sm.project, key) + req := &secretmanagerpb.AccessSecretVersionRequest{ + Name: resource, + } + version, err := sm.client.AccessSecretVersion(ctx, req) + if err != nil { + return "", gerrors.Wrap(err) + } + return string(version.Payload.GetData()), nil +} + +func getSecretKey(bucket string, repoData *models.RepoData, name string) string { + repoPath := repoData.RepoDataPath("-") + repoPath = strings.ReplaceAll(repoPath, ".", "-") + return fmt.Sprintf("dstack-secrets-%s-%s-%s", bucket, repoPath, name) +} + +func getCreadentialsKey(bucket string, repoData *models.RepoData) string { + repoPath := repoData.RepoDataPath("-") + repoPath = strings.ReplaceAll(repoPath, ".", "-") + return fmt.Sprintf("dstack-credentials-%s-%s", bucket, repoPath) +} diff --git a/runner/internal/models/backend.go b/runner/internal/models/backend.go index aa7ef65160..ab6adaf9eb 100644 --- a/runner/internal/models/backend.go +++ b/runner/internal/models/backend.go @@ -1,6 +1,9 @@ package models -import "fmt" +import ( + "fmt" + "strings" +) type Resource struct { CPUs int `yaml:"cpus,omitempty"` @@ -82,12 +85,14 @@ type GPU struct { Name string `yaml:"name,omitempty"` MemoryMiB int `yaml:"memory_mib,omitempty"` } + type State struct { Job *Job `yaml:"job"` RequestID string `yaml:"request_id"` Resources Resource `yaml:"resources"` RunnerID string `yaml:"runner_id"` } + type GitCredentials struct { Protocol string `json:"protocol"` OAuthToken *string `json:"oauth_token,omitempty"` @@ -95,15 +100,26 @@ type GitCredentials struct { Passphrase *string `json:"passphrase,omitempty"` } +type RepoData struct { + RepoHost string + RepoUserName string + RepoName string +} + func (j *Job) RepoHostNameWithPort() string { if j.RepoPort == 0 { return j.RepoHostName } return fmt.Sprintf("%s:%d", j.RepoHostName, j.RepoPort) } + func (d *Dep) RepoHostNameWithPort() string { if d.RepoPort == 0 { return d.RepoHostName } return fmt.Sprintf("%s:%d", d.RepoHostName, d.RepoPort) } + +func (rd *RepoData) RepoDataPath(sep string) string { + return strings.Join([]string{rd.RepoHost, rd.RepoUserName, rd.RepoName}, sep) +} From cce3e41c3c58fd1db160e3b3c8df6f41664ea759 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 22 Feb 2023 10:05:26 +0500 Subject: [PATCH 15/58] Fix logs after merge --- cli/dstack/backend/gcp/logs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/dstack/backend/gcp/logs.py b/cli/dstack/backend/gcp/logs.py index 40ed9b561c..3c875b4d87 100644 --- a/cli/dstack/backend/gcp/logs.py +++ b/cli/dstack/backend/gcp/logs.py @@ -3,7 +3,7 @@ from google.cloud import logging from dstack.backend.base import jobs -from dstack.backend.base.logs import replace_logs_host +from dstack.backend.base.logs import fix_urls from dstack.backend.base.storage import Storage from dstack.core.log_event import LogEvent, LogEventSource from dstack.core.repo import RepoAddress @@ -36,7 +36,7 @@ def log_entry_to_log_event( job_id = log_entry.payload["job_id"] log = log_entry.payload["log"] job = jobs.get_job(storage, repo_address, job_id) - log = replace_logs_host(log, job) + log = fix_urls(log.encode(), job).decode() return LogEvent( event_id=log_entry.insert_id, timestamp=log_entry.timestamp, From 8283805a7bf4d93dc9e9650ed5afecef6d226971 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 22 Feb 2023 10:37:22 +0500 Subject: [PATCH 16/58] Return gcp instance status for deleted instances --- cli/dstack/backend/gcp/compute.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cli/dstack/backend/gcp/compute.py b/cli/dstack/backend/gcp/compute.py index 6f206cc5f7..e37941fbd9 100644 --- a/cli/dstack/backend/gcp/compute.py +++ b/cli/dstack/backend/gcp/compute.py @@ -269,7 +269,10 @@ def _get_instance_status(project_id: str, zone: str, instance_name: str) -> Requ zone=zone, ) client = compute_v1.InstancesClient() - instance = client.get(get_instance_request) + try: + instance = client.get(get_instance_request) + except google.api_core.exceptions.NotFound: + return RequestStatus.TERMINATED if instance.status in ["PROVISIONING", "STAGING", "RUNNING"]: return RequestStatus.RUNNING return RequestStatus.TERMINATED From 4e1e5f1c0d8b8bdf1e905d4cf0395a3ab0c59623 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 22 Feb 2023 10:38:27 +0500 Subject: [PATCH 17/58] Implement gcp compute for runner --- runner/go.mod | 2 ++ runner/internal/backend/gcp/compute.go | 47 ++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 runner/internal/backend/gcp/compute.go diff --git a/runner/go.mod b/runner/go.mod index c7ccae5505..c74b57a0d4 100644 --- a/runner/go.mod +++ b/runner/go.mod @@ -23,6 +23,7 @@ require ( ) require ( + cloud.google.com/go/compute v1.14.0 cloud.google.com/go/secretmanager v1.10.0 cloud.google.com/go/storage v1.29.0 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.15 @@ -38,6 +39,7 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.1 google.golang.org/api v0.106.0 + google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.20.3 ) diff --git a/runner/internal/backend/gcp/compute.go b/runner/internal/backend/gcp/compute.go new file mode 100644 index 0000000000..e686f28ea0 --- /dev/null +++ b/runner/internal/backend/gcp/compute.go @@ -0,0 +1,47 @@ +package gcp + +import ( + "context" + + compute "cloud.google.com/go/compute/apiv1" + "cloud.google.com/go/compute/apiv1/computepb" + "github.com/dstackai/dstack/runner/internal/gerrors" + "github.com/dstackai/dstack/runner/internal/log" +) + +type GCPCompute struct { + instancesClient *compute.InstancesClient + project string + zone string +} + +func NewGCPCompute(project, zone string) *GCPCompute { + ctx := context.TODO() + instancesClient, err := compute.NewInstancesRESTClient(ctx) + if err != nil { + return nil + } + return &GCPCompute{ + instancesClient: instancesClient, + project: project, + zone: zone, + } +} + +func (gcompute *GCPCompute) TerminateInstance(ctx context.Context, instanceID string) error { + log.Trace(ctx, "Terminate instance", "ID", instanceID) + req := &computepb.DeleteInstanceRequest{ + Instance: instanceID, + Project: gcompute.project, + Zone: gcompute.zone, + } + op, err := gcompute.instancesClient.Delete(ctx, req) + if err != nil { + return gerrors.Wrap(err) + } + err = op.Wait(ctx) + if err != nil { + return gerrors.Wrap(err) + } + return nil +} From df2733ad49bf249080b9bee7bc7f3d9290043d01 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 22 Feb 2023 14:43:09 +0500 Subject: [PATCH 18/58] Implement gcp logging for runner --- runner/go.mod | 2 +- runner/go.sum | 2 + runner/internal/backend/gcp/logging.go | 72 ++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 runner/internal/backend/gcp/logging.go diff --git a/runner/go.mod b/runner/go.mod index c74b57a0d4..335015e400 100644 --- a/runner/go.mod +++ b/runner/go.mod @@ -24,6 +24,7 @@ require ( require ( cloud.google.com/go/compute v1.14.0 + cloud.google.com/go/logging v1.6.1 cloud.google.com/go/secretmanager v1.10.0 cloud.google.com/go/storage v1.29.0 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.15 @@ -39,7 +40,6 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.1 google.golang.org/api v0.106.0 - google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.20.3 ) diff --git a/runner/go.sum b/runner/go.sum index e397246e9c..4923afd9e9 100644 --- a/runner/go.sum +++ b/runner/go.sum @@ -220,6 +220,7 @@ cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiP cloud.google.com/go/language v1.8.0/go.mod h1:qYPVHf7SPoNNiCL2Dr0FfEFNil1qi3pQEyygwpgVKB8= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= +cloud.google.com/go/logging v1.6.1 h1:ZBsZK+JG+oCDT+vaxwqF2egKNRjz8soXiS6Xv79benI= cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw= cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= @@ -1316,6 +1317,7 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/runner/internal/backend/gcp/logging.go b/runner/internal/backend/gcp/logging.go new file mode 100644 index 0000000000..2911cba124 --- /dev/null +++ b/runner/internal/backend/gcp/logging.go @@ -0,0 +1,72 @@ +package gcp + +import ( + "context" + "fmt" + "strings" + "time" + + "cloud.google.com/go/logging" +) + +const FLUSH_INTERVAL = 250 * time.Millisecond + +type LogMesage struct { + JobID string `json:"job_id"` + Log string `json:"log"` + Source string `json:"source"` +} + +type GCPLogger struct { + client *logging.Client + logger *logging.Logger + jobID string +} + +func NewGCPLogger(ctx context.Context, project, jobID, logGroup, logName string) *GCPLogger { + client, err := logging.NewClient(ctx, project) + if err != nil { + return nil + } + logID := getLogID(logGroup, logName) + logger := client.Logger(logID) + return &GCPLogger{ + client: client, + jobID: jobID, + logger: logger, + } +} + +func (glogger *GCPLogger) Launch(ctx context.Context) { + ticker := time.NewTicker(FLUSH_INTERVAL) + go glogger.flushLogs(ctx, ticker) +} + +func (glogger *GCPLogger) Write(b []byte) (int, error) { + msg := LogMesage{ + JobID: glogger.jobID, + Log: string(b), + Source: "stdout", + } + entry := logging.Entry{Payload: msg} + glogger.logger.Log(entry) + return len(b), nil +} + +func (glogger *GCPLogger) flushLogs(ctx context.Context, ticker *time.Ticker) { + for { + select { + case <-ctx.Done(): + glogger.client.Close() + return + case <-ticker.C: + glogger.logger.Flush() + } + } +} + +func getLogID(logGroup, logName string) string { + logGroup = strings.ReplaceAll(logGroup, "/", "-") + logGroup = strings.TrimLeft(logGroup, "-") + return fmt.Sprintf("%s-%s", logGroup, logName) +} From 05cd91376dedcf998d155710e76cdb2d0360e1fb Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 22 Feb 2023 15:44:41 +0500 Subject: [PATCH 19/58] Implement metadata for gcp storage --- cli/dstack/backend/gcp/storage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/dstack/backend/gcp/storage.py b/cli/dstack/backend/gcp/storage.py index bcbeacb88e..d9c46d044d 100644 --- a/cli/dstack/backend/gcp/storage.py +++ b/cli/dstack/backend/gcp/storage.py @@ -18,6 +18,7 @@ def configure(self): def put_object(self, key: str, content: str, metadata: Optional[Dict] = None): blob = self.bucket.blob(key) + blob.metadata = metadata blob.upload_from_string(content) def get_object(self, key: str) -> Optional[str]: From 69df2cb5229dbeaab4eddaf8f8945a0bb557499a Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 22 Feb 2023 16:00:00 +0500 Subject: [PATCH 20/58] Implement metadata for gcp storage in runner --- runner/internal/backend/gcp/storage.go | 27 ++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/runner/internal/backend/gcp/storage.go b/runner/internal/backend/gcp/storage.go index bd82807c9d..d9cdcee7c0 100644 --- a/runner/internal/backend/gcp/storage.go +++ b/runner/internal/backend/gcp/storage.go @@ -3,6 +3,7 @@ package gcp import ( "bytes" "context" + "errors" "io" "cloud.google.com/go/storage" @@ -10,10 +11,12 @@ import ( "google.golang.org/api/iterator" ) +var ErrTagNotFound = errors.New("tag not found") + type GCPStorage struct { - client *storage.Client - bucket *storage.BucketHandle - project string + client *storage.Client + bucket *storage.BucketHandle + project string bucketName string } @@ -25,9 +28,9 @@ func NewGCPStorage(project, bucketName string) (*GCPStorage, error) { return nil, gerrors.New("Cannot access bucket") } return &GCPStorage{ - client: client, - bucket: bucket, - project: project, + client: client, + bucket: bucket, + project: project, bucketName: bucketName, }, nil } @@ -86,3 +89,15 @@ func (gstorage *GCPStorage) RenameFile(ctx context.Context, oldKey, newKey strin err = src.Delete(ctx) return gerrors.Wrap(err) } + +func (gstorage *GCPStorage) GetMetadata(ctx context.Context, key, tag string) (string, error) { + obj := gstorage.bucket.Object(key) + attrs, err := obj.Attrs(ctx) + if err != nil { + return "", gerrors.Wrap(err) + } + if value, ok := attrs.Metadata[tag]; ok { + return value, nil + } + return "", gerrors.Wrap(ErrTagNotFound) +} From 556fa2434f378b0f5b31c13726e7a117571b77b1 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 22 Feb 2023 16:42:17 +0500 Subject: [PATCH 21/58] Add job helper methods --- runner/internal/models/backend.go | 37 +++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/runner/internal/models/backend.go b/runner/internal/models/backend.go index ab6adaf9eb..5c6d8a940b 100644 --- a/runner/internal/models/backend.go +++ b/runner/internal/models/backend.go @@ -101,9 +101,9 @@ type GitCredentials struct { } type RepoData struct { - RepoHost string + RepoHost string RepoUserName string - RepoName string + RepoName string } func (j *Job) RepoHostNameWithPort() string { @@ -113,6 +113,39 @@ func (j *Job) RepoHostNameWithPort() string { return fmt.Sprintf("%s:%d", j.RepoHostName, j.RepoPort) } +func (j *Job) JobFilepath() string { + return fmt.Sprintf("jobs/%s/%s/%s/%s;", j.RepoHostNameWithPort(), j.RepoUserName, j.RepoName, j.JobID) +} + +func (j *Job) JobHeadFilepathPrefix() string { + return fmt.Sprintf("jobs/%s/%s/%s/l;%s;", j.RepoHostNameWithPort(), j.RepoUserName, j.RepoName, j.JobID) +} + +func (j *Job) JobHeadFilepath() string { + appsSlice := make([]string, len(j.Apps)) + for _, app := range j.Apps { + appsSlice = append(appsSlice, app.Name) + } + artifactSlice := make([]string, len(j.Artifacts)) + for _, art := range j.Artifacts { + artifactSlice = append(artifactSlice, art.Path) + } + return fmt.Sprintf( + "jobs/%s/%s/%s/l;%s;%s;%s;%d;%s;%s;%s;%s", + j.RepoHostNameWithPort(), + j.RepoUserName, + j.RepoName, + j.JobID, + j.ProviderName, + j.LocalRepoUserName, + j.SubmittedAt, + j.Status, + strings.Join(artifactSlice, ","), + strings.Join(appsSlice, ","), + j.TagName, + ) +} + func (d *Dep) RepoHostNameWithPort() string { if d.RepoPort == 0 { return d.RepoHostName From 97247c60807980b8caf51cc0e068826b0ce74a02 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 22 Feb 2023 17:02:34 +0500 Subject: [PATCH 22/58] Fix removeprefix after merge --- cli/dstack/backend/gcp/storage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cli/dstack/backend/gcp/storage.py b/cli/dstack/backend/gcp/storage.py index d9c46d044d..49efb19fdc 100644 --- a/cli/dstack/backend/gcp/storage.py +++ b/cli/dstack/backend/gcp/storage.py @@ -6,6 +6,7 @@ from dstack.backend.base.storage import SIGNED_URL_EXPIRATION, CloudStorage from dstack.core.storage import StorageFile +from dstack.utils.common import removeprefix class GCPStorage(CloudStorage): @@ -46,7 +47,7 @@ def list_files(self, dirpath: str) -> List[StorageFile]: files = [] for blob in blobs: file = StorageFile( - filepath=blob.name.removeprefix(prefix), + filepath=removeprefix(blob.name, prefix), filesize_in_bytes=blob.size, ) files.append(file) From 6c91d9bed1d9f8f5921a198a9a825a1f14ee8e2e Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 22 Feb 2023 17:28:32 +0500 Subject: [PATCH 23/58] Implement GCPBackend methods in runner: Init, Job, UpdateState, CheckStop, Shutdown, CreateLogger --- runner/internal/backend/gcp/backend.go | 189 +++++++++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 runner/internal/backend/gcp/backend.go diff --git a/runner/internal/backend/gcp/backend.go b/runner/internal/backend/gcp/backend.go new file mode 100644 index 0000000000..91a0dfad7d --- /dev/null +++ b/runner/internal/backend/gcp/backend.go @@ -0,0 +1,189 @@ +package gcp + +import ( + + // "errors" + // "fmt" + + // "os" + // "path" + // "strings" + // "time" + + // "github.com/dstackai/dstack/runner/consts" + "context" + "errors" + "fmt" + "io" + + "github.com/dstackai/dstack/runner/internal/artifacts" + "github.com/dstackai/dstack/runner/internal/gerrors" + "gopkg.in/yaml.v2" + + // "github.com/dstackai/dstack/runner/internal/artifacts/simple" + + // "github.com/dstackai/dstack/runner/internal/common" + "github.com/dstackai/dstack/runner/internal/log" + "github.com/dstackai/dstack/runner/internal/models" +) + +type GCPBackend struct { + project string + zone string + bucket string + storage *GCPStorage + compute *GCPCompute + secretManager *GCPSecretManager + runnerID string + state *models.State + artifacts []artifacts.Artifacter +} + +// func init() { +// backend.RegisterBackend("aws", func(ctx context.Context, pathConfig string) (backend.Backend, error) { +// // TODO read config +// // log.Trace(ctx, "Read config file", "path", pathConfig) +// // theConfig, err := ioutil.ReadFile(pathConfig) +// // if err != nil { +// // return nil, gerrors.Wrap(err) +// // } +// // log.Trace(ctx, "Unmarshal config") +// // err = yaml.Unmarshal(theConfig, &file) +// // if err != nil { +// // return nil, gerrors.Wrap(err) +// // } +// return New("dstack", "us-central1-a", "dstack-bucket"), nil +// }) +// } + +func New(project, zone, bucket string) *GCPBackend { + storage, err := NewGCPStorage(project, bucket) + if err != nil { + return nil + } + compute := NewGCPCompute(project, zone) + if compute == nil { + return nil + } + secretManager := NewGCPSecretManager(project, bucket) + if secretManager == nil { + return nil + } + return &GCPBackend{ + project: project, + zone: zone, + bucket: bucket, + storage: storage, + compute: compute, + secretManager: secretManager, + } +} + +func (gbackend *GCPBackend) Init(ctx context.Context, ID string) error { + gbackend.runnerID = ID + runnerFilepath := fmt.Sprintf("runners/%s.yaml", ID) + contents, err := gbackend.storage.GetFile(ctx, runnerFilepath) + if err != nil { + return gerrors.Wrap(err) + } + err = yaml.Unmarshal(contents, &gbackend.state) + if err != nil { + return gerrors.Wrap(err) + } + if gbackend.state == nil { + return gerrors.New("State is empty. Data not loading") + } + return nil +} + +func (gbackend *GCPBackend) Job(ctx context.Context) *models.Job { + log.Trace(ctx, "Getting job from state") + log.Trace(ctx, "Get job", "job ID", gbackend.state.Job.JobID) + return gbackend.state.Job +} + +func (gbackend *GCPBackend) UpdateState(ctx context.Context) error { + log.Trace(ctx, "Fetching list jobs", "Repo username", gbackend.state.Job.RepoUserName, "Repo name", gbackend.state.Job.RepoName, "Job ID", gbackend.state.Job.JobID) + files, err := gbackend.storage.ListFile(ctx, gbackend.state.Job.JobHeadFilepathPrefix()) + if err != nil { + return gerrors.Wrap(err) + } + for _, file := range files { + log.Trace(ctx, "Deleting file job", "Bucket", gbackend.bucket, "Path", file) + err = gbackend.storage.DeleteFile(ctx, file) + if err != nil { + return gerrors.Wrap(err) + } + } + log.Trace(ctx, "Marshaling job") + contents, err := yaml.Marshal(&gbackend.state.Job) + if err != nil { + return gerrors.Wrap(err) + } + jobFilepath := gbackend.state.Job.JobFilepath() + log.Trace(ctx, "Write to file job", "Path", jobFilepath) + err = gbackend.storage.PutFile(ctx, jobFilepath, contents) + if err != nil { + return gerrors.Wrap(err) + } + jobHeadFilepath := gbackend.state.Job.JobHeadFilepath() + log.Trace(ctx, "Write to file lock job", "Path", jobHeadFilepath) + err = gbackend.storage.PutFile(ctx, jobHeadFilepath, []byte{}) + if err != nil { + return gerrors.Wrap(err) + } + return nil +} + +func (gbackend *GCPBackend) CheckStop(ctx context.Context) (bool, error) { + runnerFilepath := fmt.Sprintf("runners/%s.yaml", gbackend.runnerID) + log.Trace(ctx, "Reading metadata from state file", "path", runnerFilepath) + status, err := gbackend.storage.GetMetadata(ctx, runnerFilepath, "status") + if err != nil && !errors.Is(err, ErrTagNotFound) { + return false, gerrors.Wrap(err) + } + if status == "stopping" { + log.Trace(ctx, "Status equals stopping") + return true, nil + } + return false, nil +} + +func (gbackend *GCPBackend) Shutdown(ctx context.Context) error { + return gbackend.compute.TerminateInstance(ctx, gbackend.state.RequestID) +} + +func (gbackend *GCPBackend) Requirements(ctx context.Context) models.Requirements { + log.Trace(ctx, "Getting requirements") + return gbackend.state.Job.Requirements +} + +func (gbackend *GCPBackend) MasterJob(ctx context.Context) *models.Job { + // TODO + return nil +} + +func (gbackend *GCPBackend) CreateLogger(ctx context.Context, logGroup, logName string) io.Writer { + logger := NewGCPLogger(ctx, gbackend.project, gbackend.state.Job.JobID, logGroup, logName) + if logger == nil { + log.Error(ctx, "Failed to create logger") + return nil + } + logger.Launch(ctx) + return logger +} + +func (gbackend *GCPBackend) ListSubDir(ctx context.Context, dir string) ([]string, error) { + // TODO + return nil, nil +} + +func (gbackend *GCPBackend) GetJobByPath(ctx context.Context, path string) (*models.Job, error) { + // TODO + return nil, nil +} + +func (gbackend *GCPBackend) Bucket(ctx context.Context) string { + log.Trace(ctx, "Getting bucket") + return gbackend.bucket +} From 0e8ff04a9e229e14c0a5c396b0956ca15b605116 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Thu, 23 Feb 2023 10:40:33 +0500 Subject: [PATCH 24/58] Implement Secrets() for gcp backend in runner --- runner/internal/backend/gcp/backend.go | 23 +++++++++++++++++++++++ runner/internal/backend/gcp/secret.go | 9 ++++++++- runner/internal/models/backend.go | 12 ++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/runner/internal/backend/gcp/backend.go b/runner/internal/backend/gcp/backend.go index 91a0dfad7d..ac810629d2 100644 --- a/runner/internal/backend/gcp/backend.go +++ b/runner/internal/backend/gcp/backend.go @@ -15,6 +15,7 @@ import ( "errors" "fmt" "io" + "strings" "github.com/dstackai/dstack/runner/internal/artifacts" "github.com/dstackai/dstack/runner/internal/gerrors" @@ -187,3 +188,25 @@ func (gbackend *GCPBackend) Bucket(ctx context.Context) string { log.Trace(ctx, "Getting bucket") return gbackend.bucket } + +func (gbackend *GCPBackend) Secrets(ctx context.Context) (map[string]string, error) { + log.Trace(ctx, "Getting secrets") + prefix := gbackend.state.Job.JobRepoData().SecretsPrefix() + secretFilenames, err := gbackend.storage.ListFile(ctx, prefix) + if err != nil { + return nil, gerrors.Wrap(err) + } + secrets := make(map[string]string, 0) + for _, secretFilename := range secretFilenames { + secretName := strings.ReplaceAll(secretFilename, prefix, "") + secretValue, err := gbackend.secretManager.FetchSecret(ctx, gbackend.state.Job.JobRepoData(), secretName) + if err != nil { + if errors.Is(err, ErrSecretNotFound) { + continue + } + return nil, gerrors.Wrap(err) + } + secrets[secretName] = secretValue + } + return secrets, nil +} diff --git a/runner/internal/backend/gcp/secret.go b/runner/internal/backend/gcp/secret.go index 37e904cb3b..8653b20d7e 100644 --- a/runner/internal/backend/gcp/secret.go +++ b/runner/internal/backend/gcp/secret.go @@ -3,6 +3,7 @@ package gcp import ( "context" "encoding/json" + "errors" "fmt" "strings" @@ -12,6 +13,8 @@ import ( "github.com/dstackai/dstack/runner/internal/models" ) +var ErrSecretNotFound = errors.New("secret not found") + type GCPSecretManager struct { client *secretmanager.Client project string @@ -54,7 +57,11 @@ func (sm *GCPSecretManager) getSecretValue(ctx context.Context, key string) (str } version, err := sm.client.AccessSecretVersion(ctx, req) if err != nil { - return "", gerrors.Wrap(err) + if strings.Contains(err.Error(), "NotFound") { + return "", gerrors.Wrap(ErrSecretNotFound) + } else { + return "", gerrors.Wrap(err) + } } return string(version.Payload.GetData()), nil } diff --git a/runner/internal/models/backend.go b/runner/internal/models/backend.go index 5c6d8a940b..d420537881 100644 --- a/runner/internal/models/backend.go +++ b/runner/internal/models/backend.go @@ -113,6 +113,14 @@ func (j *Job) RepoHostNameWithPort() string { return fmt.Sprintf("%s:%d", j.RepoHostName, j.RepoPort) } +func (j *Job) JobRepoData() *RepoData { + return &RepoData{ + RepoHost: j.RepoHostNameWithPort(), + RepoUserName: j.RepoUserName, + RepoName: j.RepoName, + } +} + func (j *Job) JobFilepath() string { return fmt.Sprintf("jobs/%s/%s/%s/%s;", j.RepoHostNameWithPort(), j.RepoUserName, j.RepoName, j.JobID) } @@ -156,3 +164,7 @@ func (d *Dep) RepoHostNameWithPort() string { func (rd *RepoData) RepoDataPath(sep string) string { return strings.Join([]string{rd.RepoHost, rd.RepoUserName, rd.RepoName}, sep) } + +func (rd *RepoData) SecretsPrefix() string { + return fmt.Sprintf("secrets/%s/%s/%s/l;", rd.RepoHost, rd.RepoUserName, rd.RepoName) +} From b41c8185e544d1aa9aca5e3cfd68387b1622d13c Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Thu, 23 Feb 2023 11:57:14 +0500 Subject: [PATCH 25/58] Implement basic gcp support in runner --- runner/internal/backend/gcp/backend.go | 86 ++++++++++++++++---------- runner/internal/backend/gcp/logging.go | 19 ++++-- runner/main.go | 1 + 3 files changed, 68 insertions(+), 38 deletions(-) diff --git a/runner/internal/backend/gcp/backend.go b/runner/internal/backend/gcp/backend.go index ac810629d2..9b21b67008 100644 --- a/runner/internal/backend/gcp/backend.go +++ b/runner/internal/backend/gcp/backend.go @@ -1,16 +1,6 @@ package gcp import ( - - // "errors" - // "fmt" - - // "os" - // "path" - // "strings" - // "time" - - // "github.com/dstackai/dstack/runner/consts" "context" "errors" "fmt" @@ -18,14 +8,11 @@ import ( "strings" "github.com/dstackai/dstack/runner/internal/artifacts" + "github.com/dstackai/dstack/runner/internal/backend" "github.com/dstackai/dstack/runner/internal/gerrors" - "gopkg.in/yaml.v2" - - // "github.com/dstackai/dstack/runner/internal/artifacts/simple" - - // "github.com/dstackai/dstack/runner/internal/common" "github.com/dstackai/dstack/runner/internal/log" "github.com/dstackai/dstack/runner/internal/models" + "gopkg.in/yaml.v2" ) type GCPBackend struct { @@ -35,27 +22,28 @@ type GCPBackend struct { storage *GCPStorage compute *GCPCompute secretManager *GCPSecretManager + logging *GCPLogging runnerID string state *models.State artifacts []artifacts.Artifacter } -// func init() { -// backend.RegisterBackend("aws", func(ctx context.Context, pathConfig string) (backend.Backend, error) { -// // TODO read config -// // log.Trace(ctx, "Read config file", "path", pathConfig) -// // theConfig, err := ioutil.ReadFile(pathConfig) -// // if err != nil { -// // return nil, gerrors.Wrap(err) -// // } -// // log.Trace(ctx, "Unmarshal config") -// // err = yaml.Unmarshal(theConfig, &file) -// // if err != nil { -// // return nil, gerrors.Wrap(err) -// // } -// return New("dstack", "us-central1-a", "dstack-bucket"), nil -// }) -// } +func init() { + backend.RegisterBackend("gcp", func(ctx context.Context, pathConfig string) (backend.Backend, error) { + // TODO read config + // log.Trace(ctx, "Read config file", "path", pathConfig) + // theConfig, err := ioutil.ReadFile(pathConfig) + // if err != nil { + // return nil, gerrors.Wrap(err) + // } + // log.Trace(ctx, "Unmarshal config") + // err = yaml.Unmarshal(theConfig, &file) + // if err != nil { + // return nil, gerrors.Wrap(err) + // } + return New("dstack", "us-central1-a", "dstack-bucket"), nil + }) +} func New(project, zone, bucket string) *GCPBackend { storage, err := NewGCPStorage(project, bucket) @@ -70,6 +58,7 @@ func New(project, zone, bucket string) *GCPBackend { if secretManager == nil { return nil } + logging := NewGCPLogging(project) return &GCPBackend{ project: project, zone: zone, @@ -77,6 +66,7 @@ func New(project, zone, bucket string) *GCPBackend { storage: storage, compute: compute, secretManager: secretManager, + logging: logging, } } @@ -151,7 +141,28 @@ func (gbackend *GCPBackend) CheckStop(ctx context.Context) (bool, error) { } func (gbackend *GCPBackend) Shutdown(ctx context.Context) error { - return gbackend.compute.TerminateInstance(ctx, gbackend.state.RequestID) + err := gbackend.compute.TerminateInstance(ctx, gbackend.state.RequestID) + if err != nil { + return err + } + err = gbackend.compute.instancesClient.Close() + if err != nil { + return err + } + err = gbackend.storage.client.Close() + if err != nil { + return err + } + err = gbackend.secretManager.client.Close() + if err != nil { + return err + } + return gbackend.logging.client.Close() +} + +func (gbackend *GCPBackend) GetArtifact(ctx context.Context, runName, localPath, remotePath string, mount bool) artifacts.Artifacter { + // TODO + return nil } func (gbackend *GCPBackend) Requirements(ctx context.Context) models.Requirements { @@ -165,7 +176,7 @@ func (gbackend *GCPBackend) MasterJob(ctx context.Context) *models.Job { } func (gbackend *GCPBackend) CreateLogger(ctx context.Context, logGroup, logName string) io.Writer { - logger := NewGCPLogger(ctx, gbackend.project, gbackend.state.Job.JobID, logGroup, logName) + logger := gbackend.logging.NewGCPLogger(ctx, gbackend.state.Job.JobID, logGroup, logName) if logger == nil { log.Error(ctx, "Failed to create logger") return nil @@ -210,3 +221,12 @@ func (gbackend *GCPBackend) Secrets(ctx context.Context) (map[string]string, err } return secrets, nil } + +func (gbackend *GCPBackend) GitCredentials(ctx context.Context) *models.GitCredentials { + log.Trace(ctx, "Getting credentials") + creds, err := gbackend.secretManager.FetchCredentials(ctx, gbackend.state.Job.JobRepoData()) + if err != nil { + return nil + } + return creds +} diff --git a/runner/internal/backend/gcp/logging.go b/runner/internal/backend/gcp/logging.go index 2911cba124..c9f9ded794 100644 --- a/runner/internal/backend/gcp/logging.go +++ b/runner/internal/backend/gcp/logging.go @@ -17,21 +17,28 @@ type LogMesage struct { Source string `json:"source"` } -type GCPLogger struct { +type GCPLogging struct { client *logging.Client +} + +type GCPLogger struct { logger *logging.Logger jobID string } -func NewGCPLogger(ctx context.Context, project, jobID, logGroup, logName string) *GCPLogger { +func NewGCPLogging(project string) *GCPLogging { + ctx := context.TODO() client, err := logging.NewClient(ctx, project) if err != nil { return nil } + return &GCPLogging{client: client} +} + +func (glogging *GCPLogging) NewGCPLogger(ctx context.Context, jobID, logGroup, logName string) *GCPLogger { logID := getLogID(logGroup, logName) - logger := client.Logger(logID) + logger := glogging.client.Logger(logID) return &GCPLogger{ - client: client, jobID: jobID, logger: logger, } @@ -57,7 +64,9 @@ func (glogger *GCPLogger) flushLogs(ctx context.Context, ticker *time.Ticker) { for { select { case <-ctx.Done(): - glogger.client.Close() + // Backend will Close() the client on Shutdown(), flushing the logs. + // We don't want to Close() from this goroutine since google libraries may + // send audit logs after ctx.Done(). return case <-ticker.C: glogger.logger.Flush() diff --git a/runner/main.go b/runner/main.go index 5518e659eb..325e912f53 100644 --- a/runner/main.go +++ b/runner/main.go @@ -20,6 +20,7 @@ import ( "github.com/dstackai/dstack/runner/consts" "github.com/dstackai/dstack/runner/internal/backend" + _ "github.com/dstackai/dstack/runner/internal/backend/gcp" _ "github.com/dstackai/dstack/runner/internal/backend/local" _ "github.com/dstackai/dstack/runner/internal/backend/s3" "github.com/dstackai/dstack/runner/internal/common" From 771db5d83ff08350c1b83ce663a25f4ac95ca5e1 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Thu, 23 Feb 2023 12:26:57 +0500 Subject: [PATCH 26/58] Fix job filepath --- runner/internal/models/backend.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/internal/models/backend.go b/runner/internal/models/backend.go index d420537881..fa935cfdb8 100644 --- a/runner/internal/models/backend.go +++ b/runner/internal/models/backend.go @@ -122,7 +122,7 @@ func (j *Job) JobRepoData() *RepoData { } func (j *Job) JobFilepath() string { - return fmt.Sprintf("jobs/%s/%s/%s/%s;", j.RepoHostNameWithPort(), j.RepoUserName, j.RepoName, j.JobID) + return fmt.Sprintf("jobs/%s/%s/%s/%s.yaml", j.RepoHostNameWithPort(), j.RepoUserName, j.RepoName, j.JobID) } func (j *Job) JobHeadFilepathPrefix() string { From 92ca454608b8f2b0708a6e9f8adcefbbfcc60de6 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Thu, 23 Feb 2023 12:53:36 +0500 Subject: [PATCH 27/58] Do not wait for gcp instance to terminate --- runner/internal/backend/gcp/compute.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/runner/internal/backend/gcp/compute.go b/runner/internal/backend/gcp/compute.go index e686f28ea0..3e461d7fdd 100644 --- a/runner/internal/backend/gcp/compute.go +++ b/runner/internal/backend/gcp/compute.go @@ -35,11 +35,7 @@ func (gcompute *GCPCompute) TerminateInstance(ctx context.Context, instanceID st Project: gcompute.project, Zone: gcompute.zone, } - op, err := gcompute.instancesClient.Delete(ctx, req) - if err != nil { - return gerrors.Wrap(err) - } - err = op.Wait(ctx) + _, err := gcompute.instancesClient.Delete(ctx, req) if err != nil { return gerrors.Wrap(err) } From 8c194609e99f3869e3d4f651d3f114a2b936f5ba Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Thu, 23 Feb 2023 12:54:06 +0500 Subject: [PATCH 28/58] Clean up gcp cli --- cli/dstack/backend/base/runs.py | 3 ++- cli/dstack/backend/gcp/compute.py | 5 ----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/cli/dstack/backend/base/runs.py b/cli/dstack/backend/base/runs.py index 54aa8da728..1b6b8b5adc 100644 --- a/cli/dstack/backend/base/runs.py +++ b/cli/dstack/backend/base/runs.py @@ -57,7 +57,8 @@ def get_run_heads( else: run = runs_by_id[run_id] _update_run(storage, compute, run, job_head, include_request_heads) - return sorted(list(runs_by_id.values()), key=lambda r: r.submitted_at, reverse=True) + run_heads = list(sorted(runs_by_id.values(), key=lambda r: r.submitted_at, reverse=True)) + return run_heads def _create_run( diff --git a/cli/dstack/backend/gcp/compute.py b/cli/dstack/backend/gcp/compute.py index e37941fbd9..90f53834b0 100644 --- a/cli/dstack/backend/gcp/compute.py +++ b/cli/dstack/backend/gcp/compute.py @@ -251,14 +251,9 @@ def _create_instance( request.project = project_id request.instance_resource = instance - # Wait for the create operation to complete. - print(f"Creating the {instance_name} instance in {zone}...") - operation = instance_client.insert(request=request) - gcp_utils.wait_for_extended_operation(operation, "instance creation") - print(f"Instance {instance_name} created.") return instance_client.get(project=project_id, zone=zone, instance=instance_name) From ec94134ce4c2740b5c9db7c4b1165b6ab68b6512 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Thu, 23 Feb 2023 15:05:49 +0500 Subject: [PATCH 29/58] Implement basic gcp support with local runner --- cli/dstack/backend/base/jobs.py | 2 +- cli/dstack/backend/gcp/logs.py | 13 +++++++++---- runner/internal/backend/gcp/compute.go | 4 ++++ 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/cli/dstack/backend/base/jobs.py b/cli/dstack/backend/base/jobs.py index 064373239e..c230c9d054 100644 --- a/cli/dstack/backend/base/jobs.py +++ b/cli/dstack/backend/base/jobs.py @@ -150,7 +150,7 @@ def run_job( runner = None try: - job.runner_id = uuid.uuid4().hex + job.runner_id = "testrunner" # uuid.uuid4().hex update_job(storage, job) instance_type = compute.get_instance_type(job) if instance_type is None: diff --git a/cli/dstack/backend/gcp/logs.py b/cli/dstack/backend/gcp/logs.py index 3c875b4d87..f9e798a6ba 100644 --- a/cli/dstack/backend/gcp/logs.py +++ b/cli/dstack/backend/gcp/logs.py @@ -21,14 +21,18 @@ def poll_logs( repo_address: RepoAddress, run_name: str, ) -> Generator[LogEvent, None, None]: - log_name = f"dstack-{self.bucket_name}-{run_name}" + log_name = _get_log_name(self.bucket_name, repo_address, run_name) logger = self.logging_client.logger(log_name) log_entries = logger.list_entries() for log_entry in log_entries: - yield log_entry_to_log_event(storage, repo_address, log_entry) + yield _log_entry_to_log_event(storage, repo_address, log_entry) -def log_entry_to_log_event( +def _get_log_name(bucket_name: str, repo_address: RepoAddress, run_name) -> str: + return f"dstack-jobs-{bucket_name}-{repo_address.path('-')}-{run_name}" + + +def _log_entry_to_log_event( storage: Storage, repo_address: RepoAddress, log_entry: logging.LogEntry, @@ -37,9 +41,10 @@ def log_entry_to_log_event( log = log_entry.payload["log"] job = jobs.get_job(storage, repo_address, job_id) log = fix_urls(log.encode(), job).decode() + timestamp = int(log_entry.timestamp.timestamp()) return LogEvent( event_id=log_entry.insert_id, - timestamp=log_entry.timestamp, + timestamp=timestamp, job_id=job_id, log_message=log, log_source=LogEventSource.STDOUT diff --git a/runner/internal/backend/gcp/compute.go b/runner/internal/backend/gcp/compute.go index 3e461d7fdd..8160657b5f 100644 --- a/runner/internal/backend/gcp/compute.go +++ b/runner/internal/backend/gcp/compute.go @@ -2,6 +2,7 @@ package gcp import ( "context" + "strings" compute "cloud.google.com/go/compute/apiv1" "cloud.google.com/go/compute/apiv1/computepb" @@ -37,6 +38,9 @@ func (gcompute *GCPCompute) TerminateInstance(ctx context.Context, instanceID st } _, err := gcompute.instancesClient.Delete(ctx, req) if err != nil { + if strings.Contains(err.Error(), "Error 404") { + return nil + } return gerrors.Wrap(err) } return nil From c72fc0dc30f0a3c775f208a026959d3651a11cad Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Thu, 23 Feb 2023 17:00:00 +0500 Subject: [PATCH 30/58] Fix log.Error() call --- runner/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/main.go b/runner/main.go index 325e912f53..ba9187a91a 100644 --- a/runner/main.go +++ b/runner/main.go @@ -120,7 +120,7 @@ func start(logLevel int, httpPort int, configDir string) { go func() { if err = ex.Run(ctxSig); err != nil { - log.Error(logCtx, "dstack-runner ended with an error: ", err) + log.Error(logCtx, "dstack-runner ended with an error: ", "err", err) } wg.Done() }() From b83322d58bcaa2ffb4408ea2a2e5bb159fafe2bf Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Thu, 23 Feb 2023 17:01:15 +0500 Subject: [PATCH 31/58] Implement gcp artifacter upload --- cli/dstack/backend/base/jobs.py | 2 +- runner/internal/backend/gcp/artifact.go | 60 +++++++++++++++++++++++++ runner/internal/backend/gcp/backend.go | 8 ++-- runner/internal/backend/gcp/storage.go | 55 +++++++++++++++++++++++ 4 files changed, 121 insertions(+), 4 deletions(-) create mode 100644 runner/internal/backend/gcp/artifact.go diff --git a/cli/dstack/backend/base/jobs.py b/cli/dstack/backend/base/jobs.py index c230c9d054..064373239e 100644 --- a/cli/dstack/backend/base/jobs.py +++ b/cli/dstack/backend/base/jobs.py @@ -150,7 +150,7 @@ def run_job( runner = None try: - job.runner_id = "testrunner" # uuid.uuid4().hex + job.runner_id = uuid.uuid4().hex update_job(storage, job) instance_type = compute.get_instance_type(job) if instance_type is None: diff --git a/runner/internal/backend/gcp/artifact.go b/runner/internal/backend/gcp/artifact.go new file mode 100644 index 0000000000..2d8752b602 --- /dev/null +++ b/runner/internal/backend/gcp/artifact.go @@ -0,0 +1,60 @@ +package gcp + +import ( + "context" + "errors" + "os" + "path" + "path/filepath" + + "github.com/docker/docker/api/types/mount" + "github.com/dstackai/dstack/runner/internal/log" +) + +type GCPArtifacter struct { + storage *GCPStorage + workDir string + pathLocal string + pathRemote string +} + +func NewGCPArtifacter(storage *GCPStorage, workDir, pathLocal, pathRemote string) *GCPArtifacter { + err := os.MkdirAll(path.Join(workDir, pathLocal), 0o755) + if err != nil { + return nil + } + return &GCPArtifacter{ + storage: storage, + workDir: workDir, + pathLocal: pathLocal, + pathRemote: pathRemote, + } +} + +func (gart *GCPArtifacter) BeforeRun(ctx context.Context) error { + log.Trace(ctx, "Upload artifact", "artifact", gart.pathLocal) + return gart.storage.DownloadDir(ctx, gart.pathRemote, path.Join(gart.workDir, gart.pathLocal)) +} + +func (gart *GCPArtifacter) AfterRun(ctx context.Context) error { + log.Trace(ctx, "Upload artifact", "artifact", gart.pathLocal) + return gart.storage.UploadDir(ctx, path.Join(gart.workDir, gart.pathLocal), gart.pathRemote) +} + +func (gart *GCPArtifacter) DockerBindings(workDir string) ([]mount.Mount, error) { + cleanPath := filepath.Clean(gart.pathLocal) + if path.IsAbs(cleanPath) && path.Dir(cleanPath) == cleanPath { + return nil, errors.New("directory needs to be a non-root path") + } + dir := gart.pathLocal + if !filepath.IsAbs(gart.pathLocal) { + dir = path.Join(workDir, gart.pathLocal) + } + return []mount.Mount{ + { + Type: mount.TypeBind, + Source: path.Join(gart.workDir, gart.pathLocal), + Target: dir, + }, + }, nil +} diff --git a/runner/internal/backend/gcp/backend.go b/runner/internal/backend/gcp/backend.go index 9b21b67008..6b38d59f5d 100644 --- a/runner/internal/backend/gcp/backend.go +++ b/runner/internal/backend/gcp/backend.go @@ -5,10 +5,13 @@ import ( "errors" "fmt" "io" + "path" "strings" + "github.com/dstackai/dstack/runner/consts" "github.com/dstackai/dstack/runner/internal/artifacts" "github.com/dstackai/dstack/runner/internal/backend" + "github.com/dstackai/dstack/runner/internal/common" "github.com/dstackai/dstack/runner/internal/gerrors" "github.com/dstackai/dstack/runner/internal/log" "github.com/dstackai/dstack/runner/internal/models" @@ -25,7 +28,6 @@ type GCPBackend struct { logging *GCPLogging runnerID string state *models.State - artifacts []artifacts.Artifacter } func init() { @@ -161,8 +163,8 @@ func (gbackend *GCPBackend) Shutdown(ctx context.Context) error { } func (gbackend *GCPBackend) GetArtifact(ctx context.Context, runName, localPath, remotePath string, mount bool) artifacts.Artifacter { - // TODO - return nil + workDir := path.Join(common.HomeDir(), consts.USER_ARTIFACTS_PATH, runName) + return NewGCPArtifacter(gbackend.storage, workDir, localPath, remotePath) } func (gbackend *GCPBackend) Requirements(ctx context.Context) models.Requirements { diff --git a/runner/internal/backend/gcp/storage.go b/runner/internal/backend/gcp/storage.go index d9cdcee7c0..25d1d0c469 100644 --- a/runner/internal/backend/gcp/storage.go +++ b/runner/internal/backend/gcp/storage.go @@ -5,9 +5,15 @@ import ( "context" "errors" "io" + "io/fs" + "os" + "path" + "path/filepath" + "strings" "cloud.google.com/go/storage" "github.com/dstackai/dstack/runner/internal/gerrors" + "github.com/dstackai/dstack/runner/internal/log" "google.golang.org/api/iterator" ) @@ -101,3 +107,52 @@ func (gstorage *GCPStorage) GetMetadata(ctx context.Context, key, tag string) (s } return "", gerrors.Wrap(ErrTagNotFound) } + +func (gstorage *GCPStorage) UploadDir(ctx context.Context, src, dst string) error { + // TODO upload in parallel + for file := range walkFiles(ctx, src) { + key := path.Join(dst, strings.TrimPrefix(file, src)) + gstorage.uploadFile(ctx, file, key) + } + return nil +} + +func (gstorage *GCPStorage) DownloadDir(ctx context.Context, src, dst string) error { + return nil +} + +func walkFiles(ctx context.Context, local string) chan string { + files := make(chan string) + go func() { + defer close(files) + err := filepath.Walk(local, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + files <- path + return nil + }) + if err != nil { + log.Error(ctx, "Error while walking files", "err", err) + } + }() + return files +} + +func (gstorage *GCPStorage) uploadFile(ctx context.Context, src, dst string) error { + f, err := os.Open(src) + if err != nil { + return gerrors.Wrap(err) + } + obj := gstorage.bucket.Object(dst) + writer := obj.NewWriter(ctx) + _, err = io.Copy(writer, f) + if err != nil { + return gerrors.Wrap(err) + } + return writer.Close() +} + +func (gstorage *GCPStorage) downloadFile(ctx context.Context, src, dst string) error { + return nil +} From 191588dbf68eda8e976aef447236fcb4c45bdd11 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Fri, 24 Feb 2023 10:53:57 +0500 Subject: [PATCH 32/58] Implement gcp artifacter download --- runner/internal/backend/gcp/backend.go | 18 +++++++++++++----- runner/internal/backend/gcp/storage.go | 24 ++++++++++++++++++++++++ runner/internal/backend/local/backend.go | 2 +- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/runner/internal/backend/gcp/backend.go b/runner/internal/backend/gcp/backend.go index 6b38d59f5d..de23683e77 100644 --- a/runner/internal/backend/gcp/backend.go +++ b/runner/internal/backend/gcp/backend.go @@ -91,7 +91,7 @@ func (gbackend *GCPBackend) Init(ctx context.Context, ID string) error { func (gbackend *GCPBackend) Job(ctx context.Context) *models.Job { log.Trace(ctx, "Getting job from state") - log.Trace(ctx, "Get job", "job ID", gbackend.state.Job.JobID) + log.Trace(ctx, "Get job", "ID", gbackend.state.Job.JobID) return gbackend.state.Job } @@ -188,13 +188,21 @@ func (gbackend *GCPBackend) CreateLogger(ctx context.Context, logGroup, logName } func (gbackend *GCPBackend) ListSubDir(ctx context.Context, dir string) ([]string, error) { - // TODO - return nil, nil + return gbackend.storage.ListFile(ctx, dir) } func (gbackend *GCPBackend) GetJobByPath(ctx context.Context, path string) (*models.Job, error) { - // TODO - return nil, nil + log.Trace(ctx, "Fetching job by path", "Path", path) + content, err := gbackend.storage.GetFile(ctx, path) + if err != nil { + return nil, gerrors.Wrap(err) + } + job := new(models.Job) + err = yaml.Unmarshal(content, &job) + if err != nil { + return nil, gerrors.Wrap(err) + } + return job, nil } func (gbackend *GCPBackend) Bucket(ctx context.Context) string { diff --git a/runner/internal/backend/gcp/storage.go b/runner/internal/backend/gcp/storage.go index 25d1d0c469..aa23256275 100644 --- a/runner/internal/backend/gcp/storage.go +++ b/runner/internal/backend/gcp/storage.go @@ -118,6 +118,14 @@ func (gstorage *GCPStorage) UploadDir(ctx context.Context, src, dst string) erro } func (gstorage *GCPStorage) DownloadDir(ctx context.Context, src, dst string) error { + files, err := gstorage.ListFile(ctx, src) + if err != nil { + return gerrors.Wrap(err) + } + for _, file := range files { + dstFilepath := path.Join(dst, strings.TrimPrefix(file, src)) + gstorage.downloadFile(ctx, file, dstFilepath) + } return nil } @@ -154,5 +162,21 @@ func (gstorage *GCPStorage) uploadFile(ctx context.Context, src, dst string) err } func (gstorage *GCPStorage) downloadFile(ctx context.Context, src, dst string) error { + os.MkdirAll(filepath.Dir(dst), 0o755) + obj := gstorage.bucket.Object(src) + reader, err := obj.NewReader(ctx) + if err != nil { + return gerrors.Wrap(err) + } + defer reader.Close() + file, err := os.Create(dst) + if err != nil { + return gerrors.Wrap(err) + } + defer file.Close() + _, err = io.Copy(file, reader) + if err != nil { + return gerrors.Wrap(err) + } return nil } diff --git a/runner/internal/backend/local/backend.go b/runner/internal/backend/local/backend.go index 78aaa92936..4b2595ac6e 100644 --- a/runner/internal/backend/local/backend.go +++ b/runner/internal/backend/local/backend.go @@ -180,7 +180,7 @@ func (l Local) Job(ctx context.Context) *models.Job { log.Trace(ctx, "State not exist") return new(models.Job) } - log.Trace(ctx, "Get job", "job ID", l.state.Job.JobID) + log.Trace(ctx, "Get job", "ID", l.state.Job.JobID) return l.state.Job } From 15dee4a1fad2f36de30ca58c4b2e76b1675072e6 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Fri, 24 Feb 2023 12:57:00 +0500 Subject: [PATCH 33/58] Fix gcp logs and storage concurrency issues --- cli/dstack/backend/gcp/logs.py | 19 ++++++++++++++++--- cli/dstack/backend/gcp/storage.py | 8 ++++---- runner/internal/backend/gcp/backend.go | 25 ++++++++++--------------- runner/internal/backend/gcp/logging.go | 1 + runner/internal/backend/gcp/storage.go | 10 ++++++++-- 5 files changed, 39 insertions(+), 24 deletions(-) diff --git a/cli/dstack/backend/gcp/logs.py b/cli/dstack/backend/gcp/logs.py index f9e798a6ba..0fb76940d6 100644 --- a/cli/dstack/backend/gcp/logs.py +++ b/cli/dstack/backend/gcp/logs.py @@ -1,3 +1,4 @@ +import time from typing import Generator from google.cloud import logging @@ -8,6 +9,9 @@ from dstack.core.log_event import LogEvent, LogEventSource from dstack.core.repo import RepoAddress +POLL_LOGS_ATTEMPTS = 5 +POLL_LOGS_WAIT_TIME = 2 + class GCPLogging: def __init__(self, project_id: str, bucket_name: str): @@ -23,9 +27,18 @@ def poll_logs( ) -> Generator[LogEvent, None, None]: log_name = _get_log_name(self.bucket_name, repo_address, run_name) logger = self.logging_client.logger(log_name) - log_entries = logger.list_entries() - for log_entry in log_entries: - yield _log_entry_to_log_event(storage, repo_address, log_entry) + # Hack: It takes some time for logs to become available after runner writes them. + # So we try reading logs multiple times. + # The proper solution would be for the runner to ensure logs availability before marking job as Done. + found_log = False + for _ in range(POLL_LOGS_ATTEMPTS): + log_entries = logger.list_entries() + for log_entry in log_entries: + found_log = True + yield _log_entry_to_log_event(storage, repo_address, log_entry) + if found_log: + break + time.sleep(POLL_LOGS_WAIT_TIME) def _get_log_name(bucket_name: str, repo_address: RepoAddress, run_name) -> str: diff --git a/cli/dstack/backend/gcp/storage.py b/cli/dstack/backend/gcp/storage.py index 49efb19fdc..d9e381cc5c 100644 --- a/cli/dstack/backend/gcp/storage.py +++ b/cli/dstack/backend/gcp/storage.py @@ -23,11 +23,11 @@ def put_object(self, key: str, content: str, metadata: Optional[Dict] = None): blob.upload_from_string(content) def get_object(self, key: str) -> Optional[str]: - blob = self.bucket.get_blob(key) - if blob is None: + blob = self.bucket.blob(key) + try: + return blob.download_as_text() + except exceptions.NotFound: return None - with blob.open() as f: - return f.read() def delete_object(self, key: str): try: diff --git a/runner/internal/backend/gcp/backend.go b/runner/internal/backend/gcp/backend.go index de23683e77..6c424e9083 100644 --- a/runner/internal/backend/gcp/backend.go +++ b/runner/internal/backend/gcp/backend.go @@ -96,18 +96,6 @@ func (gbackend *GCPBackend) Job(ctx context.Context) *models.Job { } func (gbackend *GCPBackend) UpdateState(ctx context.Context) error { - log.Trace(ctx, "Fetching list jobs", "Repo username", gbackend.state.Job.RepoUserName, "Repo name", gbackend.state.Job.RepoName, "Job ID", gbackend.state.Job.JobID) - files, err := gbackend.storage.ListFile(ctx, gbackend.state.Job.JobHeadFilepathPrefix()) - if err != nil { - return gerrors.Wrap(err) - } - for _, file := range files { - log.Trace(ctx, "Deleting file job", "Bucket", gbackend.bucket, "Path", file) - err = gbackend.storage.DeleteFile(ctx, file) - if err != nil { - return gerrors.Wrap(err) - } - } log.Trace(ctx, "Marshaling job") contents, err := yaml.Marshal(&gbackend.state.Job) if err != nil { @@ -119,12 +107,19 @@ func (gbackend *GCPBackend) UpdateState(ctx context.Context) error { if err != nil { return gerrors.Wrap(err) } - jobHeadFilepath := gbackend.state.Job.JobHeadFilepath() - log.Trace(ctx, "Write to file lock job", "Path", jobHeadFilepath) - err = gbackend.storage.PutFile(ctx, jobHeadFilepath, []byte{}) + log.Trace(ctx, "Fetching list jobs", "Repo username", gbackend.state.Job.RepoUserName, "Repo name", gbackend.state.Job.RepoName, "Job ID", gbackend.state.Job.JobID) + files, err := gbackend.storage.ListFile(ctx, gbackend.state.Job.JobHeadFilepathPrefix()) if err != nil { return gerrors.Wrap(err) } + jobHeadFilepath := gbackend.state.Job.JobHeadFilepath() + for _, file := range files[:1] { + log.Trace(ctx, "Renaming file job", "From", file, "To", jobHeadFilepath) + err = gbackend.storage.RenameFile(ctx, file, jobHeadFilepath) + if err != nil { + return gerrors.Wrap(err) + } + } return nil } diff --git a/runner/internal/backend/gcp/logging.go b/runner/internal/backend/gcp/logging.go index c9f9ded794..a3145d6bff 100644 --- a/runner/internal/backend/gcp/logging.go +++ b/runner/internal/backend/gcp/logging.go @@ -67,6 +67,7 @@ func (glogger *GCPLogger) flushLogs(ctx context.Context, ticker *time.Ticker) { // Backend will Close() the client on Shutdown(), flushing the logs. // We don't want to Close() from this goroutine since google libraries may // send audit logs after ctx.Done(). + glogger.logger.Flush() return case <-ticker.C: glogger.logger.Flush() diff --git a/runner/internal/backend/gcp/storage.go b/runner/internal/backend/gcp/storage.go index aa23256275..e0712c5577 100644 --- a/runner/internal/backend/gcp/storage.go +++ b/runner/internal/backend/gcp/storage.go @@ -49,7 +49,10 @@ func (gstorage *GCPStorage) GetFile(ctx context.Context, key string) ([]byte, er } defer reader.Close() buffer := new(bytes.Buffer) - io.Copy(buffer, reader) + _, err = io.Copy(buffer, reader) + if err != nil { + return nil, gerrors.Wrap(err) + } return buffer.Bytes(), nil } @@ -57,7 +60,10 @@ func (gstorage *GCPStorage) PutFile(ctx context.Context, key string, contents [] obj := gstorage.bucket.Object(key) writer := obj.NewWriter(ctx) reader := bytes.NewReader(contents) - io.Copy(writer, reader) + _, err := io.Copy(writer, reader) + if err != nil { + return gerrors.Wrap(err) + } return writer.Close() } From a022749a0be74096aedca9bad42c2d1c783194d0 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Mon, 27 Feb 2023 09:45:13 +0500 Subject: [PATCH 34/58] Read gcp config file in runner --- runner/internal/backend/gcp/backend.go | 31 ++++++++++++++++---------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/runner/internal/backend/gcp/backend.go b/runner/internal/backend/gcp/backend.go index 6c424e9083..505fd301bc 100644 --- a/runner/internal/backend/gcp/backend.go +++ b/runner/internal/backend/gcp/backend.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "io/ioutil" "path" "strings" @@ -30,20 +31,26 @@ type GCPBackend struct { state *models.State } +type GCPConfigFile struct { + Project string `yaml:"project"` + Zone string `yaml:"zone"` + Bucket string `yaml:"bucket"` +} + func init() { backend.RegisterBackend("gcp", func(ctx context.Context, pathConfig string) (backend.Backend, error) { - // TODO read config - // log.Trace(ctx, "Read config file", "path", pathConfig) - // theConfig, err := ioutil.ReadFile(pathConfig) - // if err != nil { - // return nil, gerrors.Wrap(err) - // } - // log.Trace(ctx, "Unmarshal config") - // err = yaml.Unmarshal(theConfig, &file) - // if err != nil { - // return nil, gerrors.Wrap(err) - // } - return New("dstack", "us-central1-a", "dstack-bucket"), nil + configFile := GCPConfigFile{} + log.Trace(ctx, "Read config file", "path", pathConfig) + fileContent, err := ioutil.ReadFile(pathConfig) + if err != nil { + return nil, gerrors.Wrap(err) + } + log.Trace(ctx, "Unmarshal config") + err = yaml.Unmarshal(fileContent, &configFile) + if err != nil { + return nil, gerrors.Wrap(err) + } + return New(configFile.Project, configFile.Zone, configFile.Bucket), nil }) } From b33350a24d28cda35cef9d8f69f36bca05a952de Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Mon, 27 Feb 2023 10:50:47 +0500 Subject: [PATCH 35/58] Run build workflow --- .github/workflows/manual_gcp_image_stgn.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/manual_gcp_image_stgn.yml b/.github/workflows/manual_gcp_image_stgn.yml index 5226092c76..231196a7ad 100644 --- a/.github/workflows/manual_gcp_image_stgn.yml +++ b/.github/workflows/manual_gcp_image_stgn.yml @@ -1,6 +1,7 @@ name: Manual GCP image build for STGN -on: [workflow_dispatch] +on: + push: jobs: gcp-image-build: From db004d9f6f51ee490aac33d9c5446d1ef6ad095e Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Mon, 27 Feb 2023 11:26:27 +0500 Subject: [PATCH 36/58] Run main build workflow --- .github/workflows/build.yml | 1 + .github/workflows/manual_gcp_image_stgn.yml | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f4c6a99700..0a13132325 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,6 +7,7 @@ on: pull_request: branches: - 'master' + workflow_dispatch: jobs: python-lint: diff --git a/.github/workflows/manual_gcp_image_stgn.yml b/.github/workflows/manual_gcp_image_stgn.yml index 231196a7ad..5226092c76 100644 --- a/.github/workflows/manual_gcp_image_stgn.yml +++ b/.github/workflows/manual_gcp_image_stgn.yml @@ -1,7 +1,6 @@ name: Manual GCP image build for STGN -on: - push: +on: [workflow_dispatch] jobs: gcp-image-build: From 67af930275f7d637ea7216283b4c245835dd7c2a Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Mon, 27 Feb 2023 11:42:56 +0500 Subject: [PATCH 37/58] Add gcloud requirements to setup.py --- setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.py b/setup.py index 985aec026a..666f3fc399 100644 --- a/setup.py +++ b/setup.py @@ -76,6 +76,10 @@ def get_long_description(): "jinja2", "pygtail", "packaging", + "google-cloud-storage", + "google-cloud-compute", + "google-cloud-secret-manager", + "google-cloud-logging", ], classifiers=[ "Development Status :: 2 - Pre-Alpha", From f311327898a03e1f1c76f5ac9fc3fcc1c77e9812 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Mon, 27 Feb 2023 11:45:37 +0500 Subject: [PATCH 38/58] Run manual_gcp_image_stgn workflow --- .github/workflows/manual_gcp_image_stgn.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/manual_gcp_image_stgn.yml b/.github/workflows/manual_gcp_image_stgn.yml index 5226092c76..e1a3fb7a82 100644 --- a/.github/workflows/manual_gcp_image_stgn.yml +++ b/.github/workflows/manual_gcp_image_stgn.yml @@ -1,6 +1,7 @@ name: Manual GCP image build for STGN -on: [workflow_dispatch] +on: + push jobs: gcp-image-build: From 4822c010997f3511c5983802fc881e23fa3298b7 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Mon, 27 Feb 2023 12:30:53 +0500 Subject: [PATCH 39/58] Handle file renaming to the same name in gcp storage --- runner/internal/backend/gcp/storage.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/runner/internal/backend/gcp/storage.go b/runner/internal/backend/gcp/storage.go index e0712c5577..62d8935217 100644 --- a/runner/internal/backend/gcp/storage.go +++ b/runner/internal/backend/gcp/storage.go @@ -91,6 +91,9 @@ func (gstorage *GCPStorage) DeleteFile(ctx context.Context, key string) error { } func (gstorage *GCPStorage) RenameFile(ctx context.Context, oldKey, newKey string) error { + if newKey == oldKey { + return nil + } src := gstorage.bucket.Object(oldKey) dst := gstorage.bucket.Object(newKey) copier := dst.CopierFrom(src) From 74e6fe8b4795fb048dbfaa95a4771a5f977d3736 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Mon, 27 Feb 2023 12:34:12 +0500 Subject: [PATCH 40/58] Make runner work on gcp --- cli/dstack/backend/gcp/compute.py | 65 ++++++++++++++++++++++++++----- cli/dstack/backend/gcp/config.py | 9 ++++- 2 files changed, 64 insertions(+), 10 deletions(-) diff --git a/cli/dstack/backend/gcp/compute.py b/cli/dstack/backend/gcp/compute.py index 90f53834b0..ad05313235 100644 --- a/cli/dstack/backend/gcp/compute.py +++ b/cli/dstack/backend/gcp/compute.py @@ -5,6 +5,8 @@ import google.api_core.exceptions from google.cloud import compute_v1 +from dstack import version +from dstack.backend.aws.runners import _serialize_runner_yaml from dstack.backend.base.compute import Compute from dstack.backend.gcp import utils as gcp_utils from dstack.backend.gcp.config import GCPConfig @@ -51,8 +53,10 @@ def run_instance(self, job: Job, instance_type: InstanceType) -> str: instance = _launch_instance( project_id=self.gcp_config.project_id, zone=self.gcp_config.zone, - image_name="stgn-dstack-5", + image_name=_get_image_name(), instance_name=_get_instance_name(job), + user_data_script=_get_user_data_script(self.gcp_config, job, instance_type), + service_account=self.gcp_config.service_account, ) return instance.name @@ -69,16 +73,49 @@ def cancel_spot_request(self, request_id: str): ) +def _get_image_name() -> Optional[str]: + if version.__is_release__: + image_prefix = "dstack-" + else: + image_prefix = "stgn-dstack-" + list_request = compute_v1.ListImagesRequest() + list_request.project = "dstack" + list_request.order_by = "creationTimestamp desc" + images_client = compute_v1.ImagesClient() + images = images_client.list(list_request) + for image in images: + # Specifying both a list filter and sort order is not currently supported in compute_v1 + # so we don't use list_request.filter to filter images + if image.name.startswith(image_prefix): + return image.name + return None + + def _get_instance_name(job: Job) -> str: # TODO support multiple jobs per run return f"dstack-{job.run_name}" +def _get_user_data_script(gcp_config: GCPConfig, job: Job, instance_type: InstanceType) -> str: + config_content = gcp_config.serialize_yaml().replace("\n", "\\n") + runner_content = _serialize_runner_yaml(job.runner_id, instance_type.resources, 3000, 4000) + return f"""#!/bin/sh +mkdir -p /root/.dstack/ +echo '{config_content}' > /root/.dstack/config.yaml +echo '{runner_content}' > /root/.dstack/runner.yaml +EXTERNAL_IP=`curl -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip` +echo "hostname: $EXTERNAL_IP" >> /root/.dstack/runner.yaml +HOME=/root nohup dstack-runner --log-level 6 start --http-port 4000 +""" + + def _launch_instance( project_id: str, zone: str, image_name: str, instance_name: str, + user_data_script: str, + service_account: str, ) -> compute_v1.Instance: disk = _disk_from_image( disk_type=f"zones/{zone}/diskTypes/pd-balanced", @@ -92,6 +129,9 @@ def _launch_instance( zone=zone, instance_name=instance_name, disks=[disk], + user_data_script=user_data_script, + service_account=service_account, + external_access=True, ) return instance @@ -151,6 +191,8 @@ def _create_instance( instance_termination_action: str = "STOP", custom_hostname: str = None, delete_protection: bool = False, + user_data_script: Optional[str] = None, + service_account: Optional[str] = None, ) -> compute_v1.Instance: """ Send an instance creation request to the Compute Engine API and wait for it to complete. @@ -245,6 +287,19 @@ def _create_instance( # Set the delete protection bit instance.deletion_protection = True + if user_data_script is not None: + instance.metadata = compute_v1.Metadata( + items=[compute_v1.Items(key="user-data", value=user_data_script)] + ) + + if service_account is not None: + instance.service_accounts = [ + compute_v1.ServiceAccount( + email=service_account, + scopes=["https://www.googleapis.com/auth/cloud-platform"], + ) + ] + # Prepare the request to insert an instance. request = compute_v1.InsertInstanceRequest() request.zone = zone @@ -292,11 +347,3 @@ def _delete_instance(project_id: str, zone: str, instance_name: str): client.delete(delete_request) except google.api_core.exceptions.NotFound: pass - - -def main(): - pass - - -if __name__ == "__main__": - main() diff --git a/cli/dstack/backend/gcp/config.py b/cli/dstack/backend/gcp/config.py index 0ea4036365..63fef9207e 100644 --- a/cli/dstack/backend/gcp/config.py +++ b/cli/dstack/backend/gcp/config.py @@ -1,6 +1,8 @@ from pathlib import Path from typing import Dict +import yaml + from dstack.core.config import BackendConfig, get_config_path @@ -12,6 +14,7 @@ def name(self): def __init__( self, ): + self.service_account = "my-dstack-sa@dstack.iam.gserviceaccount.com" self.project_id = "dstack" self.zone = "us-central1-a" self.bucket_name = "dstack-bucket" @@ -28,7 +31,11 @@ def save(self, path: Path = get_config_path()): def serialize(self) -> Dict: return { - "project_id": self.project_id, + "backend": self.name, + "project": self.project_id, "zone": self.zone, "bucket": self.bucket_name, } + + def serialize_yaml(self) -> str: + return yaml.dump(self.serialize()) From 8a0f521488963fee57b11219180b6e328b7be761 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Mon, 27 Feb 2023 12:54:08 +0500 Subject: [PATCH 41/58] Run manual_gcp_image_stgn workflow --- .github/workflows/manual_gcp_image_stgn.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/manual_gcp_image_stgn.yml b/.github/workflows/manual_gcp_image_stgn.yml index e1a3fb7a82..e4022dae9d 100644 --- a/.github/workflows/manual_gcp_image_stgn.yml +++ b/.github/workflows/manual_gcp_image_stgn.yml @@ -1,7 +1,9 @@ name: Manual GCP image build for STGN on: - push + workflow_dispatch: + push: + jobs: gcp-image-build: From 99bb4bb2770e90a90ba24692cf0a7e77d8f8b6af Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Mon, 27 Feb 2023 15:51:01 +0500 Subject: [PATCH 42/58] Create gcp firewall rule --- .github/workflows/manual_gcp_image_stgn.yml | 1 - cli/dstack/api/backend.py | 2 +- cli/dstack/backend/gcp/compute.py | 50 +++++++++++++++++++++ 3 files changed, 51 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_gcp_image_stgn.yml b/.github/workflows/manual_gcp_image_stgn.yml index e4022dae9d..39af40d8b8 100644 --- a/.github/workflows/manual_gcp_image_stgn.yml +++ b/.github/workflows/manual_gcp_image_stgn.yml @@ -2,7 +2,6 @@ name: Manual GCP image build for STGN on: workflow_dispatch: - push: jobs: diff --git a/cli/dstack/api/backend.py b/cli/dstack/api/backend.py index 14b438f0ed..d2187c2e65 100644 --- a/cli/dstack/api/backend.py +++ b/cli/dstack/api/backend.py @@ -10,7 +10,7 @@ DEFAULT_REMOTE = "aws" DEFAULT = "local" -backends_classes = [GCPBackend, LocalBackend] +backends_classes = [AwsBackend, GCPBackend, LocalBackend] if not version.__is_release__: backends_classes.append(HubBackend) diff --git a/cli/dstack/backend/gcp/compute.py b/cli/dstack/backend/gcp/compute.py index ad05313235..95e14ee9fb 100644 --- a/cli/dstack/backend/gcp/compute.py +++ b/cli/dstack/backend/gcp/compute.py @@ -15,6 +15,8 @@ from dstack.core.request import RequestHead, RequestStatus from dstack.core.runners import Resources, Runner +DSTACK_INSTANCE_TAG = "dstack-runner-instance" + class GCPCompute(Compute): def __init__( @@ -117,6 +119,10 @@ def _launch_instance( user_data_script: str, service_account: str, ) -> compute_v1.Instance: + try: + _create_firewall_rules(project_id=project_id) + except google.api_core.exceptions.Conflict: + pass disk = _disk_from_image( disk_type=f"zones/{zone}/diskTypes/pd-balanced", disk_size_gb=20, @@ -300,6 +306,8 @@ def _create_instance( ) ] + instance.tags = compute_v1.Tags(items=[DSTACK_INSTANCE_TAG]) + # Prepare the request to insert an instance. request = compute_v1.InsertInstanceRequest() request.zone = zone @@ -312,6 +320,48 @@ def _create_instance( return instance_client.get(project=project_id, zone=zone, instance=instance_name) +def _create_firewall_rules( + project_id: str, + network: str = "global/networks/default", +): + """ + Creates a simple firewall rule allowing for incoming HTTP and HTTPS access from the entire Internet. + + Args: + project_id: project ID or project number of the Cloud project you want to use. + firewall_rule_name: name of the rule that is created. + network: name of the network the rule will be applied to. Available name formats: + * https://www.googleapis.com/compute/v1/projects/{project_id}/global/networks/{network} + * projects/{project_id}/global/networks/{network} + * global/networks/{network} + + Returns: + A Firewall object. + """ + firewall_rule = compute_v1.Firewall() + firewall_rule.name = "dstack-runner-allow-incoming" + firewall_rule.direction = "INGRESS" + + allowed_ports_tcp = compute_v1.Allowed() + allowed_ports_tcp.I_p_protocol = "tcp" + allowed_ports_tcp.ports = ["3000-4000"] + + allowed_ports_udp = compute_v1.Allowed() + allowed_ports_udp.I_p_protocol = "udp" + allowed_ports_udp.ports = ["3000-4000"] + + firewall_rule.allowed = [allowed_ports_tcp, allowed_ports_udp] + firewall_rule.source_ranges = ["0.0.0.0/0"] + firewall_rule.network = network + firewall_rule.description = "Allowing TCP/UDP traffic on ports 3000-4000 from Internet." + + firewall_rule.target_tags = [DSTACK_INSTANCE_TAG] + + firewall_client = compute_v1.FirewallsClient() + operation = firewall_client.insert(project=project_id, firewall_resource=firewall_rule) + gcp_utils.wait_for_extended_operation(operation, "firewall rule creation") + + def _get_instance_status(project_id: str, zone: str, instance_name: str) -> RequestStatus: get_instance_request = compute_v1.GetInstanceRequest( instance=instance_name, From dbd5c63fb1886855830719e8e49265e85bb50900 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Mon, 27 Feb 2023 17:26:27 +0500 Subject: [PATCH 43/58] Switch to gcp auth via service account --- cli/dstack/backend/gcp/__init__.py | 23 ++++++++++++-- cli/dstack/backend/gcp/compute.py | 49 ++++++++++++++++++++---------- cli/dstack/backend/gcp/config.py | 2 +- cli/dstack/backend/gcp/logs.py | 9 ++++-- cli/dstack/backend/gcp/secrets.py | 7 +++-- cli/dstack/backend/gcp/storage.py | 10 +++--- 6 files changed, 70 insertions(+), 30 deletions(-) diff --git a/cli/dstack/backend/gcp/__init__.py b/cli/dstack/backend/gcp/__init__.py index d789870d4b..89654bd62c 100644 --- a/cli/dstack/backend/gcp/__init__.py +++ b/cli/dstack/backend/gcp/__init__.py @@ -1,8 +1,10 @@ +import os import warnings from pathlib import Path from typing import Generator, List, Optional from google.auth._default import _CLOUD_SDK_CREDENTIALS_WARNING +from google.oauth2 import service_account from dstack.backend.base import CloudBackend from dstack.backend.base import artifacts as base_artifacts @@ -32,21 +34,36 @@ def __init__(self, config: Optional[GCPConfig] = None): if config is None: config = GCPConfig.load() if config is None: - self._loaded = False return self.config = config + + if self.config.credentials is not None: + credentials = service_account.Credentials.from_service_account_info( + self.config.credentials + ) + else: + credentials_file = os.getenv("GOOGLE_APPLICATION_CREDENTIALS") + if not credentials_file: + return + credentials = service_account.Credentials.from_service_account_file(credentials_file) + self._storage = GCPStorage( - project_id=self.config.project_id, bucket_name=self.config.bucket_name + project_id=self.config.project_id, + bucket_name=self.config.bucket_name, + credentials=credentials, ) - self._compute = GCPCompute(gcp_config=self.config) + self._compute = GCPCompute(gcp_config=self.config, credentials=credentials) self._secrets_manager = GCPSecretsManager( project_id=self.config.project_id, bucket_name=self.config.bucket_name, + credentials=credentials, ) self._logging = GCPLogging( project_id=self.config.project_id, bucket_name=self.config.bucket_name, + credentials=credentials, ) + self.configure() self._loaded = True diff --git a/cli/dstack/backend/gcp/compute.py b/cli/dstack/backend/gcp/compute.py index 95e14ee9fb..dfd8d9a2e9 100644 --- a/cli/dstack/backend/gcp/compute.py +++ b/cli/dstack/backend/gcp/compute.py @@ -4,6 +4,7 @@ import google.api_core.exceptions from google.cloud import compute_v1 +from google.oauth2 import service_account from dstack import version from dstack.backend.aws.runners import _serialize_runner_yaml @@ -22,8 +23,13 @@ class GCPCompute(Compute): def __init__( self, gcp_config: GCPConfig, + credentials: Optional[service_account.Credentials], ): self.gcp_config = gcp_config + self.credentials = credentials + self.instances_client = compute_v1.InstancesClient(credentials=self.credentials) + self.images_client = compute_v1.ImagesClient(credentials=self.credentials) + self.firewalls_client = compute_v1.FirewallsClient(credentials=self.credentials) def get_request_head(self, job: Job, request_id: Optional[str]) -> RequestHead: if request_id is None: @@ -33,6 +39,7 @@ def get_request_head(self, job: Job, request_id: Optional[str]) -> RequestHead: message="request_id is not specified", ) instance_status = _get_instance_status( + client=self.instances_client, project_id=self.gcp_config.project_id, zone=self.gcp_config.zone, instance_name=request_id, @@ -53,29 +60,33 @@ def get_instance_type(self, job: Job) -> Optional[InstanceType]: def run_instance(self, job: Job, instance_type: InstanceType) -> str: instance = _launch_instance( + instances_client=self.instances_client, + firewalls_client=self.firewalls_client, project_id=self.gcp_config.project_id, zone=self.gcp_config.zone, - image_name=_get_image_name(), + image_name=_get_image_name(images_client=self.images_client), instance_name=_get_instance_name(job), user_data_script=_get_user_data_script(self.gcp_config, job, instance_type), - service_account=self.gcp_config.service_account, + service_account=self.credentials.service_account_email, ) return instance.name def terminate_instance(self, request_id: str): _terminate_instance( + client=self.instances_client, gcp_config=self.gcp_config, instance_name=request_id, ) def cancel_spot_request(self, request_id: str): _terminate_instance( + client=self.instances_client, gcp_config=self.gcp_config, instance_name=request_id, ) -def _get_image_name() -> Optional[str]: +def _get_image_name(images_client: compute_v1.ImagesClient) -> Optional[str]: if version.__is_release__: image_prefix = "dstack-" else: @@ -83,7 +94,6 @@ def _get_image_name() -> Optional[str]: list_request = compute_v1.ListImagesRequest() list_request.project = "dstack" list_request.order_by = "creationTimestamp desc" - images_client = compute_v1.ImagesClient() images = images_client.list(list_request) for image in images: # Specifying both a list filter and sort order is not currently supported in compute_v1 @@ -112,6 +122,8 @@ def _get_user_data_script(gcp_config: GCPConfig, job: Job, instance_type: Instan def _launch_instance( + instances_client: compute_v1.InstancesClient, + firewalls_client: compute_v1.FirewallsClient, project_id: str, zone: str, image_name: str, @@ -120,7 +132,7 @@ def _launch_instance( service_account: str, ) -> compute_v1.Instance: try: - _create_firewall_rules(project_id=project_id) + _create_firewall_rules(firewalls_client=firewalls_client, project_id=project_id) except google.api_core.exceptions.Conflict: pass disk = _disk_from_image( @@ -131,6 +143,7 @@ def _launch_instance( auto_delete=False, ) instance = _create_instance( + instances_client=instances_client, project_id=project_id, zone=zone, instance_name=instance_name, @@ -181,6 +194,7 @@ def _disk_from_image( def _create_instance( + instances_client: compute_v1.InstancesClient, project_id: str, zone: str, instance_name: str, @@ -240,8 +254,6 @@ def _create_instance( Returns: Instance object. """ - instance_client = compute_v1.InstancesClient() - # Use the network interface provided in the network_link argument. network_interface = compute_v1.NetworkInterface() network_interface.name = network_link @@ -314,13 +326,14 @@ def _create_instance( request.project = project_id request.instance_resource = instance - operation = instance_client.insert(request=request) + operation = instances_client.insert(request=request) gcp_utils.wait_for_extended_operation(operation, "instance creation") - return instance_client.get(project=project_id, zone=zone, instance=instance_name) + return instances_client.get(project=project_id, zone=zone, instance=instance_name) def _create_firewall_rules( + firewalls_client: compute_v1.FirewallsClient, project_id: str, network: str = "global/networks/default", ): @@ -357,18 +370,18 @@ def _create_firewall_rules( firewall_rule.target_tags = [DSTACK_INSTANCE_TAG] - firewall_client = compute_v1.FirewallsClient() - operation = firewall_client.insert(project=project_id, firewall_resource=firewall_rule) + operation = firewalls_client.insert(project=project_id, firewall_resource=firewall_rule) gcp_utils.wait_for_extended_operation(operation, "firewall rule creation") -def _get_instance_status(project_id: str, zone: str, instance_name: str) -> RequestStatus: +def _get_instance_status( + client: compute_v1.InstancesClient, project_id: str, zone: str, instance_name: str +) -> RequestStatus: get_instance_request = compute_v1.GetInstanceRequest( instance=instance_name, project=project_id, zone=zone, ) - client = compute_v1.InstancesClient() try: instance = client.get(get_instance_request) except google.api_core.exceptions.NotFound: @@ -378,21 +391,25 @@ def _get_instance_status(project_id: str, zone: str, instance_name: str) -> Requ return RequestStatus.TERMINATED -def _terminate_instance(gcp_config: GCPConfig, instance_name: str): +def _terminate_instance( + client: compute_v1.InstancesClient, gcp_config: GCPConfig, instance_name: str +): _delete_instance( + client=client, instance_name=instance_name, project_id=gcp_config.project_id, zone=gcp_config.zone, ) -def _delete_instance(project_id: str, zone: str, instance_name: str): +def _delete_instance( + client: compute_v1.InstancesClient, project_id: str, zone: str, instance_name: str +): delete_request = compute_v1.DeleteInstanceRequest( instance=instance_name, project=project_id, zone=zone, ) - client = compute_v1.InstancesClient() try: client.delete(delete_request) except google.api_core.exceptions.NotFound: diff --git a/cli/dstack/backend/gcp/config.py b/cli/dstack/backend/gcp/config.py index 63fef9207e..1c2192501d 100644 --- a/cli/dstack/backend/gcp/config.py +++ b/cli/dstack/backend/gcp/config.py @@ -14,7 +14,7 @@ def name(self): def __init__( self, ): - self.service_account = "my-dstack-sa@dstack.iam.gserviceaccount.com" + self.credentials = None self.project_id = "dstack" self.zone = "us-central1-a" self.bucket_name = "dstack-bucket" diff --git a/cli/dstack/backend/gcp/logs.py b/cli/dstack/backend/gcp/logs.py index 0fb76940d6..120d57033a 100644 --- a/cli/dstack/backend/gcp/logs.py +++ b/cli/dstack/backend/gcp/logs.py @@ -1,7 +1,8 @@ import time -from typing import Generator +from typing import Generator, Optional from google.cloud import logging +from google.oauth2 import service_account from dstack.backend.base import jobs from dstack.backend.base.logs import fix_urls @@ -14,10 +15,12 @@ class GCPLogging: - def __init__(self, project_id: str, bucket_name: str): + def __init__( + self, project_id: str, bucket_name: str, credentials: Optional[service_account.Credentials] + ): self.project_id = project_id self.bucket_name = bucket_name - self.logging_client = logging.Client(project=project_id) + self.logging_client = logging.Client(project=project_id, credentials=credentials) def poll_logs( self, diff --git a/cli/dstack/backend/gcp/secrets.py b/cli/dstack/backend/gcp/secrets.py index 24f5a49906..f6d634ac9f 100644 --- a/cli/dstack/backend/gcp/secrets.py +++ b/cli/dstack/backend/gcp/secrets.py @@ -2,6 +2,7 @@ from google.api_core import exceptions from google.cloud import secretmanager +from google.oauth2 import service_account from dstack.backend.base.secrets import SecretsManager from dstack.core.repo import RepoAddress @@ -9,10 +10,12 @@ class GCPSecretsManager(SecretsManager): - def __init__(self, project_id: str, bucket_name: str): + def __init__( + self, project_id: str, bucket_name: str, credentials: Optional[service_account.Credentials] + ): self.project_id = project_id self.bucket_name = bucket_name - self.secrets_client = secretmanager.SecretManagerServiceClient() + self.secrets_client = secretmanager.SecretManagerServiceClient(credentials=credentials) def get_secret(self, repo_address: RepoAddress, secret_name: str) -> Optional[Secret]: secret_value = self._get_secret_value( diff --git a/cli/dstack/backend/gcp/storage.py b/cli/dstack/backend/gcp/storage.py index d9e381cc5c..33a581fbff 100644 --- a/cli/dstack/backend/gcp/storage.py +++ b/cli/dstack/backend/gcp/storage.py @@ -3,6 +3,7 @@ from typing import Callable, Dict, List, Optional from google.cloud import exceptions, storage +from google.oauth2 import service_account from dstack.backend.base.storage import SIGNED_URL_EXPIRATION, CloudStorage from dstack.core.storage import StorageFile @@ -10,9 +11,11 @@ class GCPStorage(CloudStorage): - def __init__(self, project_id: str, bucket_name: str): - self.storage_client = self._get_client(project_id) + def __init__( + self, project_id: str, bucket_name: str, credentials: Optional[service_account.Credentials] + ): self.bucket_name = bucket_name + self.storage_client = storage.Client(project=project_id, credentials=credentials) def configure(self): self.bucket = self._get_or_create_bucket(self.bucket_name) @@ -76,9 +79,6 @@ def get_signed_upload_url(self, key: str) -> str: ) return url - def _get_client(self, project_id: str) -> storage.Client: - return storage.Client(project=project_id) - def _get_or_create_bucket(self, bucket_name: str): try: return self.storage_client.create_bucket(bucket_name) From 1a4429991f5924e5bc1d7dccfcdbaf1c35c38af2 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 28 Feb 2023 12:48:45 +0500 Subject: [PATCH 44/58] Configure gcp via cli --- cli/dstack/api/config.py | 6 +- cli/dstack/backend/aws/config.py | 2 - cli/dstack/backend/gcp/__init__.py | 6 +- cli/dstack/backend/gcp/config.py | 206 ++++++++++++++++++++++++++--- cli/dstack/backend/hub/config.py | 2 - cli/dstack/core/config.py | 6 - cli/requirements.txt | 1 + setup.py | 1 + 8 files changed, 193 insertions(+), 37 deletions(-) diff --git a/cli/dstack/api/config.py b/cli/dstack/api/config.py index e356b7289f..42bf6e351e 100644 --- a/cli/dstack/api/config.py +++ b/cli/dstack/api/config.py @@ -2,10 +2,11 @@ from dstack import version from dstack.backend.aws.config import AWSConfig +from dstack.backend.gcp.config import GCPConfigurator from dstack.backend.hub.config import HUBConfig from dstack.core.config import BackendConfig -configs_classes = [AWSConfig] +configs_classes = [AWSConfig, GCPConfigurator] if not version.__is_release__: configs_classes.append(HUBConfig) @@ -19,7 +20,6 @@ def dict_config() -> Dict[str, BackendConfig]: configs = [cls() for cls in configs_classes] names = {} for config in configs: - if config.configured: - names[config.name] = config + names[config.name] = config return names diff --git a/cli/dstack/backend/aws/config.py b/cli/dstack/backend/aws/config.py index 2b1a01289f..11ad897116 100644 --- a/cli/dstack/backend/aws/config.py +++ b/cli/dstack/backend/aws/config.py @@ -31,8 +31,6 @@ class AWSConfig(BackendConfig): NAME = "aws" - _configured = True - bucket_name = None region_name = None profile_name = None diff --git a/cli/dstack/backend/gcp/__init__.py b/cli/dstack/backend/gcp/__init__.py index 89654bd62c..a3e21ea168 100644 --- a/cli/dstack/backend/gcp/__init__.py +++ b/cli/dstack/backend/gcp/__init__.py @@ -14,7 +14,7 @@ from dstack.backend.base import secrets as base_secrets from dstack.backend.base import tags as base_tags from dstack.backend.gcp.compute import GCPCompute -from dstack.backend.gcp.config import GCPConfig +from dstack.backend.gcp.config import GCPConfig, GCPConfigurator from dstack.backend.gcp.logs import GCPLogging from dstack.backend.gcp.secrets import GCPSecretsManager from dstack.backend.gcp.storage import GCPStorage @@ -32,7 +32,7 @@ class GCPBackend(CloudBackend): def __init__(self, config: Optional[GCPConfig] = None): if config is None: - config = GCPConfig.load() + config = GCPConfigurator.load() if config is None: return self.config = config @@ -43,7 +43,7 @@ def __init__(self, config: Optional[GCPConfig] = None): ) else: credentials_file = os.getenv("GOOGLE_APPLICATION_CREDENTIALS") - if not credentials_file: + if credentials_file is None: return credentials = service_account.Credentials.from_service_account_file(credentials_file) diff --git a/cli/dstack/backend/gcp/config.py b/cli/dstack/backend/gcp/config.py index 1c2192501d..29482bc941 100644 --- a/cli/dstack/backend/gcp/config.py +++ b/cli/dstack/backend/gcp/config.py @@ -1,37 +1,33 @@ +import os from pathlib import Path -from typing import Dict +from typing import Dict, Optional import yaml +from google.cloud import compute_v1, exceptions, storage +from google.oauth2 import service_account +from rich.prompt import Confirm, Prompt +from simple_term_menu import TerminalMenu +from dstack.cli.common import ask_choice, console from dstack.core.config import BackendConfig, get_config_path -class GCPConfig(BackendConfig): - @property - def name(self): - return "gcp" - +class GCPConfig: def __init__( self, + project_id: str, + zone: str, + bucket_name: str, + credentials: Optional[Dict] = None, ): - self.credentials = None - self.project_id = "dstack" - self.zone = "us-central1-a" - self.bucket_name = "dstack-bucket" - - def configure(self): - pass - - @classmethod - def load(cls, path: Path = get_config_path()): - return cls() - - def save(self, path: Path = get_config_path()): - pass + self.project_id = project_id + self.zone = zone + self.bucket_name = bucket_name + self.credentials = credentials def serialize(self) -> Dict: return { - "backend": self.name, + "backend": "gcp", "project": self.project_id, "zone": self.zone, "bucket": self.bucket_name, @@ -39,3 +35,171 @@ def serialize(self) -> Dict: def serialize_yaml(self) -> str: return yaml.dump(self.serialize()) + + @classmethod + def deserialize(cls, data: Dict) -> "GCPConfig": + return cls(project_id=data["project"], zone=data["zone"], bucket_name=data["bucket"]) + + @classmethod + def deserialize_yaml(cls, yaml_content: str) -> "GCPConfig": + return cls.deserialize(yaml.load(yaml_content, yaml.FullLoader)) + + +class GCPConfigurator(BackendConfig): + @property + def name(self): + return "gcp" + + @classmethod + def load(cls, path: Path = get_config_path()) -> GCPConfig: + with open(path) as f: + return GCPConfig.deserialize_yaml(f.read()) + + def save(self, config: GCPConfig, path: Path = get_config_path()): + with open(path, "w+") as f: + f.write(config.serialize_yaml()) + + def configure(self): + credentials_file = os.getenv("GOOGLE_APPLICATION_CREDENTIALS") + if credentials_file is None: + console.print( + "[red bold]✗[/red bold] GOOGLE_APPLICATION_CREDENTIALS is not set." + " It should point to the service account credentials file." + " Learn more about service account credentials: https://cloud.google.com/iam/docs/creating-managing-service-account-keys" + ) + exit(1) + + zone = None + bucket_name = None + try: + config = self.load() + except Exception: + pass + else: + zone = config.zone + bucket_name = config.bucket_name + + self.credentials = self._get_credentials(credentials_file) + self.project_id = self.credentials.project_id + + self.zone = self._ask_zone(zone) + self.bucket_name = self._ask_bucket(default_bucket=bucket_name) + + config = GCPConfig( + project_id=self.project_id, + zone=self.zone, + bucket_name=bucket_name, + ) + self.save(config) + console.print(f"[grey58]OK[/]") + + def _get_credentials(self, credentials_file: str) -> service_account.Credentials: + try: + credentials = service_account.Credentials.from_service_account_file(credentials_file) + storage_client = storage.Client(credentials=credentials) + storage_client.list_buckets(max_results=1) + except Exception as e: + console.print( + f"[red bold]✗[/red bold] Error while checking GCP credentials:\n{e}", + ) + exit(1) + return credentials + + def _ask_zone(self, default_zone: Optional[str]) -> str: + regions_client = compute_v1.RegionsClient(credentials=self.credentials) + zones_client = compute_v1.ZonesClient(credentials=self.credentials) + default_region = None + if default_zone is not None: + get_zone_request = compute_v1.GetZoneRequest( + project=self.project_id, zone=default_zone + ) + zone = zones_client.get(get_zone_request) + default_region = self._get_resource_name(zone.region) + list_regions_request = compute_v1.ListRegionsRequest(project=self.project_id) + regions = regions_client.list(list_regions_request) + region_names = [r.name for r in regions] + if default_region is None: + default_region = region_names[0] + region_name = ask_choice( + "Choose GCP region", + region_names, + region_names, + default_region, + ) + region = {r.name: r for r in regions}[region_name] + zone_names = [self._get_resource_name(z) for z in region.zones] + if default_zone not in zone_names: + default_zone = zone_names[0] + zone = ask_choice( + "Choose GCP zone", + zone_names, + zone_names, + default_zone, + ) + return zone + + def _ask_bucket(self, default_bucket: Optional[str]) -> str: + console.print( + "[sea_green3 bold]?[/sea_green3 bold] [bold]Choose storage bucket[/bold] " + "[gray46]Use arrows to move, type to filter[/gray46]" + ) + if default_bucket is None: + default_bucket = f"dstack-{self.project_id}" + bucket_options = [f"Default [{default_bucket}]", "Custom..."] + bucket_menu = TerminalMenu( + bucket_options, + menu_cursor_style=["fg_red", "bold"], + menu_highlight_style=["fg_red", "bold"], + search_highlight_style=["fg_purple"], + ) + bucket_index = bucket_menu.show() + bucket_title = bucket_options[bucket_index].replace("[", "\\[") + console.print(f"[sea_green3 bold]✓[/sea_green3 bold] [grey74]{bucket_title}[/grey74]") + if bucket_index == 1: + return self._ask_bucket_name(default_bucket) + if self._validate_bucket(default_bucket): + return default_bucket + return self._ask_bucket_name(default_bucket) + + def _ask_bucket_name(self, default_bucket: Optional[str]) -> str: + bucket_name = Prompt.ask( + "[sea_green3 bold]?[/sea_green3 bold] [bold]Enter bucket name[/bold]", + default=default_bucket, + ) + if self._validate_bucket(bucket_name): + return bucket_name + return self._ask_bucket_name(default_bucket) + + def _validate_bucket(self, bucket_name: str) -> bool: + storage_client = storage.Client(project=self.project_id, credentials=self.credentials) + try: + storage_client.get_bucket(bucket_name) + except (ValueError, exceptions.BadRequest) as e: + console.print( + "[red bold]✗[/red bold] Bucket name is not valid." + " See bucket naming rules: https://cloud.google.com/storage/docs/buckets" + ) + return False + except exceptions.NotFound: + if Confirm.ask( + f"[sea_green3 bold]?[/sea_green3 bold] " + f"[red bold]The bucket doesn't exist. Create it?[/red bold]", + default="y", + ): + # We create multi-region buckets by default + location = self._get_zone_location(self.zone) + storage_client.create_bucket(bucket_name, location=location) + return True + else: + return False + return True + + def _get_resource_name(self, resource_path: str) -> str: + return resource_path.rsplit(sep="/", maxsplit=1)[1] + + def _get_zone_location(self, zone: str) -> str: + if zone.startswith("asia"): + return "ASIA" + if zone.startswith("eu"): + return "EU" + return "US" diff --git a/cli/dstack/backend/hub/config.py b/cli/dstack/backend/hub/config.py index 99429b217b..dcf66fe5d6 100644 --- a/cli/dstack/backend/hub/config.py +++ b/cli/dstack/backend/hub/config.py @@ -18,8 +18,6 @@ class HUBConfig(BackendConfig): NAME = "hub" - _configured = True - def __init__(self): super().__init__() self.host = os.getenv("DSTACK_HUB_HOST") or "127.0.0.1" diff --git a/cli/dstack/core/config.py b/cli/dstack/core/config.py index fd2c5d3d79..cf9b73d753 100644 --- a/cli/dstack/core/config.py +++ b/cli/dstack/core/config.py @@ -15,8 +15,6 @@ def get_dstack_dir(): class BackendConfig(ABC): NAME = "" - _configured = False - @property def name(self): return self.NAME or "" @@ -37,7 +35,3 @@ def load_json(self, json_data): @abstractmethod def configure(self): pass - - @property - def configured(self): - return self._configured diff --git a/cli/requirements.txt b/cli/requirements.txt index bfb32cb19d..21124bce23 100644 --- a/cli/requirements.txt +++ b/cli/requirements.txt @@ -25,6 +25,7 @@ google-cloud-storage google-cloud-compute google-cloud-secret-manager google-cloud-logging +google-api-python-client aiosqlite aiosqlite packaging diff --git a/setup.py b/setup.py index 666f3fc399..f00ce0d761 100644 --- a/setup.py +++ b/setup.py @@ -80,6 +80,7 @@ def get_long_description(): "google-cloud-compute", "google-cloud-secret-manager", "google-cloud-logging", + "google-api-python-client", ], classifiers=[ "Development Status :: 2 - Pre-Alpha", From 35babaf76b9ed32dbe80b32c0c5e4c13dbe35693 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 28 Feb 2023 12:50:47 +0500 Subject: [PATCH 45/58] Remove google-api-python-client from dependencies --- cli/requirements.txt | 1 - setup.py | 1 - 2 files changed, 2 deletions(-) diff --git a/cli/requirements.txt b/cli/requirements.txt index 21124bce23..bfb32cb19d 100644 --- a/cli/requirements.txt +++ b/cli/requirements.txt @@ -25,7 +25,6 @@ google-cloud-storage google-cloud-compute google-cloud-secret-manager google-cloud-logging -google-api-python-client aiosqlite aiosqlite packaging diff --git a/setup.py b/setup.py index f00ce0d761..666f3fc399 100644 --- a/setup.py +++ b/setup.py @@ -80,7 +80,6 @@ def get_long_description(): "google-cloud-compute", "google-cloud-secret-manager", "google-cloud-logging", - "google-api-python-client", ], classifiers=[ "Development Status :: 2 - Pre-Alpha", From f01f7cc79f7c74336d20a05ca17a6759cb9f0a79 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 28 Feb 2023 15:08:51 +0500 Subject: [PATCH 46/58] Store gcp credentials file in config.yaml --- cli/dstack/backend/gcp/__init__.py | 4 +++ cli/dstack/backend/gcp/config.py | 51 +++++++++++++++++++++--------- 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/cli/dstack/backend/gcp/__init__.py b/cli/dstack/backend/gcp/__init__.py index a3e21ea168..934d9f4081 100644 --- a/cli/dstack/backend/gcp/__init__.py +++ b/cli/dstack/backend/gcp/__init__.py @@ -41,6 +41,10 @@ def __init__(self, config: Optional[GCPConfig] = None): credentials = service_account.Credentials.from_service_account_info( self.config.credentials ) + elif self.config.credentials_file is not None: + credentials = service_account.Credentials.from_service_account_file( + self.config.credentials_file + ) else: credentials_file = os.getenv("GOOGLE_APPLICATION_CREDENTIALS") if credentials_file is None: diff --git a/cli/dstack/backend/gcp/config.py b/cli/dstack/backend/gcp/config.py index 29482bc941..67762aea0f 100644 --- a/cli/dstack/backend/gcp/config.py +++ b/cli/dstack/backend/gcp/config.py @@ -18,27 +18,41 @@ def __init__( project_id: str, zone: str, bucket_name: str, + credentials_file: Optional[str] = None, credentials: Optional[Dict] = None, ): self.project_id = project_id self.zone = zone self.bucket_name = bucket_name + self.credentials_file = credentials_file self.credentials = credentials def serialize(self) -> Dict: - return { + res = { "backend": "gcp", "project": self.project_id, "zone": self.zone, "bucket": self.bucket_name, } + if self.credentials_file is not None: + res["credentials_file"] = self.credentials_file + return res def serialize_yaml(self) -> str: return yaml.dump(self.serialize()) @classmethod def deserialize(cls, data: Dict) -> "GCPConfig": - return cls(project_id=data["project"], zone=data["zone"], bucket_name=data["bucket"]) + project_id = data["project"] + zone = data["zone"] + bucket_name = data["bucket"] + credentials_file = data.get("credentials_file") + return cls( + project_id=project_id, + zone=zone, + bucket_name=bucket_name, + credentials_file=credentials_file, + ) @classmethod def deserialize_yaml(cls, yaml_content: str) -> "GCPConfig": @@ -60,17 +74,9 @@ def save(self, config: GCPConfig, path: Path = get_config_path()): f.write(config.serialize_yaml()) def configure(self): - credentials_file = os.getenv("GOOGLE_APPLICATION_CREDENTIALS") - if credentials_file is None: - console.print( - "[red bold]✗[/red bold] GOOGLE_APPLICATION_CREDENTIALS is not set." - " It should point to the service account credentials file." - " Learn more about service account credentials: https://cloud.google.com/iam/docs/creating-managing-service-account-keys" - ) - exit(1) - zone = None bucket_name = None + credentials_file = None try: config = self.load() except Exception: @@ -78,8 +84,11 @@ def configure(self): else: zone = config.zone bucket_name = config.bucket_name + credentials_file = config.credentials_file - self.credentials = self._get_credentials(credentials_file) + self.credentials_file = self._ask_credentials_file(credentials_file) + + self.credentials = self._get_credentials(self.credentials_file) self.project_id = self.credentials.project_id self.zone = self._ask_zone(zone) @@ -88,12 +97,24 @@ def configure(self): config = GCPConfig( project_id=self.project_id, zone=self.zone, - bucket_name=bucket_name, + bucket_name=self.bucket_name, + credentials_file=self.credentials_file, ) self.save(config) console.print(f"[grey58]OK[/]") - def _get_credentials(self, credentials_file: str) -> service_account.Credentials: + def _ask_credentials_file(self, default_credentials_file: Optional[str]) -> str: + if default_credentials_file is None: + default_credentials_file = os.getenv("GOOGLE_APPLICATION_CREDENTIALS") + credentials_file = Prompt.ask( + "[sea_green3 bold]?[/sea_green3 bold] [bold]Enter path to credentials file[/bold]", + default=default_credentials_file, + ) + if not credentials_file or self._get_credentials(credentials_file) is None: + return self._ask_credentials_file(default_credentials_file) + return credentials_file + + def _get_credentials(self, credentials_file: str) -> Optional[service_account.Credentials]: try: credentials = service_account.Credentials.from_service_account_file(credentials_file) storage_client = storage.Client(credentials=credentials) @@ -102,7 +123,7 @@ def _get_credentials(self, credentials_file: str) -> service_account.Credentials console.print( f"[red bold]✗[/red bold] Error while checking GCP credentials:\n{e}", ) - exit(1) + return None return credentials def _ask_zone(self, default_zone: Optional[str]) -> str: From 59b8d7a8a7823fe1d869f1861f01c4479f7b0d6c Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 28 Feb 2023 15:52:49 +0500 Subject: [PATCH 47/58] Test gcp images release build --- .../workflows/manual_gcp_image_release.yml | 44 ++++++++++++ .github/workflows/manual_gcp_image_stgn.yml | 7 +- runner/ami/packer/gcp-image-cuda.json | 69 +++++++++++++++++++ 3 files changed, 117 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/manual_gcp_image_release.yml create mode 100644 runner/ami/packer/gcp-image-cuda.json diff --git a/.github/workflows/manual_gcp_image_release.yml b/.github/workflows/manual_gcp_image_release.yml new file mode 100644 index 0000000000..6454bc3fee --- /dev/null +++ b/.github/workflows/manual_gcp_image_release.yml @@ -0,0 +1,44 @@ +name: Manual GCP image build for release + +on: + workflow_dispatch: + push: + + +jobs: + gcp-images-build: + defaults: + run: + working-directory: runner + strategy: + matrix: + variants: [ gcp-image, gcp-image-cuda ] + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + steps: + - uses: actions/checkout@v3 + - name: 'Authenticate to Google Cloud' + uses: 'google-github-actions/auth@v1' + with: + workload_identity_provider: 'projects/531508670106/locations/global/workloadIdentityPools/github-identity-pool/providers/github-id-provider' + service_account: 'github-actions@dstack.iam.gserviceaccount.com' + create_credentials_file: true + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v1' + - name: Download packer + run: | + wget https://releases.hashicorp.com/packer/1.8.0/packer_1.8.0_linux_amd64.zip + unzip packer_1.8.0_linux_amd64.zip + chmod +x packer + cp -R ami/packer/* . + - name: Run packer + run: | + VERSION=${GITHUB_REF#refs/tags/} + VERSION=${VERSION//./-} + ./packer build -var build_id=$VERSION -var build_prefix='' -var dstack_stage=PROD ${{ matrix.variants }}.json + - name: Publish images + run: | + gcloud compute images add-iam-policy-binding dstack-$VERSION --member='allAuthenticatedUsers' --role='roles/compute.imageUser' + gcloud compute images add-iam-policy-binding dstack-cuda-$VERSION --member='allAuthenticatedUsers' --role='roles/compute.imageUser' diff --git a/.github/workflows/manual_gcp_image_stgn.yml b/.github/workflows/manual_gcp_image_stgn.yml index 39af40d8b8..729cf8193d 100644 --- a/.github/workflows/manual_gcp_image_stgn.yml +++ b/.github/workflows/manual_gcp_image_stgn.yml @@ -5,13 +5,13 @@ on: jobs: - gcp-image-build: + gcp-images-build: defaults: run: working-directory: runner strategy: matrix: - variants: [ gcp-image ] + variants: [ gcp-image, gcp-image-cuda ] runs-on: ubuntu-latest permissions: contents: 'read' @@ -36,7 +36,8 @@ jobs: run: | VERSION=${{ github.run_number }} ./packer build -var build_id=$VERSION ${{ matrix.variants }}.json - - name: Publish image + - name: Publish images run: | VERSION=${{ github.run_number }} gcloud compute images add-iam-policy-binding stgn-dstack-$VERSION --member='allAuthenticatedUsers' --role='roles/compute.imageUser' + gcloud compute images add-iam-policy-binding stgn-dstack-cuda-$VERSION --member='allAuthenticatedUsers' --role='roles/compute.imageUser' diff --git a/runner/ami/packer/gcp-image-cuda.json b/runner/ami/packer/gcp-image-cuda.json new file mode 100644 index 0000000000..e8749189e8 --- /dev/null +++ b/runner/ami/packer/gcp-image-cuda.json @@ -0,0 +1,69 @@ +{ + "variables": { + "build_id": "{{timestamp}}", + "build_prefix": "stgn-", + "docker_version": "20.10.17", + "dstack_stage": "STGN" + }, + "builders": [ + { + "type": "googlecompute", + "project_id": "dstack", + "source_image": "ubuntu-1804-bionic-v20220616", + "image_name": "{{user `build_prefix`}}dstack-cuda-{{user `build_id`}}", + "instance_name": "{{user `build_prefix`}}dstack-cuda-{{user `build_id`}}", + "ssh_username": "ubuntu", + "zone": "us-central1-a" + } + ], + "provisioners": [ + { + "type": "shell", + "inline": ["cloud-init status --long --wait"] + }, + { + "type": "shell", + "scripts": [ + "provisioners/kernel/apt-upgrade.sh", + "provisioners/kernel/apt-daily.sh", + "provisioners/kernel/apt-packages.sh", + "provisioners/kernel/kernel-tuning.sh" + ] + }, + { + "type": "shell", + "environment_vars": ["DSTACK_STAGE={{user `dstack_stage`}}"], + "script": "provisioners/get-dstack-runner.sh" + }, + { + "type": "file", + "source": "provisioners/install-docker.sh", + "destination": "/tmp/install-docker.sh" + }, + { + "type": "file", + "source": "provisioners/run-docker", + "destination": "/tmp/run-docker" + }, + { + "type": "shell", + "inline": [ + "cd /tmp", + "chmod +x install-docker.sh", + "./install-docker.sh --version {{user `docker_version`}}"] + }, + { + "type": "shell", + "environment_vars": ["CUDA_DRIVERS_VERSION=455.45.01-1"], + "script": "provisioners/cuda.sh" + }, + { + "type": "shell", + "script": "provisioners/docker-image-with-cuda.sh" + }, + { + "type": "shell", + "inline": ["/usr/local/bin/dstack-runner --version"] + } + ] +} From fb9a006424012091bb9cc9df29132e3abdab6820 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 28 Feb 2023 15:55:55 +0500 Subject: [PATCH 48/58] Test gcp images staging build --- .../workflows/manual_gcp_image_release.yml | 18 +++++---- .github/workflows/manual_gcp_image_stgn.yml | 14 ++++--- .github/workflows/release.yml | 37 +++++++++++++++++++ 3 files changed, 57 insertions(+), 12 deletions(-) diff --git a/.github/workflows/manual_gcp_image_release.yml b/.github/workflows/manual_gcp_image_release.yml index 6454bc3fee..635653dc28 100644 --- a/.github/workflows/manual_gcp_image_release.yml +++ b/.github/workflows/manual_gcp_image_release.yml @@ -2,7 +2,7 @@ name: Manual GCP image build for release on: workflow_dispatch: - push: + # push: jobs: @@ -12,7 +12,12 @@ jobs: working-directory: runner strategy: matrix: - variants: [ gcp-image, gcp-image-cuda ] + variant: [ gcp-image, gcp-image-cuda ] + include: + - variant: gcp-image + image_prefix: dstack + - variant: gcp-image-cuda + image_prefix: dstack-cuda runs-on: ubuntu-latest permissions: contents: 'read' @@ -35,10 +40,9 @@ jobs: cp -R ami/packer/* . - name: Run packer run: | - VERSION=${GITHUB_REF#refs/tags/} + VERSION=${GITHUB_REF#refs/heads/} VERSION=${VERSION//./-} - ./packer build -var build_id=$VERSION -var build_prefix='' -var dstack_stage=PROD ${{ matrix.variants }}.json - - name: Publish images + ./packer build -var build_id=$VERSION -var build_prefix='' -var dstack_stage=PROD ${{ matrix.variant }}.json + - name: Publish image run: | - gcloud compute images add-iam-policy-binding dstack-$VERSION --member='allAuthenticatedUsers' --role='roles/compute.imageUser' - gcloud compute images add-iam-policy-binding dstack-cuda-$VERSION --member='allAuthenticatedUsers' --role='roles/compute.imageUser' + gcloud compute images add-iam-policy-binding ${{ matrix.image_prefix }}-$VERSION --member='allAuthenticatedUsers' --role='roles/compute.imageUser' diff --git a/.github/workflows/manual_gcp_image_stgn.yml b/.github/workflows/manual_gcp_image_stgn.yml index 729cf8193d..899cd3aeb4 100644 --- a/.github/workflows/manual_gcp_image_stgn.yml +++ b/.github/workflows/manual_gcp_image_stgn.yml @@ -2,6 +2,7 @@ name: Manual GCP image build for STGN on: workflow_dispatch: + push: jobs: @@ -11,7 +12,12 @@ jobs: working-directory: runner strategy: matrix: - variants: [ gcp-image, gcp-image-cuda ] + variant: [ gcp-image, gcp-image-cuda ] + include: + - variant: gcp-image + image_prefix: stgn-dstack + - variant: gcp-image-cuda + image_prefix: stgn-dstack-cuda runs-on: ubuntu-latest permissions: contents: 'read' @@ -35,9 +41,7 @@ jobs: - name: Run packer run: | VERSION=${{ github.run_number }} - ./packer build -var build_id=$VERSION ${{ matrix.variants }}.json + ./packer build -var build_id=$VERSION ${{ matrix.variant }}.json - name: Publish images run: | - VERSION=${{ github.run_number }} - gcloud compute images add-iam-policy-binding stgn-dstack-$VERSION --member='allAuthenticatedUsers' --role='roles/compute.imageUser' - gcloud compute images add-iam-policy-binding stgn-dstack-cuda-$VERSION --member='allAuthenticatedUsers' --role='roles/compute.imageUser' + gcloud compute images add-iam-policy-binding ${{ matrix.image_prefix }}-$VERSION --member='allAuthenticatedUsers' --role='roles/compute.imageUser' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b63cc5b6ce..b06757ec22 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -195,6 +195,43 @@ jobs: run: | VERSION=${GITHUB_REF#refs/tags/} ./packer build -var build_id=$VERSION -var build_prefix='' -var-file=prod-var.json ${{ matrix.variants }}.json + + gcp-images-build: + defaults: + run: + working-directory: runner + strategy: + matrix: + variants: [ gcp-image, gcp-image-cuda ] + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + steps: + - uses: actions/checkout@v3 + - name: 'Authenticate to Google Cloud' + uses: 'google-github-actions/auth@v1' + with: + workload_identity_provider: 'projects/531508670106/locations/global/workloadIdentityPools/github-identity-pool/providers/github-id-provider' + service_account: 'github-actions@dstack.iam.gserviceaccount.com' + create_credentials_file: true + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v1' + - name: Download packer + run: | + wget https://releases.hashicorp.com/packer/1.8.0/packer_1.8.0_linux_amd64.zip + unzip packer_1.8.0_linux_amd64.zip + chmod +x packer + cp -R ami/packer/* . + - name: Run packer + run: | + VERSION=${GITHUB_REF#refs/tags/} + VERSION=${VERSION//./-} + ./packer build -var build_id=$VERSION -var build_prefix='' -var dstack_stage=PROD ${{ matrix.variants }}.json + - name: Publish images + run: | + gcloud compute images add-iam-policy-binding dstack-$VERSION --member='allAuthenticatedUsers' --role='roles/compute.imageUser' + gcloud compute images add-iam-policy-binding dstack-cuda-$VERSION --member='allAuthenticatedUsers' --role='roles/compute.imageUser' update-get-dstack-tag: runs-on: ubuntu-latest From 7844c13d3a3b85abdd09bf97f810b5d09f2821e3 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 28 Feb 2023 16:10:00 +0500 Subject: [PATCH 49/58] Test gcp images release build 2 --- .github/workflows/manual_gcp_image_release.yml | 2 +- .github/workflows/manual_gcp_image_stgn.yml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/manual_gcp_image_release.yml b/.github/workflows/manual_gcp_image_release.yml index 635653dc28..66d9608f05 100644 --- a/.github/workflows/manual_gcp_image_release.yml +++ b/.github/workflows/manual_gcp_image_release.yml @@ -2,7 +2,7 @@ name: Manual GCP image build for release on: workflow_dispatch: - # push: + push: jobs: diff --git a/.github/workflows/manual_gcp_image_stgn.yml b/.github/workflows/manual_gcp_image_stgn.yml index 899cd3aeb4..75bbb0b04b 100644 --- a/.github/workflows/manual_gcp_image_stgn.yml +++ b/.github/workflows/manual_gcp_image_stgn.yml @@ -2,7 +2,6 @@ name: Manual GCP image build for STGN on: workflow_dispatch: - push: jobs: From 647b883be2ccc1627d406d0a95bff1e64f9f45ae Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 28 Feb 2023 16:49:05 +0500 Subject: [PATCH 50/58] Test gcp images staging build 2 --- .../workflows/manual_gcp_image_release.yml | 48 ------------------- .github/workflows/manual_gcp_image_stgn.yml | 3 +- .github/workflows/release.yml | 16 +++++-- 3 files changed, 13 insertions(+), 54 deletions(-) delete mode 100644 .github/workflows/manual_gcp_image_release.yml diff --git a/.github/workflows/manual_gcp_image_release.yml b/.github/workflows/manual_gcp_image_release.yml deleted file mode 100644 index 66d9608f05..0000000000 --- a/.github/workflows/manual_gcp_image_release.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Manual GCP image build for release - -on: - workflow_dispatch: - push: - - -jobs: - gcp-images-build: - defaults: - run: - working-directory: runner - strategy: - matrix: - variant: [ gcp-image, gcp-image-cuda ] - include: - - variant: gcp-image - image_prefix: dstack - - variant: gcp-image-cuda - image_prefix: dstack-cuda - runs-on: ubuntu-latest - permissions: - contents: 'read' - id-token: 'write' - steps: - - uses: actions/checkout@v3 - - name: 'Authenticate to Google Cloud' - uses: 'google-github-actions/auth@v1' - with: - workload_identity_provider: 'projects/531508670106/locations/global/workloadIdentityPools/github-identity-pool/providers/github-id-provider' - service_account: 'github-actions@dstack.iam.gserviceaccount.com' - create_credentials_file: true - - name: 'Set up Cloud SDK' - uses: 'google-github-actions/setup-gcloud@v1' - - name: Download packer - run: | - wget https://releases.hashicorp.com/packer/1.8.0/packer_1.8.0_linux_amd64.zip - unzip packer_1.8.0_linux_amd64.zip - chmod +x packer - cp -R ami/packer/* . - - name: Run packer - run: | - VERSION=${GITHUB_REF#refs/heads/} - VERSION=${VERSION//./-} - ./packer build -var build_id=$VERSION -var build_prefix='' -var dstack_stage=PROD ${{ matrix.variant }}.json - - name: Publish image - run: | - gcloud compute images add-iam-policy-binding ${{ matrix.image_prefix }}-$VERSION --member='allAuthenticatedUsers' --role='roles/compute.imageUser' diff --git a/.github/workflows/manual_gcp_image_stgn.yml b/.github/workflows/manual_gcp_image_stgn.yml index 75bbb0b04b..c2bbfba8e7 100644 --- a/.github/workflows/manual_gcp_image_stgn.yml +++ b/.github/workflows/manual_gcp_image_stgn.yml @@ -2,7 +2,7 @@ name: Manual GCP image build for STGN on: workflow_dispatch: - + push: jobs: gcp-images-build: @@ -43,4 +43,5 @@ jobs: ./packer build -var build_id=$VERSION ${{ matrix.variant }}.json - name: Publish images run: | + VERSION=${{ github.run_number }} gcloud compute images add-iam-policy-binding ${{ matrix.image_prefix }}-$VERSION --member='allAuthenticatedUsers' --role='roles/compute.imageUser' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b06757ec22..6808952079 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -202,7 +202,12 @@ jobs: working-directory: runner strategy: matrix: - variants: [ gcp-image, gcp-image-cuda ] + variant: [ gcp-image, gcp-image-cuda ] + include: + - variant: gcp-image + image_prefix: dstack + - variant: gcp-image-cuda + image_prefix: dstack-cuda runs-on: ubuntu-latest permissions: contents: 'read' @@ -227,11 +232,12 @@ jobs: run: | VERSION=${GITHUB_REF#refs/tags/} VERSION=${VERSION//./-} - ./packer build -var build_id=$VERSION -var build_prefix='' -var dstack_stage=PROD ${{ matrix.variants }}.json - - name: Publish images + ./packer build -var build_id=$VERSION -var build_prefix='' -var dstack_stage=PROD ${{ matrix.variant }}.json + - name: Publish image run: | - gcloud compute images add-iam-policy-binding dstack-$VERSION --member='allAuthenticatedUsers' --role='roles/compute.imageUser' - gcloud compute images add-iam-policy-binding dstack-cuda-$VERSION --member='allAuthenticatedUsers' --role='roles/compute.imageUser' + VERSION=${GITHUB_REF#refs/tags/} + VERSION=${VERSION//./-} + gcloud compute images add-iam-policy-binding ${{ matrix.image_prefix }}-$VERSION --member='allAuthenticatedUsers' --role='roles/compute.imageUser' update-get-dstack-tag: runs-on: ubuntu-latest From 21d5edfff295b602e1296e280d3ef7bc0587019d Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 28 Feb 2023 17:17:21 +0500 Subject: [PATCH 51/58] Add nocuda prefix to image without cuda --- .github/workflows/manual_gcp_image_stgn.yml | 2 +- .github/workflows/release.yml | 2 +- cli/dstack/api/backend.py | 12 ++++++++---- cli/dstack/backend/gcp/compute.py | 15 ++++++++++++--- runner/ami/packer/gcp-image.json | 4 ++-- 5 files changed, 24 insertions(+), 11 deletions(-) diff --git a/.github/workflows/manual_gcp_image_stgn.yml b/.github/workflows/manual_gcp_image_stgn.yml index c2bbfba8e7..a4b08724f4 100644 --- a/.github/workflows/manual_gcp_image_stgn.yml +++ b/.github/workflows/manual_gcp_image_stgn.yml @@ -14,7 +14,7 @@ jobs: variant: [ gcp-image, gcp-image-cuda ] include: - variant: gcp-image - image_prefix: stgn-dstack + image_prefix: stgn-dstack-nocuda - variant: gcp-image-cuda image_prefix: stgn-dstack-cuda runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6808952079..4313ab63be 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -205,7 +205,7 @@ jobs: variant: [ gcp-image, gcp-image-cuda ] include: - variant: gcp-image - image_prefix: dstack + image_prefix: dstack-nocuda - variant: gcp-image-cuda image_prefix: dstack-cuda runs-on: ubuntu-latest diff --git a/cli/dstack/api/backend.py b/cli/dstack/api/backend.py index d2187c2e65..6f5fcaa9ef 100644 --- a/cli/dstack/api/backend.py +++ b/cli/dstack/api/backend.py @@ -29,6 +29,10 @@ def list_backends(all_backend: bool = False) -> List[Backend]: return l +def list_remote_backends() -> List[RemoteBackend]: + return [b for b in list_backends() if b.type is BackendType.REMOTE] + + def dict_backends(all_backend: bool = False) -> Dict[str, Backend]: return {backend.name: backend for backend in list_backends(all_backend=all_backend)} @@ -38,10 +42,10 @@ def get_backend_by_name(name: str) -> Optional[Backend]: def get_current_remote_backend() -> Optional[RemoteBackend]: - for backend in list_backends(): - if backend.type is BackendType.REMOTE: - return backend - return None + remote_backends = list_remote_backends() + if len(remote_backends) == 0: + return None + return remote_backends[0] def get_local_backend() -> LocalBackend: diff --git a/cli/dstack/backend/gcp/compute.py b/cli/dstack/backend/gcp/compute.py index dfd8d9a2e9..b4cb57dc25 100644 --- a/cli/dstack/backend/gcp/compute.py +++ b/cli/dstack/backend/gcp/compute.py @@ -14,7 +14,7 @@ from dstack.core.instance import InstanceType from dstack.core.job import Job, Requirements from dstack.core.request import RequestHead, RequestStatus -from dstack.core.runners import Resources, Runner +from dstack.core.runners import Gpu, Resources, Runner DSTACK_INSTANCE_TAG = "dstack-runner-instance" @@ -64,7 +64,10 @@ def run_instance(self, job: Job, instance_type: InstanceType) -> str: firewalls_client=self.firewalls_client, project_id=self.gcp_config.project_id, zone=self.gcp_config.zone, - image_name=_get_image_name(images_client=self.images_client), + image_name=_get_image_name( + images_client=self.images_client, + instance_type=instance_type, + ), instance_name=_get_instance_name(job), user_data_script=_get_user_data_script(self.gcp_config, job, instance_type), service_account=self.credentials.service_account_email, @@ -86,11 +89,17 @@ def cancel_spot_request(self, request_id: str): ) -def _get_image_name(images_client: compute_v1.ImagesClient) -> Optional[str]: +def _get_image_name( + images_client: compute_v1.ImagesClient, instance_type: InstanceType +) -> Optional[str]: if version.__is_release__: image_prefix = "dstack-" else: image_prefix = "stgn-dstack-" + if len(instance_type.resources.gpus) > 0: + image_prefix += "cuda-" + else: + image_prefix += "nocuda-" list_request = compute_v1.ListImagesRequest() list_request.project = "dstack" list_request.order_by = "creationTimestamp desc" diff --git a/runner/ami/packer/gcp-image.json b/runner/ami/packer/gcp-image.json index b4fafe903b..a042a8f352 100644 --- a/runner/ami/packer/gcp-image.json +++ b/runner/ami/packer/gcp-image.json @@ -10,8 +10,8 @@ "type": "googlecompute", "project_id": "dstack", "source_image": "ubuntu-1804-bionic-v20220616", - "image_name": "{{user `build_prefix`}}dstack-{{user `build_id`}}", - "instance_name": "{{user `build_prefix`}}dstack-{{user `build_id`}}", + "image_name": "{{user `build_prefix`}}dstack-nocuda-{{user `build_id`}}", + "instance_name": "{{user `build_prefix`}}dstack-nocuda-{{user `build_id`}}", "ssh_username": "ubuntu", "zone": "us-central1-a" } From bf84ef877bfedb22f21bee3eb4369cc0722d3aed Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 28 Feb 2023 18:04:50 +0500 Subject: [PATCH 52/58] Use configured remote backend as default remote --- cli/dstack/api/backend.py | 3 --- cli/dstack/backend/gcp/config.py | 13 ++++++++----- cli/dstack/cli/commands/run/__init__.py | 26 ++++++++++++------------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/cli/dstack/api/backend.py b/cli/dstack/api/backend.py index 6f5fcaa9ef..beaa6610ba 100644 --- a/cli/dstack/api/backend.py +++ b/cli/dstack/api/backend.py @@ -7,9 +7,6 @@ from dstack.backend.hub import HubBackend from dstack.backend.local import LocalBackend -DEFAULT_REMOTE = "aws" -DEFAULT = "local" - backends_classes = [AwsBackend, GCPBackend, LocalBackend] if not version.__is_release__: backends_classes.append(HubBackend) diff --git a/cli/dstack/backend/gcp/config.py b/cli/dstack/backend/gcp/config.py index 67762aea0f..014f7205d1 100644 --- a/cli/dstack/backend/gcp/config.py +++ b/cli/dstack/backend/gcp/config.py @@ -42,10 +42,13 @@ def serialize_yaml(self) -> str: return yaml.dump(self.serialize()) @classmethod - def deserialize(cls, data: Dict) -> "GCPConfig": - project_id = data["project"] - zone = data["zone"] - bucket_name = data["bucket"] + def deserialize(cls, data: Dict) -> Optional["GCPConfig"]: + try: + project_id = data["project"] + zone = data["zone"] + bucket_name = data["bucket"] + except KeyError: + return None credentials_file = data.get("credentials_file") return cls( project_id=project_id, @@ -55,7 +58,7 @@ def deserialize(cls, data: Dict) -> "GCPConfig": ) @classmethod - def deserialize_yaml(cls, yaml_content: str) -> "GCPConfig": + def deserialize_yaml(cls, yaml_content: str) -> Optional["GCPConfig"]: return cls.deserialize(yaml.load(yaml_content, yaml.FullLoader)) diff --git a/cli/dstack/cli/commands/run/__init__.py b/cli/dstack/cli/commands/run/__init__.py index c01fe6c14a..e85d16693f 100644 --- a/cli/dstack/cli/commands/run/__init__.py +++ b/cli/dstack/cli/commands/run/__init__.py @@ -16,7 +16,7 @@ from websocket import WebSocketApp from dstack import providers -from dstack.api.backend import DEFAULT, DEFAULT_REMOTE, get_backend_by_name +from dstack.api.backend import get_backend_by_name, get_current_remote_backend, get_local_backend from dstack.api.logs import poll_logs from dstack.api.repo import load_repo_data from dstack.api.run import list_runs_with_merged_backends @@ -254,16 +254,20 @@ def _command(self, args: Namespace): exit(1) try: repo_data = load_repo_data() - backend_name = DEFAULT + backend = get_local_backend() if args.remote is not None: if len(args.remote) == 0: - backend_name = DEFAULT_REMOTE + remote_backend = get_current_remote_backend() + if remote_backend is None: + console.print(f"No remote configured. Run `dstack config`.") + exit(1) else: - backend_name = args.remote[0] - backend = get_backend_by_name(backend_name) - if backend is None: - console.print(f"Backend '{backend_name}' not configured") - exit(1) + remote_backend = get_backend_by_name(args.remote[0]) + if remote_backend is None: + console.print(f"Backend '{args.remote[0]}' is not configured") + exit(1) + backend = remote_backend + ( provider_name, provider_args, @@ -281,11 +285,7 @@ def _command(self, args: Namespace): if args.tag_name: tag_head = backend.get_tag_head(repo_data, args.tag_name) if tag_head: - # if args.yes or Confirm.ask(f"[red]The tag '{args.tag_name}' already exists. " - # f"Do you want to override it?[/]"): backend.delete_tag_head(repo_data, tag_head) - # else: - # return jobs = provider.submit_jobs(backend, args.tag_name) backend.update_repo_last_run_at( repo_data, last_run_at=int(round(time.time() * 1000)) @@ -294,7 +294,7 @@ def _command(self, args: Namespace): if not args.detach: poll_run(repo_data, jobs, backend) else: - sys.exit(f"Call 'dstack init' first") + sys.exit(f"Call `dstack init` first") except ValidationError as e: sys.exit( f"There a syntax error in one of the files inside the {os.getcwd()}/.dstack/workflows directory:\n\n{e}" From ccd7837fa73eee6a828d7df52635a7a1a5306341 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 28 Feb 2023 18:07:03 +0500 Subject: [PATCH 53/58] Support interruptible isntances for gcp --- .github/workflows/manual_gcp_image_stgn.yml | 1 - cli/dstack/backend/gcp/compute.py | 18 +++--------------- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/.github/workflows/manual_gcp_image_stgn.yml b/.github/workflows/manual_gcp_image_stgn.yml index a4b08724f4..51d22623c4 100644 --- a/.github/workflows/manual_gcp_image_stgn.yml +++ b/.github/workflows/manual_gcp_image_stgn.yml @@ -2,7 +2,6 @@ name: Manual GCP image build for STGN on: workflow_dispatch: - push: jobs: gcp-images-build: diff --git a/cli/dstack/backend/gcp/compute.py b/cli/dstack/backend/gcp/compute.py index b4cb57dc25..43e616d438 100644 --- a/cli/dstack/backend/gcp/compute.py +++ b/cli/dstack/backend/gcp/compute.py @@ -71,6 +71,7 @@ def run_instance(self, job: Job, instance_type: InstanceType) -> str: instance_name=_get_instance_name(job), user_data_script=_get_user_data_script(self.gcp_config, job, instance_type), service_account=self.credentials.service_account_email, + interruptible=job.requirements.interruptible, ) return instance.name @@ -139,6 +140,7 @@ def _launch_instance( instance_name: str, user_data_script: str, service_account: str, + interruptible: bool, ) -> compute_v1.Instance: try: _create_firewall_rules(firewalls_client=firewalls_client, project_id=project_id) @@ -160,6 +162,7 @@ def _launch_instance( user_data_script=user_data_script, service_account=service_account, external_access=True, + spot=interruptible, ) return instance @@ -215,7 +218,6 @@ def _create_instance( external_access: bool = False, external_ipv4: str = None, accelerators: List[compute_v1.AcceleratorConfig] = None, - preemptible: bool = False, spot: bool = False, instance_termination_action: str = "STOP", custom_hostname: str = None, @@ -251,8 +253,6 @@ def _create_instance( This setting requires `external_access` to be set to True to work. accelerators: a list of AcceleratorConfig objects describing the accelerators that will be attached to the new instance. - preemptible: boolean value indicating if the new instance should be preemptible - or not. Preemptible VMs have been deprecated and you should now use Spot VMs. spot: boolean value indicating if the new instance should be a Spot VM or not. instance_termination_action: What action should be taken once a Spot VM is terminated. Possible values: "STOP", "DELETE" @@ -263,7 +263,6 @@ def _create_instance( Returns: Instance object. """ - # Use the network interface provided in the network_link argument. network_interface = compute_v1.NetworkInterface() network_interface.name = network_link if subnetwork_link: @@ -281,7 +280,6 @@ def _create_instance( access.nat_i_p = external_ipv4 network_interface.access_configs = [access] - # Collect information into the Instance object. instance = compute_v1.Instance() instance.network_interfaces = [network_interface] instance.name = instance_name @@ -294,24 +292,15 @@ def _create_instance( if accelerators: instance.guest_accelerators = accelerators - if preemptible: - # Set the preemptible setting - warnings.warn("Preemptible VMs are being replaced by Spot VMs.", DeprecationWarning) - instance.scheduling = compute_v1.Scheduling() - instance.scheduling.preemptible = True - if spot: - # Set the Spot VM setting instance.scheduling = compute_v1.Scheduling() instance.scheduling.provisioning_model = compute_v1.Scheduling.ProvisioningModel.SPOT.name instance.scheduling.instance_termination_action = instance_termination_action if custom_hostname is not None: - # Set the custom hostname for the instance instance.hostname = custom_hostname if delete_protection: - # Set the delete protection bit instance.deletion_protection = True if user_data_script is not None: @@ -329,7 +318,6 @@ def _create_instance( instance.tags = compute_v1.Tags(items=[DSTACK_INSTANCE_TAG]) - # Prepare the request to insert an instance. request = compute_v1.InsertInstanceRequest() request.zone = zone request.project = project_id From 281e97fd0b32b376dc7613d263c9f9ad34211fee Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 1 Mar 2023 12:53:34 +0500 Subject: [PATCH 54/58] Choose gcp machine type --- cli/dstack/backend/aws/compute.py | 8 +- cli/dstack/backend/aws/runners.py | 102 ++++--------------------- cli/dstack/backend/base/compute.py | 83 ++++++++++++++++++++- cli/dstack/backend/gcp/compute.py | 115 ++++++++++++++++++++++------- 4 files changed, 191 insertions(+), 117 deletions(-) diff --git a/cli/dstack/backend/aws/compute.py b/cli/dstack/backend/aws/compute.py index 9b8fb2e5d7..1b9396d208 100644 --- a/cli/dstack/backend/aws/compute.py +++ b/cli/dstack/backend/aws/compute.py @@ -32,13 +32,13 @@ def get_request_head(self, job: Job, request_id: Optional[str]) -> RequestHead: ) def get_instance_type(self, job: Job) -> Optional[InstanceType]: - return runners._get_instance_type( + return runners.get_instance_type( ec2_client=self.ec2_client, requirements=job.requirements, ) def run_instance(self, job: Job, instance_type: InstanceType) -> str: - return runners._run_instance_retry( + return runners.run_instance_retry( ec2_client=self.ec2_client, iam_client=self.iam_client, bucket_name=self.bucket_name, @@ -52,13 +52,13 @@ def run_instance(self, job: Job, instance_type: InstanceType) -> str: ) def terminate_instance(self, request_id: str): - runners._terminate_instance( + runners.terminate_instance( ec2_client=self.ec2_client, request_id=request_id, ) def cancel_spot_request(self, request_id: str): - runners._cancel_spot_request( + runners.cancel_spot_request( ec2_client=self.ec2_client, request_id=request_id, ) diff --git a/cli/dstack/backend/aws/runners.py b/cli/dstack/backend/aws/runners.py index 2d32f4b53b..c545f0a48f 100644 --- a/cli/dstack/backend/aws/runners.py +++ b/cli/dstack/backend/aws/runners.py @@ -1,21 +1,28 @@ import json import time -from functools import cmp_to_key, reduce +from functools import reduce from typing import List, Optional, Tuple -import yaml from botocore.client import BaseClient from dstack import version +from dstack.backend.base.compute import choose_instance_type from dstack.core.instance import InstanceType from dstack.core.job import Job, JobStatus, Requirements from dstack.core.repo import RepoAddress from dstack.core.request import RequestHead, RequestStatus -from dstack.core.runners import Gpu, Resources, Runner +from dstack.core.runners import Gpu, Resources CREATE_INSTANCE_RETRY_RATE_SECS = 3 +def get_instance_type( + ec2_client: BaseClient, requirements: Optional[Requirements] +) -> Optional[InstanceType]: + instance_types = _get_instance_types(ec2_client) + return choose_instance_type(instance_types, requirements) + + def _get_instance_types(ec2_client: BaseClient) -> List[InstanceType]: response = None instance_types = [] @@ -54,86 +61,7 @@ def _get_instance_types(ec2_client: BaseClient) -> List[InstanceType]: ), ) ) - - def compare(i1, i2): - r1_gpu_total_memory_mib = sum(map(lambda g: g.memory_mib, i1.resources.gpus or [])) - r2_gpu_total_memory_mib = sum(map(lambda g: g.memory_mib, i2.resources.gpus or [])) - if r1_gpu_total_memory_mib < r2_gpu_total_memory_mib: - return -1 - elif r1_gpu_total_memory_mib > r2_gpu_total_memory_mib: - return 1 - if i1.resources.cpus < i2.resources.cpus: - return -1 - elif i1.resources.cpus > i2.resources.cpus: - return 1 - if i1.resources.memory_mib < i2.resources.memory_mib: - return -1 - elif i1.resources.memory_mib > i2.resources.memory_mib: - return 1 - return 0 - - return sorted(instance_types, key=cmp_to_key(compare)) - - -def _matches(resources: Resources, requirements: Optional[Requirements]) -> bool: - if not requirements: - return True - if requirements.cpus and requirements.cpus > resources.cpus: - return False - if requirements.memory_mib and requirements.memory_mib > resources.memory_mib: - return False - if requirements.gpus: - gpu_count = requirements.gpus.count or 1 - if gpu_count > len(resources.gpus or []): - return False - if requirements.gpus.name and gpu_count > len( - list(filter(lambda gpu: gpu.name == requirements.gpus.name, resources.gpus or [])) - ): - return False - if requirements.gpus.memory_mib and gpu_count > len( - list( - filter( - lambda gpu: gpu.memory_mib >= requirements.gpus.memory_mib, - resources.gpus or [], - ) - ) - ): - return False - if requirements.interruptible and not resources.interruptible: - return False - return True - - -def _get_instance_type( - ec2_client: BaseClient, requirements: Optional[Requirements] -) -> Optional[InstanceType]: - instance_types = _get_instance_types(ec2_client) - - instance_type = next( - ( - instance_type - for instance_type in instance_types - if _matches(instance_type.resources, requirements) - ), - None, - ) - interruptible = False - if requirements and requirements.interruptible: - interruptible = True - return ( - InstanceType( - instance_name=instance_type.instance_name, - resources=Resources( - cpus=instance_type.resources.cpus, - memory_mib=instance_type.resources.memory_mib, - gpus=instance_type.resources.gpus, - interruptible=interruptible, - local=False, - ), - ) - if instance_type - else None - ) + return instance_types def _get_security_group_id(ec2_client: BaseClient, bucket_name: str, subnet_id: Optional[str]): @@ -491,7 +419,7 @@ def _run_instance( return request_id -def _run_instance_retry( +def run_instance_retry( ec2_client: BaseClient, iam_client: BaseClient, bucket_name: str, @@ -525,7 +453,7 @@ def _run_instance_retry( ): if attempts > 0: time.sleep(CREATE_INSTANCE_RETRY_RATE_SECS) - return _run_instance_retry( + return run_instance_retry( ec2_client, iam_client, bucket_name, @@ -544,7 +472,7 @@ def _run_instance_retry( raise e -def _cancel_spot_request(ec2_client: BaseClient, request_id: str): +def cancel_spot_request(ec2_client: BaseClient, request_id: str): ec2_client.cancel_spot_instance_requests(SpotInstanceRequestIds=[request_id]) response = ec2_client.describe_instances( Filters=[ @@ -557,7 +485,7 @@ def _cancel_spot_request(ec2_client: BaseClient, request_id: str): ) -def _terminate_instance(ec2_client: BaseClient, request_id: str): +def terminate_instance(ec2_client: BaseClient, request_id: str): try: ec2_client.terminate_instances(InstanceIds=[request_id]) except Exception as e: diff --git a/cli/dstack/backend/base/compute.py b/cli/dstack/backend/base/compute.py index 057e579b95..b8f7ec99ca 100644 --- a/cli/dstack/backend/base/compute.py +++ b/cli/dstack/backend/base/compute.py @@ -1,9 +1,11 @@ from abc import ABC, abstractmethod -from typing import Optional, Tuple +from functools import cmp_to_key +from typing import List, Optional, Tuple from dstack.core.instance import InstanceType from dstack.core.job import Job, Requirements from dstack.core.request import RequestHead +from dstack.core.runners import Resources class Compute(ABC): @@ -26,3 +28,82 @@ def terminate_instance(self, request_id: str): @abstractmethod def cancel_spot_request(self, request_id: str): pass + + +def choose_instance_type( + instance_types: List[InstanceType], + requirements: Optional[Requirements], +) -> Optional[InstanceType]: + instance_types = _sort_instance_types(instance_types) + eligible_instance_types = [ + instance_type + for instance_type in instance_types + if _matches_requirements(instance_type.resources, requirements) + ] + if len(eligible_instance_types) == 0: + return None + instance_type = eligible_instance_types[0] + interruptible = False + if requirements and requirements.interruptible: + interruptible = True + return InstanceType( + instance_name=instance_type.instance_name, + resources=Resources( + cpus=instance_type.resources.cpus, + memory_mib=instance_type.resources.memory_mib, + gpus=instance_type.resources.gpus, + interruptible=interruptible, + local=False, + ), + ) + + +def _sort_instance_types(instance_types: List[InstanceType]) -> List[InstanceType]: + return sorted(instance_types, key=cmp_to_key(_compare_instance_types)) + + +def _compare_instance_types(i1, i2): + r1_gpu_total_memory_mib = sum(map(lambda g: g.memory_mib, i1.resources.gpus or [])) + r2_gpu_total_memory_mib = sum(map(lambda g: g.memory_mib, i2.resources.gpus or [])) + if r1_gpu_total_memory_mib < r2_gpu_total_memory_mib: + return -1 + elif r1_gpu_total_memory_mib > r2_gpu_total_memory_mib: + return 1 + if i1.resources.cpus < i2.resources.cpus: + return -1 + elif i1.resources.cpus > i2.resources.cpus: + return 1 + if i1.resources.memory_mib < i2.resources.memory_mib: + return -1 + elif i1.resources.memory_mib > i2.resources.memory_mib: + return 1 + return 0 + + +def _matches_requirements(resources: Resources, requirements: Optional[Requirements]) -> bool: + if not requirements: + return True + if requirements.cpus and requirements.cpus > resources.cpus: + return False + if requirements.memory_mib and requirements.memory_mib > resources.memory_mib: + return False + if requirements.gpus: + gpu_count = requirements.gpus.count or 1 + if gpu_count > len(resources.gpus or []): + return False + if requirements.gpus.name and gpu_count > len( + list(filter(lambda gpu: gpu.name == requirements.gpus.name, resources.gpus or [])) + ): + return False + if requirements.gpus.memory_mib and gpu_count > len( + list( + filter( + lambda gpu: gpu.memory_mib >= requirements.gpus.memory_mib, + resources.gpus or [], + ) + ) + ): + return False + if requirements.interruptible and not resources.interruptible: + return False + return True diff --git a/cli/dstack/backend/gcp/compute.py b/cli/dstack/backend/gcp/compute.py index 43e616d438..f59a09dc5e 100644 --- a/cli/dstack/backend/gcp/compute.py +++ b/cli/dstack/backend/gcp/compute.py @@ -1,5 +1,4 @@ import re -import warnings from typing import Any, List, Optional, Tuple import google.api_core.exceptions @@ -8,7 +7,7 @@ from dstack import version from dstack.backend.aws.runners import _serialize_runner_yaml -from dstack.backend.base.compute import Compute +from dstack.backend.base.compute import Compute, choose_instance_type from dstack.backend.gcp import utils as gcp_utils from dstack.backend.gcp.config import GCPConfig from dstack.core.instance import InstanceType @@ -30,6 +29,7 @@ def __init__( self.instances_client = compute_v1.InstancesClient(credentials=self.credentials) self.images_client = compute_v1.ImagesClient(credentials=self.credentials) self.firewalls_client = compute_v1.FirewallsClient(credentials=self.credentials) + self.machine_types_client = compute_v1.MachineTypesClient(credentials=self.credentials) def get_request_head(self, job: Job, request_id: Optional[str]) -> RequestHead: if request_id is None: @@ -51,12 +51,13 @@ def get_request_head(self, job: Job, request_id: Optional[str]) -> RequestHead: ) def get_instance_type(self, job: Job) -> Optional[InstanceType]: - return InstanceType( - instance_name="n1-standard-1", - resources=Resources( - cpus=1, memory_mib=3750, gpus=[], interruptible=False, local=False - ), + instance_types = _get_instance_types( + machine_types_client=self.machine_types_client, + project_id=self.gcp_config.project_id, + zone=self.gcp_config.zone, ) + instance_type = choose_instance_type(instance_types, job.requirements) + return instance_type def run_instance(self, job: Job, instance_type: InstanceType) -> str: instance = _launch_instance( @@ -64,6 +65,7 @@ def run_instance(self, job: Job, instance_type: InstanceType) -> str: firewalls_client=self.firewalls_client, project_id=self.gcp_config.project_id, zone=self.gcp_config.zone, + machine_type=instance_type.instance_name, image_name=_get_image_name( images_client=self.images_client, instance_type=instance_type, @@ -71,7 +73,7 @@ def run_instance(self, job: Job, instance_type: InstanceType) -> str: instance_name=_get_instance_name(job), user_data_script=_get_user_data_script(self.gcp_config, job, instance_type), service_account=self.credentials.service_account_email, - interruptible=job.requirements.interruptible, + interruptible=instance_type.resources.interruptible, ) return instance.name @@ -90,6 +92,82 @@ def cancel_spot_request(self, request_id: str): ) +def _get_instance_status( + client: compute_v1.InstancesClient, project_id: str, zone: str, instance_name: str +) -> RequestStatus: + get_instance_request = compute_v1.GetInstanceRequest( + instance=instance_name, + project=project_id, + zone=zone, + ) + try: + instance = client.get(get_instance_request) + except google.api_core.exceptions.NotFound: + return RequestStatus.TERMINATED + if instance.status in ["PROVISIONING", "STAGING", "RUNNING"]: + return RequestStatus.RUNNING + return RequestStatus.TERMINATED + + +def _get_instance_types( + machine_types_client: compute_v1.MachineTypesClient, + project_id: str, + zone: str, +) -> List[InstanceType]: + machine_families = ["n1-*", "e2-*", "c2-*", "m1-*", "a2-*"] + machine_types = _list_machine_types( + machine_types_client=machine_types_client, + project_id=project_id, + zone=zone, + machine_families=machine_families, + ) + return [_machine_type_to_instance_type(mt) for mt in machine_types] + + +def _list_machine_types( + machine_types_client: compute_v1.MachineTypesClient, + project_id: str, + zone: str, + machine_families: List[str], +) -> List[compute_v1.MachineType]: + list_machine_types_request = compute_v1.ListMachineTypesRequest() + list_machine_types_request.project = project_id + list_machine_types_request.zone = zone + list_machine_types_request.filter = " OR ".join(f"(name = {mf})" for mf in machine_families) + machine_types = machine_types_client.list(list_machine_types_request) + return [mt for mt in machine_types if not mt.deprecated.state == "DEPRECATED"] + + +def _machine_type_to_instance_type(machine_type: compute_v1.MachineType) -> InstanceType: + gpus = [] + for acc in machine_type.accelerators: + gpus.extend( + [ + Gpu( + name=acc.guest_accelerator_type, + memory_mib=_get_gpu_memory(acc.guest_accelerator_type), + ) + for _ in range(acc.guest_accelerator_count) + ] + ) + return InstanceType( + instance_name=machine_type.name, + resources=Resources( + cpus=machine_type.guest_cpus, + memory_mib=machine_type.memory_mb, + gpus=gpus, + interruptible=True, + local=False, + ), + ) + + +def _get_gpu_memory(gpu_name: str) -> int: + if gpu_name == "nvidia-a100-80gb": + return 80 * 1024 + return 40 * 1024 + + def _get_image_name( images_client: compute_v1.ImagesClient, instance_type: InstanceType ) -> Optional[str]: @@ -137,6 +215,7 @@ def _launch_instance( project_id: str, zone: str, image_name: str, + machine_type: str, instance_name: str, user_data_script: str, service_account: str, @@ -157,6 +236,7 @@ def _launch_instance( instances_client=instances_client, project_id=project_id, zone=zone, + machine_type=machine_type, instance_name=instance_name, disks=[disk], user_data_script=user_data_script, @@ -297,6 +377,8 @@ def _create_instance( instance.scheduling.provisioning_model = compute_v1.Scheduling.ProvisioningModel.SPOT.name instance.scheduling.instance_termination_action = instance_termination_action + instance.scheduling.on_host_maintenance = "TERMINATE" + if custom_hostname is not None: instance.hostname = custom_hostname @@ -371,23 +453,6 @@ def _create_firewall_rules( gcp_utils.wait_for_extended_operation(operation, "firewall rule creation") -def _get_instance_status( - client: compute_v1.InstancesClient, project_id: str, zone: str, instance_name: str -) -> RequestStatus: - get_instance_request = compute_v1.GetInstanceRequest( - instance=instance_name, - project=project_id, - zone=zone, - ) - try: - instance = client.get(get_instance_request) - except google.api_core.exceptions.NotFound: - return RequestStatus.TERMINATED - if instance.status in ["PROVISIONING", "STAGING", "RUNNING"]: - return RequestStatus.RUNNING - return RequestStatus.TERMINATED - - def _terminate_instance( client: compute_v1.InstancesClient, gcp_config: GCPConfig, instance_name: str ): From 7fb2c587dd123cff8a70164f8bb51697ee506de5 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 1 Mar 2023 17:05:40 +0500 Subject: [PATCH 55/58] Support gcp gpu instances --- cli/dstack/backend/gcp/compute.py | 216 ++++++++++++++++++++++++++++-- 1 file changed, 204 insertions(+), 12 deletions(-) diff --git a/cli/dstack/backend/gcp/compute.py b/cli/dstack/backend/gcp/compute.py index f59a09dc5e..b0b55153fc 100644 --- a/cli/dstack/backend/gcp/compute.py +++ b/cli/dstack/backend/gcp/compute.py @@ -18,6 +18,15 @@ DSTACK_INSTANCE_TAG = "dstack-runner-instance" +_supported_accelerators = [ + {"accelerator_name": "nvidia-a100-80gb", "gpu_name": "A100", "memory_mb": 1024 * 80}, + {"accelerator_name": "nvidia-tesla-a100", "gpu_name": "A100", "memory_mb": 1024 * 40}, + {"accelerator_name": "nvidia-tesla-v100", "gpu_name": "V100", "memory_mb": 1024 * 16}, + {"accelerator_name": "nvidia-tesla-p100", "gpu_name": "P100", "memory_mb": 1024 * 16}, + {"accelerator_name": "nvidia-tesla-k80", "gpu_name": "K80", "memory_mb": 1024 * 12}, +] + + class GCPCompute(Compute): def __init__( self, @@ -30,6 +39,9 @@ def __init__( self.images_client = compute_v1.ImagesClient(credentials=self.credentials) self.firewalls_client = compute_v1.FirewallsClient(credentials=self.credentials) self.machine_types_client = compute_v1.MachineTypesClient(credentials=self.credentials) + self.accelerator_types_client = compute_v1.AcceleratorTypesClient( + credentials=self.credentials + ) def get_request_head(self, job: Job, request_id: Optional[str]) -> RequestHead: if request_id is None: @@ -51,13 +63,13 @@ def get_request_head(self, job: Job, request_id: Optional[str]) -> RequestHead: ) def get_instance_type(self, job: Job) -> Optional[InstanceType]: - instance_types = _get_instance_types( + return _choose_instance_type( machine_types_client=self.machine_types_client, + accelerator_types_client=self.accelerator_types_client, project_id=self.gcp_config.project_id, zone=self.gcp_config.zone, + requirements=job.requirements, ) - instance_type = choose_instance_type(instance_types, job.requirements) - return instance_type def run_instance(self, job: Job, instance_type: InstanceType) -> str: instance = _launch_instance( @@ -74,6 +86,11 @@ def run_instance(self, job: Job, instance_type: InstanceType) -> str: user_data_script=_get_user_data_script(self.gcp_config, job, instance_type), service_account=self.credentials.service_account_email, interruptible=instance_type.resources.interruptible, + accelerators=_get_accelerator_configs( + project_id=self.gcp_config.project_id, + zone=self.gcp_config.zone, + instance_type=instance_type, + ), ) return instance.name @@ -109,12 +126,105 @@ def _get_instance_status( return RequestStatus.TERMINATED -def _get_instance_types( +def _choose_instance_type( + machine_types_client: compute_v1.MachineTypesClient, + accelerator_types_client: compute_v1.AcceleratorTypesClient, + project_id: str, + zone: str, + requirements: Optional[Requirements], +) -> Optional[InstanceType]: + if requirements is None or requirements.gpus is None: + return _get_nongpu_instance_type( + machine_types_client=machine_types_client, + project_id=project_id, + zone=zone, + requirements=requirements, + ) + return _get_gpu_instance_type( + machine_types_client=machine_types_client, + accelerator_types_client=accelerator_types_client, + project_id=project_id, + zone=zone, + requirements=requirements, + ) + + +def _get_nongpu_instance_type( + machine_types_client: compute_v1.MachineTypesClient, + project_id: str, + zone: str, + requirements: Requirements, +) -> List[InstanceType]: + machine_families = ["e2-medium", "e2-standard-*", "e2-highmem-*", "e2-highcpu-*", "m1-*"] + instance_types = _list_instance_types( + machine_types_client=machine_types_client, + project_id=project_id, + zone=zone, + machine_families=machine_families, + ) + return choose_instance_type(instance_types, requirements) + + +def _get_gpu_instance_type( + machine_types_client: compute_v1.MachineTypesClient, + accelerator_types_client: compute_v1.AcceleratorTypesClient, + project_id: str, + zone: str, + requirements: Requirements, +) -> InstanceType: + # To create a GPU instance in GCP, we need to create a n1-* instance + # and attach an accelerator to it. The only exception are a2-* instances + # that already come with A100 accelerators. + instance_types_without_gpus = _list_instance_types( + machine_types_client=machine_types_client, + project_id=project_id, + zone=zone, + machine_families=["n1-*"], + ) + instance_types_without_gpus = [ + it + for it in instance_types_without_gpus + if it.instance_name not in ["n1-standard-1", "n1-highcpu-2"] + ] + instance_type = choose_instance_type( + instance_types=instance_types_without_gpus, + requirements=Requirements( + cpus=requirements.cpus, + memory_mib=requirements.memory_mib, + shm_size_mib=requirements.shm_size_mib, + interruptible=requirements.interruptible, + local=requirements.local, + gpus=None, + ), + ) + instance_types_with_gpus = [] + if instance_type is not None: + instance_types_with_gpus.extend( + _add_gpus_to_instance_type( + accelerator_types_client=accelerator_types_client, + project_id=project_id, + zone=zone, + instance_type=instance_type, + requirements=requirements, + ) + ) + instance_types_with_gpus.extend( + _list_instance_types( + machine_types_client=machine_types_client, + project_id=project_id, + zone=zone, + machine_families=["a2-*"], + ) + ) + return choose_instance_type(instance_types_with_gpus, requirements) + + +def _list_instance_types( machine_types_client: compute_v1.MachineTypesClient, project_id: str, zone: str, + machine_families: List[str], ) -> List[InstanceType]: - machine_families = ["n1-*", "e2-*", "c2-*", "m1-*", "a2-*"] machine_types = _list_machine_types( machine_types_client=machine_types_client, project_id=project_id, @@ -144,7 +254,7 @@ def _machine_type_to_instance_type(machine_type: compute_v1.MachineType) -> Inst gpus.extend( [ Gpu( - name=acc.guest_accelerator_type, + name=_accelerator_name_to_gpu_name(acc.guest_accelerator_type), memory_mib=_get_gpu_memory(acc.guest_accelerator_type), ) for _ in range(acc.guest_accelerator_count) @@ -162,10 +272,75 @@ def _machine_type_to_instance_type(machine_type: compute_v1.MachineType) -> Inst ) -def _get_gpu_memory(gpu_name: str) -> int: - if gpu_name == "nvidia-a100-80gb": - return 80 * 1024 - return 40 * 1024 +def _get_gpu_memory(accelerator_name: str) -> int: + for acc in _supported_accelerators: + if acc["accelerator_name"] == accelerator_name: + return acc["memory_mb"] + + +def _add_gpus_to_instance_type( + accelerator_types_client: compute_v1.AcceleratorTypesClient, + project_id: str, + zone: str, + instance_type: InstanceType, + requirements: Requirements, +) -> bool: + instance_types = [] + accelerator_types = _list_accelerator_types( + accelerator_types_client=accelerator_types_client, + project_id=project_id, + zone=zone, + accelerator_families=["nvidia-tesla-v100", "nvidia-tesla-k80", "nvidia-tesla-p100"], + ) + for at in accelerator_types: + for gpu_count in range(1, at.maximum_cards_per_instance): + instance_types.append( + InstanceType( + instance_name=instance_type.instance_name, + resources=Resources( + cpus=instance_type.resources.cpus, + memory_mib=instance_type.resources.memory_mib, + interruptible=instance_type.resources.interruptible, + local=instance_type.resources.local, + gpus=[ + Gpu( + name=_accelerator_name_to_gpu_name(at.name), + memory_mib=_get_gpu_memory(at.name), + ) + for _ in range(gpu_count) + ], + ), + ) + ) + return instance_types + + +def _list_accelerator_types( + accelerator_types_client: compute_v1.AcceleratorTypesClient, + project_id: str, + zone: str, + accelerator_families: List[str], +) -> List[compute_v1.AcceleratorType]: + list_accelerator_types_request = compute_v1.ListAcceleratorTypesRequest() + list_accelerator_types_request.project = project_id + list_accelerator_types_request.zone = zone + list_accelerator_types_request.filter = " OR ".join( + f"(name = {af})" for af in accelerator_families + ) + accelerator_types = accelerator_types_client.list(list_accelerator_types_request) + return [at for at in accelerator_types] + + +def _accelerator_name_to_gpu_name(accelerator_name: str) -> str: + for acc in _supported_accelerators: + if acc["accelerator_name"] == accelerator_name: + return acc["gpu_name"] + + +def _gpu_to_accelerator_name(gpu: Gpu) -> str: + for acc in _supported_accelerators: + if acc["gpu_name"] == gpu.name and acc["memory_mb"] == gpu.memory_mib: + return acc["accelerator_name"] def _get_image_name( @@ -209,6 +384,20 @@ def _get_user_data_script(gcp_config: GCPConfig, job: Job, instance_type: Instan """ +def _get_accelerator_configs( + project_id: str, zone: str, instance_type: InstanceType +) -> List[compute_v1.AcceleratorConfig]: + if instance_type.instance_name.startswith("a2") or len(instance_type.resources.gpus) == 0: + return [] + accelerator_config = compute_v1.AcceleratorConfig() + accelerator_config.accelerator_count = len(instance_type.resources.gpus) + accelerator_name = _gpu_to_accelerator_name(instance_type.resources.gpus[0]) + accelerator_config.accelerator_type = ( + f"projects/{project_id}/zones/{zone}/acceleratorTypes/{accelerator_name}" + ) + return [accelerator_config] + + def _launch_instance( instances_client: compute_v1.InstancesClient, firewalls_client: compute_v1.FirewallsClient, @@ -220,6 +409,7 @@ def _launch_instance( user_data_script: str, service_account: str, interruptible: bool, + accelerators: List[compute_v1.AcceleratorConfig], ) -> compute_v1.Instance: try: _create_firewall_rules(firewalls_client=firewalls_client, project_id=project_id) @@ -243,6 +433,7 @@ def _launch_instance( service_account=service_account, external_access=True, spot=interruptible, + accelerators=accelerators, ) return instance @@ -372,13 +563,14 @@ def _create_instance( if accelerators: instance.guest_accelerators = accelerators + if accelerators or "a2-" in machine_type: + instance.scheduling.on_host_maintenance = "TERMINATE" + if spot: instance.scheduling = compute_v1.Scheduling() instance.scheduling.provisioning_model = compute_v1.Scheduling.ProvisioningModel.SPOT.name instance.scheduling.instance_termination_action = instance_termination_action - instance.scheduling.on_host_maintenance = "TERMINATE" - if custom_hostname is not None: instance.hostname = custom_hostname From 6cb882af3701bb2f4cb78540c0d1da5edef2d42e Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 1 Mar 2023 17:32:34 +0500 Subject: [PATCH 56/58] Warn about mount artifacts not supported on gcp --- cli/dstack/backend/gcp/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cli/dstack/backend/gcp/__init__.py b/cli/dstack/backend/gcp/__init__.py index 934d9f4081..794c39acd3 100644 --- a/cli/dstack/backend/gcp/__init__.py +++ b/cli/dstack/backend/gcp/__init__.py @@ -18,6 +18,7 @@ from dstack.backend.gcp.logs import GCPLogging from dstack.backend.gcp.secrets import GCPSecretsManager from dstack.backend.gcp.storage import GCPStorage +from dstack.cli.common import console from dstack.core.artifact import Artifact from dstack.core.job import Job, JobHead from dstack.core.log_event import LogEvent @@ -82,6 +83,9 @@ def create_run(self, repo_address: RepoAddress) -> str: return base_runs.create_run(self._storage, repo_address, self.type) def create_job(self, job: Job): + if job.artifact_specs and any(art_spec.mount for art_spec in job.artifact_specs): + console.print("Mount artifacts are not currently supported for 'gcp' backend") + exit(1) base_jobs.create_job(self._storage, job) def get_job(self, repo_address: RepoAddress, job_id: str) -> Optional[Job]: From fc15b7f75af8b88cd1c710d5b8bd828673fbebde Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 1 Mar 2023 18:12:26 +0500 Subject: [PATCH 57/58] Set ssd size to 100gb --- cli/dstack/backend/gcp/compute.py | 2 +- cli/dstack/backend/gcp/utils.py | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/cli/dstack/backend/gcp/compute.py b/cli/dstack/backend/gcp/compute.py index b0b55153fc..ed9c53d5b8 100644 --- a/cli/dstack/backend/gcp/compute.py +++ b/cli/dstack/backend/gcp/compute.py @@ -417,7 +417,7 @@ def _launch_instance( pass disk = _disk_from_image( disk_type=f"zones/{zone}/diskTypes/pd-balanced", - disk_size_gb=20, + disk_size_gb=100, boot=True, source_image=f"projects/dstack/global/images/{image_name}", auto_delete=False, diff --git a/cli/dstack/backend/gcp/utils.py b/cli/dstack/backend/gcp/utils.py index eb5e634c56..404ad6606a 100644 --- a/cli/dstack/backend/gcp/utils.py +++ b/cli/dstack/backend/gcp/utils.py @@ -43,9 +43,4 @@ def wait_for_extended_operation( print(f"Operation ID: {operation.name}", file=sys.stderr, flush=True) raise operation.exception() or RuntimeError(operation.error_message) - if operation.warnings: - print(f"Warnings during {verbose_name}:\n", file=sys.stderr, flush=True) - for warning in operation.warnings: - print(f" - {warning.code}: {warning.message}", file=sys.stderr, flush=True) - return result From 67a1f86f9487f00ddae1860e5b38d58a9c5fb4f0 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Thu, 2 Mar 2023 09:10:33 +0500 Subject: [PATCH 58/58] Remove duplicated aiosqlite from requirements --- cli/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/cli/requirements.txt b/cli/requirements.txt index 1bd3007caf..31e2c47f18 100644 --- a/cli/requirements.txt +++ b/cli/requirements.txt @@ -27,5 +27,4 @@ google-cloud-compute google-cloud-secret-manager google-cloud-logging aiosqlite -aiosqlite packaging