From df8e61f52465ae0b646ed6b9fc05d4d7e227bb59 Mon Sep 17 00:00:00 2001 From: Anand J Date: Sun, 13 Oct 2024 14:13:20 -0400 Subject: [PATCH 01/15] Update sdk/config.py --- centml/sdk/api.py | 2 +- centml/sdk/auth.py | 2 +- centml/sdk/config.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/centml/sdk/api.py b/centml/sdk/api.py index 02281cbd..90434937 100644 --- a/centml/sdk/api.py +++ b/centml/sdk/api.py @@ -10,7 +10,7 @@ @contextlib.contextmanager def get_api(): configuration = platform_api_client.Configuration( - host=settings.PLATFORM_API_URL, access_token=auth.get_centml_token() + host=settings.CENTML_PLATFORM_API_URL, access_token=auth.get_centml_token() ) with platform_api_client.ApiClient(configuration) as api_client: diff --git a/centml/sdk/auth.py b/centml/sdk/auth.py index 65dc834d..71069ea4 100644 --- a/centml/sdk/auth.py +++ b/centml/sdk/auth.py @@ -9,7 +9,7 @@ def refresh_centml_token(refresh_token): - api_key = settings.FIREBASE_API_KEY + api_key = settings.CENTML_FIREBASE_API_KEY cred = requests.post( f"https://securetoken.googleapis.com/v1/token?key={api_key}", diff --git a/centml/sdk/config.py b/centml/sdk/config.py index 409edc2a..f26bb299 100644 --- a/centml/sdk/config.py +++ b/centml/sdk/config.py @@ -3,14 +3,14 @@ class Config(BaseSettings): - CENTML_WEB_URL: str = "https://main.d1tz9z8hgabab9.amplifyapp.com/" + CENTML_WEB_URL: str = "https://app.centml.com/" CENTML_CONFIG_PATH: str = os.path.expanduser("~/.centml") CENTML_CRED_FILE: str = "credential" CENTML_CRED_FILE_PATH: str = CENTML_CONFIG_PATH + "/" + CENTML_CRED_FILE - PLATFORM_API_URL: str = "https://api.centml.org" + CENTML_PLATFORM_API_URL: str = "https://api.centml.com" - FIREBASE_API_KEY: str = "AIzaSyBXSNjruNdtypqUt_CPhB8QNl8Djfh5RXI" + CENTML_FIREBASE_API_KEY: str = "AIzaSyChPXy41cIAxS_Nd8oaYKyP_oKkIucobtY" settings = Config() From 0109a5f6afb1e68406038026b53b15009c885e92 Mon Sep 17 00:00:00 2001 From: Anand J Date: Sun, 13 Oct 2024 14:21:39 -0400 Subject: [PATCH 02/15] Rename platform_api_client to platform_api_python_client --- centml/cli/cluster.py | 17 ++++++++--------- centml/sdk/api.py | 16 ++++++++-------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/centml/cli/cluster.py b/centml/cli/cluster.py index cf3b7b1a..15e97e5a 100644 --- a/centml/cli/cluster.py +++ b/centml/cli/cluster.py @@ -2,9 +2,8 @@ from typing import Dict import click from tabulate import tabulate -import platform_api_client -from platform_api_client.models.endpoint_ready_state import EndpointReadyState -from platform_api_client.models.deployment_status import DeploymentStatus +import platform_api_python_client +from platform_api_python_client.models.deployment_status import DeploymentStatus from centml.sdk import api @@ -60,8 +59,8 @@ def convert(self, value, param, ctx): hardware_pricing_tier_instance = HardwarePricingTier() depl_type_map = { - "inference": platform_api_client.DeploymentType.INFERENCE, - "compute": platform_api_client.DeploymentType.COMPUTE, + "inference": platform_api_python_client.DeploymentType.INFERENCE, + "compute": platform_api_python_client.DeploymentType.COMPUTE, } @@ -124,9 +123,9 @@ def ls(type): @click.argument("type", type=click.Choice(list(depl_type_map.keys()))) @click.argument("id", type=int) def get(type, id): - if type == platform_api_client.DeploymentType.INFERENCE: + if type == platform_api_python_client.DeploymentType.INFERENCE: deployment = api.get_inference(id) - elif type == platform_api_client.DeploymentType.COMPUTE: + elif type == platform_api_python_client.DeploymentType.COMPUTE: deployment = api.get_compute(id) else: sys.exit("Please enter correct deployment type") @@ -150,7 +149,7 @@ def get(type, id): ) click.echo("Additional deployment configurations:") - if type == platform_api_client.DeploymentType.INFERENCE: + if type == platform_api_python_client.DeploymentType.INFERENCE: click.echo( tabulate( [ @@ -164,7 +163,7 @@ def get(type, id): disable_numparse=True, ) ) - elif type == platform_api_client.DeploymentType.COMPUTE: + elif type == platform_api_python_client.DeploymentType.COMPUTE: click.echo( tabulate( [ diff --git a/centml/sdk/api.py b/centml/sdk/api.py index 90434937..0f9eb1a0 100644 --- a/centml/sdk/api.py +++ b/centml/sdk/api.py @@ -1,6 +1,6 @@ import contextlib -import platform_api_client -from platform_api_client.models.deployment_status import DeploymentStatus +import platform_api_python_client +from platform_api_python_client.models.deployment_status import DeploymentStatus from centml.sdk import auth from centml.sdk.config import settings @@ -9,12 +9,12 @@ @contextlib.contextmanager def get_api(): - configuration = platform_api_client.Configuration( + configuration = platform_api_python_client.Configuration( host=settings.CENTML_PLATFORM_API_URL, access_token=auth.get_centml_token() ) - with platform_api_client.ApiClient(configuration) as api_client: - api_instance = platform_api_client.EXTERNALApi(api_client) + with platform_api_python_client.ApiClient(configuration) as api_client: + api_instance = platform_api_python_client.EXTERNALApi(api_client) yield api_instance @@ -56,7 +56,7 @@ def create_inference( # Handle automatic download of client private secrets client_certs.save_pem_file(name, triplet.client_private_key, triplet.client_certificate) with get_api() as api: - req = platform_api_client.CreateInferenceDeploymentRequest( + req = platform_api_python_client.CreateInferenceDeploymentRequest( name=name, image_url=image, port=port, @@ -75,7 +75,7 @@ def create_inference( def create_compute(name, image, username, password, ssh_key, hw_to_id_map): with get_api() as api: - req = platform_api_client.CreateComputeDeploymentRequest( + req = platform_api_python_client.CreateComputeDeploymentRequest( name=name, image_url=image, hardware_instance_id=hw_to_id_map, @@ -88,7 +88,7 @@ def create_compute(name, image, username, password, ssh_key, hw_to_id_map): def update_status(id, new_status): with get_api() as api: - status_req = platform_api_client.DeploymentStatusRequest(status=new_status) + status_req = platform_api_python_client.DeploymentStatusRequest(status=new_status) api.update_deployment_status_deployments_status_deployment_id_put(id, status_req) From a8a30431125e50ff8255e345fa5ee712e8d8110a Mon Sep 17 00:00:00 2001 From: Anand J Date: Sun, 13 Oct 2024 14:54:04 -0400 Subject: [PATCH 03/15] Set requires_python to <3.12 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index f60249d5..ca76469d 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,7 @@ name='centml', version='0.1.0', packages=find_packages(), + requires_python="<3.12" entry_points={ "console_scripts": [ "centml = centml.cli:cli", From 010c6d643c35f6e5fa55215ecd51e6de38cb02c2 Mon Sep 17 00:00:00 2001 From: Anand J Date: Sun, 13 Oct 2024 14:54:52 -0400 Subject: [PATCH 04/15] Add .swp to gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index d2e251e8..c7e480ea 100644 --- a/.gitignore +++ b/.gitignore @@ -161,3 +161,6 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +# vim +.*.swp From 2e159fcf63a455cd46ae03806fb219775396c89c Mon Sep 17 00:00:00 2001 From: Anand J Date: Sun, 13 Oct 2024 15:04:00 -0400 Subject: [PATCH 05/15] Update requirements and python requires version --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index f2bc96f2..992ce9f6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,4 @@ cryptography==43.0.1 prometheus-client>=0.20.0 scipy>=1.6.0 scikit-learn>=1.5.1 -platform_api_client @ git+https://github.com/CentML/platform_api_python_client.git@main +platform_api_python_client @ git+https://github.com/CentML/platform_api_python_client.git@main diff --git a/setup.py b/setup.py index ca76469d..93ced9fb 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ name='centml', version='0.1.0', packages=find_packages(), - requires_python="<3.12" + python_requires="<3.12", entry_points={ "console_scripts": [ "centml = centml.cli:cli", From 05b2c49410b71db192ba98e20face4d398849416 Mon Sep 17 00:00:00 2001 From: Anand J Date: Sun, 13 Oct 2024 15:16:01 -0400 Subject: [PATCH 06/15] Add sdk in centml import --- centml/__init__.py | 6 +++++- centml/sdk/__init__.py | 6 ++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/centml/__init__.py b/centml/__init__.py index 334e3b0a..25860391 100644 --- a/centml/__init__.py +++ b/centml/__init__.py @@ -1,3 +1,7 @@ from .compile import compile +from . import sdk -__all__ = ["compile"] +__all__ = [ + "compile", + "sdk", +] diff --git a/centml/sdk/__init__.py b/centml/sdk/__init__.py index e69de29b..22542871 100644 --- a/centml/sdk/__init__.py +++ b/centml/sdk/__init__.py @@ -0,0 +1,6 @@ +from . import api, auth + +__all__ = [ + "api", + "auth", +] From 90f0c65f9310a187406bc40af5e43717e8af622a Mon Sep 17 00:00:00 2001 From: Anand J Date: Sun, 13 Oct 2024 15:44:56 -0400 Subject: [PATCH 07/15] Move compiler to another file --- centml/__init__.py | 6 ------ centml/compiler/__init__.py | 11 ++++------- centml/{compile.py => compiler/main.py} | 0 3 files changed, 4 insertions(+), 13 deletions(-) rename centml/{compile.py => compiler/main.py} (100%) diff --git a/centml/__init__.py b/centml/__init__.py index 25860391..8b137891 100644 --- a/centml/__init__.py +++ b/centml/__init__.py @@ -1,7 +1 @@ -from .compile import compile -from . import sdk -__all__ = [ - "compile", - "sdk", -] diff --git a/centml/compiler/__init__.py b/centml/compiler/__init__.py index 9b8289cd..247852f2 100644 --- a/centml/compiler/__init__.py +++ b/centml/compiler/__init__.py @@ -1,8 +1,5 @@ -import torch._dynamo +from centml.compiler.main import compile -from centml.compiler.backend import centml_dynamo_backend - - -# Register centml compiler backend to torch dynamo -if "centml" not in torch._dynamo.list_backends(): - torch._dynamo.register_backend(compiler_fn=centml_dynamo_backend, name="centml") +all = [ + "compile", +] diff --git a/centml/compile.py b/centml/compiler/main.py similarity index 100% rename from centml/compile.py rename to centml/compiler/main.py From 2ba84ace6f59eb200251a0ea856ea77eb947a876 Mon Sep 17 00:00:00 2001 From: Anand J Date: Sun, 13 Oct 2024 15:49:16 -0400 Subject: [PATCH 08/15] Move torch to requirements-dev.txt --- requirements-dev.txt | 3 +-- requirements.txt | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 03abdc6f..6179c207 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,6 @@ -r requirements.txt +torch==2.3.1 black>=23.10.0 pylint>=3.0.1 pytest>=7.4.0 @@ -11,5 +12,3 @@ parameterized>=0.9.0 mypy==1.5.1 types-requests==2.31.0.2 types-tabulate>=0.9.0 -prometheus-client>=0.20.0 - diff --git a/requirements.txt b/requirements.txt index 992ce9f6..d33e1467 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -torch==2.3.1 fastapi>=0.103.0 uvicorn>=0.23.0 python-multipart>=0.0.6 From aa83b71d883eca501b0a401b89a65b2ea42030c1 Mon Sep 17 00:00:00 2001 From: Anand J Date: Sun, 13 Oct 2024 19:47:36 -0400 Subject: [PATCH 09/15] Fix update deployment --- centml/cli/cluster.py | 195 ++++++++++++++---------------------------- centml/cli/main.py | 2 +- centml/sdk/api.py | 27 +++--- 3 files changed, 78 insertions(+), 146 deletions(-) diff --git a/centml/cli/cluster.py b/centml/cli/cluster.py index 15e97e5a..b0a6a5a8 100644 --- a/centml/cli/cluster.py +++ b/centml/cli/cluster.py @@ -4,6 +4,7 @@ from tabulate import tabulate import platform_api_python_client from platform_api_python_client.models.deployment_status import DeploymentStatus +from platform_api_python_client.models.health_status import HealthStatus from centml.sdk import api @@ -20,48 +21,25 @@ def convert(self, value, param, ctx): return None # to avoid warning from lint for inconsistent return statements -def get_hw_to_id_map(): - response = api.get_hardware_instances() - - # Convert to list of dictionaries - instances = [item.to_dict() for item in response] +def get_hw_to_id_map(cluster_id): + response = api.get_hardware_instances(cluster_id) # Initialize hashmap for hardware to id or vice versa mapping hw_to_id_map: Dict[str, int] = {} - id_to_hw_map: Dict[str, str] = {} + id_to_hw_map: Dict[int, HardwareInstanceResponse] = {} - for item in instances: - hw_to_id_map[item["name"]] = item["id"] - id_to_hw_map[item["id"]] = item["name"] + for hw in response: + hw_to_id_map[hw.name] = hw.id + id_to_hw_map[hw.id] = hw return hw_to_id_map, id_to_hw_map -# # Hardware pricing tier that loads choices dynamically -class HardwarePricingTier(click.ParamType): - def __init__(self): - self.hw_to_id_map = None - self.id_to_hw_map = None - self.choices = None - - def initialize_maps(self): - if self.hw_to_id_map is None or self.id_to_hw_map is None or self.choices is None: - self.hw_to_id_map, self.id_to_hw_map = get_hw_to_id_map() - self.choices = list(self.hw_to_id_map.keys()) - - def convert(self, value, param, ctx): - # calling initialize_maps to defer api call during initialization phase - self.initialize_maps() - if value not in self.choices: - self.fail(f"{value} is not a valid choice. Available choices are: {', '.join(self.choices)}", param, ctx) - return value - - -hardware_pricing_tier_instance = HardwarePricingTier() - -depl_type_map = { - "inference": platform_api_python_client.DeploymentType.INFERENCE, - "compute": platform_api_python_client.DeploymentType.COMPUTE, +depl_name_to_type_map = { + "inference": platform_api_python_client.DeploymentType.INFERENCE_V2, + "compute": platform_api_python_client.DeploymentType.COMPUTE_V2, + "cserve": platform_api_python_client.DeploymentType.CSERVE, } +depl_type_to_name_map = {v: k for k, v in depl_name_to_type_map.items()} def format_ssh_key(ssh_key): @@ -70,32 +48,17 @@ def format_ssh_key(ssh_key): return ssh_key[:10] + '...' -def get_ready_status(api_status, service_status): +def get_ready_status(deployment): + api_status = deployment.status + service_status = api.get_status(deployment.id).service_status if deployment.status == DeploymentStatus.ACTIVE else None + status_styles = { (DeploymentStatus.PAUSED, None): ("paused", "yellow", "black"), (DeploymentStatus.DELETED, None): ("deleted", "white", "black"), - (DeploymentStatus.ACTIVE, EndpointReadyState.READY): ("ready", "green", "black"), - (DeploymentStatus.ACTIVE, EndpointReadyState.NOT_READY): ("starting", "black", "white"), - (DeploymentStatus.ACTIVE, EndpointReadyState.NOT_FOUND): ("not found", "cyan"), - (DeploymentStatus.ACTIVE, EndpointReadyState.FOUND_MULTIPLE): ("found multiple", "black", "white"), - (DeploymentStatus.ACTIVE, EndpointReadyState.INGRESS_RULE_NOT_FOUND): ( - "ingress rule not found", - "black", - "white", - ), - (DeploymentStatus.ACTIVE, EndpointReadyState.CONDITION_NOT_FOUND): ("condition not found", "black", "white"), - (DeploymentStatus.ACTIVE, EndpointReadyState.INGRESS_NOT_CONFIGURED): ( - "ingress not configured", - "black", - "white", - ), - (DeploymentStatus.ACTIVE, EndpointReadyState.CONTAINER_MISSING): ("container missing", "black", "white"), - (DeploymentStatus.ACTIVE, EndpointReadyState.PROGRESS_DEADLINE_EXCEEDED): ( - "progress deadline exceeded", - "black", - "white", - ), - (DeploymentStatus.ACTIVE, EndpointReadyState.REVISION_MISSING): ("revision missing", "black", "white"), + (DeploymentStatus.ACTIVE, HealthStatus.HEALTHY): ("ready", "green", "black"), + (DeploymentStatus.ACTIVE, HealthStatus.PROGRESSING): ("starting", "black", "white"), + (DeploymentStatus.ACTIVE, HealthStatus.DEGRADED): ("starting", "black", "white"), + (DeploymentStatus.ACTIVE, HealthStatus.MISSING): ("not found", "cyan"), } style = status_styles.get((api_status, service_status), ("unknown", "black", "white")) @@ -104,10 +67,14 @@ def get_ready_status(api_status, service_status): @click.command(help="List all deployments") -@click.argument("type", default="all") +@click.argument("type", type=click.Choice(depl_name_to_type_map.keys()), required=False, default=None) def ls(type): - depl_type = depl_type_map[type] if type in depl_type_map else None - rows = api.get(depl_type) + depl_type = depl_name_to_type_map[type] if type in depl_name_to_type_map else None + deployments = api.get(depl_type) + rows = [ + [d.id, d.name, depl_type_to_name_map[d.type], d.status.value, d.created_at.strftime("%Y-%m-%d %H:%M:%S")] + for d in deployments + ] click.echo( tabulate( @@ -120,28 +87,33 @@ def ls(type): @click.command(help="Get deployment details") -@click.argument("type", type=click.Choice(list(depl_type_map.keys()))) +@click.argument("type", type=click.Choice(depl_name_to_type_map.keys())) @click.argument("id", type=int) def get(type, id): - if type == platform_api_python_client.DeploymentType.INFERENCE: + depl_type = depl_name_to_type_map[type] + + if depl_type == platform_api_python_client.DeploymentType.INFERENCE_V2: deployment = api.get_inference(id) - elif type == platform_api_python_client.DeploymentType.COMPUTE: + elif depl_type == platform_api_python_client.DeploymentType.COMPUTE_V2: deployment = api.get_compute(id) + elif depl_type == platform_api_python_client.DeploymentType.CSERVE: + deployment = api.get_cserve(id) else: sys.exit("Please enter correct deployment type") - state = api.get_status(id) - ready_status = get_ready_status(deployment.status, state.service_status) - click.echo(f"The current status of Deployment #{id} is: {ready_status}.") + ready_status = get_ready_status(deployment) + _, id_to_hw_map = get_hw_to_id_map(deployment.cluster_id) + hw = id_to_hw_map[deployment.hardware_instance_id] click.echo( tabulate( [ ("Name", deployment.name), - ("Image", deployment.image_url), + ("Status", ready_status), ("Endpoint", deployment.endpoint_url), ("Created at", deployment.created_at.strftime("%Y-%m-%d %H:%M:%S")), - ("Hardware", hardware_pricing_tier_instance.id_to_hw_map[deployment.hardware_instance_id]), + ("Hardware", f"{hw.name} ({hw.num_gpu}x {hw.gpu_type})"), + ("Cost", f"{hw.cost_per_hr/100} credits/hr"), ], tablefmt="rounded_outline", disable_numparse=True, @@ -149,27 +121,40 @@ def get(type, id): ) click.echo("Additional deployment configurations:") - if type == platform_api_python_client.DeploymentType.INFERENCE: + if depl_type == platform_api_python_client.DeploymentType.INFERENCE_V2: click.echo( tabulate( [ - ("Port", deployment.port), + ("Image", deployment.image_url), + ("Container port", deployment.container_port), ("Healthcheck", deployment.healthcheck or "/"), - ("Replicas", {"min": deployment.min_replicas, "max": deployment.max_replicas}), + ("Replicas", {"min": deployment.min_scale, "max": deployment.max_scale}), ("Environment variables", deployment.env_vars or "None"), - ("Max concurrency", deployment.timeout or "None"), + ("Max concurrency", deployment.concurrency or "None"), + ], + tablefmt="rounded_outline", + disable_numparse=True, + ) + ) + elif depl_type == platform_api_python_client.DeploymentType.COMPUTE_V2: + click.echo( + tabulate( + [ + ("Username", "centml"), + ("SSH key", format_ssh_key(deployment.ssh_public_key)), ], tablefmt="rounded_outline", disable_numparse=True, ) ) - elif type == platform_api_python_client.DeploymentType.COMPUTE: + elif depl_type == platform_api_python_client.DeploymentType.CSERVE: click.echo( tabulate( [ - ("Port", deployment.port), - ("Username", deployment.username or "None"), - ("SSH key", format_ssh_key(deployment.ssh_key)), + ("Hugging face model", deployment.model), + ("Parallelism", {"tensor": deployment.tensor_parallel_size, "pipeline": deployment.pipeline_parallel_size}), + ("Replicas", {"min": deployment.min_scale, "max": deployment.max_scale}), + ("Max concurrency", deployment.concurrency or "None"), ], tablefmt="rounded_outline", disable_numparse=True, @@ -182,74 +167,22 @@ def create(): pass -@create.command(name="inference", help="Create an inference deployment") -@click.option("--name", "-n", prompt="Name", help="Name of the deployment") -@click.option("--image", "-i", prompt="Image", help="Container image") -@click.option("--hardware", "-h", prompt="Hardware", type=hardware_pricing_tier_instance, help="Hardware instance type") -@click.option("--port", "-p", prompt="Port", type=int, help="Port to expose") -@click.option("--env", type=InferenceEnvType(), help="Environment variables in the format KEY=VALUE", multiple=True) -@click.option("--min_replicas", default="1", prompt="Min replicas", type=click.IntRange(1, 10)) -@click.option("--max_replicas", default="1", prompt="Max replicas", type=click.IntRange(1, 10)) -@click.option("--health", default="/", prompt="Health check", help="Health check endpoint") -@click.option("--is_private", default=False, type=bool, prompt="Is private?", help="Is private endpoint?") -@click.option("--timeout", prompt="Max concurrency", default=0, type=int) -@click.option("--command", type=str, required=False, default=None, help="Define a command for a container") -@click.option("--command_args", multiple=True, type=str, default=None, help="List of command arguments") -def create_inference( - name, image, hardware, port, env, min_replicas, max_replicas, health, is_private, timeout, command, command_args -): - click.echo("Creating inference deployment with the following options:") - - # Call the API function for creating inference deployment - resp = api.create_inference( - name, - image, - port, - is_private, - hardware_pricing_tier_instance.hw_to_id_map[hardware], - health, - min_replicas, - max_replicas, - env, - command, - command_args, - timeout, - ) - - click.echo(f"Inference deployment #{resp.id} created at https://{resp.endpoint_url}/") - - -@create.command(name="compute", help="Create a compute deployment") -@click.option("--name", "-n", prompt="Name", help="Name of the deployment") -@click.option("--image", "-i", prompt="Image", help="Container image") -@click.option("--hardware", "-h", prompt="Hardware", type=hardware_pricing_tier_instance, help="Hardware instance type") -@click.option("--username", prompt="Username", type=str, help="Username") -@click.option("--password", prompt="Password", hide_input=True, type=str, help="password") -@click.option("--ssh_key", prompt="Add ssh key", default="", type=str, help="Would you like to add an SSH key?") -def create_compute(name, image, hardware, username, password, ssh_key): - click.echo("Creating inference deployment with the following options:") - - # Call the API function for creating infrence deployment - resp = api.create_compute( - name, image, username, password, ssh_key, hardware_pricing_tier_instance.hw_to_id_map[hardware] - ) - - click.echo(f"Compute deployment #{resp.id} created at https://{resp.endpoint_url}/") - - @click.command(help="Delete a deployment") @click.argument("id", type=int) def delete(id): api.delete(id) + click.echo("Deployment has been deleted") @click.command(help="Pause a deployment") @click.argument("id", type=int) def pause(id): api.pause(id) + click.echo("Deployment has been paused") @click.command(help="Resume a deployment") @click.argument("id", type=int) def resume(id): api.resume(id) + click.echo("Deployment has been resumed") diff --git a/centml/cli/main.py b/centml/cli/main.py index dec73eea..216d8637 100644 --- a/centml/cli/main.py +++ b/centml/cli/main.py @@ -27,7 +27,7 @@ def ccluster(): ccluster.add_command(ls) ccluster.add_command(get) -ccluster.add_command(create) +#ccluster.add_command(create) ccluster.add_command(delete) ccluster.add_command(pause) ccluster.add_command(resume) diff --git a/centml/sdk/api.py b/centml/sdk/api.py index 0f9eb1a0..ff9ed72f 100644 --- a/centml/sdk/api.py +++ b/centml/sdk/api.py @@ -21,30 +21,29 @@ def get_api(): def get(depl_type): with get_api() as api: - results = api.get_deployments_deployments_get(type=depl_type).results + results = api.get_deployments_deployments_v2_get(type=depl_type).results deployments = sorted(results, reverse=True, key=lambda d: d.created_at) - - rows = [ - [d.id, d.name, d.type.value, d.status.value, d.created_at.strftime("%Y-%m-%d %H:%M:%S")] - for d in deployments - ] - - return rows + return deployments def get_status(id): with get_api() as api: - return api.get_deployment_status_deployments_status_deployment_id_get(id) + return api.get_deployment_status_deployments_v2_status_deployment_id_get(id) def get_inference(id): with get_api() as api: - return api.get_inference_deployment_deployments_inference_deployment_id_get(id) + return api.get_inference_deployment_deployments_v2_inference_deployment_id_get(id) def get_compute(id): with get_api() as api: - return api.get_compute_deployment_deployments_compute_deployment_id_get(id) + return api.get_compute_deployment_deployments_v2_compute_deployment_id_get(id) + + +def get_cserve(id): + with get_api() as api: + return api.get_cserve_deployment_deployments_v2_cserve_deployment_id_get(id) def create_inference( @@ -89,7 +88,7 @@ def create_compute(name, image, username, password, ssh_key, hw_to_id_map): def update_status(id, new_status): with get_api() as api: status_req = platform_api_python_client.DeploymentStatusRequest(status=new_status) - api.update_deployment_status_deployments_status_deployment_id_put(id, status_req) + api.update_deployment_status_deployments_v2_status_deployment_id_put(id, status_req) def delete(id): @@ -104,6 +103,6 @@ def resume(id): update_status(id, DeploymentStatus.ACTIVE) -def get_hardware_instances(): +def get_hardware_instances(cluster_id): with get_api() as api: - return api.get_hardware_instances_hardware_instances_get().results + return api.get_hardware_instances_hardware_instances_v2_get(cluster_id).results From 2aca22e87ba766c0533b3d1adc90ed44253a70f4 Mon Sep 17 00:00:00 2001 From: Anand J Date: Sun, 13 Oct 2024 21:12:18 -0400 Subject: [PATCH 10/15] A bunch of refactoring --- centml/cli/cluster.py | 226 +++++++++++++++++++++-------------------- centml/cli/main.py | 2 +- centml/sdk/__init__.py | 6 +- centml/sdk/api.py | 149 ++++++++++++--------------- 4 files changed, 181 insertions(+), 202 deletions(-) diff --git a/centml/cli/cluster.py b/centml/cli/cluster.py index b0a6a5a8..791843a3 100644 --- a/centml/cli/cluster.py +++ b/centml/cli/cluster.py @@ -1,28 +1,38 @@ import sys +from functools import wraps from typing import Dict import click from tabulate import tabulate -import platform_api_python_client -from platform_api_python_client.models.deployment_status import DeploymentStatus -from platform_api_python_client.models.health_status import HealthStatus -from centml.sdk import api +from centml.sdk import ( + DeploymentType, + DeploymentStatus, + HealthStatus, + ApiException, +) +from centml.sdk.api import get_centml_client -# Custom class to parse key-value pairs for env variables for inference deployment -class InferenceEnvType(click.ParamType): - name = "key_value" +depl_name_to_type_map = { + "inference": DeploymentType.INFERENCE_V2, + "compute": DeploymentType.COMPUTE_V2, + "cserve": DeploymentType.CSERVE, +} +depl_type_to_name_map = {v: k for k, v in depl_name_to_type_map.items()} - def convert(self, value, param, ctx): + +def handle_exception(func): + @wraps(func) + def wrapper(*args, **kwargs): try: - key, val = value.split('=', 1) - return key, val - except ValueError: - self.fail(f"{value} is not a valid key=value pair", param, ctx) - return None # to avoid warning from lint for inconsistent return statements + return func(*args, **kwargs) + except ApiException as e: + click.echo(f"Error: {e.reason}") + + return wrapper -def get_hw_to_id_map(cluster_id): - response = api.get_hardware_instances(cluster_id) +def _get_hw_to_id_map(cclient, cluster_id): + response = cclient.get_hardware_instances(cluster_id) # Initialize hashmap for hardware to id or vice versa mapping hw_to_id_map: Dict[str, int] = {} @@ -34,23 +44,15 @@ def get_hw_to_id_map(cluster_id): return hw_to_id_map, id_to_hw_map -depl_name_to_type_map = { - "inference": platform_api_python_client.DeploymentType.INFERENCE_V2, - "compute": platform_api_python_client.DeploymentType.COMPUTE_V2, - "cserve": platform_api_python_client.DeploymentType.CSERVE, -} -depl_type_to_name_map = {v: k for k, v in depl_name_to_type_map.items()} - - -def format_ssh_key(ssh_key): +def _format_ssh_key(ssh_key): if not ssh_key: return "No SSH Key Found" return ssh_key[:10] + '...' -def get_ready_status(deployment): +def _get_ready_status(cclient, deployment): api_status = deployment.status - service_status = api.get_status(deployment.id).service_status if deployment.status == DeploymentStatus.ACTIVE else None + service_status = cclient.get_status(deployment.id).service_status if deployment.status == DeploymentStatus.ACTIVE else None status_styles = { (DeploymentStatus.PAUSED, None): ("paused", "yellow", "black"), @@ -69,120 +71,124 @@ def get_ready_status(deployment): @click.command(help="List all deployments") @click.argument("type", type=click.Choice(depl_name_to_type_map.keys()), required=False, default=None) def ls(type): - depl_type = depl_name_to_type_map[type] if type in depl_name_to_type_map else None - deployments = api.get(depl_type) - rows = [ - [d.id, d.name, depl_type_to_name_map[d.type], d.status.value, d.created_at.strftime("%Y-%m-%d %H:%M:%S")] - for d in deployments - ] - - click.echo( - tabulate( - rows, - headers=["ID", "Name", "Type", "Status", "Created at"], - tablefmt="rounded_outline", - disable_numparse=True, + with get_centml_client() as cclient: + depl_type = depl_name_to_type_map[type] if type in depl_name_to_type_map else None + deployments = cclient.get(depl_type) + rows = [ + [d.id, d.name, depl_type_to_name_map[d.type], d.status.value, d.created_at.strftime("%Y-%m-%d %H:%M:%S")] + for d in deployments + ] + + click.echo( + tabulate( + rows, + headers=["ID", "Name", "Type", "Status", "Created at"], + tablefmt="rounded_outline", + disable_numparse=True, + ) ) - ) @click.command(help="Get deployment details") @click.argument("type", type=click.Choice(depl_name_to_type_map.keys())) @click.argument("id", type=int) +@handle_exception def get(type, id): - depl_type = depl_name_to_type_map[type] - - if depl_type == platform_api_python_client.DeploymentType.INFERENCE_V2: - deployment = api.get_inference(id) - elif depl_type == platform_api_python_client.DeploymentType.COMPUTE_V2: - deployment = api.get_compute(id) - elif depl_type == platform_api_python_client.DeploymentType.CSERVE: - deployment = api.get_cserve(id) - else: - sys.exit("Please enter correct deployment type") - - ready_status = get_ready_status(deployment) - _, id_to_hw_map = get_hw_to_id_map(deployment.cluster_id) - hw = id_to_hw_map[deployment.hardware_instance_id] - - click.echo( - tabulate( - [ - ("Name", deployment.name), - ("Status", ready_status), - ("Endpoint", deployment.endpoint_url), - ("Created at", deployment.created_at.strftime("%Y-%m-%d %H:%M:%S")), - ("Hardware", f"{hw.name} ({hw.num_gpu}x {hw.gpu_type})"), - ("Cost", f"{hw.cost_per_hr/100} credits/hr"), - ], - tablefmt="rounded_outline", - disable_numparse=True, - ) - ) + with get_centml_client() as cclient: + depl_type = depl_name_to_type_map[type] + + if depl_type == DeploymentType.INFERENCE_V2: + deployment = cclient.get_inference(id) + elif depl_type == DeploymentType.COMPUTE_V2: + deployment = cclient.get_compute(id) + elif depl_type == DeploymentType.CSERVE: + deployment = cclient.get_cserve(id) + else: + sys.exit("Please enter correct deployment type") + + ready_status = _get_ready_status(cclient, deployment) + _, id_to_hw_map = _get_hw_to_id_map(cclient, deployment.cluster_id) + hw = id_to_hw_map[deployment.hardware_instance_id] - click.echo("Additional deployment configurations:") - if depl_type == platform_api_python_client.DeploymentType.INFERENCE_V2: click.echo( tabulate( [ - ("Image", deployment.image_url), - ("Container port", deployment.container_port), - ("Healthcheck", deployment.healthcheck or "/"), - ("Replicas", {"min": deployment.min_scale, "max": deployment.max_scale}), - ("Environment variables", deployment.env_vars or "None"), - ("Max concurrency", deployment.concurrency or "None"), + ("Name", deployment.name), + ("Status", ready_status), + ("Endpoint", deployment.endpoint_url), + ("Created at", deployment.created_at.strftime("%Y-%m-%d %H:%M:%S")), + ("Hardware", f"{hw.name} ({hw.num_gpu}x {hw.gpu_type})"), + ("Cost", f"{hw.cost_per_hr/100} credits/hr"), ], tablefmt="rounded_outline", disable_numparse=True, ) ) - elif depl_type == platform_api_python_client.DeploymentType.COMPUTE_V2: - click.echo( - tabulate( - [ - ("Username", "centml"), - ("SSH key", format_ssh_key(deployment.ssh_public_key)), - ], - tablefmt="rounded_outline", - disable_numparse=True, + + click.echo("Additional deployment configurations:") + if depl_type == DeploymentType.INFERENCE_V2: + click.echo( + tabulate( + [ + ("Image", deployment.image_url), + ("Container port", deployment.container_port), + ("Healthcheck", deployment.healthcheck or "/"), + ("Replicas", {"min": deployment.min_scale, "max": deployment.max_scale}), + ("Environment variables", deployment.env_vars or "None"), + ("Max concurrency", deployment.concurrency or "None"), + ], + tablefmt="rounded_outline", + disable_numparse=True, + ) ) - ) - elif depl_type == platform_api_python_client.DeploymentType.CSERVE: - click.echo( - tabulate( - [ - ("Hugging face model", deployment.model), - ("Parallelism", {"tensor": deployment.tensor_parallel_size, "pipeline": deployment.pipeline_parallel_size}), - ("Replicas", {"min": deployment.min_scale, "max": deployment.max_scale}), - ("Max concurrency", deployment.concurrency or "None"), - ], - tablefmt="rounded_outline", - disable_numparse=True, + elif depl_type == DeploymentType.COMPUTE_V2: + click.echo( + tabulate( + [ + ("Username", "centml"), + ("SSH key", format_ssh_key(deployment.ssh_public_key)), + ], + tablefmt="rounded_outline", + disable_numparse=True, + ) + ) + elif depl_type == DeploymentType.CSERVE: + click.echo( + tabulate( + [ + ("Hugging face model", deployment.model), + ("Parallelism", {"tensor": deployment.tensor_parallel_size, "pipeline": deployment.pipeline_parallel_size}), + ("Replicas", {"min": deployment.min_scale, "max": deployment.max_scale}), + ("Max concurrency", deployment.concurrency or "None"), + ], + tablefmt="rounded_outline", + disable_numparse=True, + ) ) - ) - - -@click.group(help="Create a new deployment") -def create(): - pass @click.command(help="Delete a deployment") @click.argument("id", type=int) +@handle_exception def delete(id): - api.delete(id) - click.echo("Deployment has been deleted") + with get_centml_client() as cclient: + cclient.delete(id) + click.echo("Deployment has been deleted") @click.command(help="Pause a deployment") @click.argument("id", type=int) +@handle_exception def pause(id): - api.pause(id) - click.echo("Deployment has been paused") + with get_centml_client() as cclient: + cclient.pause(id) + click.echo("Deployment has been paused") @click.command(help="Resume a deployment") @click.argument("id", type=int) +@handle_exception def resume(id): - api.resume(id) - click.echo("Deployment has been resumed") + with get_centml_client() as cclient: + cclient.resume(id) + click.echo("Deployment has been resumed") diff --git a/centml/cli/main.py b/centml/cli/main.py index 216d8637..a438088a 100644 --- a/centml/cli/main.py +++ b/centml/cli/main.py @@ -1,7 +1,7 @@ import click from centml.cli.login import login, logout -from centml.cli.cluster import ls, get, create, delete, pause, resume +from centml.cli.cluster import ls, get, delete, pause, resume @click.group() diff --git a/centml/sdk/__init__.py b/centml/sdk/__init__.py index 22542871..2bed9e77 100644 --- a/centml/sdk/__init__.py +++ b/centml/sdk/__init__.py @@ -1,6 +1,2 @@ +from platform_api_python_client import * from . import api, auth - -__all__ = [ - "api", - "auth", -] diff --git a/centml/sdk/api.py b/centml/sdk/api.py index ff9ed72f..d4c95883 100644 --- a/centml/sdk/api.py +++ b/centml/sdk/api.py @@ -1,108 +1,85 @@ -import contextlib +from contextlib import contextmanager + import platform_api_python_client -from platform_api_python_client.models.deployment_status import DeploymentStatus +from platform_api_python_client.rest import ApiException +from platform_api_python_client import ( + DeploymentStatus, + CreateInferenceDeploymentV2Request, + CreateComputeDeploymentV2Request, + CreateCServeDeploymentRequest, +) from centml.sdk import auth from centml.sdk.config import settings from centml.sdk.utils import client_certs -@contextlib.contextmanager -def get_api(): - configuration = platform_api_python_client.Configuration( - host=settings.CENTML_PLATFORM_API_URL, access_token=auth.get_centml_token() - ) +class CentMLClient: + def __init__(self, api): + self._api = api - with platform_api_python_client.ApiClient(configuration) as api_client: - api_instance = platform_api_python_client.EXTERNALApi(api_client) + def get(self, depl_type): + results = self._api.get_deployments_deployments_v2_get(type=depl_type).results + deployments = sorted(results, reverse=True, key=lambda d: d.created_at) + return deployments - yield api_instance + def get_status(self, id): + return self._api.get_deployment_status_deployments_v2_status_deployment_id_get(id) + def get_inference(self, id): + return self._api.get_inference_deployment_deployments_v2_inference_deployment_id_get(id) -def get(depl_type): - with get_api() as api: - results = api.get_deployments_deployments_v2_get(type=depl_type).results - deployments = sorted(results, reverse=True, key=lambda d: d.created_at) - return deployments + def get_compute(self, id): + return self._api.get_compute_deployment_deployments_v2_compute_deployment_id_get(id) + def get_cserve(self, id): + return self._api.get_cserve_deployment_deployments_v2_cserve_deployment_id_get(id) -def get_status(id): - with get_api() as api: - return api.get_deployment_status_deployments_v2_status_deployment_id_get(id) - - -def get_inference(id): - with get_api() as api: - return api.get_inference_deployment_deployments_v2_inference_deployment_id_get(id) - - -def get_compute(id): - with get_api() as api: - return api.get_compute_deployment_deployments_v2_compute_deployment_id_get(id) - - -def get_cserve(id): - with get_api() as api: - return api.get_cserve_deployment_deployments_v2_cserve_deployment_id_get(id) - - -def create_inference( - name, image, port, is_private, hw_to_id_map, health, min_replicas, max_replicas, env, command, command_args, timeout -): - triplet = None - if is_private: - triplet = client_certs.generate_ca_client_triplet(name) - # Handle automatic download of client private secrets - client_certs.save_pem_file(name, triplet.client_private_key, triplet.client_certificate) - with get_api() as api: - req = platform_api_python_client.CreateInferenceDeploymentRequest( - name=name, - image_url=image, - port=port, - hardware_instance_id=hw_to_id_map, - healthcheck=health, - min_replicas=min_replicas, - max_replicas=max_replicas, - env_vars=dict(env) if dict(env) else None, - command=[command] if command else None, - command_args=(list(command_args) if command and len(list(command_args)) > 0 else None), - timeout=timeout, - endpoint_certificate_authority=triplet.certificate_authority if triplet else None, - ) - return api.create_inference_deployment_deployments_inference_post(req) - - -def create_compute(name, image, username, password, ssh_key, hw_to_id_map): - with get_api() as api: - req = platform_api_python_client.CreateComputeDeploymentRequest( - name=name, - image_url=image, - hardware_instance_id=hw_to_id_map, - username=username, - password=password, - ssh_key=ssh_key if ssh_key else None, - ) - return api.create_compute_deployment_deployments_compute_post(req) - - -def update_status(id, new_status): - with get_api() as api: + def create_inference( + self, + request: CreateInferenceDeploymentV2Request, + ): + return self._api.create_inference_deployment_deployments_v2_inference_post(request) + + def create_compute( + self, + request: CreateComputeDeploymentV2Request, + ): + return self._api.create_compute_deployment_deployments_compute_post(request) + + def create_cserve( + self, + request: CreateCServeDeploymentRequest, + ): + return self._api.create_cserve_deployment_deployments_v2_cserve_post(request) + + def _update_status(self, id, new_status): status_req = platform_api_python_client.DeploymentStatusRequest(status=new_status) - api.update_deployment_status_deployments_v2_status_deployment_id_put(id, status_req) + self._api.update_deployment_status_deployments_v2_status_deployment_id_put(id, status_req) + def delete(self, id): + self._update_status(id, DeploymentStatus.DELETED) -def delete(id): - update_status(id, DeploymentStatus.DELETED) + def pause(self, id): + self._update_status(id, DeploymentStatus.PAUSED) + def resume(self, id): + self._update_status(id, DeploymentStatus.ACTIVE) -def pause(id): - update_status(id, DeploymentStatus.PAUSED) + def get_clusters(self): + return self._api.get_clusters_clusters_get() + def get_hardware_instances(self, cluster_id): + return self._api.get_hardware_instances_hardware_instances_v2_get(cluster_id).results -def resume(id): - update_status(id, DeploymentStatus.ACTIVE) +@contextmanager +def get_centml_client(): + configuration = platform_api_python_client.Configuration( + host=settings.CENTML_PLATFORM_API_URL, access_token=auth.get_centml_token() + ) + + with platform_api_python_client.ApiClient(configuration) as api_client: + api_instance = platform_api_python_client.EXTERNALApi(api_client) -def get_hardware_instances(cluster_id): - with get_api() as api: - return api.get_hardware_instances_hardware_instances_v2_get(cluster_id).results + yield CentMLClient(api_instance) From f968b8db63fce3b601510f7fceda2a9d35844a8f Mon Sep 17 00:00:00 2001 From: Anand J Date: Sun, 13 Oct 2024 21:26:47 -0400 Subject: [PATCH 11/15] Lint fix --- centml/cli/cluster.py | 22 ++++++++++------------ centml/cli/main.py | 1 - centml/compiler/__init__.py | 4 +--- centml/sdk/api.py | 17 +++-------------- 4 files changed, 14 insertions(+), 30 deletions(-) diff --git a/centml/cli/cluster.py b/centml/cli/cluster.py index 791843a3..7eceb3c5 100644 --- a/centml/cli/cluster.py +++ b/centml/cli/cluster.py @@ -3,12 +3,7 @@ from typing import Dict import click from tabulate import tabulate -from centml.sdk import ( - DeploymentType, - DeploymentStatus, - HealthStatus, - ApiException, -) +from centml.sdk import DeploymentType, DeploymentStatus, HealthStatus, ApiException, HardwareInstanceResponse from centml.sdk.api import get_centml_client @@ -27,6 +22,7 @@ def wrapper(*args, **kwargs): return func(*args, **kwargs) except ApiException as e: click.echo(f"Error: {e.reason}") + return None return wrapper @@ -52,7 +48,9 @@ def _format_ssh_key(ssh_key): def _get_ready_status(cclient, deployment): api_status = deployment.status - service_status = cclient.get_status(deployment.id).service_status if deployment.status == DeploymentStatus.ACTIVE else None + service_status = ( + cclient.get_status(deployment.id).service_status if deployment.status == DeploymentStatus.ACTIVE else None + ) status_styles = { (DeploymentStatus.PAUSED, None): ("paused", "yellow", "black"), @@ -144,10 +142,7 @@ def get(type, id): elif depl_type == DeploymentType.COMPUTE_V2: click.echo( tabulate( - [ - ("Username", "centml"), - ("SSH key", format_ssh_key(deployment.ssh_public_key)), - ], + [("Username", "centml"), ("SSH key", _format_ssh_key(deployment.ssh_public_key))], tablefmt="rounded_outline", disable_numparse=True, ) @@ -157,7 +152,10 @@ def get(type, id): tabulate( [ ("Hugging face model", deployment.model), - ("Parallelism", {"tensor": deployment.tensor_parallel_size, "pipeline": deployment.pipeline_parallel_size}), + ( + "Parallelism", + {"tensor": deployment.tensor_parallel_size, "pipeline": deployment.pipeline_parallel_size}, + ), ("Replicas", {"min": deployment.min_scale, "max": deployment.max_scale}), ("Max concurrency", deployment.concurrency or "None"), ], diff --git a/centml/cli/main.py b/centml/cli/main.py index a438088a..64dfa645 100644 --- a/centml/cli/main.py +++ b/centml/cli/main.py @@ -27,7 +27,6 @@ def ccluster(): ccluster.add_command(ls) ccluster.add_command(get) -#ccluster.add_command(create) ccluster.add_command(delete) ccluster.add_command(pause) ccluster.add_command(resume) diff --git a/centml/compiler/__init__.py b/centml/compiler/__init__.py index 247852f2..03d6a075 100644 --- a/centml/compiler/__init__.py +++ b/centml/compiler/__init__.py @@ -1,5 +1,3 @@ from centml.compiler.main import compile -all = [ - "compile", -] +all = ["compile"] diff --git a/centml/sdk/api.py b/centml/sdk/api.py index d4c95883..4a918d97 100644 --- a/centml/sdk/api.py +++ b/centml/sdk/api.py @@ -1,7 +1,6 @@ from contextlib import contextmanager import platform_api_python_client -from platform_api_python_client.rest import ApiException from platform_api_python_client import ( DeploymentStatus, CreateInferenceDeploymentV2Request, @@ -11,7 +10,6 @@ from centml.sdk import auth from centml.sdk.config import settings -from centml.sdk.utils import client_certs class CentMLClient: @@ -35,22 +33,13 @@ def get_compute(self, id): def get_cserve(self, id): return self._api.get_cserve_deployment_deployments_v2_cserve_deployment_id_get(id) - def create_inference( - self, - request: CreateInferenceDeploymentV2Request, - ): + def create_inference(self, request: CreateInferenceDeploymentV2Request): return self._api.create_inference_deployment_deployments_v2_inference_post(request) - def create_compute( - self, - request: CreateComputeDeploymentV2Request, - ): + def create_compute(self, request: CreateComputeDeploymentV2Request): return self._api.create_compute_deployment_deployments_compute_post(request) - def create_cserve( - self, - request: CreateCServeDeploymentRequest, - ): + def create_cserve(self, request: CreateCServeDeploymentRequest): return self._api.create_cserve_deployment_deployments_v2_cserve_post(request) def _update_status(self, id, new_status): From 222b9e3e31bd77e8334825bb906ca9c13a1c7a36 Mon Sep 17 00:00:00 2001 From: Anand J Date: Sun, 13 Oct 2024 21:38:06 -0400 Subject: [PATCH 12/15] lint and format test fix --- centml/__init__.py | 2 +- centml/cli/cluster.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/centml/__init__.py b/centml/__init__.py index 8b137891..94818772 100644 --- a/centml/__init__.py +++ b/centml/__init__.py @@ -1 +1 @@ - +# centml diff --git a/centml/cli/cluster.py b/centml/cli/cluster.py index 7eceb3c5..989f3249 100644 --- a/centml/cli/cluster.py +++ b/centml/cli/cluster.py @@ -67,7 +67,7 @@ def _get_ready_status(cclient, deployment): @click.command(help="List all deployments") -@click.argument("type", type=click.Choice(depl_name_to_type_map.keys()), required=False, default=None) +@click.argument("type", type=click.Choice(list(depl_name_to_type_map.keys())), required=False, default=None) def ls(type): with get_centml_client() as cclient: depl_type = depl_name_to_type_map[type] if type in depl_name_to_type_map else None @@ -88,7 +88,7 @@ def ls(type): @click.command(help="Get deployment details") -@click.argument("type", type=click.Choice(depl_name_to_type_map.keys())) +@click.argument("type", type=click.Choice(list(depl_name_to_type_map.keys()))) @click.argument("id", type=int) @handle_exception def get(type, id): From 56860daab85b758d41768665faa66dd6481d1854 Mon Sep 17 00:00:00 2001 From: Anand J Date: Sun, 13 Oct 2024 21:58:37 -0400 Subject: [PATCH 13/15] Changes in test --- centml/__init__.py | 2 +- tests/test_backend.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/centml/__init__.py b/centml/__init__.py index 94818772..551291ce 100644 --- a/centml/__init__.py +++ b/centml/__init__.py @@ -1 +1 @@ -# centml +from centml.compiler.main import compile diff --git a/tests/test_backend.py b/tests/test_backend.py index d55aca4f..8bd08ad3 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -5,6 +5,7 @@ import torch from parameterized import parameterized_class from torch.fx import GraphModule +import centml from centml.compiler.backend import Runner from centml.compiler.config import CompilationStatus, settings from .test_helpers import MODEL_SUITE @@ -42,12 +43,12 @@ def test_no_serialized_model(self): @patch("centml.compiler.backend.get_backend_compiled_forward_path", side_effect=Exception("Exiting early")) def test_model_id_consistency(self, mock_get_path): # self.model and self.inputs come from @parameterized_class - model_compiled_1 = torch.compile(self.model, backend="centml") + model_compiled_1 = centml.compile(self.model) model_compiled_1(self.inputs) hash_1 = mock_get_path.call_args[0][0] torch._dynamo.reset() # Reset the dynamo cache to force recompilation - model_compiled_2 = torch.compile(self.model, backend="centml") + model_compiled_2 = centml.compile(self.model) model_compiled_2(self.inputs) hash_2 = mock_get_path.call_args[0][0] torch._dynamo.reset() @@ -68,12 +69,12 @@ def get_modified_model(model): return modified # self.model and self.inputs come from @parameterized_class - model_compiled_1 = torch.compile(self.model, backend="centml") + model_compiled_1 = centml.compile(self.model) model_compiled_1(self.inputs) hash_1 = mock_get_path.call_args[0][0] model_2 = get_modified_model(self.model) - model_compiled_2 = torch.compile(model_2, backend="centml") + model_compiled_2 = centml.compile(model_2) model_compiled_2(self.inputs) hash_2 = mock_get_path.call_args[0][0] @@ -232,7 +233,7 @@ def call_remote_compilation(self): with patch("threading.Thread.start", new=start_func), patch( "centml.compiler.backend.Runner.__call__", new=self.model.forward ): - compiled_model = torch.compile(self.model, backend="centml") + compiled_model = centml.compile(self.model) compiled_model(self.inputs) torch._dynamo.reset() From 8dc93e5fca16d82c7aa2b4933772a663701131cc Mon Sep 17 00:00:00 2001 From: Anand J Date: Sun, 13 Oct 2024 22:13:38 -0400 Subject: [PATCH 14/15] pip build fix --- centml/compiler/main.py | 9 +++++---- requirements-dev.txt | 2 -- requirements.txt | 12 ------------ setup.py | 25 ++++++++++++++----------- 4 files changed, 19 insertions(+), 29 deletions(-) delete mode 100644 requirements.txt diff --git a/centml/compiler/main.py b/centml/compiler/main.py index e8021ffe..d3efee1e 100644 --- a/centml/compiler/main.py +++ b/centml/compiler/main.py @@ -1,11 +1,7 @@ import builtins from typing import Callable, Dict, Optional, Union -import torch - -from centml.compiler.backend import centml_dynamo_backend from centml.compiler.config import OperationMode, settings -from centml.compiler.prediction.backend import centml_prediction_backend, get_gauge def compile( @@ -17,8 +13,11 @@ def compile( options: Optional[Dict[str, Union[str, builtins.int, builtins.bool]]] = None, disable: builtins.bool = False, ) -> Callable: + import torch if settings.CENTML_MODE == OperationMode.REMOTE_COMPILATION: + from centml.compiler.backend import centml_dynamo_backend + # Return the remote-compiled model compiled_model = torch.compile( model, @@ -31,6 +30,8 @@ def compile( ) return compiled_model elif settings.CENTML_MODE == OperationMode.PREDICTION: + from centml.compiler.prediction.backend import centml_prediction_backend, get_gauge + # Proceed with prediction workflow compiled_model = torch.compile( model, diff --git a/requirements-dev.txt b/requirements-dev.txt index 6179c207..da65db1b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,3 @@ --r requirements.txt - torch==2.3.1 black>=23.10.0 pylint>=3.0.1 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index d33e1467..00000000 --- a/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -fastapi>=0.103.0 -uvicorn>=0.23.0 -python-multipart>=0.0.6 -pydantic-settings==2.0.* -Requests==2.32.2 -tabulate>=0.9.0 -pyjwt>=2.8.0 -cryptography==43.0.1 -prometheus-client>=0.20.0 -scipy>=1.6.0 -scikit-learn>=1.5.1 -platform_api_python_client @ git+https://github.com/CentML/platform_api_python_client.git@main diff --git a/setup.py b/setup.py index 93ced9fb..199ca854 100644 --- a/setup.py +++ b/setup.py @@ -1,15 +1,5 @@ from setuptools import setup, find_packages -REQUIRES = [] -with open('requirements.txt') as f: - for line in f: - line, _, _ = line.partition('#') - line = line.strip() - if not line or line.startswith('setuptools'): - continue - - REQUIRES.append(line) - setup( name='centml', version='0.1.0', @@ -21,6 +11,19 @@ "ccluster = centml.cli:ccluster", ], }, - install_requires=REQUIRES + install_requires=[ + "fastapi>=0.103.0", + "uvicorn>=0.23.0", + "python-multipart>=0.0.6", + "pydantic-settings==2.0.*", + "Requests==2.32.2", + "tabulate>=0.9.0", + "pyjwt>=2.8.0", + "cryptography==43.0.1", + "prometheus-client>=0.20.0", + "scipy>=1.6.0", + "scikit-learn>=1.5.1", + "platform_api_python_client @ git+https://github.com/CentML/platform_api_python_client.git@v0.1-rc1", + ], ) From 75921d8f6779810e72b60d823a8d26c4b2b515d0 Mon Sep 17 00:00:00 2001 From: Anand J Date: Sun, 13 Oct 2024 22:16:09 -0400 Subject: [PATCH 15/15] Add publish.yml --- .github/workflows/publish.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..b734509a --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,35 @@ +name: publish + +on: + release: + types: [published] + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + id-token: write + steps: + - uses: actions/checkout@v3 + - name: setup-python + uses: actions/setup-python@v3 + with: + python-version: "3.11" + architecture: "x64" + - name: install pypa/build + run: >- + python -m + pip install + build + --user + - name: build sdist(tarball) and bdist(wheel) to dist/ + run: >- # = python -m build . works the same way by default + python -m + build + --sdist + --wheel + --outdir dist/ + - name: publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://upload.pypi.org/legacy/