From a65f8c51e38b56f379c36a25631e583187cd28f4 Mon Sep 17 00:00:00 2001 From: Chuyang Deng Date: Wed, 22 Jul 2020 17:34:05 -0700 Subject: [PATCH 1/3] change: deprecate unused parameters --- doc/algorithms/randomcutforest.rst | 2 +- src/sagemaker/amazon/kmeans.py | 12 ----- src/sagemaker/amazon/randomcutforest.py | 13 ----- src/sagemaker/chainer/estimator.py | 1 - src/sagemaker/estimator.py | 16 ------- src/sagemaker/model.py | 7 --- src/sagemaker/mxnet/estimator.py | 1 - src/sagemaker/pytorch/estimator.py | 1 - src/sagemaker/rl/estimator.py | 1 - src/sagemaker/sklearn/estimator.py | 1 - src/sagemaker/workflow/airflow.py | 3 -- src/sagemaker/xgboost/estimator.py | 1 - tests/component/test_mxnet_estimator.py | 1 - tests/component/test_tf_estimator.py | 1 - tests/integ/test_airflow_config.py | 2 - tests/integ/test_data_capture_config.py | 1 - tests/integ/test_kmeans.py | 3 -- tests/integ/test_multidatamodel.py | 1 - .../sagemaker/model/test_framework_model.py | 4 -- .../sagemaker/tensorflow/test_estimator.py | 3 -- .../tensorflow/test_estimator_attach.py | 4 -- tests/unit/test_airflow.py | 11 ----- tests/unit/test_amazon_estimator.py | 1 - tests/unit/test_chainer.py | 8 ---- tests/unit/test_estimator.py | 47 ------------------- tests/unit/test_job.py | 1 - tests/unit/test_kmeans.py | 4 +- tests/unit/test_linear_learner.py | 2 +- tests/unit/test_multidatamodel.py | 1 - tests/unit/test_mxnet.py | 8 ---- tests/unit/test_pipeline_model.py | 1 - tests/unit/test_pytorch.py | 7 --- tests/unit/test_randomcutforest.py | 10 +--- tests/unit/test_rl.py | 5 -- tests/unit/test_sklearn.py | 7 --- tests/unit/test_xgboost.py | 8 ---- 36 files changed, 5 insertions(+), 195 deletions(-) diff --git a/doc/algorithms/randomcutforest.rst b/doc/algorithms/randomcutforest.rst index b723319a52..bfdef51558 100644 --- a/doc/algorithms/randomcutforest.rst +++ b/doc/algorithms/randomcutforest.rst @@ -8,7 +8,7 @@ The Amazon SageMaker Random Cut Forest algorithm. :undoc-members: :show-inheritance: :inherited-members: - :exclude-members: image_uri, num_trees, num_samples_per_tree, eval_metrics, feature_dim, MINI_BATCH_SIZE + :exclude-members: image_uri, num_trees, num_samples_per_tree, feature_dim, MINI_BATCH_SIZE .. autoclass:: sagemaker.RandomCutForestModel diff --git a/src/sagemaker/amazon/kmeans.py b/src/sagemaker/amazon/kmeans.py index bc771292b1..3e321a33d0 100644 --- a/src/sagemaker/amazon/kmeans.py +++ b/src/sagemaker/amazon/kmeans.py @@ -43,11 +43,6 @@ class KMeans(AmazonAlgorithmEstimatorBase): ) epochs = hp("epochs", gt(0), "An integer greater-than 0", int) center_factor = hp("extra_center_factor", gt(0), "An integer greater-than 0", int) - eval_metrics = hp( - name="eval_metrics", - validation_message='A comma separated list of "msd" or "ssd"', - data_type=list, - ) def __init__( self, @@ -63,7 +58,6 @@ def __init__( half_life_time_size=None, epochs=None, center_factor=None, - eval_metrics=None, **kwargs ): """A k-means clustering @@ -130,11 +124,6 @@ def __init__( center_factor (int): The algorithm will create ``num_clusters * extra_center_factor`` as it runs and reduce the number of centers to ``k`` when finalizing - eval_metrics (list): JSON list of metrics types to be used for - reporting the score for the model. Allowed values are "msd" - Means Square Error, "ssd": Sum of square distance. If test data - is provided, the score shall be reported in terms of all - requested metrics. **kwargs: base class keyword argument values. .. tip:: @@ -153,7 +142,6 @@ def __init__( self.half_life_time_size = half_life_time_size self.epochs = epochs self.center_factor = center_factor - self.eval_metrics = eval_metrics def create_model(self, vpc_config_override=VPC_CONFIG_DEFAULT, **kwargs): """Return a :class:`~sagemaker.amazon.kmeans.KMeansModel` referencing diff --git a/src/sagemaker/amazon/randomcutforest.py b/src/sagemaker/amazon/randomcutforest.py index 6a141150e8..eacfe08305 100644 --- a/src/sagemaker/amazon/randomcutforest.py +++ b/src/sagemaker/amazon/randomcutforest.py @@ -31,12 +31,6 @@ class RandomCutForest(AmazonAlgorithmEstimatorBase): repo_version = 1 MINI_BATCH_SIZE = 1000 - eval_metrics = hp( - name="eval_metrics", - validation_message='A comma separated list of "accuracy" or "precision_recall_fscore"', - data_type=list, - ) - num_trees = hp("num_trees", (ge(50), le(1000)), "An integer in [50, 1000]", int) num_samples_per_tree = hp( "num_samples_per_tree", (ge(1), le(2048)), "An integer in [1, 2048]", int @@ -50,7 +44,6 @@ def __init__( instance_type, num_samples_per_tree=None, num_trees=None, - eval_metrics=None, **kwargs ): """RandomCutForest is :class:`Estimator` used for anomaly detection. @@ -99,11 +92,6 @@ def __init__( build each tree in the forest. The total number of samples drawn from the train dataset is num_trees * num_samples_per_tree. num_trees (int): Optional. The number of trees used in the forest. - eval_metrics (list): Optional. JSON list of metrics types to be used - for reporting the score for the model. Allowed values are - "accuracy", "precision_recall_fscore": positive and negative - precision, recall, and f1 scores. If test data is provided, the - score shall be reported in terms of all requested metrics. **kwargs: base class keyword argument values. .. tip:: @@ -116,7 +104,6 @@ def __init__( super(RandomCutForest, self).__init__(role, instance_count, instance_type, **kwargs) self.num_samples_per_tree = num_samples_per_tree self.num_trees = num_trees - self.eval_metrics = eval_metrics def create_model(self, vpc_config_override=VPC_CONFIG_DEFAULT, **kwargs): """Return a :class:`~sagemaker.amazon.RandomCutForestModel` referencing diff --git a/src/sagemaker/chainer/estimator.py b/src/sagemaker/chainer/estimator.py index 1fb1115832..e435caee1a 100644 --- a/src/sagemaker/chainer/estimator.py +++ b/src/sagemaker/chainer/estimator.py @@ -216,7 +216,6 @@ def create_model( role or self.role, entry_point or self._model_entry_point(), source_dir=(source_dir or self._model_source_dir()), - enable_cloudwatch_metrics=self.enable_cloudwatch_metrics, container_log_level=self.container_log_level, code_location=self.code_location, py_version=self.py_version, diff --git a/src/sagemaker/estimator.py b/src/sagemaker/estimator.py index fe8d537016..a107e27bf2 100644 --- a/src/sagemaker/estimator.py +++ b/src/sagemaker/estimator.py @@ -17,7 +17,6 @@ import logging import os import uuid -import warnings from abc import ABCMeta from abc import abstractmethod @@ -46,7 +45,6 @@ from sagemaker.model import ( SCRIPT_PARAM_NAME, DIR_PARAM_NAME, - CLOUDWATCH_METRICS_PARAM_NAME, CONTAINER_LOG_LEVEL_PARAM_NAME, JOB_NAME_PARAM_NAME, SAGEMAKER_REGION_PARAM_NAME, @@ -1433,7 +1431,6 @@ def __init__( entry_point, source_dir=None, hyperparameters=None, - enable_cloudwatch_metrics=False, container_log_level=logging.INFO, code_location=None, image_uri=None, @@ -1491,9 +1488,6 @@ def __init__( SageMaker. For convenience, this accepts other types for keys and values, but ``str()`` will be called to convert them before training. - enable_cloudwatch_metrics (bool): [DEPRECATED] Now there are - cloudwatch metrics emitted by all SageMaker training jobs. This - will be ignored for now and removed in a further release. container_log_level (int): Log level to use within the container (default: logging.INFO). Valid values are defined in the Python logging module. @@ -1624,12 +1618,6 @@ def __init__( self.dependencies = dependencies or [] self.uploaded_code = None - if enable_cloudwatch_metrics: - warnings.warn( - "enable_cloudwatch_metrics is now deprecated and will be removed in the future.", - DeprecationWarning, - ) - self.enable_cloudwatch_metrics = False self.container_log_level = container_log_level self.code_location = code_location self.image_uri = image_uri @@ -1687,7 +1675,6 @@ def _prepare_for_training(self, job_name=None): # Modify hyperparameters in-place to point to the right code directory and script URIs self._hyperparameters[DIR_PARAM_NAME] = code_dir self._hyperparameters[SCRIPT_PARAM_NAME] = script - self._hyperparameters[CLOUDWATCH_METRICS_PARAM_NAME] = self.enable_cloudwatch_metrics self._hyperparameters[CONTAINER_LOG_LEVEL_PARAM_NAME] = self.container_log_level self._hyperparameters[JOB_NAME_PARAM_NAME] = self._current_job_name self._hyperparameters[SAGEMAKER_REGION_PARAM_NAME] = self.sagemaker_session.boto_region_name @@ -1798,9 +1785,6 @@ class constructor init_params["hyperparameters"].get(SCRIPT_PARAM_NAME) ) init_params["source_dir"] = json.loads(init_params["hyperparameters"].get(DIR_PARAM_NAME)) - init_params["enable_cloudwatch_metrics"] = json.loads( - init_params["hyperparameters"].get(CLOUDWATCH_METRICS_PARAM_NAME) - ) init_params["container_log_level"] = json.loads( init_params["hyperparameters"].get(CONTAINER_LOG_LEVEL_PARAM_NAME) ) diff --git a/src/sagemaker/model.py b/src/sagemaker/model.py index e030bb3439..05e1aef067 100644 --- a/src/sagemaker/model.py +++ b/src/sagemaker/model.py @@ -599,7 +599,6 @@ def delete_model(self): SCRIPT_PARAM_NAME = "sagemaker_program" DIR_PARAM_NAME = "sagemaker_submit_directory" -CLOUDWATCH_METRICS_PARAM_NAME = "sagemaker_enable_cloudwatch_metrics" CONTAINER_LOG_LEVEL_PARAM_NAME = "sagemaker_container_log_level" JOB_NAME_PARAM_NAME = "sagemaker_job_name" MODEL_SERVER_WORKERS_PARAM_NAME = "sagemaker_model_server_workers" @@ -624,7 +623,6 @@ def __init__( predictor_cls=None, env=None, name=None, - enable_cloudwatch_metrics=False, container_log_level=logging.INFO, code_location=None, sagemaker_session=None, @@ -682,9 +680,6 @@ def __init__( when hosted in SageMaker (default: None). name (str): The model name. If None, a default model name will be selected on each ``deploy``. - enable_cloudwatch_metrics (bool): Whether training and hosting - containers will generate CloudWatch metrics under the - AWS/SageMakerContainer namespace (default: False). container_log_level (int): Log level to use within the container (default: logging.INFO). Valid values are defined in the Python logging module. @@ -792,7 +787,6 @@ def __init__( self.source_dir = source_dir self.dependencies = dependencies or [] self.git_config = git_config - self.enable_cloudwatch_metrics = enable_cloudwatch_metrics self.container_log_level = container_log_level if code_location: self.bucket, self.key_prefix = fw_utils.parse_s3_url(code_location) @@ -890,7 +884,6 @@ def _framework_env_vars(self): return { SCRIPT_PARAM_NAME.upper(): script_name, DIR_PARAM_NAME.upper(): dir_name, - CLOUDWATCH_METRICS_PARAM_NAME.upper(): str(self.enable_cloudwatch_metrics).lower(), CONTAINER_LOG_LEVEL_PARAM_NAME.upper(): str(self.container_log_level), SAGEMAKER_REGION_PARAM_NAME.upper(): self.sagemaker_session.boto_region_name, } diff --git a/src/sagemaker/mxnet/estimator.py b/src/sagemaker/mxnet/estimator.py index e93d8fd51a..6e03308dcd 100644 --- a/src/sagemaker/mxnet/estimator.py +++ b/src/sagemaker/mxnet/estimator.py @@ -225,7 +225,6 @@ def create_model( framework_version=self.framework_version, py_version=self.py_version, source_dir=(source_dir or self._model_source_dir()), - enable_cloudwatch_metrics=self.enable_cloudwatch_metrics, container_log_level=self.container_log_level, code_location=self.code_location, model_server_workers=model_server_workers, diff --git a/src/sagemaker/pytorch/estimator.py b/src/sagemaker/pytorch/estimator.py index fc5eebe352..4451fef0e2 100644 --- a/src/sagemaker/pytorch/estimator.py +++ b/src/sagemaker/pytorch/estimator.py @@ -179,7 +179,6 @@ def create_model( framework_version=self.framework_version, py_version=self.py_version, source_dir=(source_dir or self._model_source_dir()), - enable_cloudwatch_metrics=self.enable_cloudwatch_metrics, container_log_level=self.container_log_level, code_location=self.code_location, model_server_workers=model_server_workers, diff --git a/src/sagemaker/rl/estimator.py b/src/sagemaker/rl/estimator.py index 6774b2a5ad..f0af0424ed 100644 --- a/src/sagemaker/rl/estimator.py +++ b/src/sagemaker/rl/estimator.py @@ -244,7 +244,6 @@ def create_model( source_dir=source_dir, code_location=self.code_location, dependencies=dependencies, - enable_cloudwatch_metrics=self.enable_cloudwatch_metrics, ) extended_args.update(base_args) diff --git a/src/sagemaker/sklearn/estimator.py b/src/sagemaker/sklearn/estimator.py index 15980b325a..66a4e25e27 100644 --- a/src/sagemaker/sklearn/estimator.py +++ b/src/sagemaker/sklearn/estimator.py @@ -201,7 +201,6 @@ def create_model( role, entry_point or self._model_entry_point(), source_dir=(source_dir or self._model_source_dir()), - enable_cloudwatch_metrics=self.enable_cloudwatch_metrics, container_log_level=self.container_log_level, code_location=self.code_location, py_version=self.py_version, diff --git a/src/sagemaker/workflow/airflow.py b/src/sagemaker/workflow/airflow.py index 09daa7187b..5874a53170 100644 --- a/src/sagemaker/workflow/airflow.py +++ b/src/sagemaker/workflow/airflow.py @@ -59,9 +59,6 @@ def prepare_framework(estimator, s3_operations): ] estimator._hyperparameters[sagemaker.model.DIR_PARAM_NAME] = code_dir estimator._hyperparameters[sagemaker.model.SCRIPT_PARAM_NAME] = script - estimator._hyperparameters[ - sagemaker.model.CLOUDWATCH_METRICS_PARAM_NAME - ] = estimator.enable_cloudwatch_metrics estimator._hyperparameters[ sagemaker.model.CONTAINER_LOG_LEVEL_PARAM_NAME ] = estimator.container_log_level diff --git a/src/sagemaker/xgboost/estimator.py b/src/sagemaker/xgboost/estimator.py index 2fdbe177fb..5b13bc95fd 100644 --- a/src/sagemaker/xgboost/estimator.py +++ b/src/sagemaker/xgboost/estimator.py @@ -163,7 +163,6 @@ def create_model( entry_point or self._model_entry_point(), framework_version=self.framework_version, source_dir=(source_dir or self._model_source_dir()), - enable_cloudwatch_metrics=self.enable_cloudwatch_metrics, container_log_level=self.container_log_level, code_location=self.code_location, py_version=self.py_version, diff --git a/tests/component/test_mxnet_estimator.py b/tests/component/test_mxnet_estimator.py index 0bcbabecd8..2c18df59bd 100644 --- a/tests/component/test_mxnet_estimator.py +++ b/tests/component/test_mxnet_estimator.py @@ -75,7 +75,6 @@ def test_deploy(sagemaker_session, tf_version): ROLE, { "Environment": { - "SAGEMAKER_ENABLE_CLOUDWATCH_METRICS": "false", "SAGEMAKER_CONTAINER_LOG_LEVEL": "20", "SAGEMAKER_SUBMIT_DIRECTORY": SOURCE_DIR, "SAGEMAKER_REGION": REGION, diff --git a/tests/component/test_tf_estimator.py b/tests/component/test_tf_estimator.py index f9571a69b4..72a8247f56 100644 --- a/tests/component/test_tf_estimator.py +++ b/tests/component/test_tf_estimator.py @@ -74,7 +74,6 @@ def test_deploy(sagemaker_session, tf_version): ROLE, { "Environment": { - "SAGEMAKER_ENABLE_CLOUDWATCH_METRICS": "false", "SAGEMAKER_CONTAINER_LOG_LEVEL": "20", "SAGEMAKER_SUBMIT_DIRECTORY": SOURCE_DIR, "SAGEMAKER_REQUIREMENTS": "", diff --git a/tests/integ/test_airflow_config.py b/tests/integ/test_airflow_config.py index 155c03ea66..97140b92d4 100644 --- a/tests/integ/test_airflow_config.py +++ b/tests/integ/test_airflow_config.py @@ -111,7 +111,6 @@ def test_kmeans_airflow_config_uploads_data_source_to_s3(sagemaker_session, cpu_ kmeans.half_life_time_size = 1 kmeans.epochs = 1 kmeans.center_factor = 1 - kmeans.eval_metrics = ["ssd", "msd"] records = kmeans.record_set(datasets.one_p_mnist()[0][:100]) @@ -386,7 +385,6 @@ def test_rcf_airflow_config_uploads_data_source_to_s3(sagemaker_session, cpu_ins instance_type=cpu_instance_type, num_trees=50, num_samples_per_tree=20, - eval_metrics=["accuracy", "precision_recall_fscore"], sagemaker_session=sagemaker_session, ) diff --git a/tests/integ/test_data_capture_config.py b/tests/integ/test_data_capture_config.py index 11e74e9695..722e9671dc 100644 --- a/tests/integ/test_data_capture_config.py +++ b/tests/integ/test_data_capture_config.py @@ -32,7 +32,6 @@ ENVIRONMENT = {"env_key_1": "env_value_1"} TAGS = [{"Key": "tag_key_1", "Value": "tag_value_1"}] NETWORK_CONFIG = NetworkConfig(enable_network_isolation=True) -ENABLE_CLOUDWATCH_METRICS = True CUSTOM_SAMPLING_PERCENTAGE = 10 CUSTOM_CAPTURE_OPTIONS = ["REQUEST"] diff --git a/tests/integ/test_kmeans.py b/tests/integ/test_kmeans.py index f04614f493..c394b88d55 100644 --- a/tests/integ/test_kmeans.py +++ b/tests/integ/test_kmeans.py @@ -12,7 +12,6 @@ # language governing permissions and limitations under the License. from __future__ import absolute_import -import json import time import pytest @@ -47,7 +46,6 @@ def test_kmeans(sagemaker_session, cpu_instance_type, training_set): kmeans.half_life_time_size = 1 kmeans.epochs = 1 kmeans.center_factor = 1 - kmeans.eval_metrics = ["ssd", "msd"] assert kmeans.hyperparameters() == dict( init_method=kmeans.init_method, @@ -59,7 +57,6 @@ def test_kmeans(sagemaker_session, cpu_instance_type, training_set): epochs=str(kmeans.epochs), extra_center_factor=str(kmeans.center_factor), k=str(kmeans.k), - eval_metrics=json.dumps(kmeans.eval_metrics), force_dense="True", ) diff --git a/tests/integ/test_multidatamodel.py b/tests/integ/test_multidatamodel.py index 415cdd1048..628903c4e5 100644 --- a/tests/integ/test_multidatamodel.py +++ b/tests/integ/test_multidatamodel.py @@ -439,7 +439,6 @@ def __rcf_training_job( instance_type=cpu_instance_type, num_trees=num_trees, num_samples_per_tree=num_samples_per_tree, - eval_metrics=["accuracy", "precision_recall_fscore"], sagemaker_session=sagemaker_session, ) diff --git a/tests/unit/sagemaker/model/test_framework_model.py b/tests/unit/sagemaker/model/test_framework_model.py index 6df43bdbe4..25dc80f265 100644 --- a/tests/unit/sagemaker/model/test_framework_model.py +++ b/tests/unit/sagemaker/model/test_framework_model.py @@ -106,7 +106,6 @@ def test_prepare_container_def(time, sagemaker_session): "SAGEMAKER_SUBMIT_DIRECTORY": "s3://mybucket/mi-2017-10-10-14-14-15/sourcedir.tar.gz", "SAGEMAKER_CONTAINER_LOG_LEVEL": "20", "SAGEMAKER_REGION": REGION, - "SAGEMAKER_ENABLE_CLOUDWATCH_METRICS": "false", }, "Image": MODEL_IMAGE, "ModelDataUrl": MODEL_DATA, @@ -125,7 +124,6 @@ def test_prepare_container_def_with_network_isolation(time, sagemaker_session): "SAGEMAKER_SUBMIT_DIRECTORY": "/opt/ml/model/code", "SAGEMAKER_CONTAINER_LOG_LEVEL": "20", "SAGEMAKER_REGION": REGION, - "SAGEMAKER_ENABLE_CLOUDWATCH_METRICS": "false", }, "Image": MODEL_IMAGE, "ModelDataUrl": MODEL_DATA, @@ -144,7 +142,6 @@ def test_prepare_container_def_no_model_defaults(sagemaker_session, tmpdir): source_dir="sd", env={"a": "a"}, name="name", - enable_cloudwatch_metrics=True, container_log_level=55, code_location="s3://cb/cp", ) @@ -155,7 +152,6 @@ def test_prepare_container_def_no_model_defaults(sagemaker_session, tmpdir): "SAGEMAKER_SUBMIT_DIRECTORY": "s3://cb/cp/name/sourcedir.tar.gz", "SAGEMAKER_CONTAINER_LOG_LEVEL": "55", "SAGEMAKER_REGION": REGION, - "SAGEMAKER_ENABLE_CLOUDWATCH_METRICS": "true", "a": "a", }, "Image": MODEL_IMAGE, diff --git a/tests/unit/sagemaker/tensorflow/test_estimator.py b/tests/unit/sagemaker/tensorflow/test_estimator.py index 44bbe8d944..06c0021a61 100644 --- a/tests/unit/sagemaker/tensorflow/test_estimator.py +++ b/tests/unit/sagemaker/tensorflow/test_estimator.py @@ -88,7 +88,6 @@ def _hyperparameters(horovod=False): "sagemaker_submit_directory": json.dumps( "s3://{}/{}/source/sourcedir.tar.gz".format(BUCKET_NAME, JOB_NAME) ), - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": str(logging.INFO), "sagemaker_job_name": json.dumps(JOB_NAME), "sagemaker_region": json.dumps("us-west-2"), @@ -220,7 +219,6 @@ def test_create_model_with_optional_params( container_log_level = '"logging.INFO"' source_dir = "s3://mybucket/source" - enable_cloudwatch_metrics = "true" tf = TensorFlow( entry_point=SCRIPT_PATH, framework_version=tensorflow_inference_version, @@ -232,7 +230,6 @@ def test_create_model_with_optional_params( container_log_level=container_log_level, base_job_name="job", source_dir=source_dir, - enable_cloudwatch_metrics=enable_cloudwatch_metrics, output_path="s3://mybucket/output", ) diff --git a/tests/unit/sagemaker/tensorflow/test_estimator_attach.py b/tests/unit/sagemaker/tensorflow/test_estimator_attach.py index d5c8f9299c..80d68d7d74 100644 --- a/tests/unit/sagemaker/tensorflow/test_estimator_attach.py +++ b/tests/unit/sagemaker/tensorflow/test_estimator_attach.py @@ -63,7 +63,6 @@ def test_attach(sagemaker_session, tensorflow_training_version, tensorflow_train "HyperParameters": { "sagemaker_submit_directory": '"s3://some/sourcedir.tar.gz"', "sagemaker_program": '"iris-dnn-classifier.py"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": '"logging.INFO"', "sagemaker_job_name": '"neo"', }, @@ -110,7 +109,6 @@ def test_attach_old_container(sagemaker_session): "HyperParameters": { "sagemaker_submit_directory": '"s3://some/sourcedir.tar.gz"', "sagemaker_program": '"iris-dnn-classifier.py"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": '"logging.INFO"', "sagemaker_job_name": '"neo"', }, @@ -156,7 +154,6 @@ def test_attach_wrong_framework(sagemaker_session): "HyperParameters": { "sagemaker_submit_directory": '"s3://some/sourcedir.tar.gz"', "sagemaker_program": '"iris-dnn-classifier.py"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": '"logging.INFO"', }, "RoleArn": "arn:aws:iam::366:role/SageMakerRole", @@ -188,7 +185,6 @@ def test_attach_custom_image(sagemaker_session): "HyperParameters": { "sagemaker_submit_directory": '"s3://some/sourcedir.tar.gz"', "sagemaker_program": '"iris-dnn-classifier.py"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": '"logging.INFO"', "sagemaker_job_name": '"neo"', }, diff --git a/tests/unit/test_airflow.py b/tests/unit/test_airflow.py index bb7afe89e4..608b89978a 100644 --- a/tests/unit/test_airflow.py +++ b/tests/unit/test_airflow.py @@ -221,7 +221,6 @@ def test_framework_training_config_required_args(retrieve_image_uri, sagemaker_s "sagemaker_submit_directory": '"s3://output/tensorflow-training-%s/source/sourcedir.tar.gz"' % TIME_STAMP, "sagemaker_program": '"script.py"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": "20", "sagemaker_job_name": '"tensorflow-training-%s"' % TIME_STAMP, "sagemaker_region": '"us-west-2"', @@ -256,7 +255,6 @@ def test_framework_training_config_all_args(retrieve_image_uri, sagemaker_sessio tf = tensorflow.TensorFlow( entry_point="{{ entry_point }}", source_dir="{{ source_dir }}", - enable_cloudwatch_metrics=False, container_log_level="{{ log_level }}", code_location="s3://{{ bucket_name }}/{{ prefix }}", hyperparameters={"epochs": 1}, @@ -321,7 +319,6 @@ def test_framework_training_config_all_args(retrieve_image_uri, sagemaker_sessio "sagemaker_submit_directory": '"s3://{{ bucket_name }}/{{ prefix }}/{{ base_job_name }}-%s/' 'source/sourcedir.tar.gz"' % TIME_STAMP, "sagemaker_program": '"{{ entry_point }}"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": '"{{ log_level }}"', "sagemaker_job_name": '"{{ base_job_name }}-%s"' % TIME_STAMP, "sagemaker_region": '"us-west-2"', @@ -586,7 +583,6 @@ def test_framework_tuning_config(retrieve_image_uri, sagemaker_session): "sagemaker_submit_directory": '"s3://output/{{ base_job_name }}-%s/source/sourcedir.tar.gz"' % TIME_STAMP, "sagemaker_program": '"{{ entry_point }}"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": "20", "sagemaker_job_name": '"{{ base_job_name }}-%s"' % TIME_STAMP, "sagemaker_region": '"us-west-2"', @@ -778,7 +774,6 @@ def test_multi_estimator_tuning_config(botocore_resolver, sagemaker_session): "StaticHyperParameters": { "batch_size": "100", "sagemaker_container_log_level": "20", - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_estimator_class_name": '"MXNet"', "sagemaker_estimator_module": '"sagemaker.mxnet.estimator"', "sagemaker_job_name": '"{{ base_job_name }}-%s"' % TIME_STAMP, @@ -935,7 +930,6 @@ def test_byo_framework_model_config(sagemaker_session): "{{ key }}": "{{ value }}", "SAGEMAKER_PROGRAM": "{{ entry_point }}", "SAGEMAKER_SUBMIT_DIRECTORY": "s3://output/model/source/sourcedir.tar.gz", - "SAGEMAKER_ENABLE_CLOUDWATCH_METRICS": "false", "SAGEMAKER_CONTAINER_LOG_LEVEL": "20", "SAGEMAKER_REGION": "us-west-2", }, @@ -980,7 +974,6 @@ def test_framework_model_config(sagemaker_session): "SAGEMAKER_PROGRAM": "{{ entry_point }}", "SAGEMAKER_SUBMIT_DIRECTORY": "s3://output/sagemaker-chainer-%s/source/sourcedir.tar.gz" % TIME_STAMP, - "SAGEMAKER_ENABLE_CLOUDWATCH_METRICS": "false", "SAGEMAKER_CONTAINER_LOG_LEVEL": "20", "SAGEMAKER_REGION": "us-west-2", "SAGEMAKER_MODEL_SERVER_WORKERS": "{{ model_server_worker }}", @@ -1072,7 +1065,6 @@ def test_model_config_from_framework_estimator(retrieve_image_uri, sagemaker_ses "SAGEMAKER_PROGRAM": "{{ entry_point }}", "SAGEMAKER_SUBMIT_DIRECTORY": "s3://output/{{ ti.xcom_pull(task_ids='task_id')['Training']" "['TrainingJobName'] }}/source/sourcedir.tar.gz", - "SAGEMAKER_ENABLE_CLOUDWATCH_METRICS": "false", "SAGEMAKER_CONTAINER_LOG_LEVEL": "20", "SAGEMAKER_REGION": "us-west-2", }, @@ -1246,7 +1238,6 @@ def test_transform_config_from_framework_estimator(retrieve_image_uri, sagemaker "SAGEMAKER_SUBMIT_DIRECTORY": "s3://output/{{ ti.xcom_pull(task_ids='task_id')" "['Training']['TrainingJobName'] }}" "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/source/sourcedir.tar.gz", - "SAGEMAKER_ENABLE_CLOUDWATCH_METRICS": "false", "SAGEMAKER_CONTAINER_LOG_LEVEL": "20", "SAGEMAKER_REGION": "us-west-2", }, @@ -1362,7 +1353,6 @@ def test_deploy_framework_model_config(sagemaker_session): "SAGEMAKER_PROGRAM": "{{ entry_point }}", "SAGEMAKER_SUBMIT_DIRECTORY": "s3://output/sagemaker-chainer-%s/source/sourcedir.tar.gz" % TIME_STAMP, - "SAGEMAKER_ENABLE_CLOUDWATCH_METRICS": "false", "SAGEMAKER_CONTAINER_LOG_LEVEL": "20", "SAGEMAKER_REGION": "us-west-2", "SAGEMAKER_MODEL_SERVER_WORKERS": "{{ model_server_worker }}", @@ -1494,7 +1484,6 @@ def test_deploy_config_from_framework_estimator(retrieve_image_uri, sagemaker_se "SAGEMAKER_PROGRAM": "{{ entry_point }}", "SAGEMAKER_SUBMIT_DIRECTORY": "s3://output/{{ ti.xcom_pull(task_ids='task_id')['Training']" "['TrainingJobName'] }}/source/sourcedir.tar.gz", - "SAGEMAKER_ENABLE_CLOUDWATCH_METRICS": "false", "SAGEMAKER_CONTAINER_LOG_LEVEL": "20", "SAGEMAKER_REGION": "us-west-2", }, diff --git a/tests/unit/test_amazon_estimator.py b/tests/unit/test_amazon_estimator.py index 1422351c1e..2341c79c5b 100644 --- a/tests/unit/test_amazon_estimator.py +++ b/tests/unit/test_amazon_estimator.py @@ -53,7 +53,6 @@ def sagemaker_session(): "sagemaker_submit_directory": '"s3://some/sourcedir.tar.gz"', "checkpoint_path": '"s3://other/1508872349"', "sagemaker_program": '"iris-dnn-classifier.py"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": '"logging.INFO"', "sagemaker_job_name": '"neo"', "training_steps": "100", diff --git a/tests/unit/test_chainer.py b/tests/unit/test_chainer.py index 2f1d8b064e..5d5a763335 100644 --- a/tests/unit/test_chainer.py +++ b/tests/unit/test_chainer.py @@ -132,7 +132,6 @@ def _create_train_job(version, py_version): }, "hyperparameters": { "sagemaker_program": json.dumps("dummy_script.py"), - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": str(logging.INFO), "sagemaker_job_name": json.dumps(JOB_NAME), "sagemaker_submit_directory": json.dumps( @@ -183,7 +182,6 @@ def test_attach_with_additional_hyperparameters( "sagemaker_submit_directory": '"s3://some/sourcedir.tar.gz"', "sagemaker_program": '"iris-dnn-classifier.py"', "sagemaker_s3_uri_training": '"sagemaker-3/integ-test-data/tf_iris"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": '"logging.INFO"', "sagemaker_job_name": '"neo"', "sagemaker_region": '"us-west-2"', @@ -264,7 +262,6 @@ def test_create_model(name_from_base, sagemaker_session, chainer_version, chaine def test_create_model_with_optional_params(sagemaker_session, chainer_version, chainer_py_version): container_log_level = '"logging.INFO"' source_dir = "s3://mybucket/source" - enable_cloudwatch_metrics = "true" chainer = Chainer( entry_point=SCRIPT_PATH, role=ROLE, @@ -276,7 +273,6 @@ def test_create_model_with_optional_params(sagemaker_session, chainer_version, c py_version=chainer_py_version, base_job_name="job", source_dir=source_dir, - enable_cloudwatch_metrics=enable_cloudwatch_metrics, ) chainer.fit(inputs="s3://mybucket/train", job_name="new_name") @@ -361,7 +357,6 @@ def test_chainer(strftime, sagemaker_session, chainer_version, chainer_py_versio TIMESTAMP ), "SAGEMAKER_PROGRAM": "dummy_script.py", - "SAGEMAKER_ENABLE_CLOUDWATCH_METRICS": "false", "SAGEMAKER_REGION": "us-west-2", "SAGEMAKER_CONTAINER_LOG_LEVEL": "20", }, @@ -444,7 +439,6 @@ def test_attach(sagemaker_session, chainer_version, chainer_py_version): "sagemaker_submit_directory": '"s3://some/sourcedir.tar.gz"', "sagemaker_program": '"iris-dnn-classifier.py"', "sagemaker_s3_uri_training": '"sagemaker-3/integ-test-data/tf_iris"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": '"logging.INFO"', "sagemaker_job_name": '"neo"', "training_steps": "100", @@ -493,7 +487,6 @@ def test_attach_wrong_framework(sagemaker_session): "sagemaker_submit_directory": '"s3://some/sourcedir.tar.gz"', "checkpoint_path": '"s3://other/1508872349"', "sagemaker_program": '"iris-dnn-classifier.py"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": '"logging.INFO"', "training_steps": "100", "sagemaker_region": '"us-west-2"', @@ -528,7 +521,6 @@ def test_attach_custom_image(sagemaker_session): "sagemaker_submit_directory": '"s3://some/sourcedir.tar.gz"', "sagemaker_program": '"iris-dnn-classifier.py"', "sagemaker_s3_uri_training": '"sagemaker-3/integ-test-data/tf_iris"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": '"logging.INFO"', "sagemaker_job_name": '"neo"', "training_steps": "100", diff --git a/tests/unit/test_estimator.py b/tests/unit/test_estimator.py index 1451a9b743..0671195634 100644 --- a/tests/unit/test_estimator.py +++ b/tests/unit/test_estimator.py @@ -72,7 +72,6 @@ "sagemaker_submit_directory": '"s3://some/sourcedir.tar.gz"', "checkpoint_path": '"s3://other/1508872349"', "sagemaker_program": '"iris-dnn-classifier.py"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": '"logging.INFO"', "sagemaker_job_name": '"neo"', "training_steps": "100", @@ -95,7 +94,6 @@ "SAGEMAKER_SUBMIT_DIRECTORY": "s3://mybucket/mi-2017-10-10-14-14-15/sourcedir.tar.gz", "SAGEMAKER_CONTAINER_LOG_LEVEL": "20", "SAGEMAKER_REGION": REGION, - "SAGEMAKER_ENABLE_CLOUDWATCH_METRICS": "false", }, "Image": MODEL_IMAGE, "ModelDataUrl": MODEL_DATA, @@ -513,7 +511,6 @@ def test_augmented_manifest(sagemaker_session): sagemaker_session=sagemaker_session, instance_count=INSTANCE_COUNT, instance_type=INSTANCE_TYPE, - enable_cloudwatch_metrics=True, ) fw.fit( inputs=TrainingInput( @@ -538,7 +535,6 @@ def test_s3_input_mode(sagemaker_session): sagemaker_session=sagemaker_session, instance_count=INSTANCE_COUNT, instance_type=INSTANCE_TYPE, - enable_cloudwatch_metrics=True, ) fw.fit(inputs=TrainingInput("s3://mybucket/train_manifest", input_mode=expected_input_mode)) @@ -553,7 +549,6 @@ def test_shuffle_config(sagemaker_session): sagemaker_session=sagemaker_session, instance_count=INSTANCE_COUNT, instance_type=INSTANCE_TYPE, - enable_cloudwatch_metrics=True, ) fw.fit(inputs=TrainingInput("s3://mybucket/train_manifest", shuffle_config=ShuffleConfig(100))) _, _, train_kwargs = sagemaker_session.train.mock_calls[0] @@ -611,7 +606,6 @@ def test_start_new_convert_hyperparameters_to_str(strftime, sagemaker_session): t.fit("s3://{}".format(uri)) expected_hyperparameters = BASE_HP.copy() - expected_hyperparameters["sagemaker_enable_cloudwatch_metrics"] = "false" expected_hyperparameters["sagemaker_container_log_level"] = str(logging.INFO) expected_hyperparameters["learning_rate"] = json.dumps(0.1) expected_hyperparameters["123"] = json.dumps([456]) @@ -636,7 +630,6 @@ def test_start_new_wait_called(strftime, sagemaker_session): t.fit("s3://{}".format(uri)) expected_hyperparameters = BASE_HP.copy() - expected_hyperparameters["sagemaker_enable_cloudwatch_metrics"] = "false" expected_hyperparameters["sagemaker_container_log_level"] = str(logging.INFO) expected_hyperparameters["sagemaker_region"] = '"us-west-2"' @@ -645,21 +638,6 @@ def test_start_new_wait_called(strftime, sagemaker_session): assert sagemaker_session.wait_for_job.assert_called_once -def test_enable_cloudwatch_metrics(sagemaker_session): - fw = DummyFramework( - entry_point=SCRIPT_PATH, - role="DummyRole", - sagemaker_session=sagemaker_session, - instance_count=INSTANCE_COUNT, - instance_type=INSTANCE_TYPE, - enable_cloudwatch_metrics=True, - ) - fw.fit(inputs=TrainingInput("s3://mybucket/train")) - - _, _, train_kwargs = sagemaker_session.train.mock_calls[0] - assert train_kwargs["hyperparameters"]["sagemaker_enable_cloudwatch_metrics"] - - def test_attach_framework(sagemaker_session, training_job_description): training_job_description["VpcConfig"] = {"Subnets": ["foo"], "SecurityGroupIds": ["bar"]} training_job_description["EnableNetworkIsolation"] = True @@ -769,7 +747,6 @@ def test_fit_verify_job_name(strftime, sagemaker_session): sagemaker_session=sagemaker_session, instance_count=INSTANCE_COUNT, instance_type=INSTANCE_TYPE, - enable_cloudwatch_metrics=True, tags=TAGS, encrypt_inter_container_traffic=True, ) @@ -777,7 +754,6 @@ def test_fit_verify_job_name(strftime, sagemaker_session): _, _, train_kwargs = sagemaker_session.train.mock_calls[0] - assert train_kwargs["hyperparameters"]["sagemaker_enable_cloudwatch_metrics"] assert train_kwargs["image_uri"] == IMAGE_URI assert train_kwargs["input_mode"] == "File" assert train_kwargs["tags"] == TAGS @@ -793,7 +769,6 @@ def test_prepare_for_training_unique_job_name_generation(sagemaker_session): sagemaker_session=sagemaker_session, instance_count=INSTANCE_COUNT, instance_type=INSTANCE_TYPE, - enable_cloudwatch_metrics=True, ) fw._prepare_for_training() first_job_name = fw._current_job_name @@ -813,7 +788,6 @@ def test_prepare_for_training_force_name(sagemaker_session): instance_count=INSTANCE_COUNT, instance_type=INSTANCE_TYPE, base_job_name="some", - enable_cloudwatch_metrics=True, ) fw._prepare_for_training(job_name="use_it") assert "use_it" == fw._current_job_name @@ -828,7 +802,6 @@ def test_prepare_for_training_force_name_generation(strftime, sagemaker_session) instance_count=INSTANCE_COUNT, instance_type=INSTANCE_TYPE, base_job_name="some", - enable_cloudwatch_metrics=True, ) fw.base_job_name = None fw._prepare_for_training() @@ -851,7 +824,6 @@ def test_git_support_with_branch_and_commit_succeed(git_clone_repo, sagemaker_se sagemaker_session=sagemaker_session, instance_count=INSTANCE_COUNT, instance_type=INSTANCE_TYPE, - enable_cloudwatch_metrics=True, ) fw.fit() git_clone_repo.assert_called_once_with(git_config, entry_point, None, []) @@ -873,7 +845,6 @@ def test_git_support_with_branch_succeed(git_clone_repo, sagemaker_session): sagemaker_session=sagemaker_session, instance_count=INSTANCE_COUNT, instance_type=INSTANCE_TYPE, - enable_cloudwatch_metrics=True, ) fw.fit() git_clone_repo.assert_called_once_with(git_config, entry_point, None, []) @@ -896,7 +867,6 @@ def test_git_support_with_dependencies_succeed(git_clone_repo, sagemaker_session sagemaker_session=sagemaker_session, instance_count=INSTANCE_COUNT, instance_type=INSTANCE_TYPE, - enable_cloudwatch_metrics=True, ) fw.fit() git_clone_repo.assert_called_once_with(git_config, entry_point, None, ["foo", "foo/bar"]) @@ -918,7 +888,6 @@ def test_git_support_without_branch_and_commit_succeed(git_clone_repo, sagemaker sagemaker_session=sagemaker_session, instance_count=INSTANCE_COUNT, instance_type=INSTANCE_TYPE, - enable_cloudwatch_metrics=True, ) fw.fit() git_clone_repo.assert_called_once_with(git_config, entry_point, None, []) @@ -934,7 +903,6 @@ def test_git_support_repo_not_provided(sagemaker_session): sagemaker_session=sagemaker_session, instance_count=INSTANCE_COUNT, instance_type=INSTANCE_TYPE, - enable_cloudwatch_metrics=True, ) with pytest.raises(ValueError) as error: fw.fit() @@ -951,7 +919,6 @@ def test_git_support_bad_repo_url_format(sagemaker_session): sagemaker_session=sagemaker_session, instance_count=INSTANCE_COUNT, instance_type=INSTANCE_TYPE, - enable_cloudwatch_metrics=True, ) with pytest.raises(ValueError) as error: fw.fit() @@ -973,7 +940,6 @@ def test_git_support_git_clone_fail(sagemaker_session): sagemaker_session=sagemaker_session, instance_count=INSTANCE_COUNT, instance_type=INSTANCE_TYPE, - enable_cloudwatch_metrics=True, ) with pytest.raises(subprocess.CalledProcessError) as error: fw.fit() @@ -995,7 +961,6 @@ def test_git_support_branch_not_exist(sagemaker_session): sagemaker_session=sagemaker_session, instance_count=INSTANCE_COUNT, instance_type=INSTANCE_TYPE, - enable_cloudwatch_metrics=True, ) with pytest.raises(subprocess.CalledProcessError) as error: fw.fit() @@ -1017,7 +982,6 @@ def test_git_support_commit_not_exist(sagemaker_session): sagemaker_session=sagemaker_session, instance_count=INSTANCE_COUNT, instance_type=INSTANCE_TYPE, - enable_cloudwatch_metrics=True, ) with pytest.raises(subprocess.CalledProcessError) as error: fw.fit() @@ -1037,7 +1001,6 @@ def test_git_support_entry_point_not_exist(sagemaker_session): sagemaker_session=sagemaker_session, instance_count=INSTANCE_COUNT, instance_type=INSTANCE_TYPE, - enable_cloudwatch_metrics=True, ) with pytest.raises(ValueError) as error: fw.fit() @@ -1058,7 +1021,6 @@ def test_git_support_source_dir_not_exist(sagemaker_session): sagemaker_session=sagemaker_session, instance_count=INSTANCE_COUNT, instance_type=INSTANCE_TYPE, - enable_cloudwatch_metrics=True, ) with pytest.raises(ValueError) as error: fw.fit() @@ -1080,7 +1042,6 @@ def test_git_support_dependencies_not_exist(sagemaker_session): sagemaker_session=sagemaker_session, instance_count=INSTANCE_COUNT, instance_type=INSTANCE_TYPE, - enable_cloudwatch_metrics=True, ) with pytest.raises(ValueError) as error: fw.fit() @@ -1111,7 +1072,6 @@ def test_git_support_with_username_password_no_2fa(git_clone_repo, sagemaker_ses sagemaker_session=sagemaker_session, instance_count=INSTANCE_COUNT, instance_type=INSTANCE_TYPE, - enable_cloudwatch_metrics=True, ) fw.fit() git_clone_repo.assert_called_once_with(git_config, entry_point, None, []) @@ -1142,7 +1102,6 @@ def test_git_support_with_token_2fa(git_clone_repo, sagemaker_session): sagemaker_session=sagemaker_session, instance_count=INSTANCE_COUNT, instance_type=INSTANCE_TYPE, - enable_cloudwatch_metrics=True, ) fw.fit() git_clone_repo.assert_called_once_with(git_config, entry_point, None, []) @@ -1167,7 +1126,6 @@ def test_git_support_ssh_no_passphrase_needed(git_clone_repo, sagemaker_session) sagemaker_session=sagemaker_session, instance_count=INSTANCE_COUNT, instance_type=INSTANCE_TYPE, - enable_cloudwatch_metrics=True, ) fw.fit() git_clone_repo.assert_called_once_with(git_config, entry_point, None, []) @@ -1190,7 +1148,6 @@ def test_git_support_ssh_passphrase_required(git_clone_repo, sagemaker_session): sagemaker_session=sagemaker_session, instance_count=INSTANCE_COUNT, instance_type=INSTANCE_TYPE, - enable_cloudwatch_metrics=True, ) with pytest.raises(subprocess.CalledProcessError) as error: fw.fit() @@ -1222,7 +1179,6 @@ def test_git_support_codecommit_with_username_and_password_succeed( sagemaker_session=sagemaker_session, instance_count=INSTANCE_COUNT, instance_type=INSTANCE_TYPE, - enable_cloudwatch_metrics=True, ) fw.fit() git_clone_repo.assert_called_once_with(git_config, entry_point, None, []) @@ -1247,7 +1203,6 @@ def test_git_support_codecommit_with_ssh_no_passphrase_needed(git_clone_repo, sa sagemaker_session=sagemaker_session, instance_count=INSTANCE_COUNT, instance_type=INSTANCE_TYPE, - # enable_cloudwatch_metrics=True, ) fw.fit() git_clone_repo.assert_called_once_with(git_config, entry_point, None, []) @@ -1263,14 +1218,12 @@ def test_init_with_source_dir_s3(strftime, sagemaker_session): sagemaker_session=sagemaker_session, instance_count=INSTANCE_COUNT, instance_type=INSTANCE_TYPE, - enable_cloudwatch_metrics=False, ) fw._prepare_for_training() expected_hyperparameters = { "sagemaker_program": SCRIPT_NAME, "sagemaker_job_name": JOB_NAME, - "sagemaker_enable_cloudwatch_metrics": False, "sagemaker_container_log_level": logging.INFO, "sagemaker_submit_directory": "s3://location", "sagemaker_region": "us-west-2", diff --git a/tests/unit/test_job.py b/tests/unit/test_job.py index a2d75b36ad..6d76f7636f 100644 --- a/tests/unit/test_job.py +++ b/tests/unit/test_job.py @@ -49,7 +49,6 @@ "SAGEMAKER_SUBMIT_DIRECTORY": "s3://mybucket/mi-2017-10-10-14-14-15/sourcedir.tar.gz", "SAGEMAKER_CONTAINER_LOG_LEVEL": "20", "SAGEMAKER_REGION": REGION, - "SAGEMAKER_ENABLE_CLOUDWATCH_METRICS": "false", }, "Image": IMAGE_NAME, "ModelDataUrl": MODEL_URI, diff --git a/tests/unit/test_kmeans.py b/tests/unit/test_kmeans.py index ea4e6bcb8e..2d76861b85 100644 --- a/tests/unit/test_kmeans.py +++ b/tests/unit/test_kmeans.py @@ -92,7 +92,6 @@ def test_all_hyperparameters(sagemaker_session): half_life_time_size=0, epochs=10, center_factor=2, - eval_metrics=["msd", "ssd"], **ALL_REQ_ARGS ) assert kmeans.hyperparameters() == dict( @@ -105,7 +104,6 @@ def test_all_hyperparameters(sagemaker_session): half_life_time_size="0", epochs="10", extra_center_factor="2", - eval_metrics='["msd", "ssd"]', force_dense="True", ) @@ -131,7 +129,7 @@ def test_required_hyper_parameters_value(sagemaker_session, required_hyper_param KMeans(sagemaker_session=sagemaker_session, **test_params) -@pytest.mark.parametrize("iterable_hyper_parameters, value", [("eval_metrics", 0)]) +@pytest.mark.parametrize("iterable_hyper_parameters, value", [("eval_metrics", [0])]) def test_iterable_hyper_parameters_type(sagemaker_session, iterable_hyper_parameters, value): with pytest.raises(TypeError): test_params = ALL_REQ_ARGS.copy() diff --git a/tests/unit/test_linear_learner.py b/tests/unit/test_linear_learner.py index 7d3a9bb1ab..a97ff55cbd 100644 --- a/tests/unit/test_linear_learner.py +++ b/tests/unit/test_linear_learner.py @@ -216,7 +216,7 @@ def test_num_classes_can_be_string_for_multiclass_classifier(sagemaker_session): LinearLearner(sagemaker_session=sagemaker_session, **test_params) -@pytest.mark.parametrize("iterable_hyper_parameters, value", [("eval_metrics", 0)]) +@pytest.mark.parametrize("iterable_hyper_parameters, value", [("eval_metrics", [0])]) def test_iterable_hyper_parameters_type(sagemaker_session, iterable_hyper_parameters, value): with pytest.raises(TypeError): test_params = ALL_REQ_ARGS.copy() diff --git a/tests/unit/test_multidatamodel.py b/tests/unit/test_multidatamodel.py index f9121764ff..ea3a37172a 100644 --- a/tests/unit/test_multidatamodel.py +++ b/tests/unit/test_multidatamodel.py @@ -175,7 +175,6 @@ def test_multi_data_model_create_with_model_arg_only(mxnet_model): def test_prepare_container_def_mxnet(sagemaker_session, mxnet_model): expected_container_env_keys = [ "SAGEMAKER_CONTAINER_LOG_LEVEL", - "SAGEMAKER_ENABLE_CLOUDWATCH_METRICS", "SAGEMAKER_PROGRAM", "SAGEMAKER_REGION", "SAGEMAKER_SUBMIT_DIRECTORY", diff --git a/tests/unit/test_mxnet.py b/tests/unit/test_mxnet.py index 8aaf1fa8c3..48a84ccb0d 100644 --- a/tests/unit/test_mxnet.py +++ b/tests/unit/test_mxnet.py @@ -127,7 +127,6 @@ def _get_train_args(job_name): }, "hyperparameters": { "sagemaker_program": json.dumps("dummy_script.py"), - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": str(logging.INFO), "sagemaker_job_name": json.dumps(job_name), "sagemaker_submit_directory": json.dumps( @@ -152,7 +151,6 @@ def _get_environment(submit_directory, model_url, image_uri): "Environment": { "SAGEMAKER_SUBMIT_DIRECTORY": submit_directory, "SAGEMAKER_PROGRAM": "dummy_script.py", - "SAGEMAKER_ENABLE_CLOUDWATCH_METRICS": "false", "SAGEMAKER_REGION": "us-west-2", "SAGEMAKER_CONTAINER_LOG_LEVEL": "20", }, @@ -227,7 +225,6 @@ def test_create_model_with_optional_params( ): container_log_level = '"logging.INFO"' source_dir = "s3://mybucket/source" - enable_cloudwatch_metrics = "true" mx = MXNet( entry_point=SCRIPT_NAME, source_dir=source_dir, @@ -239,7 +236,6 @@ def test_create_model_with_optional_params( instance_type=INSTANCE_TYPE, container_log_level=container_log_level, base_job_name="job", - enable_cloudwatch_metrics=enable_cloudwatch_metrics, ) mx.fit(inputs="s3://mybucket/train", job_name="new_name") @@ -518,7 +514,6 @@ def test_attach(sagemaker_session, mxnet_training_version, mxnet_py_version): "sagemaker_submit_directory": '"s3://some/sourcedir.tar.gz"', "sagemaker_program": '"iris-dnn-classifier.py"', "sagemaker_s3_uri_training": '"sagemaker-3/integ-test-data/tf_iris"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": '"logging.INFO"', "sagemaker_job_name": '"neo"', "training_steps": "100", @@ -568,7 +563,6 @@ def test_attach_old_container(sagemaker_session): "sagemaker_submit_directory": '"s3://some/sourcedir.tar.gz"', "sagemaker_program": '"iris-dnn-classifier.py"', "sagemaker_s3_uri_training": '"sagemaker-3/integ-test-data/tf_iris"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": '"logging.INFO"', "sagemaker_job_name": '"neo"', "training_steps": "100", @@ -617,7 +611,6 @@ def test_attach_wrong_framework(sagemaker_session): "sagemaker_submit_directory": '"s3://some/sourcedir.tar.gz"', "checkpoint_path": '"s3://other/1508872349"', "sagemaker_program": '"iris-dnn-classifier.py"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": '"logging.INFO"', "training_steps": "100", "sagemaker_region": '"us-west-2"', @@ -652,7 +645,6 @@ def test_attach_custom_image(sagemaker_session): "sagemaker_submit_directory": '"s3://some/sourcedir.tar.gz"', "sagemaker_program": '"iris-dnn-classifier.py"', "sagemaker_s3_uri_training": '"sagemaker-3/integ-test-data/tf_iris"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": '"logging.INFO"', "sagemaker_job_name": '"neo"', "training_steps": "100", diff --git a/tests/unit/test_pipeline_model.py b/tests/unit/test_pipeline_model.py index 0f25bd7d08..c53f96f3e0 100644 --- a/tests/unit/test_pipeline_model.py +++ b/tests/unit/test_pipeline_model.py @@ -93,7 +93,6 @@ def test_prepare_container_def(tfo, time, sagemaker_session): "SAGEMAKER_SUBMIT_DIRECTORY": "s3://mybucket/mi-1-2017-10-10-14-14-15/sourcedir.tar.gz", "SAGEMAKER_CONTAINER_LOG_LEVEL": "20", "SAGEMAKER_REGION": "us-west-2", - "SAGEMAKER_ENABLE_CLOUDWATCH_METRICS": "false", }, "Image": "mi-1", "ModelDataUrl": "s3://bucket/model_1.tar.gz", diff --git a/tests/unit/test_pytorch.py b/tests/unit/test_pytorch.py index 9b15566bda..8a77a9edbd 100644 --- a/tests/unit/test_pytorch.py +++ b/tests/unit/test_pytorch.py @@ -135,7 +135,6 @@ def _create_train_job(version, py_version): }, "hyperparameters": { "sagemaker_program": json.dumps("dummy_script.py"), - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": str(logging.INFO), "sagemaker_job_name": json.dumps(JOB_NAME), "sagemaker_submit_directory": json.dumps( @@ -160,7 +159,6 @@ def _get_environment(submit_directory, model_url, image_uri): "Environment": { "SAGEMAKER_SUBMIT_DIRECTORY": submit_directory, "SAGEMAKER_PROGRAM": "dummy_script.py", - "SAGEMAKER_ENABLE_CLOUDWATCH_METRICS": "false", "SAGEMAKER_REGION": "us-west-2", "SAGEMAKER_CONTAINER_LOG_LEVEL": "20", }, @@ -214,7 +212,6 @@ def test_create_model_with_optional_params( ): container_log_level = '"logging.INFO"' source_dir = "s3://mybucket/source" - enable_cloudwatch_metrics = "true" pytorch = PyTorch( entry_point=SCRIPT_PATH, framework_version=pytorch_inference_version, @@ -226,7 +223,6 @@ def test_create_model_with_optional_params( container_log_level=container_log_level, base_job_name="job", source_dir=source_dir, - enable_cloudwatch_metrics=enable_cloudwatch_metrics, ) pytorch.fit(inputs="s3://mybucket/train", job_name="new_name") @@ -434,7 +430,6 @@ def test_attach(sagemaker_session, pytorch_training_version, pytorch_training_py "sagemaker_submit_directory": '"s3://some/sourcedir.tar.gz"', "sagemaker_program": '"iris-dnn-classifier.py"', "sagemaker_s3_uri_training": '"sagemaker-3/integ-test-data/tf_iris"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": '"logging.INFO"', "sagemaker_job_name": '"neo"', "training_steps": "100", @@ -483,7 +478,6 @@ def test_attach_wrong_framework(sagemaker_session): "sagemaker_submit_directory": '"s3://some/sourcedir.tar.gz"', "checkpoint_path": '"s3://other/1508872349"', "sagemaker_program": '"iris-dnn-classifier.py"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": '"logging.INFO"', "training_steps": "100", "sagemaker_region": '"us-west-2"', @@ -518,7 +512,6 @@ def test_attach_custom_image(sagemaker_session): "sagemaker_submit_directory": '"s3://some/sourcedir.tar.gz"', "sagemaker_program": '"iris-dnn-classifier.py"', "sagemaker_s3_uri_training": '"sagemaker-3/integ-test-data/tf_iris"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": '"logging.INFO"', "sagemaker_job_name": '"neo"', "training_steps": "100", diff --git a/tests/unit/test_randomcutforest.py b/tests/unit/test_randomcutforest.py index b7e6ef1e49..d2ca399090 100644 --- a/tests/unit/test_randomcutforest.py +++ b/tests/unit/test_randomcutforest.py @@ -24,7 +24,6 @@ INSTANCE_TYPE = "ml.c4.xlarge" NUM_SAMPLES_PER_TREE = 20 NUM_TREES = 50 -EVAL_METRICS = ["accuracy", "precision_recall_fscore"] COMMON_TRAIN_ARGS = { "role": ROLE, @@ -71,7 +70,6 @@ def test_init_required_positional(sagemaker_session): INSTANCE_TYPE, NUM_SAMPLES_PER_TREE, NUM_TREES, - EVAL_METRICS, sagemaker_session=sagemaker_session, ) assert randomcutforest.role == ROLE @@ -79,7 +77,6 @@ def test_init_required_positional(sagemaker_session): assert randomcutforest.instance_type == INSTANCE_TYPE assert randomcutforest.num_trees == NUM_TREES assert randomcutforest.num_samples_per_tree == NUM_SAMPLES_PER_TREE - assert randomcutforest.eval_metrics == EVAL_METRICS def test_init_required_named(sagemaker_session): @@ -95,13 +92,10 @@ def test_all_hyperparameters(sagemaker_session): sagemaker_session=sagemaker_session, num_trees=NUM_TREES, num_samples_per_tree=NUM_SAMPLES_PER_TREE, - eval_metrics=EVAL_METRICS, **ALL_REQ_ARGS ) assert randomcutforest.hyperparameters() == dict( - num_samples_per_tree=str(NUM_SAMPLES_PER_TREE), - num_trees=str(NUM_TREES), - eval_metrics='["accuracy", "precision_recall_fscore"]', + num_samples_per_tree=str(NUM_SAMPLES_PER_TREE), num_trees=str(NUM_TREES), ) @@ -110,7 +104,7 @@ def test_image(sagemaker_session): assert image_uris.retrieve("randomcutforest", REGION) == randomcutforest.train_image() -@pytest.mark.parametrize("iterable_hyper_parameters, value", [("eval_metrics", 0)]) +@pytest.mark.parametrize("iterable_hyper_parameters, value", [("eval_metrics", [0])]) def test_iterable_hyper_parameters_type(sagemaker_session, iterable_hyper_parameters, value): with pytest.raises(TypeError): test_params = ALL_REQ_ARGS.copy() diff --git a/tests/unit/test_rl.py b/tests/unit/test_rl.py index 60aabfd534..f3743ddfea 100644 --- a/tests/unit/test_rl.py +++ b/tests/unit/test_rl.py @@ -130,7 +130,6 @@ def _create_train_job(toolkit, toolkit_version, framework): }, "hyperparameters": { "sagemaker_program": json.dumps("dummy_script.py"), - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_estimator": '"RLEstimator"', "sagemaker_container_log_level": str(logging.INFO), "sagemaker_job_name": json.dumps(job_name), @@ -341,7 +340,6 @@ def test_rl(strftime, sagemaker_session, rl_coach_mxnet_version): "Environment": { "SAGEMAKER_SUBMIT_DIRECTORY": submit_dir, "SAGEMAKER_PROGRAM": "dummy_script.py", - "SAGEMAKER_ENABLE_CLOUDWATCH_METRICS": "false", "SAGEMAKER_REGION": "us-west-2", "SAGEMAKER_CONTAINER_LOG_LEVEL": "20", }, @@ -426,7 +424,6 @@ def test_attach(sagemaker_session, rl_coach_mxnet_version): "HyperParameters": { "sagemaker_submit_directory": '"s3://some/sourcedir.tar.gz"', "sagemaker_program": '"train_coach.py"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": '"logging.INFO"', "sagemaker_job_name": '"neo"', "training_steps": "100", @@ -476,7 +473,6 @@ def test_attach_wrong_framework(sagemaker_session): "sagemaker_submit_directory": '"s3://some/sourcedir.tar.gz"', "checkpoint_path": '"s3://other/1508872349"', "sagemaker_program": '"iris-dnn-classifier.py"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": '"logging.INFO"', "training_steps": "100", "sagemaker_region": '"us-west-2"', @@ -511,7 +507,6 @@ def test_attach_custom_image(sagemaker_session): "sagemaker_submit_directory": '"s3://some/sourcedir.tar.gz"', "sagemaker_program": '"iris-dnn-classifier.py"', "sagemaker_s3_uri_training": '"sagemaker-3/integ-test-data/tf_iris"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": '"logging.INFO"', "sagemaker_job_name": '"neo"', "training_steps": "100", diff --git a/tests/unit/test_sklearn.py b/tests/unit/test_sklearn.py index 57925c70d7..c3bcb62426 100644 --- a/tests/unit/test_sklearn.py +++ b/tests/unit/test_sklearn.py @@ -121,7 +121,6 @@ def _create_train_job(version): }, "hyperparameters": { "sagemaker_program": json.dumps("dummy_script.py"), - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": str(logging.INFO), "sagemaker_job_name": json.dumps(JOB_NAME), "sagemaker_submit_directory": json.dumps( @@ -236,7 +235,6 @@ def test_create_model_from_estimator(name_from_base, sagemaker_session, sklearn_ def test_create_model_with_optional_params(sagemaker_session, sklearn_version): container_log_level = '"logging.INFO"' source_dir = "s3://mybucket/source" - enable_cloudwatch_metrics = "true" sklearn = SKLearn( entry_point=SCRIPT_PATH, role=ROLE, @@ -247,7 +245,6 @@ def test_create_model_with_optional_params(sagemaker_session, sklearn_version): py_version=PYTHON_VERSION, base_job_name="job", source_dir=source_dir, - enable_cloudwatch_metrics=enable_cloudwatch_metrics, ) sklearn.fit(inputs="s3://mybucket/train", job_name="new_name") @@ -340,7 +337,6 @@ def test_sklearn(strftime, sagemaker_session, sklearn_version): TIMESTAMP ), "SAGEMAKER_PROGRAM": "dummy_script.py", - "SAGEMAKER_ENABLE_CLOUDWATCH_METRICS": "false", "SAGEMAKER_REGION": "us-west-2", "SAGEMAKER_CONTAINER_LOG_LEVEL": "20", }, @@ -422,7 +418,6 @@ def test_attach(sagemaker_session, sklearn_version): "sagemaker_submit_directory": '"s3://some/sourcedir.tar.gz"', "sagemaker_program": '"iris-dnn-classifier.py"', "sagemaker_s3_uri_training": '"sagemaker-3/integ-test-data/tf_iris"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": '"logging.INFO"', "sagemaker_job_name": '"neo"', "training_steps": "100", @@ -472,7 +467,6 @@ def test_attach_wrong_framework(sagemaker_session): "sagemaker_submit_directory": '"s3://some/sourcedir.tar.gz"', "checkpoint_path": '"s3://other/1508872349"', "sagemaker_program": '"iris-dnn-classifier.py"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": '"logging.INFO"', "training_steps": "100", "sagemaker_region": '"us-west-2"', @@ -507,7 +501,6 @@ def test_attach_custom_image(sagemaker_session): "sagemaker_submit_directory": '"s3://some/sourcedir.tar.gz"', "sagemaker_program": '"iris-dnn-classifier.py"', "sagemaker_s3_uri_training": '"sagemaker-3/integ-test-data/tf_iris"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": '"logging.INFO"', "sagemaker_job_name": '"neo"', "training_steps": "100", diff --git a/tests/unit/test_xgboost.py b/tests/unit/test_xgboost.py index 239926de5a..42c47ccbbe 100644 --- a/tests/unit/test_xgboost.py +++ b/tests/unit/test_xgboost.py @@ -129,7 +129,6 @@ def _create_train_job(version, instance_count=1): }, "hyperparameters": { "sagemaker_program": json.dumps("dummy_script.py"), - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": str(logging.INFO), "sagemaker_job_name": json.dumps(JOB_NAME), "sagemaker_submit_directory": json.dumps( @@ -205,7 +204,6 @@ def test_create_model_from_estimator(name_from_base, sagemaker_session, xgboost_ def test_create_model_with_optional_params(sagemaker_session, xgboost_framework_version): container_log_level = '"logging.INFO"' source_dir = "s3://mybucket/source" - enable_cloudwatch_metrics = "true" xgboost = XGBoost( entry_point=SCRIPT_PATH, role=ROLE, @@ -217,7 +215,6 @@ def test_create_model_with_optional_params(sagemaker_session, xgboost_framework_ py_version=PYTHON_VERSION, base_job_name="job", source_dir=source_dir, - enable_cloudwatch_metrics=enable_cloudwatch_metrics, ) xgboost.fit(inputs="s3://mybucket/train", job_name="new_name") @@ -311,7 +308,6 @@ def test_xgboost(strftime, sagemaker_session, xgboost_framework_version): TIMESTAMP ), "SAGEMAKER_PROGRAM": "dummy_script.py", - "SAGEMAKER_ENABLE_CLOUDWATCH_METRICS": "false", "SAGEMAKER_REGION": "us-west-2", "SAGEMAKER_CONTAINER_LOG_LEVEL": "20", }, @@ -360,7 +356,6 @@ def test_distributed_training(strftime, sagemaker_session, xgboost_framework_ver TIMESTAMP ), "SAGEMAKER_PROGRAM": "dummy_script.py", - "SAGEMAKER_ENABLE_CLOUDWATCH_METRICS": "false", "SAGEMAKER_REGION": "us-west-2", "SAGEMAKER_CONTAINER_LOG_LEVEL": "20", }, @@ -409,7 +404,6 @@ def test_attach(sagemaker_session, xgboost_framework_version): "sagemaker_submit_directory": '"s3://some/sourcedir.tar.gz"', "sagemaker_program": '"iris-dnn-classifier.py"', "sagemaker_s3_uri_training": '"sagemaker-3/integ-test-data/tf_iris"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": '"logging.INFO"', "sagemaker_job_name": '"neo"', "training_steps": "100", @@ -459,7 +453,6 @@ def test_attach_wrong_framework(sagemaker_session): "sagemaker_submit_directory": '"s3://some/sourcedir.tar.gz"', "checkpoint_path": '"s3://other/1508872349"', "sagemaker_program": '"iris-dnn-classifier.py"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": '"logging.INFO"', "training_steps": "100", "sagemaker_region": '"us-west-2"', @@ -494,7 +487,6 @@ def test_attach_custom_image(sagemaker_session): "sagemaker_submit_directory": '"s3://some/sourcedir.tar.gz"', "sagemaker_program": '"iris-dnn-classifier.py"', "sagemaker_s3_uri_training": '"sagemaker-3/integ-test-data/tf_iris"', - "sagemaker_enable_cloudwatch_metrics": "false", "sagemaker_container_log_level": '"logging.INFO"', "sagemaker_job_name": '"neo"', "training_steps": "100", From b8dcc99dc1136ecc4fd95dde07b11d6896486c54 Mon Sep 17 00:00:00 2001 From: Chuyang Deng Date: Wed, 22 Jul 2020 22:56:30 -0700 Subject: [PATCH 2/3] remove eval_metrics from randomcutforest integ test --- tests/integ/test_randomcutforest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integ/test_randomcutforest.py b/tests/integ/test_randomcutforest.py index e108c26759..39358d864b 100644 --- a/tests/integ/test_randomcutforest.py +++ b/tests/integ/test_randomcutforest.py @@ -34,7 +34,6 @@ def test_randomcutforest(sagemaker_session, cpu_instance_type): instance_type=cpu_instance_type, num_trees=50, num_samples_per_tree=20, - eval_metrics=["accuracy", "precision_recall_fscore"], sagemaker_session=sagemaker_session, ) From d8575335c5e36534d6e1372bfa3bf833fff49409 Mon Sep 17 00:00:00 2001 From: Chuyang Deng Date: Thu, 23 Jul 2020 09:03:09 -0700 Subject: [PATCH 3/3] recover eval_metrics for kmeans and rcf --- doc/algorithms/randomcutforest.rst | 2 +- src/sagemaker/amazon/kmeans.py | 12 ++++++++++++ src/sagemaker/amazon/randomcutforest.py | 13 +++++++++++++ tests/integ/test_airflow_config.py | 2 ++ tests/integ/test_kmeans.py | 3 +++ tests/integ/test_multidatamodel.py | 1 + tests/integ/test_randomcutforest.py | 1 + tests/unit/test_kmeans.py | 4 +++- tests/unit/test_linear_learner.py | 2 +- tests/unit/test_randomcutforest.py | 10 ++++++++-- 10 files changed, 45 insertions(+), 5 deletions(-) diff --git a/doc/algorithms/randomcutforest.rst b/doc/algorithms/randomcutforest.rst index bfdef51558..b723319a52 100644 --- a/doc/algorithms/randomcutforest.rst +++ b/doc/algorithms/randomcutforest.rst @@ -8,7 +8,7 @@ The Amazon SageMaker Random Cut Forest algorithm. :undoc-members: :show-inheritance: :inherited-members: - :exclude-members: image_uri, num_trees, num_samples_per_tree, feature_dim, MINI_BATCH_SIZE + :exclude-members: image_uri, num_trees, num_samples_per_tree, eval_metrics, feature_dim, MINI_BATCH_SIZE .. autoclass:: sagemaker.RandomCutForestModel diff --git a/src/sagemaker/amazon/kmeans.py b/src/sagemaker/amazon/kmeans.py index 3e321a33d0..bc771292b1 100644 --- a/src/sagemaker/amazon/kmeans.py +++ b/src/sagemaker/amazon/kmeans.py @@ -43,6 +43,11 @@ class KMeans(AmazonAlgorithmEstimatorBase): ) epochs = hp("epochs", gt(0), "An integer greater-than 0", int) center_factor = hp("extra_center_factor", gt(0), "An integer greater-than 0", int) + eval_metrics = hp( + name="eval_metrics", + validation_message='A comma separated list of "msd" or "ssd"', + data_type=list, + ) def __init__( self, @@ -58,6 +63,7 @@ def __init__( half_life_time_size=None, epochs=None, center_factor=None, + eval_metrics=None, **kwargs ): """A k-means clustering @@ -124,6 +130,11 @@ def __init__( center_factor (int): The algorithm will create ``num_clusters * extra_center_factor`` as it runs and reduce the number of centers to ``k`` when finalizing + eval_metrics (list): JSON list of metrics types to be used for + reporting the score for the model. Allowed values are "msd" + Means Square Error, "ssd": Sum of square distance. If test data + is provided, the score shall be reported in terms of all + requested metrics. **kwargs: base class keyword argument values. .. tip:: @@ -142,6 +153,7 @@ def __init__( self.half_life_time_size = half_life_time_size self.epochs = epochs self.center_factor = center_factor + self.eval_metrics = eval_metrics def create_model(self, vpc_config_override=VPC_CONFIG_DEFAULT, **kwargs): """Return a :class:`~sagemaker.amazon.kmeans.KMeansModel` referencing diff --git a/src/sagemaker/amazon/randomcutforest.py b/src/sagemaker/amazon/randomcutforest.py index eacfe08305..6a141150e8 100644 --- a/src/sagemaker/amazon/randomcutforest.py +++ b/src/sagemaker/amazon/randomcutforest.py @@ -31,6 +31,12 @@ class RandomCutForest(AmazonAlgorithmEstimatorBase): repo_version = 1 MINI_BATCH_SIZE = 1000 + eval_metrics = hp( + name="eval_metrics", + validation_message='A comma separated list of "accuracy" or "precision_recall_fscore"', + data_type=list, + ) + num_trees = hp("num_trees", (ge(50), le(1000)), "An integer in [50, 1000]", int) num_samples_per_tree = hp( "num_samples_per_tree", (ge(1), le(2048)), "An integer in [1, 2048]", int @@ -44,6 +50,7 @@ def __init__( instance_type, num_samples_per_tree=None, num_trees=None, + eval_metrics=None, **kwargs ): """RandomCutForest is :class:`Estimator` used for anomaly detection. @@ -92,6 +99,11 @@ def __init__( build each tree in the forest. The total number of samples drawn from the train dataset is num_trees * num_samples_per_tree. num_trees (int): Optional. The number of trees used in the forest. + eval_metrics (list): Optional. JSON list of metrics types to be used + for reporting the score for the model. Allowed values are + "accuracy", "precision_recall_fscore": positive and negative + precision, recall, and f1 scores. If test data is provided, the + score shall be reported in terms of all requested metrics. **kwargs: base class keyword argument values. .. tip:: @@ -104,6 +116,7 @@ def __init__( super(RandomCutForest, self).__init__(role, instance_count, instance_type, **kwargs) self.num_samples_per_tree = num_samples_per_tree self.num_trees = num_trees + self.eval_metrics = eval_metrics def create_model(self, vpc_config_override=VPC_CONFIG_DEFAULT, **kwargs): """Return a :class:`~sagemaker.amazon.RandomCutForestModel` referencing diff --git a/tests/integ/test_airflow_config.py b/tests/integ/test_airflow_config.py index 97140b92d4..155c03ea66 100644 --- a/tests/integ/test_airflow_config.py +++ b/tests/integ/test_airflow_config.py @@ -111,6 +111,7 @@ def test_kmeans_airflow_config_uploads_data_source_to_s3(sagemaker_session, cpu_ kmeans.half_life_time_size = 1 kmeans.epochs = 1 kmeans.center_factor = 1 + kmeans.eval_metrics = ["ssd", "msd"] records = kmeans.record_set(datasets.one_p_mnist()[0][:100]) @@ -385,6 +386,7 @@ def test_rcf_airflow_config_uploads_data_source_to_s3(sagemaker_session, cpu_ins instance_type=cpu_instance_type, num_trees=50, num_samples_per_tree=20, + eval_metrics=["accuracy", "precision_recall_fscore"], sagemaker_session=sagemaker_session, ) diff --git a/tests/integ/test_kmeans.py b/tests/integ/test_kmeans.py index c394b88d55..f04614f493 100644 --- a/tests/integ/test_kmeans.py +++ b/tests/integ/test_kmeans.py @@ -12,6 +12,7 @@ # language governing permissions and limitations under the License. from __future__ import absolute_import +import json import time import pytest @@ -46,6 +47,7 @@ def test_kmeans(sagemaker_session, cpu_instance_type, training_set): kmeans.half_life_time_size = 1 kmeans.epochs = 1 kmeans.center_factor = 1 + kmeans.eval_metrics = ["ssd", "msd"] assert kmeans.hyperparameters() == dict( init_method=kmeans.init_method, @@ -57,6 +59,7 @@ def test_kmeans(sagemaker_session, cpu_instance_type, training_set): epochs=str(kmeans.epochs), extra_center_factor=str(kmeans.center_factor), k=str(kmeans.k), + eval_metrics=json.dumps(kmeans.eval_metrics), force_dense="True", ) diff --git a/tests/integ/test_multidatamodel.py b/tests/integ/test_multidatamodel.py index 628903c4e5..415cdd1048 100644 --- a/tests/integ/test_multidatamodel.py +++ b/tests/integ/test_multidatamodel.py @@ -439,6 +439,7 @@ def __rcf_training_job( instance_type=cpu_instance_type, num_trees=num_trees, num_samples_per_tree=num_samples_per_tree, + eval_metrics=["accuracy", "precision_recall_fscore"], sagemaker_session=sagemaker_session, ) diff --git a/tests/integ/test_randomcutforest.py b/tests/integ/test_randomcutforest.py index 39358d864b..e108c26759 100644 --- a/tests/integ/test_randomcutforest.py +++ b/tests/integ/test_randomcutforest.py @@ -34,6 +34,7 @@ def test_randomcutforest(sagemaker_session, cpu_instance_type): instance_type=cpu_instance_type, num_trees=50, num_samples_per_tree=20, + eval_metrics=["accuracy", "precision_recall_fscore"], sagemaker_session=sagemaker_session, ) diff --git a/tests/unit/test_kmeans.py b/tests/unit/test_kmeans.py index 2d76861b85..ea4e6bcb8e 100644 --- a/tests/unit/test_kmeans.py +++ b/tests/unit/test_kmeans.py @@ -92,6 +92,7 @@ def test_all_hyperparameters(sagemaker_session): half_life_time_size=0, epochs=10, center_factor=2, + eval_metrics=["msd", "ssd"], **ALL_REQ_ARGS ) assert kmeans.hyperparameters() == dict( @@ -104,6 +105,7 @@ def test_all_hyperparameters(sagemaker_session): half_life_time_size="0", epochs="10", extra_center_factor="2", + eval_metrics='["msd", "ssd"]', force_dense="True", ) @@ -129,7 +131,7 @@ def test_required_hyper_parameters_value(sagemaker_session, required_hyper_param KMeans(sagemaker_session=sagemaker_session, **test_params) -@pytest.mark.parametrize("iterable_hyper_parameters, value", [("eval_metrics", [0])]) +@pytest.mark.parametrize("iterable_hyper_parameters, value", [("eval_metrics", 0)]) def test_iterable_hyper_parameters_type(sagemaker_session, iterable_hyper_parameters, value): with pytest.raises(TypeError): test_params = ALL_REQ_ARGS.copy() diff --git a/tests/unit/test_linear_learner.py b/tests/unit/test_linear_learner.py index a97ff55cbd..955bddbef1 100644 --- a/tests/unit/test_linear_learner.py +++ b/tests/unit/test_linear_learner.py @@ -216,7 +216,7 @@ def test_num_classes_can_be_string_for_multiclass_classifier(sagemaker_session): LinearLearner(sagemaker_session=sagemaker_session, **test_params) -@pytest.mark.parametrize("iterable_hyper_parameters, value", [("eval_metrics", [0])]) +@pytest.mark.parametrize("iterable_hyper_parameters, value", [("max_iterations", [0])]) def test_iterable_hyper_parameters_type(sagemaker_session, iterable_hyper_parameters, value): with pytest.raises(TypeError): test_params = ALL_REQ_ARGS.copy() diff --git a/tests/unit/test_randomcutforest.py b/tests/unit/test_randomcutforest.py index d2ca399090..b7e6ef1e49 100644 --- a/tests/unit/test_randomcutforest.py +++ b/tests/unit/test_randomcutforest.py @@ -24,6 +24,7 @@ INSTANCE_TYPE = "ml.c4.xlarge" NUM_SAMPLES_PER_TREE = 20 NUM_TREES = 50 +EVAL_METRICS = ["accuracy", "precision_recall_fscore"] COMMON_TRAIN_ARGS = { "role": ROLE, @@ -70,6 +71,7 @@ def test_init_required_positional(sagemaker_session): INSTANCE_TYPE, NUM_SAMPLES_PER_TREE, NUM_TREES, + EVAL_METRICS, sagemaker_session=sagemaker_session, ) assert randomcutforest.role == ROLE @@ -77,6 +79,7 @@ def test_init_required_positional(sagemaker_session): assert randomcutforest.instance_type == INSTANCE_TYPE assert randomcutforest.num_trees == NUM_TREES assert randomcutforest.num_samples_per_tree == NUM_SAMPLES_PER_TREE + assert randomcutforest.eval_metrics == EVAL_METRICS def test_init_required_named(sagemaker_session): @@ -92,10 +95,13 @@ def test_all_hyperparameters(sagemaker_session): sagemaker_session=sagemaker_session, num_trees=NUM_TREES, num_samples_per_tree=NUM_SAMPLES_PER_TREE, + eval_metrics=EVAL_METRICS, **ALL_REQ_ARGS ) assert randomcutforest.hyperparameters() == dict( - num_samples_per_tree=str(NUM_SAMPLES_PER_TREE), num_trees=str(NUM_TREES), + num_samples_per_tree=str(NUM_SAMPLES_PER_TREE), + num_trees=str(NUM_TREES), + eval_metrics='["accuracy", "precision_recall_fscore"]', ) @@ -104,7 +110,7 @@ def test_image(sagemaker_session): assert image_uris.retrieve("randomcutforest", REGION) == randomcutforest.train_image() -@pytest.mark.parametrize("iterable_hyper_parameters, value", [("eval_metrics", [0])]) +@pytest.mark.parametrize("iterable_hyper_parameters, value", [("eval_metrics", 0)]) def test_iterable_hyper_parameters_type(sagemaker_session, iterable_hyper_parameters, value): with pytest.raises(TypeError): test_params = ALL_REQ_ARGS.copy()