diff --git a/examples/run_setup_tutorial.py b/examples/run_setup_tutorial.py new file mode 100644 index 000000000..b57ba367b --- /dev/null +++ b/examples/run_setup_tutorial.py @@ -0,0 +1,102 @@ +""" +========= +Run Setup +========= + +By: Jan N. van Rijn + +One of the key features of the openml-python library is that is allows to +reinstantiate flows with hyperparameter settings that were uploaded before. +This tutorial uses the concept of setups. Although setups are not extensively +described in the OpenML documentation (because most users will not directly +use them), they form a important concept within OpenML distinguishing between +hyperparameter configurations. +A setup is the combination of a flow with all its hyperparameters set. + +A key requirement for reinstantiating a flow is to have the same scikit-learn +version as the flow that was uploaded. However, this tutorial will upload the +flow (that will later be reinstantiated) itself, so it can be ran with any +scikit-learn version that is supported by this library. In this case, the +requirement of the corresponding scikit-learn versions is automatically met. + +In this tutorial we will + 1) Create a flow and use it to solve a task; + 2) Download the flow, reinstantiate the model with same hyperparameters, + and solve the same task again; + 3) We will verify that the obtained results are exactly the same. +""" +import logging +import numpy as np +import openml +import sklearn.ensemble +import sklearn.impute +import sklearn.preprocessing + + +root = logging.getLogger() +root.setLevel(logging.INFO) + +############################################################################### +# 1) Create a flow and use it to solve a task +############################################################################### + +# first, let's download the task that we are interested in +task = openml.tasks.get_task(6) + + +# we will create a fairly complex model, with many preprocessing components and +# many potential hyperparameters. Of course, the model can be as complex and as +# easy as you want it to be +model_original = sklearn.pipeline.make_pipeline( + sklearn.impute.SimpleImputer(), + sklearn.ensemble.RandomForestClassifier() +) + + +# Let's change some hyperparameters. Of course, in any good application we +# would tune them using, e.g., Random Search or Bayesian Optimization, but for +# the purpose of this tutorial we set them to some specific values that might +# or might not be optimal +hyperparameters_original = { + 'simpleimputer__strategy': 'median', + 'randomforestclassifier__criterion': 'entropy', + 'randomforestclassifier__max_features': 0.2, + 'randomforestclassifier__min_samples_leaf': 1, + 'randomforestclassifier__n_estimators': 16, + 'randomforestclassifier__random_state': 42, +} +model_original.set_params(**hyperparameters_original) + +# solve the task and upload the result (this implicitly creates the flow) +run = openml.runs.run_model_on_task( + model_original, + task, + avoid_duplicate_runs=False) +run_original = run.publish() # this implicitly uploads the flow + +############################################################################### +# 2) Download the flow, reinstantiate the model with same hyperparameters, +# and solve the same task again. +############################################################################### + +# obtain setup id (note that the setup id is assigned by the OpenML server - +# therefore it was not yet available in our local copy of the run) +run_downloaded = openml.runs.get_run(run_original.run_id) +setup_id = run_downloaded.setup_id + +# after this, we can easily reinstantiate the model +model_duplicate = openml.setups.initialize_model(setup_id) +# it will automatically have all the hyperparameters set + +# and run the task again +run_duplicate = openml.runs.run_model_on_task( + model_duplicate, task, avoid_duplicate_runs=False) + + +############################################################################### +# 3) We will verify that the obtained results are exactly the same. +############################################################################### + +# the run has stored all predictions in the field data content +np.testing.assert_array_equal(run_original.data_content, + run_duplicate.data_content) diff --git a/openml/flows/__init__.py b/openml/flows/__init__.py index 2d70e9e32..0bdcf0c86 100644 --- a/openml/flows/__init__.py +++ b/openml/flows/__init__.py @@ -1,7 +1,8 @@ -from .flow import OpenMLFlow, _copy_server_fields +from .flow import OpenMLFlow -from .sklearn_converter import sklearn_to_flow, flow_to_sklearn, _check_n_jobs +from .sklearn_converter import sklearn_to_flow, flow_to_sklearn, \ + openml_param_name_to_sklearn from .functions import get_flow, list_flows, flow_exists, assert_flows_equal -__all__ = ['OpenMLFlow', 'create_flow_from_model', 'get_flow', 'list_flows', - 'sklearn_to_flow', 'flow_to_sklearn', 'flow_exists'] +__all__ = ['OpenMLFlow', 'get_flow', 'list_flows', 'sklearn_to_flow', + 'flow_to_sklearn', 'flow_exists', 'openml_param_name_to_sklearn'] diff --git a/openml/flows/flow.py b/openml/flows/flow.py index 0c70fc9bc..98e9e75ee 100644 --- a/openml/flows/flow.py +++ b/openml/flows/flow.py @@ -359,6 +359,60 @@ def publish(self): (flow_id, message)) return self + def get_structure(self, key_item): + """ + Returns for each sub-component of the flow the path of identifiers that + should be traversed to reach this component. The resulting dict maps a + key (identifying a flow by either its id, name or fullname) to the + parameter prefix. + + Parameters + ---------- + key_item: str + The flow attribute that will be used to identify flows in the + structure. Allowed values {flow_id, name} + + Returns + ------- + dict[str, List[str]] + The flow structure + """ + if key_item not in ['flow_id', 'name']: + raise ValueError('key_item should be in {flow_id, name}') + structure = dict() + for key, sub_flow in self.components.items(): + sub_structure = sub_flow.get_structure(key_item) + for flow_name, flow_sub_structure in sub_structure.items(): + structure[flow_name] = [key] + flow_sub_structure + structure[getattr(self, key_item)] = [] + return structure + + def get_subflow(self, structure): + """ + Returns a subflow from the tree of dependencies. + + Parameters + ---------- + structure: list[str] + A list of strings, indicating the location of the subflow + + Returns + ------- + OpenMLFlow + The OpenMLFlow that corresponds to the structure + """ + if len(structure) < 1: + raise ValueError('Please provide a structure list of size >= 1') + sub_identifier = structure[0] + if sub_identifier not in self.components: + raise ValueError('Flow %s does not contain component with ' + 'identifier %s' % (self.name, sub_identifier)) + if len(structure) == 1: + return self.components[sub_identifier] + else: + structure.pop(0) + return self.components[sub_identifier].get_subflow(structure) + def push_tag(self, tag): """Annotates this flow with a tag on the server. diff --git a/openml/flows/sklearn_converter.py b/openml/flows/sklearn_converter.py index 82b5895fa..869ab70a7 100644 --- a/openml/flows/sklearn_converter.py +++ b/openml/flows/sklearn_converter.py @@ -11,7 +11,6 @@ import six import warnings import sys -import inspect import numpy as np import scipy.stats.distributions @@ -177,6 +176,37 @@ def flow_to_sklearn(o, components=None, initialize_with_defaults=False): return rval +def openml_param_name_to_sklearn(openml_parameter, flow): + """ + Converts the name of an OpenMLParameter into the sklean name, given a flow. + + Parameters + ---------- + openml_parameter: OpenMLParameter + The parameter under consideration + + flow: OpenMLFlow + The flow that provides context. + + Returns + ------- + sklearn_parameter_name: str + The name the parameter will have once used in scikit-learn + """ + if not isinstance(openml_parameter, openml.setups.OpenMLParameter): + raise ValueError('openml_parameter should be an instance of ' + 'OpenMLParameter') + if not isinstance(flow, OpenMLFlow): + raise ValueError('flow should be an instance of OpenMLFlow') + + flow_structure = flow.get_structure('name') + if openml_parameter.flow_name not in flow_structure: + raise ValueError('Obtained OpenMLParameter and OpenMLFlow do not ' + 'correspond. ') + name = openml_parameter.flow_name # for PEP8 + return '__'.join(flow_structure[name] + [openml_parameter.parameter_name]) + + def _serialize_model(model): """Create an OpenMLFlow. diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 3d42196b0..9dcb96a42 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -17,8 +17,9 @@ import openml._api_calls from ..exceptions import PyOpenMLError from .. import config -from ..flows import sklearn_to_flow, get_flow, flow_exists, _check_n_jobs, \ - _copy_server_fields, OpenMLFlow +from openml.flows.sklearn_converter import _check_n_jobs +from openml.flows.flow import _copy_server_fields +from ..flows import sklearn_to_flow, get_flow, flow_exists, OpenMLFlow from ..setups import setup_exists, initialize_model from ..exceptions import OpenMLCacheException, OpenMLServerException from ..tasks import OpenMLTask diff --git a/openml/setups/__init__.py b/openml/setups/__init__.py index 1c07274bb..a8b4a8863 100644 --- a/openml/setups/__init__.py +++ b/openml/setups/__init__.py @@ -1,4 +1,5 @@ -from .setup import OpenMLSetup +from .setup import OpenMLSetup, OpenMLParameter from .functions import get_setup, list_setups, setup_exists, initialize_model -__all__ = ['get_setup', 'list_setups', 'setup_exists', 'initialize_model'] \ No newline at end of file +__all__ = ['OpenMLSetup', 'OpenMLParameter', 'get_setup', 'list_setups', + 'setup_exists', 'initialize_model'] diff --git a/openml/setups/functions.py b/openml/setups/functions.py index fb58dc1ab..bec528846 100644 --- a/openml/setups/functions.py +++ b/openml/setups/functions.py @@ -211,44 +211,16 @@ def initialize_model(setup_id): # transform an openml setup object into # a dict of dicts, structured: flow_id maps to dict of # parameter_names mapping to parameter_value - setup = get_setup(setup_id) - parameters = {} - for _param in setup.parameters: - _flow_id = setup.parameters[_param].flow_id - _param_name = setup.parameters[_param].parameter_name - _param_value = setup.parameters[_param].value - if _flow_id not in parameters: - parameters[_flow_id] = {} - parameters[_flow_id][_param_name] = _param_value - - def _reconstruct_flow(_flow, _params): - # recursively set the values of flow parameters (and subflows) to - # the specific values from a setup. _params is a dict of - # dicts, mapping from flow id to param name to param value - # (obtained by using the subfunction _to_dict_of_dicts) - for _param in _flow.parameters: - # It can happen that no parameters of a flow are in a setup, - # then the flow_id is not in _params; usually happens for a - # sklearn.pipeline.Pipeline object, where the steps parameter is - # not in the setup - if _flow.flow_id not in _params: - continue - # It is not guaranteed that a setup on OpenML has all parameter - # settings of a flow, thus a param must not be in _params! - if _param not in _params[_flow.flow_id]: - continue - _flow.parameters[_param] = _params[_flow.flow_id][_param] - for _identifier in _flow.components: - _flow.components[_identifier] = _reconstruct_flow(_flow.components[_identifier], _params) - return _flow - - # now we 'abuse' the parameter object by passing in the - # parameters obtained from the setup flow = openml.flows.get_flow(setup.flow_id) - flow = _reconstruct_flow(flow, parameters) - - return openml.flows.flow_to_sklearn(flow) + model = openml.flows.flow_to_sklearn(flow) + hyperparameters = { + openml.flows.openml_param_name_to_sklearn(hp, flow): + openml.flows.flow_to_sklearn(hp.value) + for hp in setup.parameters.values() + } + model.set_params(**hyperparameters) + return model def _to_dict(flow_id, openml_parameter_settings): @@ -288,10 +260,11 @@ def _create_setup_from_xml(result_dict): def _create_setup_parameter_from_xml(result_dict): - return OpenMLParameter(int(result_dict['oml:id']), - int(result_dict['oml:flow_id']), - result_dict['oml:full_name'], - result_dict['oml:parameter_name'], - result_dict['oml:data_type'], - result_dict['oml:default_value'], - result_dict['oml:value']) + return OpenMLParameter(input_id=int(result_dict['oml:id']), + flow_id=int(result_dict['oml:flow_id']), + flow_name=result_dict['oml:flow_name'], + full_name=result_dict['oml:full_name'], + parameter_name=result_dict['oml:parameter_name'], + data_type=result_dict['oml:data_type'], + default_value=result_dict['oml:default_value'], + value=result_dict['oml:value']) diff --git a/openml/setups/setup.py b/openml/setups/setup.py index 05ab3647f..d5579b30c 100644 --- a/openml/setups/setup.py +++ b/openml/setups/setup.py @@ -29,27 +29,32 @@ def __init__(self, setup_id, flow_id, parameters): class OpenMLParameter(object): """Parameter object (used in setup). - Parameters - ---------- - id : int - The input id from the openml database - flow id : int - The flow to which this parameter is associated - full_name : str - The name of the flow and parameter combined - parameter_name : str - The name of the parameter - data_type : str - The datatype of the parameter. generally unused for sklearn flows - default_value : str - The default value. For sklearn parameters, this is unknown and a - default value is selected arbitrarily - value : str - If the parameter was set, the value that it was set to. + Parameters + ---------- + input_id : int + The input id from the openml database + flow id : int + The flow to which this parameter is associated + flow name : str + The name of the flow (no version number) to which this parameter + is associated + full_name : str + The name of the flow and parameter combined + parameter_name : str + The name of the parameter + data_type : str + The datatype of the parameter. generally unused for sklearn flows + default_value : str + The default value. For sklearn parameters, this is unknown and a + default value is selected arbitrarily + value : str + If the parameter was set, the value that it was set to. """ - def __init__(self, id, flow_id, full_name, parameter_name, data_type, default_value, value): - self.id = id + def __init__(self, input_id, flow_id, flow_name, full_name, parameter_name, + data_type, default_value, value): + self.id = input_id self.flow_id = flow_id + self.flow_name = flow_name self.full_name = full_name self.parameter_name = parameter_name self.data_type = data_type diff --git a/openml/tasks/functions.py b/openml/tasks/functions.py index de01ac052..d5b0b0ac5 100644 --- a/openml/tasks/functions.py +++ b/openml/tasks/functions.py @@ -172,7 +172,7 @@ def _list_tasks(task_type_id=None, **kwargs): - Survival Analysis: 7 - Subgroup Discovery: 8 kwargs: dict, optional - Legal filter operators: tag, data_tag, status, limit, + Legal filter operators: tag, task_id (list), data_tag, status, limit, offset, data_id, data_name, number_instances, number_features, number_classes, number_missing_values. Returns @@ -184,6 +184,8 @@ def _list_tasks(task_type_id=None, **kwargs): api_call += "/type/%d" % int(task_type_id) if kwargs is not None: for operator, value in kwargs.items(): + if operator == 'task_id': + value = ','.join([str(int(i)) for i in value]) api_call += "/%s/%s" % (operator, value) return __list_tasks(api_call) @@ -387,8 +389,8 @@ def _create_task_from_xml(xml): common_kwargs['estimation_procedure_type'] = inputs[ "estimation_procedure"][ - "oml:estimation_procedure"]["oml:type"], - common_kwargs['estimation_parameters'] = estimation_parameters, + "oml:estimation_procedure"]["oml:type"] + common_kwargs['estimation_parameters'] = estimation_parameters common_kwargs['target_name'] = inputs[ "source_data"]["oml:data_set"]["oml:target_feature"] common_kwargs['data_splits_url'] = inputs["estimation_procedure"][ diff --git a/tests/files/org/openml/test/setups/1/description.xml b/tests/files/org/openml/test/setups/1/description.xml index ee234e4ff..5717ad9f5 100644 --- a/tests/files/org/openml/test/setups/1/description.xml +++ b/tests/files/org/openml/test/setups/1/description.xml @@ -4,6 +4,7 @@ 3432 60 + weka.J48 weka.J48(1)_C C option @@ -13,6 +14,7 @@ 3435 60 + weka.J48 weka.J48(1)_M M option diff --git a/tests/test_flows/test_flow.py b/tests/test_flows/test_flow.py index 39c03fee1..f3f63b67d 100644 --- a/tests/test_flows/test_flow.py +++ b/tests/test_flows/test_flow.py @@ -74,6 +74,30 @@ def test_get_flow(self): self.assertEqual(subflow_3.parameters['L'], '-1') self.assertEqual(len(subflow_3.components), 0) + def test_get_structure(self): + # also responsible for testing: flow.get_subflow + # We need to use the production server here because 4024 is not the + # test server + openml.config.server = self.production_server + + flow = openml.flows.get_flow(4024) + flow_structure_name = flow.get_structure('name') + flow_structure_id = flow.get_structure('flow_id') + # components: root (filteredclassifier), multisearch, loginboost, + # reptree + self.assertEqual(len(flow_structure_name), 4) + self.assertEqual(len(flow_structure_id), 4) + + for sub_flow_name, structure in flow_structure_name.items(): + if len(structure) > 0: # skip root element + subflow = flow.get_subflow(structure) + self.assertEqual(subflow.name, sub_flow_name) + + for sub_flow_id, structure in flow_structure_id.items(): + if len(structure) > 0: # skip root element + subflow = flow.get_subflow(structure) + self.assertEqual(subflow.flow_id, sub_flow_id) + def test_tagging(self): flow_list = openml.flows.list_flows(size=1) flow_id = list(flow_list.keys())[0] diff --git a/tests/test_flows/test_sklearn.py b/tests/test_flows/test_sklearn.py index b4cf524b7..03960e6ef 100644 --- a/tests/test_flows/test_sklearn.py +++ b/tests/test_flows/test_sklearn.py @@ -106,14 +106,17 @@ def test_serialize_model(self, check_dependencies_mock): ('presort', 'false'), ('random_state', 'null'), ('splitter', '"best"'))) + structure_fixture = {'sklearn.tree.tree.DecisionTreeClassifier': []} serialization = sklearn_to_flow(model) + structure = serialization.get_structure('name') self.assertEqual(serialization.name, fixture_name) self.assertEqual(serialization.class_name, fixture_name) self.assertEqual(serialization.description, fixture_description) self.assertEqual(serialization.parameters, fixture_parameters) self.assertEqual(serialization.dependencies, version_fixture) + self.assertDictEqual(structure, structure_fixture) new_model = flow_to_sklearn(serialization) @@ -160,14 +163,17 @@ def test_serialize_model_clustering(self, check_dependencies_mock): ('random_state', 'null'), ('tol', '0.0001'), ('verbose', '0'))) + fixture_structure = {'sklearn.cluster.k_means_.KMeans': []} serialization = sklearn_to_flow(model) + structure = serialization.get_structure('name') self.assertEqual(serialization.name, fixture_name) self.assertEqual(serialization.class_name, fixture_name) self.assertEqual(serialization.description, fixture_description) self.assertEqual(serialization.parameters, fixture_parameters) self.assertEqual(serialization.dependencies, version_fixture) + self.assertDictEqual(structure, fixture_structure) new_model = flow_to_sklearn(serialization) @@ -190,8 +196,13 @@ def test_serialize_model_with_subcomponent(self): fixture_subcomponent_name = 'sklearn.tree.tree.DecisionTreeClassifier' fixture_subcomponent_class_name = 'sklearn.tree.tree.DecisionTreeClassifier' fixture_subcomponent_description = 'Automatically created scikit-learn flow.' + fixture_structure = { + fixture_name: [], + 'sklearn.tree.tree.DecisionTreeClassifier': ['base_estimator'] + } - serialization = sklearn_to_flow(model) + serialization = sklearn_to_flow(model) + structure = serialization.get_structure('name') self.assertEqual(serialization.name, fixture_name) self.assertEqual(serialization.class_name, fixture_class_name) @@ -206,6 +217,7 @@ def test_serialize_model_with_subcomponent(self): fixture_subcomponent_class_name) self.assertEqual(serialization.components['base_estimator'].description, fixture_subcomponent_description) + self.assertDictEqual(structure, fixture_structure) new_model = flow_to_sklearn(serialization) @@ -233,11 +245,18 @@ def test_serialize_pipeline(self): 'scaler=sklearn.preprocessing.data.StandardScaler,' \ 'dummy=sklearn.dummy.DummyClassifier)' fixture_description = 'Automatically created scikit-learn flow.' + fixture_structure = { + fixture_name: [], + 'sklearn.preprocessing.data.StandardScaler': ['scaler'], + 'sklearn.dummy.DummyClassifier': ['dummy'] + } serialization = sklearn_to_flow(model) + structure = serialization.get_structure('name') self.assertEqual(serialization.name, fixture_name) self.assertEqual(serialization.description, fixture_description) + self.assertDictEqual(structure, fixture_structure) # Comparing the pipeline # The parameters only have the name of base objects(not the whole flow) @@ -295,11 +314,18 @@ def test_serialize_pipeline_clustering(self): 'scaler=sklearn.preprocessing.data.StandardScaler,' \ 'clusterer=sklearn.cluster.k_means_.KMeans)' fixture_description = 'Automatically created scikit-learn flow.' + fixture_structure = { + fixture_name: [], + 'sklearn.preprocessing.data.StandardScaler': ['scaler'], + 'sklearn.cluster.k_means_.KMeans': ['clusterer'] + } serialization = sklearn_to_flow(model) + structure = serialization.get_structure('name') self.assertEqual(serialization.name, fixture_name) self.assertEqual(serialization.description, fixture_description) + self.assertDictEqual(structure, fixture_structure) # Comparing the pipeline # The parameters only have the name of base objects(not the whole flow) @@ -362,9 +388,17 @@ def test_serialize_column_transformer(self): 'numeric=sklearn.preprocessing.data.StandardScaler,' \ 'nominal=sklearn.preprocessing._encoders.OneHotEncoder)' fixture_description = 'Automatically created scikit-learn flow.' + fixture_structure = { + fixture: [], + 'sklearn.preprocessing.data.StandardScaler': ['numeric'], + 'sklearn.preprocessing._encoders.OneHotEncoder': ['nominal'] + } + serialization = sklearn_to_flow(model) + structure = serialization.get_structure('name') self.assertEqual(serialization.name, fixture) self.assertEqual(serialization.description, fixture_description) + self.assertDictEqual(structure, fixture_structure) # del serialization.model new_model = flow_to_sklearn(serialization) self.assertEqual(type(new_model), type(model)) @@ -393,11 +427,24 @@ def test_serialize_column_transformer_pipeline(self): 'numeric=sklearn.preprocessing.data.StandardScaler,'\ 'nominal=sklearn.preprocessing._encoders.OneHotEncoder),'\ 'classifier=sklearn.tree.tree.DecisionTreeClassifier)' + fixture_structure = { + 'sklearn.preprocessing.data.StandardScaler': + ['transformer', 'numeric'], + 'sklearn.preprocessing._encoders.OneHotEncoder': + ['transformer', 'nominal'], + 'sklearn.compose._column_transformer.ColumnTransformer(numeric=' + 'sklearn.preprocessing.data.StandardScaler,nominal=sklearn.' + 'preprocessing._encoders.OneHotEncoder)': ['transformer'], + 'sklearn.tree.tree.DecisionTreeClassifier': ['classifier'], + fixture_name: [], + } fixture_description = 'Automatically created scikit-learn flow.' serialization = sklearn_to_flow(model) + structure = serialization.get_structure('name') self.assertEqual(serialization.name, fixture_name) self.assertEqual(serialization.description, fixture_description) + self.assertDictEqual(structure, fixture_structure) # del serialization.model new_model = flow_to_sklearn(serialization) self.assertEqual(type(new_model), type(model)) @@ -415,15 +462,23 @@ def test_serialize_feature_union(self): fu = sklearn.pipeline.FeatureUnion( transformer_list=[('ohe', ohe), ('scaler', scaler)]) serialization = sklearn_to_flow(fu) + structure = serialization.get_structure('name') # OneHotEncoder was moved to _encoders module in 0.20 module_name_encoder = ('_encoders' if LooseVersion(sklearn.__version__) >= "0.20" else 'data') - self.assertEqual(serialization.name, - 'sklearn.pipeline.FeatureUnion(' - 'ohe=sklearn.preprocessing.{}.OneHotEncoder,' - 'scaler=sklearn.preprocessing.data.StandardScaler)' - .format(module_name_encoder)) + fixture_name = ('sklearn.pipeline.FeatureUnion(' + 'ohe=sklearn.preprocessing.{}.OneHotEncoder,' + 'scaler=sklearn.preprocessing.data.StandardScaler)' + .format(module_name_encoder)) + fixture_structure = { + fixture_name: [], + 'sklearn.preprocessing.{}.' + 'OneHotEncoder'.format(module_name_encoder): ['ohe'], + 'sklearn.preprocessing.data.StandardScaler': ['scaler'] + } + self.assertEqual(serialization.name, fixture_name) + self.assertDictEqual(structure, fixture_structure) new_model = flow_to_sklearn(serialization) self.assertEqual(type(new_model), type(fu)) @@ -510,19 +565,31 @@ def test_serialize_complex_flow(self): rs = sklearn.model_selection.RandomizedSearchCV( estimator=model, param_distributions=parameter_grid, cv=cv) serialized = sklearn_to_flow(rs) + structure = serialized.get_structure('name') # OneHotEncoder was moved to _encoders module in 0.20 module_name_encoder = ('_encoders' if LooseVersion(sklearn.__version__) >= "0.20" else 'data') - fixture_name = \ - ('sklearn.model_selection._search.RandomizedSearchCV(' - 'estimator=sklearn.pipeline.Pipeline(' - 'ohe=sklearn.preprocessing.{}.OneHotEncoder,' - 'scaler=sklearn.preprocessing.data.StandardScaler,' - 'boosting=sklearn.ensemble.weight_boosting.AdaBoostClassifier(' - 'base_estimator=sklearn.tree.tree.DecisionTreeClassifier)))'. - format(module_name_encoder)) + ohe_name = 'sklearn.preprocessing.%s.OneHotEncoder' % \ + module_name_encoder + scaler_name = 'sklearn.preprocessing.data.StandardScaler' + tree_name = 'sklearn.tree.tree.DecisionTreeClassifier' + boosting_name = 'sklearn.ensemble.weight_boosting.AdaBoostClassifier' \ + '(base_estimator=%s)' % tree_name + pipeline_name = 'sklearn.pipeline.Pipeline(ohe=%s,scaler=%s,' \ + 'boosting=%s)' % (ohe_name, scaler_name, boosting_name) + fixture_name = 'sklearn.model_selection._search.RandomizedSearchCV' \ + '(estimator=%s)' % pipeline_name + fixture_structure = { + ohe_name: ['estimator', 'ohe'], + scaler_name: ['estimator', 'scaler'], + tree_name: ['estimator', 'boosting', 'base_estimator'], + boosting_name: ['estimator', 'boosting'], + pipeline_name: ['estimator'], + fixture_name: [] + } self.assertEqual(serialized.name, fixture_name) + self.assertEqual(structure, fixture_structure) # now do deserialization deserialized = flow_to_sklearn(serialized) @@ -923,3 +990,38 @@ def test_deserialize_complex_with_defaults(self): # equals function for this assert_flows_equal(openml.flows.sklearn_to_flow(pipe_orig), openml.flows.sklearn_to_flow(pipe_deserialized)) + + def test_openml_param_name_to_sklearn(self): + scaler = sklearn.preprocessing.StandardScaler(with_mean=False) + boosting = sklearn.ensemble.AdaBoostClassifier( + base_estimator=sklearn.tree.DecisionTreeClassifier()) + model = sklearn.pipeline.Pipeline(steps=[ + ('scaler', scaler), ('boosting', boosting)]) + flow = openml.flows.sklearn_to_flow(model) + task = openml.tasks.get_task(115) + run = openml.runs.run_flow_on_task(flow, task) + run = run.publish() + run = openml.runs.get_run(run.run_id) + setup = openml.setups.get_setup(run.setup_id) + + # make sure to test enough parameters + self.assertGreater(len(setup.parameters), 15) + + for parameter in setup.parameters.values(): + sklearn_name = openml.flows.openml_param_name_to_sklearn( + parameter, flow) + + # test the inverse. Currently, OpenML stores the hyperparameter + # fullName as flow.name + flow.version + parameter.name on the + # server (but this behaviour is not documented and might or might + # not change in the future. Hence, we won't offer this + # transformation functionality in the main package yet.) + splitted = sklearn_name.split("__") + if len(splitted) > 1: # if len is 1, it is part of root flow + subflow = flow.get_subflow(splitted[0:-1]) + else: + subflow = flow + openml_name = "%s(%s)_%s" % (subflow.name, + subflow.version, + splitted[-1]) + self.assertEqual(parameter.full_name, openml_name) diff --git a/tests/test_setups/test_setup_functions.py b/tests/test_setups/test_setup_functions.py index 928874837..35f43422e 100644 --- a/tests/test_setups/test_setup_functions.py +++ b/tests/test_setups/test_setup_functions.py @@ -162,7 +162,6 @@ def test_get_cached_setup(self): openml.config.cache_directory = self.static_cache_dir openml.setups.functions._get_cached_setup(1) - def test_get_uncached_setup(self): openml.config.cache_directory = self.static_cache_dir with self.assertRaises(openml.exceptions.OpenMLCacheException):