From 4ecd35c48830c9f88990fb3e1987e9c764643475 Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Fri, 5 Oct 2018 16:08:18 -0400 Subject: [PATCH 01/28] fixes minor indentation problems --- openml/setups/functions.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/openml/setups/functions.py b/openml/setups/functions.py index c329eab52..7e7c296f8 100644 --- a/openml/setups/functions.py +++ b/openml/setups/functions.py @@ -186,7 +186,7 @@ def __list_setups(api_call): def initialize_model(setup_id): - ''' + """ Initialized a model based on a setup_id (i.e., using the exact same parameter settings) @@ -199,7 +199,7 @@ def initialize_model(setup_id): ------- model : sklearn model the scikitlearn model with all parameters initailized - ''' + """ # transform an openml setup object into # a dict of dicts, structured: flow_id maps to dict of @@ -256,9 +256,9 @@ def _to_dict(flow_id, openml_parameter_settings): def _create_setup_from_xml(result_dict): - ''' - Turns an API xml result into a OpenMLSetup object - ''' + """ + Turns an API xml result into a OpenMLSetup object + """ setup_id = int(result_dict['oml:setup_parameters']['oml:setup_id']) flow_id = int(result_dict['oml:setup_parameters']['oml:flow_id']) parameters = {} @@ -279,6 +279,7 @@ def _create_setup_from_xml(result_dict): return OpenMLSetup(setup_id, flow_id, parameters) + def _create_setup_parameter_from_xml(result_dict): return OpenMLParameter(int(result_dict['oml:id']), int(result_dict['oml:flow_id']), From 1eafe18166e1dceb07c6a06058eaa1ee0f18216e Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Fri, 5 Oct 2018 16:39:48 -0400 Subject: [PATCH 02/28] initial commit --- openml/setups/__init__.py | 2 +- openml/setups/sklearn_converter.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 openml/setups/sklearn_converter.py diff --git a/openml/setups/__init__.py b/openml/setups/__init__.py index 1c07274bb..56f0a0b43 100644 --- a/openml/setups/__init__.py +++ b/openml/setups/__init__.py @@ -1,4 +1,4 @@ -from .setup import OpenMLSetup +from .setup import OpenMLSetup, OpenMLParameter from .functions import get_setup, list_setups, setup_exists, initialize_model __all__ = ['get_setup', 'list_setups', 'setup_exists', 'initialize_model'] \ No newline at end of file diff --git a/openml/setups/sklearn_converter.py b/openml/setups/sklearn_converter.py new file mode 100644 index 000000000..fd1e158bd --- /dev/null +++ b/openml/setups/sklearn_converter.py @@ -0,0 +1,22 @@ +from openml.flows import OpenMLFlow +from openml.setups import OpenMLParameter + + +def openml_param_name_to_sklearn(openml_parameter, flow): + """ + Converts the name of an OpenMLParameter into the sklean name, given a flow. + Note that the same parameter might have a different name in various flows + (e.g., the parameter ) + + Parameters + ---------- + openml_parameter: OpenMLParameter + The parameter under consideration + + flow: OpenMLFlow + The flow that provides context. + """ + if not isinstance(openml_parameter, OpenMLParameter): + raise ValueError('openml_parameter should be an instance of OpenMLParameter') + if not isinstance(flow, OpenMLFlow): + raise ValueError('flow should be an instance of OpenMLFlow') From 37e115cf45b202846d600bdeea8a694d19cdb701 Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Sat, 6 Oct 2018 18:50:28 -0400 Subject: [PATCH 03/28] adds a function to deduce the flow structure --- openml/flows/__init__.py | 5 +- openml/flows/sklearn_converter.py | 34 +++++++++++ openml/setups/functions.py | 15 ++--- openml/setups/setup.py | 3 +- tests/test_flows/test_sklearn.py | 95 ++++++++++++++++++++++++++----- 5 files changed, 128 insertions(+), 24 deletions(-) diff --git a/openml/flows/__init__.py b/openml/flows/__init__.py index 2d70e9e32..1482e2f69 100644 --- a/openml/flows/__init__.py +++ b/openml/flows/__init__.py @@ -1,7 +1,8 @@ from .flow import OpenMLFlow, _copy_server_fields -from .sklearn_converter import sklearn_to_flow, flow_to_sklearn, _check_n_jobs +from .sklearn_converter import sklearn_to_flow, flow_to_sklearn, \ + flow_structure, _check_n_jobs from .functions import get_flow, list_flows, flow_exists, assert_flows_equal -__all__ = ['OpenMLFlow', 'create_flow_from_model', 'get_flow', 'list_flows', +__all__ = ['OpenMLFlow', 'flow_structure', 'get_flow', 'list_flows', 'sklearn_to_flow', 'flow_to_sklearn', 'flow_exists'] diff --git a/openml/flows/sklearn_converter.py b/openml/flows/sklearn_converter.py index 82b5895fa..31b46018b 100644 --- a/openml/flows/sklearn_converter.py +++ b/openml/flows/sklearn_converter.py @@ -486,6 +486,40 @@ def _deserialize_model(flow, keep_defaults): return model_class(**parameter_dict) +def flow_structure(flow, key_item): + """ + Returns for each sub-component of the flow the path of identifiers that + should be traversed to reach this component. The resulting dict maps a key + (identifying a flow be either its id, name or fullname) to the parameter + prefix. + + Parameters + ---------- + flow: OpenMLFlow + The flow to generate the parameter prefixes for + + key_item: str + The flow attribute that will be used to identify flows in the + structure. Allowed values {id, name, fullName} + + Returns + ------- + structure: dict[str, List[str]] + The flow structure + """ + if not isinstance(flow, OpenMLFlow): + raise TypeError('flow should be of type OpenMLFlow') + if key_item not in ['id', 'name', 'fullName']: + raise ValueError('key_item should be in {id, name, fullName}') + structure = dict() + for key, sub_flow in flow.components.items(): + sub_structure = flow_structure(sub_flow, key_item) + for flow_name, flow_sub_structure in sub_structure.items(): + structure[flow_name] = [key] + flow_sub_structure + structure[getattr(flow, key_item)] = [] + return structure + + def _check_dependencies(dependencies): if not dependencies: return diff --git a/openml/setups/functions.py b/openml/setups/functions.py index 7e7c296f8..fd7001e73 100644 --- a/openml/setups/functions.py +++ b/openml/setups/functions.py @@ -281,10 +281,11 @@ def _create_setup_from_xml(result_dict): def _create_setup_parameter_from_xml(result_dict): - return OpenMLParameter(int(result_dict['oml:id']), - int(result_dict['oml:flow_id']), - result_dict['oml:full_name'], - result_dict['oml:parameter_name'], - result_dict['oml:data_type'], - result_dict['oml:default_value'], - result_dict['oml:value']) + return OpenMLParameter(id=int(result_dict['oml:id']), + flow_id=int(result_dict['oml:flow_id']), + flow_name=result_dict['oml:flow_name'], + full_name=result_dict['oml:full_name'], + parameter_name=result_dict['oml:parameter_name'], + data_type=result_dict['oml:data_type'], + default_value=result_dict['oml:default_value'], + value=result_dict['oml:value']) diff --git a/openml/setups/setup.py b/openml/setups/setup.py index 05ab3647f..85cd03b00 100644 --- a/openml/setups/setup.py +++ b/openml/setups/setup.py @@ -47,9 +47,10 @@ class OpenMLParameter(object): value : str If the parameter was set, the value that it was set to. """ - def __init__(self, id, flow_id, full_name, parameter_name, data_type, default_value, value): + def __init__(self, id, flow_id, flow_name, full_name, parameter_name, data_type, default_value, value): self.id = id self.flow_id = flow_id + self.flow_name = flow_name self.full_name = full_name self.parameter_name = parameter_name self.data_type = data_type diff --git a/tests/test_flows/test_sklearn.py b/tests/test_flows/test_sklearn.py index b4cf524b7..f0b28655d 100644 --- a/tests/test_flows/test_sklearn.py +++ b/tests/test_flows/test_sklearn.py @@ -106,14 +106,17 @@ def test_serialize_model(self, check_dependencies_mock): ('presort', 'false'), ('random_state', 'null'), ('splitter', '"best"'))) + structure_fixture = {'sklearn.tree.tree.DecisionTreeClassifier': []} serialization = sklearn_to_flow(model) + structure = openml.flows.flow_structure(serialization, 'name') self.assertEqual(serialization.name, fixture_name) self.assertEqual(serialization.class_name, fixture_name) self.assertEqual(serialization.description, fixture_description) self.assertEqual(serialization.parameters, fixture_parameters) self.assertEqual(serialization.dependencies, version_fixture) + self.assertDictEqual(structure, structure_fixture) new_model = flow_to_sklearn(serialization) @@ -160,14 +163,17 @@ def test_serialize_model_clustering(self, check_dependencies_mock): ('random_state', 'null'), ('tol', '0.0001'), ('verbose', '0'))) + fixture_structure = {'sklearn.cluster.k_means_.KMeans': []} serialization = sklearn_to_flow(model) + structure = openml.flows.flow_structure(serialization, 'name') self.assertEqual(serialization.name, fixture_name) self.assertEqual(serialization.class_name, fixture_name) self.assertEqual(serialization.description, fixture_description) self.assertEqual(serialization.parameters, fixture_parameters) self.assertEqual(serialization.dependencies, version_fixture) + self.assertDictEqual(structure, fixture_structure) new_model = flow_to_sklearn(serialization) @@ -190,8 +196,13 @@ def test_serialize_model_with_subcomponent(self): fixture_subcomponent_name = 'sklearn.tree.tree.DecisionTreeClassifier' fixture_subcomponent_class_name = 'sklearn.tree.tree.DecisionTreeClassifier' fixture_subcomponent_description = 'Automatically created scikit-learn flow.' + fixture_structure = { + fixture_name: [], + 'sklearn.tree.tree.DecisionTreeClassifier': ['base_estimator'] + } - serialization = sklearn_to_flow(model) + serialization = sklearn_to_flow(model) + structure = openml.flows.flow_structure(serialization, 'name') self.assertEqual(serialization.name, fixture_name) self.assertEqual(serialization.class_name, fixture_class_name) @@ -206,6 +217,7 @@ def test_serialize_model_with_subcomponent(self): fixture_subcomponent_class_name) self.assertEqual(serialization.components['base_estimator'].description, fixture_subcomponent_description) + self.assertDictEqual(structure, fixture_structure) new_model = flow_to_sklearn(serialization) @@ -233,11 +245,18 @@ def test_serialize_pipeline(self): 'scaler=sklearn.preprocessing.data.StandardScaler,' \ 'dummy=sklearn.dummy.DummyClassifier)' fixture_description = 'Automatically created scikit-learn flow.' + fixture_structure = { + fixture_name: [], + 'sklearn.preprocessing.data.StandardScaler': ['scaler'], + 'sklearn.dummy.DummyClassifier': ['dummy'] + } serialization = sklearn_to_flow(model) + structure = openml.flows.flow_structure(serialization, 'name') self.assertEqual(serialization.name, fixture_name) self.assertEqual(serialization.description, fixture_description) + self.assertDictEqual(structure, fixture_structure) # Comparing the pipeline # The parameters only have the name of base objects(not the whole flow) @@ -295,11 +314,18 @@ def test_serialize_pipeline_clustering(self): 'scaler=sklearn.preprocessing.data.StandardScaler,' \ 'clusterer=sklearn.cluster.k_means_.KMeans)' fixture_description = 'Automatically created scikit-learn flow.' + fixture_structure = { + fixture_name: [], + 'sklearn.preprocessing.data.StandardScaler': ['scaler'], + 'sklearn.cluster.k_means_.KMeans': ['clusterer'] + } serialization = sklearn_to_flow(model) + structure = openml.flows.flow_structure(serialization, 'name') self.assertEqual(serialization.name, fixture_name) self.assertEqual(serialization.description, fixture_description) + self.assertDictEqual(structure, fixture_structure) # Comparing the pipeline # The parameters only have the name of base objects(not the whole flow) @@ -362,9 +388,17 @@ def test_serialize_column_transformer(self): 'numeric=sklearn.preprocessing.data.StandardScaler,' \ 'nominal=sklearn.preprocessing._encoders.OneHotEncoder)' fixture_description = 'Automatically created scikit-learn flow.' + fixture_structure = { + fixture: [], + 'sklearn.preprocessing.data.StandardScaler': ['numeric'], + 'sklearn.preprocessing._encoders.OneHotEncoder': ['nominal'] + } + serialization = sklearn_to_flow(model) + structure = openml.flows.flow_structure(serialization, 'name') self.assertEqual(serialization.name, fixture) self.assertEqual(serialization.description, fixture_description) + self.assertDictEqual(structure, fixture_structure) # del serialization.model new_model = flow_to_sklearn(serialization) self.assertEqual(type(new_model), type(model)) @@ -393,11 +427,24 @@ def test_serialize_column_transformer_pipeline(self): 'numeric=sklearn.preprocessing.data.StandardScaler,'\ 'nominal=sklearn.preprocessing._encoders.OneHotEncoder),'\ 'classifier=sklearn.tree.tree.DecisionTreeClassifier)' + fixture_structure = { + 'sklearn.preprocessing.data.StandardScaler': + ['transformer', 'numeric'], + 'sklearn.preprocessing._encoders.OneHotEncoder': + ['transformer', 'nominal'], + 'sklearn.compose._column_transformer.ColumnTransformer(numeric=' + 'sklearn.preprocessing.data.StandardScaler,nominal=sklearn.' + 'preprocessing._encoders.OneHotEncoder)': ['transformer'], + 'sklearn.tree.tree.DecisionTreeClassifier': ['classifier'], + fixture_name: [], + } fixture_description = 'Automatically created scikit-learn flow.' serialization = sklearn_to_flow(model) + structure = openml.flows.flow_structure(serialization, 'name') self.assertEqual(serialization.name, fixture_name) self.assertEqual(serialization.description, fixture_description) + self.assertDictEqual(structure, fixture_structure) # del serialization.model new_model = flow_to_sklearn(serialization) self.assertEqual(type(new_model), type(model)) @@ -415,15 +462,23 @@ def test_serialize_feature_union(self): fu = sklearn.pipeline.FeatureUnion( transformer_list=[('ohe', ohe), ('scaler', scaler)]) serialization = sklearn_to_flow(fu) + structure = openml.flows.flow_structure(serialization, 'name') # OneHotEncoder was moved to _encoders module in 0.20 module_name_encoder = ('_encoders' if LooseVersion(sklearn.__version__) >= "0.20" else 'data') - self.assertEqual(serialization.name, - 'sklearn.pipeline.FeatureUnion(' - 'ohe=sklearn.preprocessing.{}.OneHotEncoder,' - 'scaler=sklearn.preprocessing.data.StandardScaler)' - .format(module_name_encoder)) + fixture_name = ('sklearn.pipeline.FeatureUnion(' + 'ohe=sklearn.preprocessing.{}.OneHotEncoder,' + 'scaler=sklearn.preprocessing.data.StandardScaler)' + .format(module_name_encoder)) + fixture_structure = { + fixture_name: [], + 'sklearn.preprocessing.{}.' + 'OneHotEncoder'.format(module_name_encoder): ['ohe'], + 'sklearn.preprocessing.data.StandardScaler': ['scaler'] + } + self.assertEqual(serialization.name, fixture_name) + self.assertDictEqual(structure, fixture_structure) new_model = flow_to_sklearn(serialization) self.assertEqual(type(new_model), type(fu)) @@ -510,19 +565,31 @@ def test_serialize_complex_flow(self): rs = sklearn.model_selection.RandomizedSearchCV( estimator=model, param_distributions=parameter_grid, cv=cv) serialized = sklearn_to_flow(rs) + structure = openml.flows.flow_structure(serialized, 'name') # OneHotEncoder was moved to _encoders module in 0.20 module_name_encoder = ('_encoders' if LooseVersion(sklearn.__version__) >= "0.20" else 'data') - fixture_name = \ - ('sklearn.model_selection._search.RandomizedSearchCV(' - 'estimator=sklearn.pipeline.Pipeline(' - 'ohe=sklearn.preprocessing.{}.OneHotEncoder,' - 'scaler=sklearn.preprocessing.data.StandardScaler,' - 'boosting=sklearn.ensemble.weight_boosting.AdaBoostClassifier(' - 'base_estimator=sklearn.tree.tree.DecisionTreeClassifier)))'. - format(module_name_encoder)) + ohe_name = 'sklearn.preprocessing.%s.OneHotEncoder' % \ + module_name_encoder + scaler_name = 'sklearn.preprocessing.data.StandardScaler' + tree_name = 'sklearn.tree.tree.DecisionTreeClassifier' + boosting_name = 'sklearn.ensemble.weight_boosting.AdaBoostClassifier' \ + '(base_estimator=%s)' % tree_name + pipeline_name = 'sklearn.pipeline.Pipeline(ohe=%s,scaler=%s,' \ + 'boosting=%s)' % (ohe_name, scaler_name, boosting_name) + fixture_name = 'sklearn.model_selection._search.RandomizedSearchCV' \ + '(estimator=%s)' % pipeline_name + fixture_structure = { + ohe_name: ['estimator', 'ohe'], + scaler_name: ['estimator', 'scaler'], + tree_name: ['estimator', 'boosting', 'base_estimator'], + boosting_name: ['estimator', 'boosting'], + pipeline_name: ['estimator'], + fixture_name: [] + } self.assertEqual(serialized.name, fixture_name) + self.assertEqual(structure, fixture_structure) # now do deserialization deserialized = flow_to_sklearn(serialized) From 910a5d326eb501a1dafba6447c99c47fe73232a1 Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Sat, 6 Oct 2018 20:15:48 -0400 Subject: [PATCH 04/28] removes sklearn converter from this PR --- openml/setups/sklearn_converter.py | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 openml/setups/sklearn_converter.py diff --git a/openml/setups/sklearn_converter.py b/openml/setups/sklearn_converter.py deleted file mode 100644 index fd1e158bd..000000000 --- a/openml/setups/sklearn_converter.py +++ /dev/null @@ -1,22 +0,0 @@ -from openml.flows import OpenMLFlow -from openml.setups import OpenMLParameter - - -def openml_param_name_to_sklearn(openml_parameter, flow): - """ - Converts the name of an OpenMLParameter into the sklean name, given a flow. - Note that the same parameter might have a different name in various flows - (e.g., the parameter ) - - Parameters - ---------- - openml_parameter: OpenMLParameter - The parameter under consideration - - flow: OpenMLFlow - The flow that provides context. - """ - if not isinstance(openml_parameter, OpenMLParameter): - raise ValueError('openml_parameter should be an instance of OpenMLParameter') - if not isinstance(flow, OpenMLFlow): - raise ValueError('flow should be an instance of OpenMLFlow') From 1a68a729961c2fc67cb07aeb5b9a32d62efe1411 Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Sat, 6 Oct 2018 22:11:27 -0400 Subject: [PATCH 05/28] added main functionality --- openml/setups/__init__.py | 1 + openml/setups/sklearn_converter.py | 31 +++++++++++++++++++++++ tests/test_setups/test_setup_functions.py | 24 ++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 openml/setups/sklearn_converter.py diff --git a/openml/setups/__init__.py b/openml/setups/__init__.py index 56f0a0b43..e25646c76 100644 --- a/openml/setups/__init__.py +++ b/openml/setups/__init__.py @@ -1,4 +1,5 @@ from .setup import OpenMLSetup, OpenMLParameter from .functions import get_setup, list_setups, setup_exists, initialize_model +from .sklearn_converter import openml_param_name_to_sklearn __all__ = ['get_setup', 'list_setups', 'setup_exists', 'initialize_model'] \ No newline at end of file diff --git a/openml/setups/sklearn_converter.py b/openml/setups/sklearn_converter.py new file mode 100644 index 000000000..cc6687d62 --- /dev/null +++ b/openml/setups/sklearn_converter.py @@ -0,0 +1,31 @@ +import openml +from openml.flows import OpenMLFlow +from openml.setups import OpenMLParameter + + +def openml_param_name_to_sklearn(openml_parameter, flow): + """ + Converts the name of an OpenMLParameter into the sklean name, given a flow. + Note that the same parameter might have a different name in various flows + (e.g., the parameter `min_num_splits` will be called `min_num_splits` in + a `DecisionTreeClassifier`, but `base_estimator__min_num_splits`) when the + `DecisionTreeClassifier` is wrapped in `AdaboostClassifier`. + + Parameters + ---------- + openml_parameter: OpenMLParameter + The parameter under consideration + + flow: OpenMLFlow + The flow that provides context. + """ + if not isinstance(openml_parameter, OpenMLParameter): + raise ValueError('openml_parameter should be an instance of ' + 'OpenMLParameter') + if not isinstance(flow, OpenMLFlow): + raise ValueError('flow should be an instance of OpenMLFlow') + + flow_structure = openml.flows.flow_structure(flow, 'name') + complete = flow_structure[openml_parameter.flow_name] + \ + [openml_parameter.parameter_name] + return '__'.join(complete) diff --git a/tests/test_setups/test_setup_functions.py b/tests/test_setups/test_setup_functions.py index 928874837..ec5507b71 100644 --- a/tests/test_setups/test_setup_functions.py +++ b/tests/test_setups/test_setup_functions.py @@ -5,6 +5,7 @@ import openml import openml.exceptions from openml.testing import TestBase +import sklearn from sklearn.ensemble import BaggingClassifier from sklearn.tree import DecisionTreeClassifier @@ -158,6 +159,29 @@ def test_setuplist_offset(self): self.assertEqual(len(all), size * 2) + def test_openml_param_name_to_sklearn(self): + scaler = sklearn.preprocessing.StandardScaler(with_mean=False) + boosting = sklearn.ensemble.AdaBoostClassifier( + base_estimator=sklearn.tree.DecisionTreeClassifier()) + model = sklearn.pipeline.Pipeline(steps=[ + ('scaler', scaler), ('boosting', boosting)]) + flow = openml.flows.sklearn_to_flow(model) + task = openml.tasks.get_task(115) + run = openml.runs.run_flow_on_task(flow, task) + run = run.publish() + run = openml.runs.get_run(run.run_id) + setup = openml.setups.get_setup(run.setup_id) + + parametername_sid = dict() + for sid, parameter in setup.parameters.items(): + parametername_sid[parameter.parameter_name] = sid + + fixture = 'boosting__base_estimator__min_samples_leaf' + sklearn_name = openml.setups.openml_param_name_to_sklearn( + setup.parameters[parametername_sid['min_samples_leaf']], flow + ) + self.assertEqual(fixture, sklearn_name) + def test_get_cached_setup(self): openml.config.cache_directory = self.static_cache_dir openml.setups.functions._get_cached_setup(1) From f31849d0b082dce0240ccf93d9148b18c8d7c943 Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Tue, 9 Oct 2018 19:13:30 -0400 Subject: [PATCH 06/28] fix code quality --- tests/test_setups/test_setup_functions.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_setups/test_setup_functions.py b/tests/test_setups/test_setup_functions.py index ec5507b71..ffe740718 100644 --- a/tests/test_setups/test_setup_functions.py +++ b/tests/test_setups/test_setup_functions.py @@ -171,7 +171,7 @@ def test_openml_param_name_to_sklearn(self): run = run.publish() run = openml.runs.get_run(run.run_id) setup = openml.setups.get_setup(run.setup_id) - + parametername_sid = dict() for sid, parameter in setup.parameters.items(): parametername_sid[parameter.parameter_name] = sid @@ -186,7 +186,6 @@ def test_get_cached_setup(self): openml.config.cache_directory = self.static_cache_dir openml.setups.functions._get_cached_setup(1) - def test_get_uncached_setup(self): openml.config.cache_directory = self.static_cache_dir with self.assertRaises(openml.exceptions.OpenMLCacheException): From edbd5565b200a1e1763fbd8401783b824968a710 Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Tue, 9 Oct 2018 19:48:55 -0400 Subject: [PATCH 07/28] adds flow name to setup test file --- tests/files/org/openml/test/setups/1/description.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/files/org/openml/test/setups/1/description.xml b/tests/files/org/openml/test/setups/1/description.xml index ee234e4ff..5717ad9f5 100644 --- a/tests/files/org/openml/test/setups/1/description.xml +++ b/tests/files/org/openml/test/setups/1/description.xml @@ -4,6 +4,7 @@ 3432 60 + weka.J48 weka.J48(1)_C C option @@ -13,6 +14,7 @@ 3435 60 + weka.J48 weka.J48(1)_M M option From 7845a742e6b8f18d0cb1624cb308dbbe1be49a42 Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Tue, 9 Oct 2018 20:56:14 -0400 Subject: [PATCH 08/28] adds functionality to return sklearn parameter name into openml flow name --- openml/flows/__init__.py | 3 +- openml/flows/flow.py | 24 +++++++++++ openml/flows/sklearn_converter.py | 6 +-- openml/setups/__init__.py | 7 +++- openml/setups/setup.py | 3 +- openml/setups/sklearn_converter.py | 49 +++++++++++++++++++---- tests/test_setups/test_setup_functions.py | 15 ++++--- 7 files changed, 84 insertions(+), 23 deletions(-) diff --git a/openml/flows/__init__.py b/openml/flows/__init__.py index 1482e2f69..44f1c604c 100644 --- a/openml/flows/__init__.py +++ b/openml/flows/__init__.py @@ -1,7 +1,6 @@ from .flow import OpenMLFlow, _copy_server_fields -from .sklearn_converter import sklearn_to_flow, flow_to_sklearn, \ - flow_structure, _check_n_jobs +from .sklearn_converter import sklearn_to_flow, flow_to_sklearn, flow_structure from .functions import get_flow, list_flows, flow_exists, assert_flows_equal __all__ = ['OpenMLFlow', 'flow_structure', 'get_flow', 'list_flows', diff --git a/openml/flows/flow.py b/openml/flows/flow.py index 0c70fc9bc..e9007bd4e 100644 --- a/openml/flows/flow.py +++ b/openml/flows/flow.py @@ -359,6 +359,30 @@ def publish(self): (flow_id, message)) return self + def get_subflow(self, structure): + """ + Returns a subflow from the tree of dependencies. + + Parameters + ---------- + structure: list[str] + A list of strings, indicating the location of the subflow + + Returns + ------- + sub_component: OpenMLFlow + The OpenMLFlow that corresponds to the structure + """ + sub_identifier = structure[0] + if sub_identifier not in self.components: + raise ValueError('Flow %s does not contain component with ' + 'identifier %s' % (self.name, sub_identifier)) + if len(structure) == 1: + return self.components[sub_identifier] + else: + structure.pop(0) + return self.components[sub_identifier].get_subflow(structure) + def push_tag(self, tag): """Annotates this flow with a tag on the server. diff --git a/openml/flows/sklearn_converter.py b/openml/flows/sklearn_converter.py index 31b46018b..043d7942f 100644 --- a/openml/flows/sklearn_converter.py +++ b/openml/flows/sklearn_converter.py @@ -500,7 +500,7 @@ def flow_structure(flow, key_item): key_item: str The flow attribute that will be used to identify flows in the - structure. Allowed values {id, name, fullName} + structure. Allowed values {id, name} Returns ------- @@ -509,8 +509,8 @@ def flow_structure(flow, key_item): """ if not isinstance(flow, OpenMLFlow): raise TypeError('flow should be of type OpenMLFlow') - if key_item not in ['id', 'name', 'fullName']: - raise ValueError('key_item should be in {id, name, fullName}') + if key_item not in ['id', 'name']: + raise ValueError('key_item should be in {id, name}') structure = dict() for key, sub_flow in flow.components.items(): sub_structure = flow_structure(sub_flow, key_item) diff --git a/openml/setups/__init__.py b/openml/setups/__init__.py index e25646c76..25613d9b4 100644 --- a/openml/setups/__init__.py +++ b/openml/setups/__init__.py @@ -1,5 +1,8 @@ from .setup import OpenMLSetup, OpenMLParameter from .functions import get_setup, list_setups, setup_exists, initialize_model -from .sklearn_converter import openml_param_name_to_sklearn +from .sklearn_converter import openml_param_name_to_sklearn, \ + sklearn_param_name_to_openml -__all__ = ['get_setup', 'list_setups', 'setup_exists', 'initialize_model'] \ No newline at end of file +__all__ = ['OpenMLSetup', 'OpenMLParameter', 'get_setup', 'list_setups', + 'setup_exists', 'initialize_model', 'openml_param_name_to_sklearn', + 'sklearn_param_name_to_openml'] diff --git a/openml/setups/setup.py b/openml/setups/setup.py index 85cd03b00..13e986f4b 100644 --- a/openml/setups/setup.py +++ b/openml/setups/setup.py @@ -47,7 +47,8 @@ class OpenMLParameter(object): value : str If the parameter was set, the value that it was set to. """ - def __init__(self, id, flow_id, flow_name, full_name, parameter_name, data_type, default_value, value): + def __init__(self, id, flow_id, flow_name, full_name, parameter_name, + data_type, default_value, value): self.id = id self.flow_id = flow_id self.flow_name = flow_name diff --git a/openml/setups/sklearn_converter.py b/openml/setups/sklearn_converter.py index cc6687d62..6bf99067b 100644 --- a/openml/setups/sklearn_converter.py +++ b/openml/setups/sklearn_converter.py @@ -6,10 +6,6 @@ def openml_param_name_to_sklearn(openml_parameter, flow): """ Converts the name of an OpenMLParameter into the sklean name, given a flow. - Note that the same parameter might have a different name in various flows - (e.g., the parameter `min_num_splits` will be called `min_num_splits` in - a `DecisionTreeClassifier`, but `base_estimator__min_num_splits`) when the - `DecisionTreeClassifier` is wrapped in `AdaboostClassifier`. Parameters ---------- @@ -18,6 +14,11 @@ def openml_param_name_to_sklearn(openml_parameter, flow): flow: OpenMLFlow The flow that provides context. + + Returns + ------- + sklearn_parameter_name: str + The name the parameter will have once used in scikit-learn """ if not isinstance(openml_parameter, OpenMLParameter): raise ValueError('openml_parameter should be an instance of ' @@ -26,6 +27,40 @@ def openml_param_name_to_sklearn(openml_parameter, flow): raise ValueError('flow should be an instance of OpenMLFlow') flow_structure = openml.flows.flow_structure(flow, 'name') - complete = flow_structure[openml_parameter.flow_name] + \ - [openml_parameter.parameter_name] - return '__'.join(complete) + if openml_parameter.flow_name not in flow_structure: + raise ValueError('Obtained OpenMLParameter and OpenMLFlow do not ' + 'correspond. ') + + return '__'.join(flow_structure[openml_parameter.flow_name] + + [openml_parameter.parameter_name]) + + +def sklearn_param_name_to_openml(sklearn_parameter_name, flow): + """ + Converts the name of a sklearn parameter into the name that it would have + in the OpenMLParameter, given a flow. + The flow needs to be downloaded from the server, such that the flow.version + field is filled. + + Parameters + ---------- + sklearn_parameter_name: str + The parameter under consideration + + flow: OpenMLFlow + The flow that provides context. + + Returns + ------- + openml_parameter_name: str + The full name that this parameter will take when retrieved from an + OpenMLParameter object from the server + """ + if not isinstance(flow, OpenMLFlow): + raise ValueError('flow should be an instance of OpenMLFlow') + splitted = sklearn_parameter_name.split('__') + subflow = flow.get_subflow(splitted[0:-1]) + if subflow.flow_id is None or subflow.version is None: + raise ValueError('For this fn, OpenMLFlow should be downloaded from ' + 'the server, rather than being initiated locally. ') + return '%s(%s)_%s' % (subflow.name, subflow.version, splitted[-1]) diff --git a/tests/test_setups/test_setup_functions.py b/tests/test_setups/test_setup_functions.py index ffe740718..e8bfa1a6c 100644 --- a/tests/test_setups/test_setup_functions.py +++ b/tests/test_setups/test_setup_functions.py @@ -160,6 +160,7 @@ def test_setuplist_offset(self): self.assertEqual(len(all), size * 2) def test_openml_param_name_to_sklearn(self): + # test is also responsible for: sklearn_param_name_to_openml scaler = sklearn.preprocessing.StandardScaler(with_mean=False) boosting = sklearn.ensemble.AdaBoostClassifier( base_estimator=sklearn.tree.DecisionTreeClassifier()) @@ -172,15 +173,13 @@ def test_openml_param_name_to_sklearn(self): run = openml.runs.get_run(run.run_id) setup = openml.setups.get_setup(run.setup_id) - parametername_sid = dict() - for sid, parameter in setup.parameters.items(): - parametername_sid[parameter.parameter_name] = sid + # make sure to test enough parameters + self.assertGreater(len(setup.parameters), 15) - fixture = 'boosting__base_estimator__min_samples_leaf' - sklearn_name = openml.setups.openml_param_name_to_sklearn( - setup.parameters[parametername_sid['min_samples_leaf']], flow - ) - self.assertEqual(fixture, sklearn_name) + for sid, parameter in setup.parameters.items(): + sklearn_name = openml.setups.openml_param_name_to_sklearn(parameter, flow) + openml_name = openml.setups.sklearn_param_name_to_openml(sklearn_name, flow) + self.assertEqual(parameter.full_name, openml_name) def test_get_cached_setup(self): openml.config.cache_directory = self.static_cache_dir From 7f4b5ace4492e6209ed6c51c5eccc02ff2f2e294 Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Tue, 9 Oct 2018 21:16:46 -0400 Subject: [PATCH 09/28] PEP8 fixes --- openml/flows/__init__.py | 6 ++++-- tests/test_setups/test_setup_functions.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/openml/flows/__init__.py b/openml/flows/__init__.py index 44f1c604c..eeee559ce 100644 --- a/openml/flows/__init__.py +++ b/openml/flows/__init__.py @@ -1,7 +1,9 @@ from .flow import OpenMLFlow, _copy_server_fields -from .sklearn_converter import sklearn_to_flow, flow_to_sklearn, flow_structure +from .sklearn_converter import sklearn_to_flow, flow_to_sklearn, _check_n_jobs, \ + flow_structure from .functions import get_flow, list_flows, flow_exists, assert_flows_equal __all__ = ['OpenMLFlow', 'flow_structure', 'get_flow', 'list_flows', - 'sklearn_to_flow', 'flow_to_sklearn', 'flow_exists'] + 'sklearn_to_flow', 'flow_to_sklearn', 'flow_exists', + '_check_n_jobs'] diff --git a/tests/test_setups/test_setup_functions.py b/tests/test_setups/test_setup_functions.py index e8bfa1a6c..0d429bb9d 100644 --- a/tests/test_setups/test_setup_functions.py +++ b/tests/test_setups/test_setup_functions.py @@ -176,7 +176,7 @@ def test_openml_param_name_to_sklearn(self): # make sure to test enough parameters self.assertGreater(len(setup.parameters), 15) - for sid, parameter in setup.parameters.items(): + for parameter in setup.parameters.values(): sklearn_name = openml.setups.openml_param_name_to_sklearn(parameter, flow) openml_name = openml.setups.sklearn_param_name_to_openml(sklearn_name, flow) self.assertEqual(parameter.full_name, openml_name) From a05018db013b7723801bf22610f354be4a001246 Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Tue, 9 Oct 2018 21:49:20 -0400 Subject: [PATCH 10/28] changed structure of PR, such that get_structure is not part of flow class. updated unit tests accordingly --- openml/flows/__init__.py | 8 ++---- openml/flows/flow.py | 29 +++++++++++++++++++ openml/flows/sklearn_converter.py | 34 ----------------------- openml/setups/__init__.py | 6 ++-- openml/setups/sklearn_converter.py | 33 +--------------------- tests/test_flows/test_flow.py | 24 ++++++++++++++++ tests/test_flows/test_sklearn.py | 18 ++++++------ tests/test_setups/test_setup_functions.py | 16 +++++++++-- 8 files changed, 81 insertions(+), 87 deletions(-) diff --git a/openml/flows/__init__.py b/openml/flows/__init__.py index eeee559ce..efacddbe6 100644 --- a/openml/flows/__init__.py +++ b/openml/flows/__init__.py @@ -1,9 +1,7 @@ from .flow import OpenMLFlow, _copy_server_fields -from .sklearn_converter import sklearn_to_flow, flow_to_sklearn, _check_n_jobs, \ - flow_structure +from .sklearn_converter import sklearn_to_flow, flow_to_sklearn, _check_n_jobs from .functions import get_flow, list_flows, flow_exists, assert_flows_equal -__all__ = ['OpenMLFlow', 'flow_structure', 'get_flow', 'list_flows', - 'sklearn_to_flow', 'flow_to_sklearn', 'flow_exists', - '_check_n_jobs'] +__all__ = ['OpenMLFlow', 'get_flow', 'list_flows', 'sklearn_to_flow', + 'flow_to_sklearn', 'flow_exists', '_check_n_jobs'] diff --git a/openml/flows/flow.py b/openml/flows/flow.py index e9007bd4e..64550dcdd 100644 --- a/openml/flows/flow.py +++ b/openml/flows/flow.py @@ -359,6 +359,35 @@ def publish(self): (flow_id, message)) return self + def get_structure(self, key_item): + """ + Returns for each sub-component of the flow the path of identifiers that + should be traversed to reach this component. The resulting dict maps a + key (identifying a flow be either its id, name or fullname) to the + parameter prefix. + + Parameters + ---------- + + key_item: str + The flow attribute that will be used to identify flows in the + structure. Allowed values {flow_id, name} + + Returns + ------- + structure: dict[str, List[str]] + The flow structure + """ + if key_item not in ['flow_id', 'name']: + raise ValueError('key_item should be in {flow_id, name}') + structure = dict() + for key, sub_flow in self.components.items(): + sub_structure = sub_flow.get_structure(key_item) + for flow_name, flow_sub_structure in sub_structure.items(): + structure[flow_name] = [key] + flow_sub_structure + structure[getattr(self, key_item)] = [] + return structure + def get_subflow(self, structure): """ Returns a subflow from the tree of dependencies. diff --git a/openml/flows/sklearn_converter.py b/openml/flows/sklearn_converter.py index 043d7942f..82b5895fa 100644 --- a/openml/flows/sklearn_converter.py +++ b/openml/flows/sklearn_converter.py @@ -486,40 +486,6 @@ def _deserialize_model(flow, keep_defaults): return model_class(**parameter_dict) -def flow_structure(flow, key_item): - """ - Returns for each sub-component of the flow the path of identifiers that - should be traversed to reach this component. The resulting dict maps a key - (identifying a flow be either its id, name or fullname) to the parameter - prefix. - - Parameters - ---------- - flow: OpenMLFlow - The flow to generate the parameter prefixes for - - key_item: str - The flow attribute that will be used to identify flows in the - structure. Allowed values {id, name} - - Returns - ------- - structure: dict[str, List[str]] - The flow structure - """ - if not isinstance(flow, OpenMLFlow): - raise TypeError('flow should be of type OpenMLFlow') - if key_item not in ['id', 'name']: - raise ValueError('key_item should be in {id, name}') - structure = dict() - for key, sub_flow in flow.components.items(): - sub_structure = flow_structure(sub_flow, key_item) - for flow_name, flow_sub_structure in sub_structure.items(): - structure[flow_name] = [key] + flow_sub_structure - structure[getattr(flow, key_item)] = [] - return structure - - def _check_dependencies(dependencies): if not dependencies: return diff --git a/openml/setups/__init__.py b/openml/setups/__init__.py index 25613d9b4..74b4e886c 100644 --- a/openml/setups/__init__.py +++ b/openml/setups/__init__.py @@ -1,8 +1,6 @@ from .setup import OpenMLSetup, OpenMLParameter from .functions import get_setup, list_setups, setup_exists, initialize_model -from .sklearn_converter import openml_param_name_to_sklearn, \ - sklearn_param_name_to_openml +from .sklearn_converter import openml_param_name_to_sklearn __all__ = ['OpenMLSetup', 'OpenMLParameter', 'get_setup', 'list_setups', - 'setup_exists', 'initialize_model', 'openml_param_name_to_sklearn', - 'sklearn_param_name_to_openml'] + 'setup_exists', 'initialize_model', 'openml_param_name_to_sklearn'] diff --git a/openml/setups/sklearn_converter.py b/openml/setups/sklearn_converter.py index 6bf99067b..c0f49fd2b 100644 --- a/openml/setups/sklearn_converter.py +++ b/openml/setups/sklearn_converter.py @@ -26,41 +26,10 @@ def openml_param_name_to_sklearn(openml_parameter, flow): if not isinstance(flow, OpenMLFlow): raise ValueError('flow should be an instance of OpenMLFlow') - flow_structure = openml.flows.flow_structure(flow, 'name') + flow_structure = flow.get_structure('name') if openml_parameter.flow_name not in flow_structure: raise ValueError('Obtained OpenMLParameter and OpenMLFlow do not ' 'correspond. ') return '__'.join(flow_structure[openml_parameter.flow_name] + [openml_parameter.parameter_name]) - - -def sklearn_param_name_to_openml(sklearn_parameter_name, flow): - """ - Converts the name of a sklearn parameter into the name that it would have - in the OpenMLParameter, given a flow. - The flow needs to be downloaded from the server, such that the flow.version - field is filled. - - Parameters - ---------- - sklearn_parameter_name: str - The parameter under consideration - - flow: OpenMLFlow - The flow that provides context. - - Returns - ------- - openml_parameter_name: str - The full name that this parameter will take when retrieved from an - OpenMLParameter object from the server - """ - if not isinstance(flow, OpenMLFlow): - raise ValueError('flow should be an instance of OpenMLFlow') - splitted = sklearn_parameter_name.split('__') - subflow = flow.get_subflow(splitted[0:-1]) - if subflow.flow_id is None or subflow.version is None: - raise ValueError('For this fn, OpenMLFlow should be downloaded from ' - 'the server, rather than being initiated locally. ') - return '%s(%s)_%s' % (subflow.name, subflow.version, splitted[-1]) diff --git a/tests/test_flows/test_flow.py b/tests/test_flows/test_flow.py index 39c03fee1..f3f63b67d 100644 --- a/tests/test_flows/test_flow.py +++ b/tests/test_flows/test_flow.py @@ -74,6 +74,30 @@ def test_get_flow(self): self.assertEqual(subflow_3.parameters['L'], '-1') self.assertEqual(len(subflow_3.components), 0) + def test_get_structure(self): + # also responsible for testing: flow.get_subflow + # We need to use the production server here because 4024 is not the + # test server + openml.config.server = self.production_server + + flow = openml.flows.get_flow(4024) + flow_structure_name = flow.get_structure('name') + flow_structure_id = flow.get_structure('flow_id') + # components: root (filteredclassifier), multisearch, loginboost, + # reptree + self.assertEqual(len(flow_structure_name), 4) + self.assertEqual(len(flow_structure_id), 4) + + for sub_flow_name, structure in flow_structure_name.items(): + if len(structure) > 0: # skip root element + subflow = flow.get_subflow(structure) + self.assertEqual(subflow.name, sub_flow_name) + + for sub_flow_id, structure in flow_structure_id.items(): + if len(structure) > 0: # skip root element + subflow = flow.get_subflow(structure) + self.assertEqual(subflow.flow_id, sub_flow_id) + def test_tagging(self): flow_list = openml.flows.list_flows(size=1) flow_id = list(flow_list.keys())[0] diff --git a/tests/test_flows/test_sklearn.py b/tests/test_flows/test_sklearn.py index f0b28655d..81d84a607 100644 --- a/tests/test_flows/test_sklearn.py +++ b/tests/test_flows/test_sklearn.py @@ -109,7 +109,7 @@ def test_serialize_model(self, check_dependencies_mock): structure_fixture = {'sklearn.tree.tree.DecisionTreeClassifier': []} serialization = sklearn_to_flow(model) - structure = openml.flows.flow_structure(serialization, 'name') + structure = serialization.get_structure('name') self.assertEqual(serialization.name, fixture_name) self.assertEqual(serialization.class_name, fixture_name) @@ -166,7 +166,7 @@ def test_serialize_model_clustering(self, check_dependencies_mock): fixture_structure = {'sklearn.cluster.k_means_.KMeans': []} serialization = sklearn_to_flow(model) - structure = openml.flows.flow_structure(serialization, 'name') + structure = serialization.get_structure('name') self.assertEqual(serialization.name, fixture_name) self.assertEqual(serialization.class_name, fixture_name) @@ -202,7 +202,7 @@ def test_serialize_model_with_subcomponent(self): } serialization = sklearn_to_flow(model) - structure = openml.flows.flow_structure(serialization, 'name') + structure = serialization.get_structure('name') self.assertEqual(serialization.name, fixture_name) self.assertEqual(serialization.class_name, fixture_class_name) @@ -252,7 +252,7 @@ def test_serialize_pipeline(self): } serialization = sklearn_to_flow(model) - structure = openml.flows.flow_structure(serialization, 'name') + structure = serialization.get_structure('name') self.assertEqual(serialization.name, fixture_name) self.assertEqual(serialization.description, fixture_description) @@ -321,7 +321,7 @@ def test_serialize_pipeline_clustering(self): } serialization = sklearn_to_flow(model) - structure = openml.flows.flow_structure(serialization, 'name') + structure = serialization.get_structure('name') self.assertEqual(serialization.name, fixture_name) self.assertEqual(serialization.description, fixture_description) @@ -395,7 +395,7 @@ def test_serialize_column_transformer(self): } serialization = sklearn_to_flow(model) - structure = openml.flows.flow_structure(serialization, 'name') + structure = serialization.get_structure('name') self.assertEqual(serialization.name, fixture) self.assertEqual(serialization.description, fixture_description) self.assertDictEqual(structure, fixture_structure) @@ -441,7 +441,7 @@ def test_serialize_column_transformer_pipeline(self): fixture_description = 'Automatically created scikit-learn flow.' serialization = sklearn_to_flow(model) - structure = openml.flows.flow_structure(serialization, 'name') + structure = serialization.get_structure('name') self.assertEqual(serialization.name, fixture_name) self.assertEqual(serialization.description, fixture_description) self.assertDictEqual(structure, fixture_structure) @@ -462,7 +462,7 @@ def test_serialize_feature_union(self): fu = sklearn.pipeline.FeatureUnion( transformer_list=[('ohe', ohe), ('scaler', scaler)]) serialization = sklearn_to_flow(fu) - structure = openml.flows.flow_structure(serialization, 'name') + structure = serialization.get_structure('name') # OneHotEncoder was moved to _encoders module in 0.20 module_name_encoder = ('_encoders' if LooseVersion(sklearn.__version__) >= "0.20" @@ -565,7 +565,7 @@ def test_serialize_complex_flow(self): rs = sklearn.model_selection.RandomizedSearchCV( estimator=model, param_distributions=parameter_grid, cv=cv) serialized = sklearn_to_flow(rs) - structure = openml.flows.flow_structure(serialized, 'name') + structure = serialized.get_structure('name') # OneHotEncoder was moved to _encoders module in 0.20 module_name_encoder = ('_encoders' if LooseVersion(sklearn.__version__) >= "0.20" diff --git a/tests/test_setups/test_setup_functions.py b/tests/test_setups/test_setup_functions.py index 0d429bb9d..48ac8ef3b 100644 --- a/tests/test_setups/test_setup_functions.py +++ b/tests/test_setups/test_setup_functions.py @@ -160,7 +160,6 @@ def test_setuplist_offset(self): self.assertEqual(len(all), size * 2) def test_openml_param_name_to_sklearn(self): - # test is also responsible for: sklearn_param_name_to_openml scaler = sklearn.preprocessing.StandardScaler(with_mean=False) boosting = sklearn.ensemble.AdaBoostClassifier( base_estimator=sklearn.tree.DecisionTreeClassifier()) @@ -177,8 +176,19 @@ def test_openml_param_name_to_sklearn(self): self.assertGreater(len(setup.parameters), 15) for parameter in setup.parameters.values(): - sklearn_name = openml.setups.openml_param_name_to_sklearn(parameter, flow) - openml_name = openml.setups.sklearn_param_name_to_openml(sklearn_name, flow) + sklearn_name = openml.setups.openml_param_name_to_sklearn( + parameter, flow) + + # test the inverse. Currently, OpenML stores the hyperparameter + # fullName as flow.name + flow.version + parameter.name on the + # server (but this behaviour is not documented and might or might + # not change in the future. Hence, we won't offer this + # transformation functionality in the main package yet.) + splitted = sklearn_name.split("__") + subflow = flow.get_subflow(splitted[0:-1]) + openml_name = "%s(%s)_%s" % (subflow.name, + subflow.version, + splitted[-1]) self.assertEqual(parameter.full_name, openml_name) def test_get_cached_setup(self): From 046beea1efcf5e9bd849197a1ccb5ba0922d98bd Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Tue, 9 Oct 2018 21:54:12 -0400 Subject: [PATCH 11/28] pep8 fix --- openml/setups/sklearn_converter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openml/setups/sklearn_converter.py b/openml/setups/sklearn_converter.py index c0f49fd2b..d49358907 100644 --- a/openml/setups/sklearn_converter.py +++ b/openml/setups/sklearn_converter.py @@ -1,4 +1,3 @@ -import openml from openml.flows import OpenMLFlow from openml.setups import OpenMLParameter From f46beba2cf6f995ac028e3d1e7bebf7f24b7671f Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Tue, 9 Oct 2018 22:40:16 -0400 Subject: [PATCH 12/28] fixes last typo --- openml/flows/flow.py | 3 ++- tests/test_setups/test_setup_functions.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/openml/flows/flow.py b/openml/flows/flow.py index 64550dcdd..efc222558 100644 --- a/openml/flows/flow.py +++ b/openml/flows/flow.py @@ -368,7 +368,6 @@ def get_structure(self, key_item): Parameters ---------- - key_item: str The flow attribute that will be used to identify flows in the structure. Allowed values {flow_id, name} @@ -402,6 +401,8 @@ def get_subflow(self, structure): sub_component: OpenMLFlow The OpenMLFlow that corresponds to the structure """ + if len(structure) < 1: + raise ValueError('Please provide a structure list of size >= 1') sub_identifier = structure[0] if sub_identifier not in self.components: raise ValueError('Flow %s does not contain component with ' diff --git a/tests/test_setups/test_setup_functions.py b/tests/test_setups/test_setup_functions.py index 48ac8ef3b..6d672f682 100644 --- a/tests/test_setups/test_setup_functions.py +++ b/tests/test_setups/test_setup_functions.py @@ -185,7 +185,10 @@ def test_openml_param_name_to_sklearn(self): # not change in the future. Hence, we won't offer this # transformation functionality in the main package yet.) splitted = sklearn_name.split("__") - subflow = flow.get_subflow(splitted[0:-1]) + if len(splitted) > 1: # if len is 1, it is part of root flow + subflow = flow.get_subflow(splitted[0:-1]) + else: + subflow = flow openml_name = "%s(%s)_%s" % (subflow.name, subflow.version, splitted[-1]) From 4dacb8ab265d0e72a1c50cc08e579272ad2b7d67 Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Wed, 10 Oct 2018 19:30:56 -0400 Subject: [PATCH 13/28] flow name doc string --- openml/setups/setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openml/setups/setup.py b/openml/setups/setup.py index 13e986f4b..24a452b16 100644 --- a/openml/setups/setup.py +++ b/openml/setups/setup.py @@ -35,6 +35,9 @@ class OpenMLParameter(object): The input id from the openml database flow id : int The flow to which this parameter is associated + flow name : str + The name of the flow (no version number) to which this parameter + is associated full_name : str The name of the flow and parameter combined parameter_name : str From 4a2c7c8f2ba1647314092d9c9c6d48e8b345fcf9 Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Thu, 11 Oct 2018 16:19:44 -0400 Subject: [PATCH 14/28] also added additional filter for task list --- openml/tasks/functions.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openml/tasks/functions.py b/openml/tasks/functions.py index 2c3532594..d9da08369 100644 --- a/openml/tasks/functions.py +++ b/openml/tasks/functions.py @@ -171,7 +171,7 @@ def _list_tasks(task_type_id=None, **kwargs): - Survival Analysis: 7 - Subgroup Discovery: 8 kwargs: dict, optional - Legal filter operators: tag, data_tag, status, limit, + Legal filter operators: tag, task_id (list), data_tag, status, limit, offset, data_id, data_name, number_instances, number_features, number_classes, number_missing_values. Returns @@ -183,6 +183,8 @@ def _list_tasks(task_type_id=None, **kwargs): api_call += "/type/%d" % int(task_type_id) if kwargs is not None: for operator, value in kwargs.items(): + if operator == 'task_id': + value = ','.join([str(int(i)) for i in value]) api_call += "/%s/%s" % (operator, value) return __list_tasks(api_call) From 2db7ddb7e7ff1df2c5deffb33534567e4843e5a8 Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Mon, 22 Oct 2018 13:05:34 -0400 Subject: [PATCH 15/28] renamed id argument of parameter object (for code quality) --- openml/setups/setup.py | 44 +++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/openml/setups/setup.py b/openml/setups/setup.py index 24a452b16..d5579b30c 100644 --- a/openml/setups/setup.py +++ b/openml/setups/setup.py @@ -29,30 +29,30 @@ def __init__(self, setup_id, flow_id, parameters): class OpenMLParameter(object): """Parameter object (used in setup). - Parameters - ---------- - id : int - The input id from the openml database - flow id : int - The flow to which this parameter is associated - flow name : str - The name of the flow (no version number) to which this parameter - is associated - full_name : str - The name of the flow and parameter combined - parameter_name : str - The name of the parameter - data_type : str - The datatype of the parameter. generally unused for sklearn flows - default_value : str - The default value. For sklearn parameters, this is unknown and a - default value is selected arbitrarily - value : str - If the parameter was set, the value that it was set to. + Parameters + ---------- + input_id : int + The input id from the openml database + flow id : int + The flow to which this parameter is associated + flow name : str + The name of the flow (no version number) to which this parameter + is associated + full_name : str + The name of the flow and parameter combined + parameter_name : str + The name of the parameter + data_type : str + The datatype of the parameter. generally unused for sklearn flows + default_value : str + The default value. For sklearn parameters, this is unknown and a + default value is selected arbitrarily + value : str + If the parameter was set, the value that it was set to. """ - def __init__(self, id, flow_id, flow_name, full_name, parameter_name, + def __init__(self, input_id, flow_id, flow_name, full_name, parameter_name, data_type, default_value, value): - self.id = id + self.id = input_id self.flow_id = flow_id self.flow_name = flow_name self.full_name = full_name From de93578e671ba97562f986ccde9de4fc6f5c9a47 Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Mon, 22 Oct 2018 13:33:37 -0400 Subject: [PATCH 16/28] fix reference to input id --- openml/setups/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openml/setups/functions.py b/openml/setups/functions.py index fd7001e73..6ede8ceb8 100644 --- a/openml/setups/functions.py +++ b/openml/setups/functions.py @@ -281,7 +281,7 @@ def _create_setup_from_xml(result_dict): def _create_setup_parameter_from_xml(result_dict): - return OpenMLParameter(id=int(result_dict['oml:id']), + return OpenMLParameter(input_id=int(result_dict['oml:id']), flow_id=int(result_dict['oml:flow_id']), flow_name=result_dict['oml:flow_name'], full_name=result_dict['oml:full_name'], From 5ac62aa728f5329dc6f890223ef19918a680330c Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Wed, 5 Dec 2018 12:05:34 -0500 Subject: [PATCH 17/28] updated reinitialize model fn --- examples/run_setup_tutorial.py | 99 ++++++++++++++++++++++++++++++++++ openml/setups/functions.py | 43 +++------------ 2 files changed, 106 insertions(+), 36 deletions(-) create mode 100644 examples/run_setup_tutorial.py diff --git a/examples/run_setup_tutorial.py b/examples/run_setup_tutorial.py new file mode 100644 index 000000000..d19b3c408 --- /dev/null +++ b/examples/run_setup_tutorial.py @@ -0,0 +1,99 @@ +""" +========= +Run Setup +========= + +By: Jan N. van Rijn + +One of the key features of the openml-python library is that is allows to +reinstantiate flows with hyperparameter settings that were uploaded before. +This tutorial uses the concept of setups. Although setups are not extensively +described in the OpenML documentation (because most users will not directly +use them) + +A key requirement for reinstantiating a flow is to have the same scikit-learn +version as the flow that was uploaded. This tutorial will upload the flow +itself, so it can be ran with any scikit-learn version that is supported by +this library. + +In this tutotial we will + 1) Create a flow and use it to solve a task + 2) Download the flow, reinstantiate the model with same hyperparameters, + and solve the same task again. + 3) We will verify that the obtained results are exactly the same. +Readers interested in reinstantiating a setup can skip part 1 and 2 and start +with part 3 immediately. +""" +import logging +import numpy as np +import openml +import sklearn.ensemble +import sklearn.preprocessing + + +root = logging.getLogger() +root.setLevel(logging.INFO) + +############################################################################### +# 1) Create a flow and use it to solve a task +############################################################################### + +# first, let's download the task that we are interested in +task = openml.tasks.get_task(6) # letter dataset + + +# we will create a fairly complex model, with many preprocessing components and +# many potential hyperparameters. Of course, the model can be as complex and as +# easy as you want it to be +model_original = sklearn.pipeline.make_pipeline( + sklearn.preprocessing.Imputer(), + sklearn.preprocessing.StandardScaler(), + sklearn.ensemble.RandomForestClassifier() +) + + +# Let's change some hyperparameters. Of course, in any good application we +# would tune them using, e.g., Random Search or SMAC, but for the purpose of +# this tutorial we set them to some specific values that might or might not be +# optimal +hyperparameters_original = { + 'imputer__strategy': 'median', + 'randomforestclassifier__random_state': 42, + 'randomforestclassifier__min_samples_leaf': 1, + 'randomforestclassifier__max_features': 0.2 +} +model_original.set_params(**hyperparameters_original) + +# solve the task and upload the result (this implicitly creates the flow) +run = openml.runs.run_model_on_task( + model_original, + task, + avoid_duplicate_runs=False) +run_original = run.publish() # this implicitly uploads the flow + +############################################################################### +# 2) Download the flow, reinstantiate the model with same hyperparameters, +# and solve the same task again. +############################################################################### + +# obtain setup id (note that the setup id is assigned by the OpenML server - +# therefore it was not yet available in our local copy of the run) +run_downloaded = openml.runs.get_run(run_original.run_id) +setup_id = run_downloaded.setup_id + +# after this, we can easily reinstantiate the model +model_duplicate = openml.setups.initialize_model(setup_id) +# it will automatically have all the hyperparameters set + +# and run the task again +run_duplicate = openml.runs.run_model_on_task( + model_duplicate, task, avoid_duplicate_runs=False) + + +############################################################################### +# 3) We will verify that the obtained results are exactly the same. +############################################################################### + +# the run has stored all predictions in the field data content +np.testing.assert_array_equal(run_original.data_content, + run_duplicate.data_content) diff --git a/openml/setups/functions.py b/openml/setups/functions.py index 16aa60a16..877503ecd 100644 --- a/openml/setups/functions.py +++ b/openml/setups/functions.py @@ -211,44 +211,15 @@ def initialize_model(setup_id): # transform an openml setup object into # a dict of dicts, structured: flow_id maps to dict of # parameter_names mapping to parameter_value - setup = get_setup(setup_id) - parameters = {} - for _param in setup.parameters: - _flow_id = setup.parameters[_param].flow_id - _param_name = setup.parameters[_param].parameter_name - _param_value = setup.parameters[_param].value - if _flow_id not in parameters: - parameters[_flow_id] = {} - parameters[_flow_id][_param_name] = _param_value - - def _reconstruct_flow(_flow, _params): - # recursively set the values of flow parameters (and subflows) to - # the specific values from a setup. _params is a dict of - # dicts, mapping from flow id to param name to param value - # (obtained by using the subfunction _to_dict_of_dicts) - for _param in _flow.parameters: - # It can happen that no parameters of a flow are in a setup, - # then the flow_id is not in _params; usually happens for a - # sklearn.pipeline.Pipeline object, where the steps parameter is - # not in the setup - if _flow.flow_id not in _params: - continue - # It is not guaranteed that a setup on OpenML has all parameter - # settings of a flow, thus a param must not be in _params! - if _param not in _params[_flow.flow_id]: - continue - _flow.parameters[_param] = _params[_flow.flow_id][_param] - for _identifier in _flow.components: - _flow.components[_identifier] = _reconstruct_flow(_flow.components[_identifier], _params) - return _flow - - # now we 'abuse' the parameter object by passing in the - # parameters obtained from the setup flow = openml.flows.get_flow(setup.flow_id) - flow = _reconstruct_flow(flow, parameters) - - return openml.flows.flow_to_sklearn(flow) + model = openml.flows.flow_to_sklearn(flow) + hyperparameters = { + openml.setups.openml_param_name_to_sklearn(hp, flow): + openml.flows.flow_to_sklearn(hp.value) for hp in setup.parameters.values() + } + model.set_params(**hyperparameters) + return model def _to_dict(flow_id, openml_parameter_settings): From 4aec3eed8bedb94ddb4369ad9376bba9fa786457 Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Wed, 5 Dec 2018 12:08:50 -0500 Subject: [PATCH 18/28] removed imputer (deprecated) --- examples/run_setup_tutorial.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/run_setup_tutorial.py b/examples/run_setup_tutorial.py index d19b3c408..5355eb3d1 100644 --- a/examples/run_setup_tutorial.py +++ b/examples/run_setup_tutorial.py @@ -28,6 +28,7 @@ import numpy as np import openml import sklearn.ensemble +import sklearn.impute import sklearn.preprocessing @@ -46,7 +47,7 @@ # many potential hyperparameters. Of course, the model can be as complex and as # easy as you want it to be model_original = sklearn.pipeline.make_pipeline( - sklearn.preprocessing.Imputer(), + sklearn.impute.SimpleImputer(), sklearn.preprocessing.StandardScaler(), sklearn.ensemble.RandomForestClassifier() ) @@ -57,7 +58,7 @@ # this tutorial we set them to some specific values that might or might not be # optimal hyperparameters_original = { - 'imputer__strategy': 'median', + 'simpleimputer__strategy': 'median', 'randomforestclassifier__random_state': 42, 'randomforestclassifier__min_samples_leaf': 1, 'randomforestclassifier__max_features': 0.2 From 243d9c05bd1055f6ecc012bd07af00ebe1d5c657 Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Wed, 5 Dec 2018 14:04:54 -0500 Subject: [PATCH 19/28] fixes PEP8 problems --- openml/setups/functions.py | 3 ++- openml/setups/sklearn_converter.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/openml/setups/functions.py b/openml/setups/functions.py index 877503ecd..79a150855 100644 --- a/openml/setups/functions.py +++ b/openml/setups/functions.py @@ -216,7 +216,8 @@ def initialize_model(setup_id): model = openml.flows.flow_to_sklearn(flow) hyperparameters = { openml.setups.openml_param_name_to_sklearn(hp, flow): - openml.flows.flow_to_sklearn(hp.value) for hp in setup.parameters.values() + openml.flows.flow_to_sklearn(hp.value) + for hp in setup.parameters.values() } model.set_params(**hyperparameters) return model diff --git a/openml/setups/sklearn_converter.py b/openml/setups/sklearn_converter.py index d49358907..003a170d4 100644 --- a/openml/setups/sklearn_converter.py +++ b/openml/setups/sklearn_converter.py @@ -30,5 +30,5 @@ def openml_param_name_to_sklearn(openml_parameter, flow): raise ValueError('Obtained OpenMLParameter and OpenMLFlow do not ' 'correspond. ') - return '__'.join(flow_structure[openml_parameter.flow_name] + + return '__'.join(flow_structure[openml_parameter.flow_name] + \ [openml_parameter.parameter_name]) From 374fcb9955ae4978611715ed81f95ccf0f9e8933 Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Wed, 5 Dec 2018 14:11:47 -0500 Subject: [PATCH 20/28] pep8 --- openml/setups/sklearn_converter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openml/setups/sklearn_converter.py b/openml/setups/sklearn_converter.py index 003a170d4..026791fe9 100644 --- a/openml/setups/sklearn_converter.py +++ b/openml/setups/sklearn_converter.py @@ -29,6 +29,6 @@ def openml_param_name_to_sklearn(openml_parameter, flow): if openml_parameter.flow_name not in flow_structure: raise ValueError('Obtained OpenMLParameter and OpenMLFlow do not ' 'correspond. ') - - return '__'.join(flow_structure[openml_parameter.flow_name] + \ - [openml_parameter.parameter_name]) + + name = openml_parameter.flow_name # for PEP8 + return '__'.join(flow_structure[name] + [openml_parameter.parameter_name]) From 7b55bea50b43ce265d9cece8464b12ea66077eec Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Wed, 5 Dec 2018 15:26:24 -0500 Subject: [PATCH 21/28] PEP8 --- openml/setups/sklearn_converter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openml/setups/sklearn_converter.py b/openml/setups/sklearn_converter.py index 026791fe9..3ae2390d1 100644 --- a/openml/setups/sklearn_converter.py +++ b/openml/setups/sklearn_converter.py @@ -29,6 +29,5 @@ def openml_param_name_to_sklearn(openml_parameter, flow): if openml_parameter.flow_name not in flow_structure: raise ValueError('Obtained OpenMLParameter and OpenMLFlow do not ' 'correspond. ') - name = openml_parameter.flow_name # for PEP8 return '__'.join(flow_structure[name] + [openml_parameter.parameter_name]) From ff7dd8829cdf93bf9df4786d2d899af427d876ca Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Thu, 6 Dec 2018 10:52:59 -0500 Subject: [PATCH 22/28] incorporated changes by Matthias --- examples/run_setup_tutorial.py | 33 +++++++++++---------- openml/flows/__init__.py | 5 ++-- openml/flows/flow.py | 6 ++-- openml/flows/sklearn_converter.py | 31 ++++++++++++++++++++ openml/setups/__init__.py | 3 +- openml/setups/sklearn_converter.py | 33 --------------------- tests/test_flows/test_sklearn.py | 35 +++++++++++++++++++++++ tests/test_setups/test_setup_functions.py | 35 ----------------------- 8 files changed, 91 insertions(+), 90 deletions(-) delete mode 100644 openml/setups/sklearn_converter.py diff --git a/examples/run_setup_tutorial.py b/examples/run_setup_tutorial.py index 5355eb3d1..9726e2a57 100644 --- a/examples/run_setup_tutorial.py +++ b/examples/run_setup_tutorial.py @@ -9,17 +9,19 @@ reinstantiate flows with hyperparameter settings that were uploaded before. This tutorial uses the concept of setups. Although setups are not extensively described in the OpenML documentation (because most users will not directly -use them) +use them), they form a important concept within OpenML distinguishing between +hyperparameter configurations. +A setup is the combination of a flow with all its hyperparameters set. A key requirement for reinstantiating a flow is to have the same scikit-learn -version as the flow that was uploaded. This tutorial will upload the flow -itself, so it can be ran with any scikit-learn version that is supported by -this library. +version as the flow that was uploaded. However, this tutorial will upload the +flow itself, so it can be ran with any scikit-learn version that is supported +by this library. -In this tutotial we will - 1) Create a flow and use it to solve a task +In this tutorial we will + 1) Create a flow and use it to solve a task; 2) Download the flow, reinstantiate the model with same hyperparameters, - and solve the same task again. + and solve the same task again; 3) We will verify that the obtained results are exactly the same. Readers interested in reinstantiating a setup can skip part 1 and 2 and start with part 3 immediately. @@ -40,7 +42,7 @@ ############################################################################### # first, let's download the task that we are interested in -task = openml.tasks.get_task(6) # letter dataset +task = openml.tasks.get_task(6) # we will create a fairly complex model, with many preprocessing components and @@ -48,20 +50,21 @@ # easy as you want it to be model_original = sklearn.pipeline.make_pipeline( sklearn.impute.SimpleImputer(), - sklearn.preprocessing.StandardScaler(), sklearn.ensemble.RandomForestClassifier() ) # Let's change some hyperparameters. Of course, in any good application we -# would tune them using, e.g., Random Search or SMAC, but for the purpose of -# this tutorial we set them to some specific values that might or might not be -# optimal +# would tune them using, e.g., Random Search or Bayesian Optimization, but for +# the purpose of this tutorial we set them to some specific values that might +# or might not be optimal hyperparameters_original = { 'simpleimputer__strategy': 'median', - 'randomforestclassifier__random_state': 42, + 'randomforestclassifier__criterion': 'entropy', + 'randomforestclassifier__max_features': 0.2, 'randomforestclassifier__min_samples_leaf': 1, - 'randomforestclassifier__max_features': 0.2 + 'randomforestclassifier__n_estimators': 16, + 'randomforestclassifier__random_state': 42, } model_original.set_params(**hyperparameters_original) @@ -96,5 +99,5 @@ ############################################################################### # the run has stored all predictions in the field data content -np.testing.assert_array_equal(run_original.data_content, +np.testing.assert_array_equal(run_original.data_content, run_duplicate.data_content) diff --git a/openml/flows/__init__.py b/openml/flows/__init__.py index efacddbe6..0cfc4b9b5 100644 --- a/openml/flows/__init__.py +++ b/openml/flows/__init__.py @@ -1,7 +1,8 @@ from .flow import OpenMLFlow, _copy_server_fields -from .sklearn_converter import sklearn_to_flow, flow_to_sklearn, _check_n_jobs +from .sklearn_converter import sklearn_to_flow, flow_to_sklearn, \ + _check_n_jobs, openml_param_name_to_sklearn from .functions import get_flow, list_flows, flow_exists, assert_flows_equal __all__ = ['OpenMLFlow', 'get_flow', 'list_flows', 'sklearn_to_flow', - 'flow_to_sklearn', 'flow_exists', '_check_n_jobs'] + 'flow_to_sklearn', 'flow_exists', 'openml_param_name_to_sklearn'] diff --git a/openml/flows/flow.py b/openml/flows/flow.py index efc222558..98e9e75ee 100644 --- a/openml/flows/flow.py +++ b/openml/flows/flow.py @@ -363,7 +363,7 @@ def get_structure(self, key_item): """ Returns for each sub-component of the flow the path of identifiers that should be traversed to reach this component. The resulting dict maps a - key (identifying a flow be either its id, name or fullname) to the + key (identifying a flow by either its id, name or fullname) to the parameter prefix. Parameters @@ -374,7 +374,7 @@ def get_structure(self, key_item): Returns ------- - structure: dict[str, List[str]] + dict[str, List[str]] The flow structure """ if key_item not in ['flow_id', 'name']: @@ -398,7 +398,7 @@ def get_subflow(self, structure): Returns ------- - sub_component: OpenMLFlow + OpenMLFlow The OpenMLFlow that corresponds to the structure """ if len(structure) < 1: diff --git a/openml/flows/sklearn_converter.py b/openml/flows/sklearn_converter.py index 82b5895fa..787d2775b 100644 --- a/openml/flows/sklearn_converter.py +++ b/openml/flows/sklearn_converter.py @@ -177,6 +177,37 @@ def flow_to_sklearn(o, components=None, initialize_with_defaults=False): return rval +def openml_param_name_to_sklearn(openml_parameter, flow): + """ + Converts the name of an OpenMLParameter into the sklean name, given a flow. + + Parameters + ---------- + openml_parameter: OpenMLParameter + The parameter under consideration + + flow: OpenMLFlow + The flow that provides context. + + Returns + ------- + sklearn_parameter_name: str + The name the parameter will have once used in scikit-learn + """ + if not isinstance(openml_parameter, openml.setups.OpenMLParameter): + raise ValueError('openml_parameter should be an instance of ' + 'OpenMLParameter') + if not isinstance(flow, OpenMLFlow): + raise ValueError('flow should be an instance of OpenMLFlow') + + flow_structure = flow.get_structure('name') + if openml_parameter.flow_name not in flow_structure: + raise ValueError('Obtained OpenMLParameter and OpenMLFlow do not ' + 'correspond. ') + name = openml_parameter.flow_name # for PEP8 + return '__'.join(flow_structure[name] + [openml_parameter.parameter_name]) + + def _serialize_model(model): """Create an OpenMLFlow. diff --git a/openml/setups/__init__.py b/openml/setups/__init__.py index 74b4e886c..a8b4a8863 100644 --- a/openml/setups/__init__.py +++ b/openml/setups/__init__.py @@ -1,6 +1,5 @@ from .setup import OpenMLSetup, OpenMLParameter from .functions import get_setup, list_setups, setup_exists, initialize_model -from .sklearn_converter import openml_param_name_to_sklearn __all__ = ['OpenMLSetup', 'OpenMLParameter', 'get_setup', 'list_setups', - 'setup_exists', 'initialize_model', 'openml_param_name_to_sklearn'] + 'setup_exists', 'initialize_model'] diff --git a/openml/setups/sklearn_converter.py b/openml/setups/sklearn_converter.py deleted file mode 100644 index 3ae2390d1..000000000 --- a/openml/setups/sklearn_converter.py +++ /dev/null @@ -1,33 +0,0 @@ -from openml.flows import OpenMLFlow -from openml.setups import OpenMLParameter - - -def openml_param_name_to_sklearn(openml_parameter, flow): - """ - Converts the name of an OpenMLParameter into the sklean name, given a flow. - - Parameters - ---------- - openml_parameter: OpenMLParameter - The parameter under consideration - - flow: OpenMLFlow - The flow that provides context. - - Returns - ------- - sklearn_parameter_name: str - The name the parameter will have once used in scikit-learn - """ - if not isinstance(openml_parameter, OpenMLParameter): - raise ValueError('openml_parameter should be an instance of ' - 'OpenMLParameter') - if not isinstance(flow, OpenMLFlow): - raise ValueError('flow should be an instance of OpenMLFlow') - - flow_structure = flow.get_structure('name') - if openml_parameter.flow_name not in flow_structure: - raise ValueError('Obtained OpenMLParameter and OpenMLFlow do not ' - 'correspond. ') - name = openml_parameter.flow_name # for PEP8 - return '__'.join(flow_structure[name] + [openml_parameter.parameter_name]) diff --git a/tests/test_flows/test_sklearn.py b/tests/test_flows/test_sklearn.py index 81d84a607..03960e6ef 100644 --- a/tests/test_flows/test_sklearn.py +++ b/tests/test_flows/test_sklearn.py @@ -990,3 +990,38 @@ def test_deserialize_complex_with_defaults(self): # equals function for this assert_flows_equal(openml.flows.sklearn_to_flow(pipe_orig), openml.flows.sklearn_to_flow(pipe_deserialized)) + + def test_openml_param_name_to_sklearn(self): + scaler = sklearn.preprocessing.StandardScaler(with_mean=False) + boosting = sklearn.ensemble.AdaBoostClassifier( + base_estimator=sklearn.tree.DecisionTreeClassifier()) + model = sklearn.pipeline.Pipeline(steps=[ + ('scaler', scaler), ('boosting', boosting)]) + flow = openml.flows.sklearn_to_flow(model) + task = openml.tasks.get_task(115) + run = openml.runs.run_flow_on_task(flow, task) + run = run.publish() + run = openml.runs.get_run(run.run_id) + setup = openml.setups.get_setup(run.setup_id) + + # make sure to test enough parameters + self.assertGreater(len(setup.parameters), 15) + + for parameter in setup.parameters.values(): + sklearn_name = openml.flows.openml_param_name_to_sklearn( + parameter, flow) + + # test the inverse. Currently, OpenML stores the hyperparameter + # fullName as flow.name + flow.version + parameter.name on the + # server (but this behaviour is not documented and might or might + # not change in the future. Hence, we won't offer this + # transformation functionality in the main package yet.) + splitted = sklearn_name.split("__") + if len(splitted) > 1: # if len is 1, it is part of root flow + subflow = flow.get_subflow(splitted[0:-1]) + else: + subflow = flow + openml_name = "%s(%s)_%s" % (subflow.name, + subflow.version, + splitted[-1]) + self.assertEqual(parameter.full_name, openml_name) diff --git a/tests/test_setups/test_setup_functions.py b/tests/test_setups/test_setup_functions.py index 6d672f682..da14983e9 100644 --- a/tests/test_setups/test_setup_functions.py +++ b/tests/test_setups/test_setup_functions.py @@ -159,41 +159,6 @@ def test_setuplist_offset(self): self.assertEqual(len(all), size * 2) - def test_openml_param_name_to_sklearn(self): - scaler = sklearn.preprocessing.StandardScaler(with_mean=False) - boosting = sklearn.ensemble.AdaBoostClassifier( - base_estimator=sklearn.tree.DecisionTreeClassifier()) - model = sklearn.pipeline.Pipeline(steps=[ - ('scaler', scaler), ('boosting', boosting)]) - flow = openml.flows.sklearn_to_flow(model) - task = openml.tasks.get_task(115) - run = openml.runs.run_flow_on_task(flow, task) - run = run.publish() - run = openml.runs.get_run(run.run_id) - setup = openml.setups.get_setup(run.setup_id) - - # make sure to test enough parameters - self.assertGreater(len(setup.parameters), 15) - - for parameter in setup.parameters.values(): - sklearn_name = openml.setups.openml_param_name_to_sklearn( - parameter, flow) - - # test the inverse. Currently, OpenML stores the hyperparameter - # fullName as flow.name + flow.version + parameter.name on the - # server (but this behaviour is not documented and might or might - # not change in the future. Hence, we won't offer this - # transformation functionality in the main package yet.) - splitted = sklearn_name.split("__") - if len(splitted) > 1: # if len is 1, it is part of root flow - subflow = flow.get_subflow(splitted[0:-1]) - else: - subflow = flow - openml_name = "%s(%s)_%s" % (subflow.name, - subflow.version, - splitted[-1]) - self.assertEqual(parameter.full_name, openml_name) - def test_get_cached_setup(self): openml.config.cache_directory = self.static_cache_dir openml.setups.functions._get_cached_setup(1) From 8d6876f7496ba9eec8928d9cff6c9ed4e78e9fe3 Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Thu, 6 Dec 2018 19:31:11 -0500 Subject: [PATCH 23/28] fix 604 --- openml/tasks/functions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openml/tasks/functions.py b/openml/tasks/functions.py index de01ac052..f9c6143ef 100644 --- a/openml/tasks/functions.py +++ b/openml/tasks/functions.py @@ -387,8 +387,8 @@ def _create_task_from_xml(xml): common_kwargs['estimation_procedure_type'] = inputs[ "estimation_procedure"][ - "oml:estimation_procedure"]["oml:type"], - common_kwargs['estimation_parameters'] = estimation_parameters, + "oml:estimation_procedure"]["oml:type"] + common_kwargs['estimation_parameters'] = estimation_parameters common_kwargs['target_name'] = inputs[ "source_data"]["oml:data_set"]["oml:target_feature"] common_kwargs['data_splits_url'] = inputs["estimation_procedure"][ From 835e78ac38811883af215de9af26e71e130f42fa Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Mon, 10 Dec 2018 12:55:04 -0500 Subject: [PATCH 24/28] bugfix --- openml/setups/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openml/setups/functions.py b/openml/setups/functions.py index 79a150855..bec528846 100644 --- a/openml/setups/functions.py +++ b/openml/setups/functions.py @@ -215,7 +215,7 @@ def initialize_model(setup_id): flow = openml.flows.get_flow(setup.flow_id) model = openml.flows.flow_to_sklearn(flow) hyperparameters = { - openml.setups.openml_param_name_to_sklearn(hp, flow): + openml.flows.openml_param_name_to_sklearn(hp, flow): openml.flows.flow_to_sklearn(hp.value) for hp in setup.parameters.values() } From d80cd3431c33806b9e2caa8c45222b6e4d401fe7 Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Mon, 10 Dec 2018 13:24:49 -0500 Subject: [PATCH 25/28] flake fix --- openml/flows/__init__.py | 2 +- tests/test_setups/test_setup_functions.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/openml/flows/__init__.py b/openml/flows/__init__.py index 0cfc4b9b5..5107e9e4a 100644 --- a/openml/flows/__init__.py +++ b/openml/flows/__init__.py @@ -1,7 +1,7 @@ from .flow import OpenMLFlow, _copy_server_fields from .sklearn_converter import sklearn_to_flow, flow_to_sklearn, \ - _check_n_jobs, openml_param_name_to_sklearn + openml_param_name_to_sklearn from .functions import get_flow, list_flows, flow_exists, assert_flows_equal __all__ = ['OpenMLFlow', 'get_flow', 'list_flows', 'sklearn_to_flow', diff --git a/tests/test_setups/test_setup_functions.py b/tests/test_setups/test_setup_functions.py index da14983e9..35f43422e 100644 --- a/tests/test_setups/test_setup_functions.py +++ b/tests/test_setups/test_setup_functions.py @@ -5,7 +5,6 @@ import openml import openml.exceptions from openml.testing import TestBase -import sklearn from sklearn.ensemble import BaggingClassifier from sklearn.tree import DecisionTreeClassifier From 428e4b6882e2c97cf185935c6e638f6d9645cd68 Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Mon, 10 Dec 2018 13:36:37 -0500 Subject: [PATCH 26/28] import error --- openml/flows/__init__.py | 2 +- openml/flows/sklearn_converter.py | 1 - openml/runs/functions.py | 5 +++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openml/flows/__init__.py b/openml/flows/__init__.py index 5107e9e4a..0bdcf0c86 100644 --- a/openml/flows/__init__.py +++ b/openml/flows/__init__.py @@ -1,4 +1,4 @@ -from .flow import OpenMLFlow, _copy_server_fields +from .flow import OpenMLFlow from .sklearn_converter import sklearn_to_flow, flow_to_sklearn, \ openml_param_name_to_sklearn diff --git a/openml/flows/sklearn_converter.py b/openml/flows/sklearn_converter.py index 787d2775b..869ab70a7 100644 --- a/openml/flows/sklearn_converter.py +++ b/openml/flows/sklearn_converter.py @@ -11,7 +11,6 @@ import six import warnings import sys -import inspect import numpy as np import scipy.stats.distributions diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 3d42196b0..9dcb96a42 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -17,8 +17,9 @@ import openml._api_calls from ..exceptions import PyOpenMLError from .. import config -from ..flows import sklearn_to_flow, get_flow, flow_exists, _check_n_jobs, \ - _copy_server_fields, OpenMLFlow +from openml.flows.sklearn_converter import _check_n_jobs +from openml.flows.flow import _copy_server_fields +from ..flows import sklearn_to_flow, get_flow, flow_exists, OpenMLFlow from ..setups import setup_exists, initialize_model from ..exceptions import OpenMLCacheException, OpenMLServerException from ..tasks import OpenMLTask From fb5dc6a3a5911e86b5ed021f21c25767a47c7989 Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Mon, 10 Dec 2018 17:32:05 -0500 Subject: [PATCH 27/28] removed sentence --- examples/run_setup_tutorial.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/run_setup_tutorial.py b/examples/run_setup_tutorial.py index 9726e2a57..7a16d480d 100644 --- a/examples/run_setup_tutorial.py +++ b/examples/run_setup_tutorial.py @@ -23,8 +23,6 @@ 2) Download the flow, reinstantiate the model with same hyperparameters, and solve the same task again; 3) We will verify that the obtained results are exactly the same. -Readers interested in reinstantiating a setup can skip part 1 and 2 and start -with part 3 immediately. """ import logging import numpy as np From 786cfcba79e88a74b3dc43e0787a5afc43e4c615 Mon Sep 17 00:00:00 2001 From: janvanrijn Date: Mon, 10 Dec 2018 17:36:12 -0500 Subject: [PATCH 28/28] updated comment --- examples/run_setup_tutorial.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/run_setup_tutorial.py b/examples/run_setup_tutorial.py index 7a16d480d..b57ba367b 100644 --- a/examples/run_setup_tutorial.py +++ b/examples/run_setup_tutorial.py @@ -15,8 +15,9 @@ A key requirement for reinstantiating a flow is to have the same scikit-learn version as the flow that was uploaded. However, this tutorial will upload the -flow itself, so it can be ran with any scikit-learn version that is supported -by this library. +flow (that will later be reinstantiated) itself, so it can be ran with any +scikit-learn version that is supported by this library. In this case, the +requirement of the corresponding scikit-learn versions is automatically met. In this tutorial we will 1) Create a flow and use it to solve a task;