diff --git a/openml/flows/__init__.py b/openml/flows/__init__.py index 2d70e9e32..0683d9ef4 100644 --- a/openml/flows/__init__.py +++ b/openml/flows/__init__.py @@ -1,7 +1,7 @@ from .flow import OpenMLFlow, _copy_server_fields -from .sklearn_converter import sklearn_to_flow, flow_to_sklearn, _check_n_jobs +from .sklearn_converter import SKLearnConverter, _check_n_jobs from .functions import get_flow, list_flows, flow_exists, assert_flows_equal __all__ = ['OpenMLFlow', 'create_flow_from_model', 'get_flow', 'list_flows', - 'sklearn_to_flow', 'flow_to_sklearn', 'flow_exists'] + 'SKLearnConverter', 'flow_exists'] diff --git a/openml/flows/abstract_converter.py b/openml/flows/abstract_converter.py new file mode 100644 index 000000000..ee429e35b --- /dev/null +++ b/openml/flows/abstract_converter.py @@ -0,0 +1,224 @@ +from collections import OrderedDict +from distutils.version import LooseVersion +import importlib +import openml +import re +import copy +import sys +import inspect + +from abc import abstractmethod + +DEPENDENCIES_PATTERN = re.compile( + '^(?P[\w\-]+)((?P==|>=|>)(?P(\d+\.)?(\d+\.)?(\d+)?(dev)?[0-9]*))?$') + +class AbstractConverter(object): + def __init__(self, model): + self._external_version = None + self._model = model + + # stores all entities that should become subcomponents + self._sub_components = OrderedDict() + # stores the keys of all subcomponents that should become + self._sub_components_explicit = set() + self._parameters = OrderedDict() + self._parameters_meta_info = OrderedDict() + + self.extract_information_from_model() + self.check_multiple_occurence_of_component_in_flow() + + @staticmethod + @abstractmethod + def from_flow(flow, components=None, initialize_with_defaults=False): + """Initializes a model based on a flow. + + Parameters + ---------- + o : mixed + the object to deserialize (can be flow object, or any serialzied + parameter value that is accepted by) + + components : dict + + + initialize_with_defaults : bool, optional (default=False) + If this flag is set, the hyperparameter values of flows will be + ignored and a flow with its defaults is returned. + + Returns + ------- + mixed + """ + print("asdf") + + @abstractmethod + def to_flow(self): + """Creates an OpenML flow of the models. + + Returns + ------- + OpenMLFlow + """ + + @abstractmethod + def extract_information_from_model(self): + """ + """ + + @abstractmethod + def check_multiple_occurence_of_component_in_flow(self): + """ + """ + to_visit_stack = [] + to_visit_stack.extend(self._sub_components.values()) + known_sub_components = set() + while len(to_visit_stack) > 0: + visitee = to_visit_stack.pop() + if visitee.name in known_sub_components: + raise ValueError('Found a second occurence of component %s when ' + 'trying to serialize %s.' % (visitee.name, self._model)) + else: + known_sub_components.add(visitee.name) + to_visit_stack.extend(visitee.components.values()) + + + @property + def external_version(self): + if self._external_version: + return self._external_version + # Create external version string for a flow, given the model and the + # already parsed dictionary of sub_components. Retrieves the external + # version of all subcomponents, which themselves already contain all + # requirements for their subcomponents. The external version string is a + # sorted concatenation of all modules which are present in this run. + model_package_name = self._model.__module__.split('.')[0] + module = importlib.import_module(model_package_name) + model_package_version_number = module.__version__ + external_version = self.format_external_version(model_package_name, + model_package_version_number) + openml_version = self.format_external_version('openml', openml.__version__) + external_versions = set() + external_versions.add(external_version) + external_versions.add(openml_version) + for visitee in self._sub_components.values(): + for external_version in visitee.external_version.split(','): + external_versions.add(external_version) + external_versions = list(sorted(external_versions)) + self._external_version = ','.join(external_versions) + return self._external_version + + @staticmethod + def format_external_version(model_package_name, model_package_version_number): + return '%s==%s' % (model_package_name, model_package_version_number) + + @staticmethod + def _get_fn_arguments_with_defaults(fn_name): + """ + Returns i) a dict with all parameter names (as key) that have a default value (as value) and ii) a set with all + parameter names that do not have a default + + Parameters + ---------- + fn_name : callable + The function of which we want to obtain the defaults + + Returns + ------- + params_with_defaults: dict + a dict mapping parameter name to the default value + params_without_defaults: dict + a set with all parameters that do not have a default value + """ + if sys.version_info[0] >= 3: + signature = inspect.getfullargspec(fn_name) + else: + signature = inspect.getargspec(fn_name) + + # len(signature.defaults) <= len(signature.args). Thus, by definition, the last entrees of signature.args + # actually have defaults. Iterate backwards over both arrays to keep them in sync + len_defaults = len(signature.defaults) if signature.defaults is not None else 0 + params_with_defaults = {signature.args[-1*i]: signature.defaults[-1*i] for i in range(1, len_defaults + 1)} + # retrieve the params without defaults + params_without_defaults = {signature.args[i] for i in range(len(signature.args) - len_defaults)} + return params_with_defaults, params_without_defaults + + @classmethod + def _deserialize_model(cls, flow, keep_defaults): + model_name = flow.class_name + cls._check_dependencies(flow.dependencies) + + parameters = flow.parameters + components = flow.components + parameter_dict = OrderedDict() + + # Do a shallow copy of the components dictionary so we can remove the + # components from this copy once we added them into the pipeline. This + # allows us to not consider them any more when looping over the + # components, but keeping the dictionary of components untouched in the + # original components dictionary. + components_ = copy.copy(components) + + for name in parameters: + value = parameters.get(name) + rval = cls.from_flow(value, components=components_, initialize_with_defaults=keep_defaults) + parameter_dict[name] = rval + + for name in components: + if name in parameter_dict: + continue + if name not in components_: + continue + value = components[name] + rval = cls.from_flow(value, **kwargs) + parameter_dict[name] = rval + + module_name = model_name.rsplit('.', 1) + model_class = getattr(importlib.import_module(module_name[0]), + module_name[1]) + + if keep_defaults: + # obtain all params with a default + param_defaults, _ = cls._get_fn_arguments_with_defaults(model_class.__init__) + + # delete the params that have a default from the dict, + # so they get initialized with their default value + # except [...] + for param in param_defaults: + # [...] the ones that also have a key in the components dict. As OpenML stores different flows for ensembles + # with different (base-)components, in OpenML terms, these are not considered hyperparameters but rather + # constants (i.e., changing them would result in a different flow) + if param not in components.keys(): + del parameter_dict[param] + return model_class(**parameter_dict) + + @classmethod + def _check_dependencies(cls, dependencies): + if not dependencies: + return + + dependencies = dependencies.split('\n') + for dependency_string in dependencies: + match = DEPENDENCIES_PATTERN.match(dependency_string) + dependency_name = match.group('name') + operation = match.group('operation') + version = match.group('version') + + module = importlib.import_module(dependency_name) + required_version = LooseVersion(version) + installed_version = LooseVersion(module.__version__) + + if operation == '==': + check = required_version == installed_version + elif operation == '>': + check = installed_version > required_version + elif operation == '>=': + check = installed_version > required_version or \ + installed_version == required_version + else: + raise NotImplementedError( + 'operation \'%s\' is not supported' % operation) + if not check: + raise ValueError('Trying to deserialize a model with dependency ' + '%s not satisfied.' % dependency_string) + + diff --git a/openml/flows/sklearn_converter.py b/openml/flows/sklearn_converter.py index e3f22a931..2efac6653 100644 --- a/openml/flows/sklearn_converter.py +++ b/openml/flows/sklearn_converter.py @@ -1,10 +1,7 @@ """Convert scikit-learn estimators into an OpenMLFlows and vice versa.""" from collections import OrderedDict -import copy -from distutils.version import LooseVersion import importlib -import inspect import json import json.decoder import re @@ -24,6 +21,7 @@ from openml.flows import OpenMLFlow from openml.exceptions import PyOpenMLError +from .abstract_converter import AbstractConverter if sys.version_info >= (3, 5): from json.decoder import JSONDecodeError @@ -31,597 +29,451 @@ JSONDecodeError = ValueError -DEPENDENCIES_PATTERN = re.compile( - '^(?P[\w\-]+)((?P==|>=|>)(?P(\d+\.)?(\d+\.)?(\d+)?(dev)?[0-9]*))?$') - - -def sklearn_to_flow(o, parent_model=None): - # TODO: assert that only on first recursion lvl `parent_model` can be None - - if _is_estimator(o): - # is the main model or a submodel - rval = _serialize_model(o) - elif isinstance(o, (list, tuple)): - # TODO: explain what type of parameter is here - rval = [sklearn_to_flow(element, parent_model) for element in o] - if isinstance(o, tuple): - rval = tuple(rval) - elif isinstance(o, (bool, int, float, six.string_types)) or o is None: - # base parameter values - rval = o - elif isinstance(o, dict): - # TODO: explain what type of parameter is here - if not isinstance(o, OrderedDict): - o = OrderedDict([(key, value) for key, value in sorted(o.items())]) - - rval = OrderedDict() - for key, value in o.items(): - if not isinstance(key, six.string_types): - raise TypeError('Can only use string as keys, you passed ' - 'type %s for value %s.' % - (type(key), str(key))) - key = sklearn_to_flow(key, parent_model) - value = sklearn_to_flow(value, parent_model) - rval[key] = value - rval = rval - elif isinstance(o, type): - # TODO: explain what type of parameter is here - rval = serialize_type(o) - elif isinstance(o, scipy.stats.distributions.rv_frozen): - rval = serialize_rv_frozen(o) - # This only works for user-defined functions (and not even partial). - # I think this is exactly what we want here as there shouldn't be any - # built-in or functool.partials in a pipeline - elif inspect.isfunction(o): - # TODO: explain what type of parameter is here - rval = serialize_function(o) - elif _is_cross_validator(o): - # TODO: explain what type of parameter is here - rval = _serialize_cross_validator(o) - else: - raise TypeError(o, type(o)) - - return rval - - -def _is_estimator(o): - return (hasattr(o, 'fit') and hasattr(o, 'get_params') and - hasattr(o, 'set_params')) - - -def _is_cross_validator(o): - return isinstance(o, sklearn.model_selection.BaseCrossValidator) - - -def flow_to_sklearn(o, components=None, initialize_with_defaults=False): - """Initializes a sklearn model based on a flow. - - Parameters - ---------- - o : mixed - the object to deserialize (can be flow object, or any serialzied - parameter value that is accepted by) - - components : dict - - - initialize_with_defaults : bool, optional (default=False) - If this flag is set, the hyperparameter values of flows will be - ignored and a flow with its defaults is returned. - - Returns - ------- - mixed - - """ +class SKLearnConverter(AbstractConverter): + def __init__(self, model): + super().__init__(model) - # First, we need to check whether the presented object is a json string. - # JSON strings are used to encoder parameter values. By passing around - # json strings for parameters, we make sure that we can flow_to_sklearn - # the parameter values to the correct type. + @staticmethod + def _serialize_function(o): + name = o.__module__ + '.' + o.__name__ + ret = OrderedDict() + ret['oml-python:serialized_object'] = 'function' + ret['value'] = name + return ret - if isinstance(o, six.string_types): + @staticmethod + def _deserialize_function(name): + module_name = name.rsplit('.', 1) try: - o = json.loads(o) - except JSONDecodeError: - pass - - if isinstance(o, dict): - # Check if the dict encodes a 'special' object, which could not - # easily converted into a string, but rather the information to - # re-create the object were stored in a dictionary. - if 'oml-python:serialized_object' in o: - serialized_type = o['oml-python:serialized_object'] - value = o['value'] - if serialized_type == 'type': - rval = deserialize_type(value) - elif serialized_type == 'rv_frozen': - rval = deserialize_rv_frozen(value) - elif serialized_type == 'function': - rval = deserialize_function(value) - elif serialized_type == 'component_reference': - value = flow_to_sklearn(value) - step_name = value['step_name'] - key = value['key'] - component = flow_to_sklearn(components[key], initialize_with_defaults=initialize_with_defaults) - # The component is now added to where it should be used - # later. It should not be passed to the constructor of the - # main flow object. - del components[key] - if step_name is None: - rval = component + function_handle = getattr(importlib.import_module(module_name[0]), + module_name[1]) + except Exception as e: + warnings.warn('Cannot load function %s due to %s.' % (name, e)) + return None + return function_handle + + @staticmethod + def _flow_to_sklearn(o, components=None, initialize_with_defaults=False): + """Initializes a sklearn model based on a flow. + + Parameters + ---------- + o : mixed + the object to deserialize (can be flow object, or any serialzied + parameter value that is accepted by) + + components : dict + + + initialize_with_defaults : bool, optional (default=False) + If this flag is set, the hyperparameter values of flows will be + ignored and a flow with its defaults is returned. + + Returns + ------- + mixed + + """ + + # First, we need to check whether the presented object is a json string. + # JSON strings are used to encoder parameter values. By passing around + # json strings for parameters, we make sure that we can flow_to_sklearn + # the parameter values to the correct type. + + if isinstance(o, six.string_types): + try: + o = json.loads(o) + except JSONDecodeError: + pass + + if isinstance(o, dict): + # Check if the dict encodes a 'special' object, which could not + # easily converted into a string, but rather the information to + # re-create the object were stored in a dictionary. + if 'oml-python:serialized_object' in o: + serialized_type = o['oml-python:serialized_object'] + value = o['value'] + if serialized_type == 'type': + rval = SKLearnConverter._deserialize_type(value) + elif serialized_type == 'rv_frozen': + rval = SKLearnConverter._deserialize_rv_frozen(value) + elif serialized_type == 'function': + rval = SKLearnConverter._deserialize_function(value) + elif serialized_type == 'component_reference': + value = SKLearnConverter._flow_to_sklearn(value) + step_name = value['step_name'] + key = value['key'] + component = SKLearnConverter._flow_to_sklearn(components[key], initialize_with_defaults=initialize_with_defaults) + # The component is now added to where it should be used + # later. It should not be passed to the constructor of the + # main flow object. + del components[key] + if step_name is None: + rval = component + else: + rval = (step_name, component) + elif serialized_type == 'cv_object': + rval = SKLearnConverter._deserialize_cross_validator(value) else: - rval = (step_name, component) - elif serialized_type == 'cv_object': - rval = _deserialize_cross_validator(value) - else: - raise ValueError('Cannot flow_to_sklearn %s' % serialized_type) + raise ValueError('Cannot flow_to_sklearn %s' % serialized_type) + else: + rval = OrderedDict((SKLearnConverter._flow_to_sklearn(key, components, initialize_with_defaults), + SKLearnConverter._flow_to_sklearn(value, components, initialize_with_defaults)) + for key, value in sorted(o.items())) + elif isinstance(o, (list, tuple)): + rval = [SKLearnConverter._flow_to_sklearn(element, components, initialize_with_defaults) for element in o] + if isinstance(o, tuple): + rval = tuple(rval) + elif isinstance(o, (bool, int, float, six.string_types)) or o is None: + rval = o + elif isinstance(o, OpenMLFlow): + rval = SKLearnConverter._deserialize_model(o, initialize_with_defaults) + else: + raise TypeError(o) + + return rval + + @staticmethod + def _sklearn_to_flow(o, parent_model=None): + """ + """ + # TODO: assert that only on first recursion lvl `parent_model` can be None + + if SKLearnConverter._is_estimator(o): + # is the main model or a submodel + rval = SKLearnConverter(o).to_flow() + elif isinstance(o, (list, tuple)): + # TODO: explain what type of parameter is here + rval = [SKLearnConverter._sklearn_to_flow(element, parent_model) for element in o] + if isinstance(o, tuple): + rval = tuple(rval) + elif isinstance(o, (bool, int, float, six.string_types)) or o is None: + # base parameter values + rval = o + elif isinstance(o, dict): + # TODO: explain what type of parameter is here + if not isinstance(o, OrderedDict): + o = OrderedDict([(key, value) for key, value in sorted(o.items())]) + + rval = OrderedDict() + for key, value in o.items(): + if not isinstance(key, six.string_types): + raise TypeError('Can only use string as keys, you passed ' + 'type %s for value %s.' % + (type(key), str(key))) + key = SKLearnConverter._sklearn_to_flow(key, parent_model) + value = SKLearnConverter._sklearn_to_flow(value, parent_model) + rval[key] = value + rval = rval + elif isinstance(o, type): + # TODO: explain what type of parameter is here + rval = SKLearnConverter._serialize_type(o) + elif isinstance(o, scipy.stats.distributions.rv_frozen): + rval = SKLearnConverter._serialize_rv_frozen(o) + # This only works for user-defined functions (and not even partial). + # I think this is exactly what we want here as there shouldn't be any + # built-in or functool.partials in a pipeline + elif inspect.isfunction(o): + # TODO: explain what type of parameter is here + rval = SKLearnConverter._serialize_function(o) + elif SKLearnConverter._is_cross_validator(o): + # TODO: explain what type of parameter is here + rval = SKLearnConverter._serialize_cross_validator(o) else: - rval = OrderedDict((flow_to_sklearn(key, components, initialize_with_defaults), - flow_to_sklearn(value, components, initialize_with_defaults)) - for key, value in sorted(o.items())) - elif isinstance(o, (list, tuple)): - rval = [flow_to_sklearn(element, components, initialize_with_defaults) for element in o] - if isinstance(o, tuple): - rval = tuple(rval) - elif isinstance(o, (bool, int, float, six.string_types)) or o is None: - rval = o - elif isinstance(o, OpenMLFlow): - rval = _deserialize_model(o, initialize_with_defaults) - else: - raise TypeError(o) + raise TypeError(o, type(o)) + return rval - return rval + @staticmethod + def _is_estimator(o): + return (hasattr(o, 'fit') and hasattr(o, 'get_params') and + hasattr(o, 'set_params')) + @staticmethod + def _is_cross_validator(o): + return isinstance(o, sklearn.model_selection.BaseCrossValidator) -def _serialize_model(model): - """Create an OpenMLFlow. + @staticmethod + def from_flow(flow, components=None, initialize_with_defaults=False): + return SKLearnConverter._flow_to_sklearn( + flow, components=components, initialize_with_defaults=initialize_with_defaults) - Calls `sklearn_to_flow` recursively to properly serialize the - parameters to strings and the components (other models) to OpenMLFlows. + def to_flow(self): + """Create an OpenMLFlow. - Parameters - ---------- - model : sklearn estimator + Calls `sklearn_to_flow` recursively to properly serialize the + parameters to strings and the components (other models) to OpenMLFlows. - Returns - ------- - OpenMLFlow + Parameters + ---------- + model : sklearn estimator - """ + Returns + ------- + OpenMLFlow - # Get all necessary information about the model objects itself - parameters, parameters_meta_info, sub_components, sub_components_explicit =\ - _extract_information_from_model(model) + """ - # Check that a component does not occur multiple times in a flow as this - # is not supported by OpenML - _check_multiple_occurence_of_component_in_flow(model, sub_components) + # Create a flow name, which contains all components in brackets, for + # example RandomizedSearchCV(Pipeline(StandardScaler,AdaBoostClassifier(DecisionTreeClassifier)),StandardScaler,AdaBoostClassifier(DecisionTreeClassifier)) + class_name = self._model.__module__ + "." + self._model.__class__.__name__ - # Create a flow name, which contains all components in brackets, for - # example RandomizedSearchCV(Pipeline(StandardScaler,AdaBoostClassifier(DecisionTreeClassifier)),StandardScaler,AdaBoostClassifier(DecisionTreeClassifier)) - class_name = model.__module__ + "." + model.__class__.__name__ + # will be part of the name (in brackets) + sub_components_names = "" + for key in self._sub_components: + if key in self._sub_components_explicit: + sub_components_names += "," + key + "=" + self._sub_components[key].name + else: + sub_components_names += "," + self._sub_components[key].name - # will be part of the name (in brackets) - sub_components_names = "" - for key in sub_components: - if key in sub_components_explicit: - sub_components_names += "," + key + "=" + sub_components[key].name + if sub_components_names: + # slice operation on string in order to get rid of leading comma + name = '%s(%s)' % (class_name, sub_components_names[1:]) else: - sub_components_names += "," + sub_components[key].name - - if sub_components_names: - # slice operation on string in order to get rid of leading comma - name = '%s(%s)' % (class_name, sub_components_names[1:]) - else: - name = class_name - - # Get the external versions of all sub-components - external_version = _get_external_version_string(model, sub_components) - - dependencies = [_format_external_version('sklearn', sklearn.__version__), - 'numpy>=1.6.1', 'scipy>=0.9'] - dependencies = '\n'.join(dependencies) - - flow = OpenMLFlow(name=name, - class_name=class_name, - description='Automatically created scikit-learn flow.', - model=model, - components=sub_components, - parameters=parameters, - parameters_meta_info=parameters_meta_info, - external_version=external_version, - tags=['openml-python', 'sklearn', 'scikit-learn', - 'python', - _format_external_version('sklearn', - sklearn.__version__).replace('==', '_'), - # TODO: add more tags based on the scikit-learn - # module a flow is in? For example automatically - # annotate a class of sklearn.svm.SVC() with the - # tag svm? - ], - language='English', - # TODO fill in dependencies! - dependencies=dependencies) - - return flow - - -def _get_external_version_string(model, sub_components): - # Create external version string for a flow, given the model and the - # already parsed dictionary of sub_components. Retrieves the external - # version of all subcomponents, which themselves already contain all - # requirements for their subcomponents. The external version string is a - # sorted concatenation of all modules which are present in this run. - model_package_name = model.__module__.split('.')[0] - module = importlib.import_module(model_package_name) - model_package_version_number = module.__version__ - external_version = _format_external_version(model_package_name, - model_package_version_number) - openml_version = _format_external_version('openml', openml.__version__) - external_versions = set() - external_versions.add(external_version) - external_versions.add(openml_version) - for visitee in sub_components.values(): - for external_version in visitee.external_version.split(','): - external_versions.add(external_version) - external_versions = list(sorted(external_versions)) - external_version = ','.join(external_versions) - return external_version - - -def _check_multiple_occurence_of_component_in_flow(model, sub_components): - to_visit_stack = [] - to_visit_stack.extend(sub_components.values()) - known_sub_components = set() - while len(to_visit_stack) > 0: - visitee = to_visit_stack.pop() - if visitee.name in known_sub_components: - raise ValueError('Found a second occurence of component %s when ' - 'trying to serialize %s.' % (visitee.name, model)) - else: - known_sub_components.add(visitee.name) - to_visit_stack.extend(visitee.components.values()) - - -def _extract_information_from_model(model): - # This function contains four "global" states and is quite long and - # complicated. If it gets to complicated to ensure it's correctness, - # it would be best to make it a class with the four "global" states being - # the class attributes and the if/elif/else in the for-loop calls to - # separate class methods - - # stores all entities that should become subcomponents - sub_components = OrderedDict() - # stores the keys of all subcomponents that should become - sub_components_explicit = set() - parameters = OrderedDict() - parameters_meta_info = OrderedDict() - - model_parameters = model.get_params(deep=False) - for k, v in sorted(model_parameters.items(), key=lambda t: t[0]): - rval = sklearn_to_flow(v, model) - - if (isinstance(rval, (list, tuple)) and len(rval) > 0 and - isinstance(rval[0], (list, tuple)) and - [type(rval[0]) == type(rval[i]) for i in range(len(rval))]): - - # Steps in a pipeline or feature union, or base classifiers in voting classifier - parameter_value = list() - reserved_keywords = set(model.get_params(deep=False).keys()) - - for sub_component_tuple in rval: - identifier, sub_component = sub_component_tuple - sub_component_type = type(sub_component_tuple) - - if identifier in reserved_keywords: - parent_model_name = model.__module__ + "." + \ - model.__class__.__name__ - raise PyOpenMLError('Found element shadowing official ' + \ - 'parameter for %s: %s' % (parent_model_name, identifier)) - - if sub_component is None: - # In a FeatureUnion it is legal to have a None step - - pv = [identifier, None] - if sub_component_type is tuple: - pv = tuple(pv) - parameter_value.append(pv) - + name = class_name + + dependencies = [self.format_external_version('sklearn', sklearn.__version__), + 'numpy>=1.6.1', 'scipy>=0.9'] + dependencies = '\n'.join(dependencies) + + return OpenMLFlow(name=name, + class_name=class_name, + description='Automatically created scikit-learn flow.', + model=self._model, + components=self._sub_components, + parameters=self._parameters, + parameters_meta_info=self._parameters_meta_info, + external_version=self.external_version, + tags=['openml-python', 'sklearn', 'scikit-learn', + 'python', + self.format_external_version( + 'sklearn', sklearn.__version__).replace('==', '_'), + # TODO: add more tags based on the scikit-learn + # module a flow is in? For example automatically + # annotate a class of sklearn.svm.SVC() with the + # tag svm? + ], + language='English', + # TODO fill in dependencies! + dependencies=dependencies) + + def extract_information_from_model(self): + # This function contains four "global" states and is quite long and + # complicated. If it gets to complicated to ensure it's correctness, + # it would be best to make it a class with the four "global" states being + # the class attributes and the if/elif/else in the for-loop calls to + # separate class methods + model_parameters = self._model.get_params(deep=False) + for k, v in sorted(model_parameters.items(), key=lambda t: t[0]): + rval = self._sklearn_to_flow(v, self._model) + + if (isinstance(rval, (list, tuple)) and len(rval) > 0 and + isinstance(rval[0], (list, tuple)) and + [type(rval[0]) == type(rval[i]) for i in range(len(rval))]): + + self._extract_sklearn_model_information(rval, k) + elif isinstance(rval, OpenMLFlow): + self._extract_openml_flow_information(rval, k) + else: + # a regular hyperparameter + if not (hasattr(rval, '__len__') and len(rval) == 0): + rval = json.dumps(rval) + self._parameters[k] = rval else: - # Add the component to the list of components, add a - # component reference as a placeholder to the list of - # parameters, which will be replaced by the real component - # when deserializing the parameter - sub_components_explicit.add(identifier) - sub_components[identifier] = sub_component - component_reference = OrderedDict() - component_reference[ - 'oml-python:serialized_object'] = 'component_reference' - cr_value = OrderedDict() - cr_value['key'] = identifier - cr_value['step_name'] = identifier - component_reference['value'] = cr_value - parameter_value.append(component_reference) - - if isinstance(rval, tuple): - parameter_value = tuple(parameter_value) - - # Here (and in the elif and else branch below) are the only - # places where we encode a value as json to make sure that all - # parameter values still have the same type after - # deserialization - parameter_value = json.dumps(parameter_value) - parameters[k] = parameter_value - - elif isinstance(rval, OpenMLFlow): - - # A subcomponent, for example the base model in - # AdaBoostClassifier - sub_components[k] = rval - sub_components_explicit.add(k) - component_reference = OrderedDict() - component_reference[ - 'oml-python:serialized_object'] = 'component_reference' - cr_value = OrderedDict() - cr_value['key'] = k - cr_value['step_name'] = None - component_reference['value'] = cr_value - component_reference = sklearn_to_flow(component_reference, model) - parameters[k] = json.dumps(component_reference) + self._parameters[k] = None - else: + self._parameters_meta_info[k] = OrderedDict((('description', None), + ('data_type', None))) - # a regular hyperparameter - if not (hasattr(rval, '__len__') and len(rval) == 0): - rval = json.dumps(rval) - parameters[k] = rval - else: - parameters[k] = None + def _extract_sklearn_model_information(self, rval, parameter_name): + # Steps in a pipeline or feature union, or base classifiers in voting classifier + parameter_value = list() + reserved_keywords = set(self._model.get_params(deep=False).keys()) - parameters_meta_info[k] = OrderedDict((('description', None), - ('data_type', None))) + for sub_component_tuple in rval: + identifier, sub_component = sub_component_tuple + sub_component_type = type(sub_component_tuple) - return parameters, parameters_meta_info, sub_components, sub_components_explicit + if identifier in reserved_keywords: + parent_model_name = self._model.__module__ + "." + \ + self._model.__class__.__name__ + raise PyOpenMLError('Found element shadowing official ' + \ + 'parameter for %s: %s' % (parent_model_name, identifier)) + if sub_component is None: + # In a FeatureUnion it is legal to have a None step -def _get_fn_arguments_with_defaults(fn_name): - """ - Returns i) a dict with all parameter names (as key) that have a default value (as value) and ii) a set with all - parameter names that do not have a default - - Parameters - ---------- - fn_name : callable - The function of which we want to obtain the defaults - - Returns - ------- - params_with_defaults: dict - a dict mapping parameter name to the default value - params_without_defaults: dict - a set with all parameters that do not have a default value - """ - if sys.version_info[0] >= 3: - signature = inspect.getfullargspec(fn_name) - else: - signature = inspect.getargspec(fn_name) - - # len(signature.defaults) <= len(signature.args). Thus, by definition, the last entrees of signature.args - # actually have defaults. Iterate backwards over both arrays to keep them in sync - len_defaults = len(signature.defaults) if signature.defaults is not None else 0 - params_with_defaults = {signature.args[-1*i]: signature.defaults[-1*i] for i in range(1, len_defaults + 1)} - # retrieve the params without defaults - params_without_defaults = {signature.args[i] for i in range(len(signature.args) - len_defaults)} - return params_with_defaults, params_without_defaults - - -def _deserialize_model(flow, keep_defaults): - - model_name = flow.class_name - _check_dependencies(flow.dependencies) - - parameters = flow.parameters - components = flow.components - parameter_dict = OrderedDict() - - # Do a shallow copy of the components dictionary so we can remove the - # components from this copy once we added them into the pipeline. This - # allows us to not consider them any more when looping over the - # components, but keeping the dictionary of components untouched in the - # original components dictionary. - components_ = copy.copy(components) - - for name in parameters: - value = parameters.get(name) - rval = flow_to_sklearn(value, components=components_, initialize_with_defaults=keep_defaults) - parameter_dict[name] = rval - - for name in components: - if name in parameter_dict: - continue - if name not in components_: - continue - value = components[name] - rval = flow_to_sklearn(value, **kwargs) - parameter_dict[name] = rval - - module_name = model_name.rsplit('.', 1) - model_class = getattr(importlib.import_module(module_name[0]), - module_name[1]) - - if keep_defaults: - # obtain all params with a default - param_defaults, _ = _get_fn_arguments_with_defaults(model_class.__init__) - - # delete the params that have a default from the dict, - # so they get initialized with their default value - # except [...] - for param in param_defaults: - # [...] the ones that also have a key in the components dict. As OpenML stores different flows for ensembles - # with different (base-)components, in OpenML terms, these are not considered hyperparameters but rather - # constants (i.e., changing them would result in a different flow) - if param not in components.keys(): - del parameter_dict[param] - return model_class(**parameter_dict) - - -def _check_dependencies(dependencies): - if not dependencies: - return - - dependencies = dependencies.split('\n') - for dependency_string in dependencies: - match = DEPENDENCIES_PATTERN.match(dependency_string) - dependency_name = match.group('name') - operation = match.group('operation') - version = match.group('version') - - module = importlib.import_module(dependency_name) - required_version = LooseVersion(version) - installed_version = LooseVersion(module.__version__) - - if operation == '==': - check = required_version == installed_version - elif operation == '>': - check = installed_version > required_version - elif operation == '>=': - check = installed_version > required_version or \ - installed_version == required_version + pv = [identifier, None] + if sub_component_type is tuple: + pv = tuple(pv) + parameter_value.append(pv) + + else: + # Add the component to the list of components, add a + # component reference as a placeholder to the list of + # parameters, which will be replaced by the real component + # when deserializing the parameter + self._sub_components_explicit.add(identifier) + self._sub_components[identifier] = sub_component + component_reference = OrderedDict() + component_reference[ + 'oml-python:serialized_object'] = 'component_reference' + cr_value = OrderedDict() + cr_value['key'] = identifier + cr_value['step_name'] = identifier + component_reference['value'] = cr_value + parameter_value.append(component_reference) + + if isinstance(rval, tuple): + parameter_value = tuple(parameter_value) + + # Here (and in the elif and else branch below) are the only + # places where we encode a value as json to make sure that all + # parameter values still have the same type after + # deserialization + self._parameters[parameter_name] = json.dumps(parameter_value) + + + def _extract_openml_flow_information(self, rval, parameter_name): + """ + + """ + # A subcomponent, for example the base model in + # AdaBoostClassifier + self._sub_components[parameter_name] = rval + self._sub_components_explicit.add(parameter_name) + component_reference = OrderedDict() + component_reference[ + 'oml-python:serialized_object'] = 'component_reference' + cr_value = OrderedDict() + cr_value['key'] = parameter_name + cr_value['step_name'] = None + component_reference['value'] = cr_value + component_reference = self._sklearn_to_flow(component_reference, self._model) + self._parameters[parameter_name] = json.dumps(component_reference) + + @staticmethod + def _serialize_cross_validator(o): + ret = OrderedDict() + parameters = OrderedDict() + + # XXX this is copied from sklearn.model_selection._split + cls = o.__class__ + init = getattr(cls.__init__, 'deprecated_original', cls.__init__) + # Ignore varargs, kw and default values and pop self + init_signature = signature(init) + # Consider the constructor parameters excluding 'self' + if init is object.__init__: + args = [] else: - raise NotImplementedError( - 'operation \'%s\' is not supported' % operation) - if not check: - raise ValueError('Trying to deserialize a model with dependency ' - '%s not satisfied.' % dependency_string) - - -def serialize_type(o): - mapping = {float: 'float', - np.float: 'np.float', - np.float32: 'np.float32', - np.float64: 'np.float64', - int: 'int', - np.int: 'np.int', - np.int32: 'np.int32', - np.int64: 'np.int64'} - ret = OrderedDict() - ret['oml-python:serialized_object'] = 'type' - ret['value'] = mapping[o] - return ret - - -def deserialize_type(o): - mapping = {'float': float, - 'np.float': np.float, - 'np.float32': np.float32, - 'np.float64': np.float64, - 'int': int, - 'np.int': np.int, - 'np.int32': np.int32, - 'np.int64': np.int64} - return mapping[o] - - -def serialize_rv_frozen(o): - args = o.args - kwds = o.kwds - a = o.a - b = o.b - dist = o.dist.__class__.__module__ + '.' + o.dist.__class__.__name__ - ret = OrderedDict() - ret['oml-python:serialized_object'] = 'rv_frozen' - ret['value'] = OrderedDict((('dist', dist), ('a', a), ('b', b), - ('args', args), ('kwds', kwds))) - return ret - - -def deserialize_rv_frozen(o): - args = o['args'] - kwds = o['kwds'] - a = o['a'] - b = o['b'] - dist_name = o['dist'] - - module_name = dist_name.rsplit('.', 1) - try: - rv_class = getattr(importlib.import_module(module_name[0]), - module_name[1]) - except: - warnings.warn('Cannot create model %s for flow.' % dist_name) - return None - - dist = scipy.stats.distributions.rv_frozen(rv_class(), *args, **kwds) - dist.a = a - dist.b = b - - return dist - - -def serialize_function(o): - name = o.__module__ + '.' + o.__name__ - ret = OrderedDict() - ret['oml-python:serialized_object'] = 'function' - ret['value'] = name - return ret - - -def deserialize_function(name): - module_name = name.rsplit('.', 1) - try: - function_handle = getattr(importlib.import_module(module_name[0]), - module_name[1]) - except Exception as e: - warnings.warn('Cannot load function %s due to %s.' % (name, e)) - return None - return function_handle - - -def _serialize_cross_validator(o): - ret = OrderedDict() - - parameters = OrderedDict() - - # XXX this is copied from sklearn.model_selection._split - cls = o.__class__ - init = getattr(cls.__init__, 'deprecated_original', cls.__init__) - # Ignore varargs, kw and default values and pop self - init_signature = signature(init) - # Consider the constructor parameters excluding 'self' - if init is object.__init__: - args = [] - else: - args = sorted([p.name for p in init_signature.parameters.values() - if p.name != 'self' and p.kind != p.VAR_KEYWORD]) - - for key in args: - # We need deprecation warnings to always be on in order to - # catch deprecated param values. - # This is set in utils/__init__.py but it gets overwritten - # when running under python3 somehow. - warnings.simplefilter("always", DeprecationWarning) + args = sorted([p.name for p in init_signature.parameters.values() + if p.name != 'self' and p.kind != p.VAR_KEYWORD]) + + for key in args: + # We need deprecation warnings to always be on in order to + # catch deprecated param values. + # This is set in utils/__init__.py but it gets overwritten + # when running under python3 somehow. + warnings.simplefilter("always", DeprecationWarning) + try: + with warnings.catch_warnings(record=True) as w: + value = getattr(o, key, None) + if len(w) and w[0].category == DeprecationWarning: + # if the parameter is deprecated, don't show it + continue + finally: + warnings.filters.pop(0) + + if not (hasattr(value, '__len__') and len(value) == 0): + value = json.dumps(value) + parameters[key] = value + else: + parameters[key] = None + + ret['oml-python:serialized_object'] = 'cv_object' + name = o.__module__ + "." + o.__class__.__name__ + value = OrderedDict([['name', name], ['parameters', parameters]]) + ret['value'] = value + + return ret + + @staticmethod + def _serialize_type(o): + mapping = {float: 'float', + np.float: 'np.float', + np.float32: 'np.float32', + np.float64: 'np.float64', + int: 'int', + np.int: 'np.int', + np.int32: 'np.int32', + np.int64: 'np.int64'} + ret = OrderedDict() + ret['oml-python:serialized_object'] = 'type' + ret['value'] = mapping[o] + return ret + + @staticmethod + def _deserialize_type(o): + mapping = {'float': float, + 'np.float': np.float, + 'np.float32': np.float32, + 'np.float64': np.float64, + 'int': int, + 'np.int': np.int, + 'np.int32': np.int32, + 'np.int64': np.int64} + return mapping[o] + + @staticmethod + def _serialize_rv_frozen(o): + args = o.args + kwds = o.kwds + a = o.a + b = o.b + dist = o.dist.__class__.__module__ + '.' + o.dist.__class__.__name__ + ret = OrderedDict() + ret['oml-python:serialized_object'] = 'rv_frozen' + ret['value'] = OrderedDict((('dist', dist), ('a', a), ('b', b), + ('args', args), ('kwds', kwds))) + return ret + + @staticmethod + def _deserialize_rv_frozen(o): + args = o['args'] + kwds = o['kwds'] + a = o['a'] + b = o['b'] + dist_name = o['dist'] + + module_name = dist_name.rsplit('.', 1) try: - with warnings.catch_warnings(record=True) as w: - value = getattr(o, key, None) - if len(w) and w[0].category == DeprecationWarning: - # if the parameter is deprecated, don't show it - continue - finally: - warnings.filters.pop(0) - - if not (hasattr(value, '__len__') and len(value) == 0): - value = json.dumps(value) - parameters[key] = value - else: - parameters[key] = None + rv_class = getattr(importlib.import_module(module_name[0]), + module_name[1]) + except: + warnings.warn('Cannot create model %s for flow.' % dist_name) + return None + + dist = scipy.stats.distributions.rv_frozen(rv_class(), *args, **kwds) + dist.a = a + dist.b = b + + return dist + + @staticmethod + def _deserialize_cross_validator(value): + model_name = value['name'] + parameters = value['parameters'] - ret['oml-python:serialized_object'] = 'cv_object' - name = o.__module__ + "." + o.__class__.__name__ - value = OrderedDict([['name', name], ['parameters', parameters]]) - ret['value'] = value + module_name = model_name.rsplit('.', 1) + model_class = getattr(importlib.import_module(module_name[0]), + module_name[1]) + for parameter in parameters: + parameters[parameter] = SKLearnConverter._flow_to_sklearn(parameters[parameter]) + return model_class(**parameters) + + +# def run_on_task(self, task): - return ret def _check_n_jobs(model): @@ -672,19 +524,3 @@ def check(param_grid, restricted_parameter_name, legal_values): # check the parameters for n_jobs return check(model.get_params(), 'n_jobs', [1, None]) - - -def _deserialize_cross_validator(value): - model_name = value['name'] - parameters = value['parameters'] - - module_name = model_name.rsplit('.', 1) - model_class = getattr(importlib.import_module(module_name[0]), - module_name[1]) - for parameter in parameters: - parameters[parameter] = flow_to_sklearn(parameters[parameter]) - return model_class(**parameters) - - -def _format_external_version(model_package_name, model_package_version_number): - return '%s==%s' % (model_package_name, model_package_version_number) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 3ecec7b5f..04517f777 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -17,7 +17,7 @@ import openml._api_calls from ..exceptions import PyOpenMLError from .. import config -from ..flows import sklearn_to_flow, get_flow, flow_exists, _check_n_jobs, \ +from ..flows import SKLearnConverter, get_flow, flow_exists, _check_n_jobs, \ _copy_server_fields, OpenMLFlow from ..setups import setup_exists, initialize_model from ..exceptions import OpenMLCacheException, OpenMLServerException @@ -33,7 +33,7 @@ def run_model_on_task(model, task, avoid_duplicate_runs=True, flow_tags=None, - seed=None, add_local_measures=True): + seed=None, add_local_measures=True, converter=SKLearnConverter): """See ``run_flow_on_task for a documentation``.""" # TODO: At some point in the future do not allow for arguments in old order (order changed 6-2018). if isinstance(model, OpenMLTask) and hasattr(task, 'fit') and hasattr(task, 'predict'): @@ -41,13 +41,10 @@ def run_model_on_task(model, task, avoid_duplicate_runs=True, flow_tags=None, "Please use the order (model, task).", DeprecationWarning) task, model = model, task - flow = sklearn_to_flow(model) - - return run_flow_on_task(task=task, flow=flow, - avoid_duplicate_runs=avoid_duplicate_runs, - flow_tags=flow_tags, seed=seed, - add_local_measures=add_local_measures) - + return converter(model).run_flow_on_task( + task, avoid_duplicate_runs=avoid_duplicate_runs, + flow_tags=flow_tags, seed=seed, + add_local_measures=add_local_measures) def run_flow_on_task(flow, task, avoid_duplicate_runs=True, flow_tags=None, seed=None, add_local_measures=True): @@ -59,7 +56,7 @@ def run_flow_on_task(flow, task, avoid_duplicate_runs=True, flow_tags=None, Parameters ---------- - model : sklearn model + flow : sklearn model A model which has a function fit(X,Y) and predict(X), all supervised estimators of scikit learn follow this definition of a model [1] [1](http://scikit-learn.org/stable/tutorial/statistical_inference/supervised_learning.html) @@ -231,7 +228,7 @@ def initialize_model_from_trace(run_id, repeat, fold, iteration=None): Parameters ---------- run_id : int - The Openml run_id. Should contain a trace file, + The Openml run_id. Should contain a trace file, otherwise a OpenMLServerException is raised repeat: int @@ -242,7 +239,7 @@ def initialize_model_from_trace(run_id, repeat, fold, iteration=None): iteration: int The iteration nr (column in trace file). If None, the - best (selected) iteration will be searched (slow), + best (selected) iteration will be searched (slow), according to the selection criteria implemented in OpenMLRunTrace.get_selected_iteration @@ -748,7 +745,7 @@ def _create_run_from_xml(xml, from_server=True): run : OpenMLRun New run object representing run_xml. """ - + def obtain_field(xml_obj, fieldname, from_server, cast=None): # this function can be used to check whether a field is present in an object. # if it is not present, either returns None or throws an error (this is @@ -769,7 +766,7 @@ def obtain_field(xml_obj, fieldname, from_server, cast=None): task_id = int(run['oml:task_id']) task_type = obtain_field(run, 'oml:task_type', from_server) - # even with the server requirement this field may be empty. + # even with the server requirement this field may be empty. if 'oml:task_evaluation_measure' in run: task_evaluation_measure = run['oml:task_evaluation_measure'] else: diff --git a/openml/setups/functions.py b/openml/setups/functions.py index c329eab52..09cf5eef8 100644 --- a/openml/setups/functions.py +++ b/openml/setups/functions.py @@ -7,7 +7,7 @@ from .. import config from .setup import OpenMLSetup, OpenMLParameter -from openml.flows import flow_exists +from openml.flows import flow_exists, SKLearnConverter from openml.exceptions import OpenMLServerNoResult import openml.utils @@ -185,7 +185,7 @@ def __list_setups(api_call): return setups -def initialize_model(setup_id): +def initialize_model(setup_id, converter=SKLearnConverter): ''' Initialized a model based on a setup_id (i.e., using the exact same parameter settings) @@ -197,8 +197,8 @@ def initialize_model(setup_id): Returns ------- - model : sklearn model - the scikitlearn model with all parameters initailized + model : an ml model + the model with all parameters initailized ''' # transform an openml setup object into @@ -240,8 +240,7 @@ def _reconstruct_flow(_flow, _params): # parameters obtained from the setup flow = openml.flows.get_flow(setup.flow_id) flow = _reconstruct_flow(flow, parameters) - - return openml.flows.flow_to_sklearn(flow) + return converter.from_flow(flow) def _to_dict(flow_id, openml_parameter_settings): diff --git a/tests/test_flows/test_sklearn.py b/tests/test_flows/test_sklearn.py index d08f63ff0..8c348f6c3 100644 --- a/tests/test_flows/test_sklearn.py +++ b/tests/test_flows/test_sklearn.py @@ -33,10 +33,10 @@ from sklearn.impute import SimpleImputer as Imputer import openml -from openml.flows import OpenMLFlow, sklearn_to_flow, flow_to_sklearn +from openml.flows import OpenMLFlow from openml.flows.functions import assert_flows_equal -from openml.flows.sklearn_converter import _format_external_version, \ - _check_dependencies, _check_n_jobs +from openml.flows.sklearn_converter import SKLearnConverter, _check_n_jobs +# _check_dependencies, _check_n_jobs from openml.exceptions import PyOpenMLError this_directory = os.path.dirname(os.path.abspath(__file__)) @@ -65,7 +65,7 @@ def setUp(self): self.X = iris.data self.y = iris.target - @mock.patch('openml.flows.sklearn_converter._check_dependencies') + @mock.patch('openml.flows.sklearn_converter.AbstractConverter._check_dependencies') def test_serialize_model(self, check_dependencies_mock): model = sklearn.tree.DecisionTreeClassifier(criterion='entropy', max_features='auto', @@ -107,7 +107,7 @@ def test_serialize_model(self, check_dependencies_mock): ('random_state', 'null'), ('splitter', '"best"'))) - serialization = sklearn_to_flow(model) + serialization = SKLearnConverter(model).to_flow() self.assertEqual(serialization.name, fixture_name) self.assertEqual(serialization.class_name, fixture_name) @@ -115,18 +115,19 @@ def test_serialize_model(self, check_dependencies_mock): self.assertEqual(serialization.parameters, fixture_parameters) self.assertEqual(serialization.dependencies, version_fixture) - new_model = flow_to_sklearn(serialization) + new_model = SKLearnConverter.from_flow(serialization) self.assertEqual(type(new_model), type(model)) self.assertIsNot(new_model, model) + print(new_model.get_params()) + print(model.get_params()) self.assertEqual(new_model.get_params(), model.get_params()) new_model.fit(self.X, self.y) - self.assertEqual(check_dependencies_mock.call_count, 1) - @mock.patch('openml.flows.sklearn_converter._check_dependencies') + @mock.patch('openml.flows.sklearn_converter.AbstractConverter._check_dependencies') def test_serialize_model_clustering(self, check_dependencies_mock): model = sklearn.cluster.KMeans() @@ -162,7 +163,7 @@ def test_serialize_model_clustering(self, check_dependencies_mock): ('tol', '0.0001'), ('verbose', '0'))) - serialization = sklearn_to_flow(model) + serialization = SKLearnConverter(model).to_flow() self.assertEqual(serialization.name, fixture_name) self.assertEqual(serialization.class_name, fixture_name) @@ -170,7 +171,7 @@ def test_serialize_model_clustering(self, check_dependencies_mock): self.assertEqual(serialization.parameters, fixture_parameters) self.assertEqual(serialization.dependencies, version_fixture) - new_model = flow_to_sklearn(serialization) + new_model = SKLearnConverter.from_flow(serialization) self.assertEqual(type(new_model), type(model)) self.assertIsNot(new_model, model) @@ -193,7 +194,7 @@ def test_serialize_model_with_subcomponent(self): fixture_subcomponent_class_name = 'sklearn.tree.tree.DecisionTreeClassifier' fixture_subcomponent_description = 'Automatically created scikit-learn flow.' - serialization = sklearn_to_flow(model) + serialization = SKLearnConverter(model).to_flow() self.assertEqual(serialization.name, fixture_name) self.assertEqual(serialization.class_name, fixture_class_name) @@ -209,7 +210,7 @@ def test_serialize_model_with_subcomponent(self): self.assertEqual(serialization.components['base_estimator'].description, fixture_subcomponent_description) - new_model = flow_to_sklearn(serialization) + new_model = SKLearnConverter.from_flow(serialization) self.assertEqual(type(new_model), type(model)) self.assertIsNot(new_model, model) @@ -236,7 +237,7 @@ def test_serialize_pipeline(self): 'dummy=sklearn.dummy.DummyClassifier)' fixture_description = 'Automatically created scikit-learn flow.' - serialization = sklearn_to_flow(model) + serialization = SKLearnConverter(model).to_flow() self.assertEqual(serialization.name, fixture_name) self.assertEqual(serialization.description, fixture_description) @@ -265,7 +266,7 @@ def test_serialize_pipeline(self): OpenMLFlow) #del serialization.model - new_model = flow_to_sklearn(serialization) + new_model = SKLearnConverter.from_flow(serialization) self.assertEqual(type(new_model), type(model)) self.assertIsNot(new_model, model) @@ -298,7 +299,7 @@ def test_serialize_pipeline_clustering(self): 'clusterer=sklearn.cluster.k_means_.KMeans)' fixture_description = 'Automatically created scikit-learn flow.' - serialization = sklearn_to_flow(model) + serialization = SKLearnConverter(model).to_flow() self.assertEqual(serialization.name, fixture_name) self.assertEqual(serialization.description, fixture_description) @@ -327,7 +328,7 @@ def test_serialize_pipeline_clustering(self): OpenMLFlow) # del serialization.model - new_model = flow_to_sklearn(serialization) + new_model = SKLearnConverter.from_flow(serialization) self.assertEqual(type(new_model), type(model)) self.assertIsNot(new_model, model) @@ -357,7 +358,7 @@ def test_serialize_feature_union(self): scaler = sklearn.preprocessing.StandardScaler() fu = sklearn.pipeline.FeatureUnion( transformer_list=[('ohe', ohe), ('scaler', scaler)]) - serialization = sklearn_to_flow(fu) + serialization = SKLearnConverter(fu).to_flow() # OneHotEncoder was moved to _encoders module in 0.20 module_name_encoder = ('_encoders' if LooseVersion(sklearn.__version__) >= "0.20" @@ -367,7 +368,7 @@ def test_serialize_feature_union(self): 'ohe=sklearn.preprocessing.{}.OneHotEncoder,' 'scaler=sklearn.preprocessing.data.StandardScaler)' .format(module_name_encoder)) - new_model = flow_to_sklearn(serialization) + new_model = SKLearnConverter.from_flow(serialization) self.assertEqual(type(new_model), type(fu)) self.assertIsNot(new_model, fu) @@ -400,12 +401,12 @@ def test_serialize_feature_union(self): new_model.fit(self.X, self.y) fu.set_params(scaler=None) - serialization = sklearn_to_flow(fu) + serialization = SKLearnConverter(fu).to_flow() self.assertEqual(serialization.name, 'sklearn.pipeline.FeatureUnion(' 'ohe=sklearn.preprocessing.{}.OneHotEncoder)' .format(module_name_encoder)) - new_model = flow_to_sklearn(serialization) + new_model = SKLearnConverter.from_flow(serialization) self.assertEqual(type(new_model), type(fu)) self.assertIsNot(new_model, fu) self.assertIs(new_model.transformer_list[1][1], None) @@ -419,8 +420,8 @@ def test_serialize_feature_union_switched_names(self): transformer_list=[('ohe', ohe), ('scaler', scaler)]) fu2 = sklearn.pipeline.FeatureUnion( transformer_list=[('scaler', ohe), ('ohe', scaler)]) - fu1_serialization = sklearn_to_flow(fu1) - fu2_serialization = sklearn_to_flow(fu2) + fu1_serialization = SKLearnConverter(fu1).to_flow() + fu2_serialization = SKLearnConverter(fu2).to_flow() # OneHotEncoder was moved to _encoders module in 0.20 module_name_encoder = ('_encoders' if LooseVersion(sklearn.__version__) >= "0.20" @@ -452,7 +453,7 @@ def test_serialize_complex_flow(self): cv = sklearn.model_selection.StratifiedKFold(n_splits=5, shuffle=True) rs = sklearn.model_selection.RandomizedSearchCV( estimator=model, param_distributions=parameter_grid, cv=cv) - serialized = sklearn_to_flow(rs) + serialized = SKLearnConverter(rs).to_flow() # OneHotEncoder was moved to _encoders module in 0.20 module_name_encoder = ('_encoders' if LooseVersion(sklearn.__version__) >= "0.20" @@ -467,10 +468,10 @@ def test_serialize_complex_flow(self): self.assertEqual(serialized.name, fixture_name) # now do deserialization - deserialized = flow_to_sklearn(serialized) + deserialized = SKLearnConverter.from_flow(serialized) # Checks that sklearn_to_flow is idempotent. - serialized2 = sklearn_to_flow(deserialized) + serialized2 = SKLearnConverter(deserialized).to_flow() self.assertNotEqual(rs, deserialized) # Would raise an exception if the flows would be unequal assert_flows_equal(serialized, serialized2) @@ -480,8 +481,8 @@ def test_serialize_type(self): int, np.int, np.int32, np.int64] for supported_type in supported_types: - serialized = sklearn_to_flow(supported_type) - deserialized = flow_to_sklearn(serialized) + serialized = SKLearnConverter._sklearn_to_flow(supported_type) + deserialized = SKLearnConverter.from_flow(serialized) self.assertEqual(deserialized, supported_type) def test_serialize_rvs(self): @@ -490,8 +491,8 @@ def test_serialize_rvs(self): scipy.stats.randint(low=-3, high=15)] for supported_rv in supported_rvs: - serialized = sklearn_to_flow(supported_rv) - deserialized = flow_to_sklearn(serialized) + serialized = SKLearnConverter._sklearn_to_flow(supported_rv) + deserialized = SKLearnConverter.from_flow(serialized) self.assertEqual(type(deserialized.dist), type(supported_rv.dist)) del deserialized.dist del supported_rv.dist @@ -499,8 +500,8 @@ def test_serialize_rvs(self): supported_rv.__dict__) def test_serialize_function(self): - serialized = sklearn_to_flow(sklearn.feature_selection.chi2) - deserialized = flow_to_sklearn(serialized) + serialized = SKLearnConverter._sklearn_to_flow(sklearn.feature_selection.chi2) + deserialized = SKLearnConverter.from_flow(serialized) self.assertEqual(deserialized, sklearn.feature_selection.chi2) def test_serialize_cvobject(self): @@ -515,10 +516,10 @@ def test_serialize_cvobject(self): ('value', OrderedDict([('name', 'sklearn.model_selection._split.LeaveOneOut'), ('parameters', OrderedDict())]))])] for method, fixture in zip(methods, fixtures): - m = sklearn_to_flow(method) + m = SKLearnConverter._sklearn_to_flow(method) self.assertEqual(m, fixture) - m_new = flow_to_sklearn(m) + m_new = SKLearnConverter.from_flow(m) self.assertIsNot(m_new, m) self.assertIsInstance(m_new, type(method)) @@ -541,8 +542,8 @@ def test_serialize_simple_parameter_grid(self): "criterion": ["gini", "entropy"]}] for grid, model in zip(grids, models): - serialized = sklearn_to_flow(grid) - deserialized = flow_to_sklearn(serialized) + serialized = SKLearnConverter._sklearn_to_flow(grid) + deserialized = SKLearnConverter.from_flow(serialized) self.assertEqual(deserialized, grid) self.assertIsNot(deserialized, grid) @@ -550,8 +551,8 @@ def test_serialize_simple_parameter_grid(self): hpo = sklearn.model_selection.GridSearchCV( param_grid=grid, estimator=model) - serialized = sklearn_to_flow(hpo) - deserialized = flow_to_sklearn(serialized) + serialized = SKLearnConverter(hpo).to_flow() + deserialized = SKLearnConverter.from_flow(serialized) self.assertEqual(hpo.param_grid, deserialized.param_grid) self.assertEqual(hpo.estimator.get_params(), deserialized.estimator.get_params()) @@ -582,8 +583,8 @@ def test_serialize_advanced_grid(self): 'reduce_dim__k': N_FEATURES_OPTIONS, 'classify__C': C_OPTIONS}] - serialized = sklearn_to_flow(grid) - deserialized = flow_to_sklearn(serialized) + serialized = SKLearnConverter._sklearn_to_flow(grid) + deserialized = SKLearnConverter.from_flow(serialized) self.assertEqual(grid[0]['reduce_dim'][0].get_params(), deserialized[0]['reduce_dim'][0].get_params()) @@ -609,8 +610,8 @@ def test_serialize_advanced_grid(self): def test_serialize_resampling(self): kfold = sklearn.model_selection.StratifiedKFold( n_splits=4, shuffle=True) - serialized = sklearn_to_flow(kfold) - deserialized = flow_to_sklearn(serialized) + serialized = SKLearnConverter._sklearn_to_flow(kfold) + deserialized = SKLearnConverter.from_flow(serialized) # Best approximation to get_params() self.assertEqual(str(deserialized), str(kfold)) self.assertIsNot(deserialized, kfold) @@ -622,8 +623,8 @@ def test_hypothetical_parameter_values(self): model = Model('true', '1', '0.1') - serialized = sklearn_to_flow(model) - deserialized = flow_to_sklearn(serialized) + serialized = SKLearnConverter(model).to_flow() + deserialized = SKLearnConverter.from_flow(serialized) self.assertEqual(deserialized.get_params(), model.get_params()) self.assertIsNot(deserialized, model) @@ -632,9 +633,9 @@ def test_gaussian_process(self): kernel = sklearn.gaussian_process.kernels.Matern() gp = sklearn.gaussian_process.GaussianProcessClassifier( kernel=kernel, optimizer=opt) - self.assertRaisesRegexp(TypeError, "Matern\(length_scale=1, nu=1.5\), " - "", - sklearn_to_flow, gp) + with self.assertRaisesRegexp(TypeError, "Matern\(length_scale=1, nu=1.5\), " + ""): + SKLearnConverter(gp).to_flow() def test_error_on_adding_component_multiple_times_to_flow(self): # this function implicitly checks @@ -644,19 +645,22 @@ def test_error_on_adding_component_multiple_times_to_flow(self): pipeline = sklearn.pipeline.Pipeline((('pca1', pca), ('pca2', pca2))) fixture = "Found a second occurence of component .*.PCA when trying " \ "to serialize Pipeline" - self.assertRaisesRegexp(ValueError, fixture, sklearn_to_flow, pipeline) + with self.assertRaisesRegexp(ValueError, fixture): + SKLearnConverter(pipeline).to_flow() fu = sklearn.pipeline.FeatureUnion((('pca1', pca), ('pca2', pca2))) fixture = "Found a second occurence of component .*.PCA when trying " \ "to serialize FeatureUnion" - self.assertRaisesRegexp(ValueError, fixture, sklearn_to_flow, fu) + with self.assertRaisesRegexp(ValueError, fixture): + SKLearnConverter(fu).to_flow() fs = sklearn.feature_selection.SelectKBest() fu2 = sklearn.pipeline.FeatureUnion((('pca1', pca), ('fs', fs))) pipeline2 = sklearn.pipeline.Pipeline((('fu', fu2), ('pca2', pca2))) fixture = "Found a second occurence of component .*.PCA when trying " \ "to serialize Pipeline" - self.assertRaisesRegexp(ValueError, fixture, sklearn_to_flow, pipeline2) + with self.assertRaisesRegexp(ValueError, fixture): + SKLearnConverter(pipeline2).to_flow() def test_subflow_version_propagated(self): this_directory = os.path.dirname(os.path.abspath(__file__)) @@ -667,22 +671,22 @@ def test_subflow_version_propagated(self): pca = sklearn.decomposition.PCA() dummy = tests.test_flows.dummy_learn.dummy_forest.DummyRegressor() pipeline = sklearn.pipeline.Pipeline((('pca', pca), ('dummy', dummy))) - flow = sklearn_to_flow(pipeline) + flow = SKLearnConverter(pipeline).to_flow() # In python2.7, the unit tests work differently on travis-ci; therefore, # I put the alternative travis-ci answer here as well. While it has a # different value, it is still correct as it is a propagation of the # subclasses' module name self.assertEqual(flow.external_version, '%s,%s,%s' % ( - _format_external_version('openml', openml.__version__), - _format_external_version('sklearn', sklearn.__version__), - _format_external_version('tests', '0.1'))) + SKLearnConverter.format_external_version('openml', openml.__version__), + SKLearnConverter.format_external_version('sklearn', sklearn.__version__), + SKLearnConverter.format_external_version('tests', '0.1'))) @mock.patch('warnings.warn') def test_check_dependencies(self, warnings_mock): dependencies = ['sklearn==0.1', 'sklearn>=99.99.99', 'sklearn>99.99.99'] for dependency in dependencies: - self.assertRaises(ValueError, _check_dependencies, dependency) + self.assertRaises(ValueError, SKLearnConverter._check_dependencies, dependency) def test_illegal_parameter_names(self): # illegal name: estimators @@ -697,7 +701,8 @@ def test_illegal_parameter_names(self): cases = [clf1, clf2] for case in cases: - self.assertRaises(PyOpenMLError, sklearn_to_flow, case) + with self.assertRaises(PyOpenMLError): + SKLearnConverter(case).to_flow() def test_illegal_parameter_names_pipeline(self): # illegal name: steps @@ -780,7 +785,7 @@ def test__get_fn_arguments_with_defaults(self): ] for fn, num_params_with_defaults in fns: - defaults, defaultless = openml.flows.sklearn_converter._get_fn_arguments_with_defaults(fn) + defaults, defaultless = SKLearnConverter._get_fn_arguments_with_defaults(fn) self.assertIsInstance(defaults, dict) self.assertIsInstance(defaultless, set) # check whether we have both defaults and defaultless params @@ -806,14 +811,14 @@ def test_deserialize_with_defaults(self): 'OneHotEncoder__sparse': False, 'Estimator__min_samples_leaf': 42} pipe_adjusted.set_params(**params) - flow = openml.flows.sklearn_to_flow(pipe_adjusted) - pipe_deserialized = openml.flows.flow_to_sklearn( + flow = SKLearnConverter(pipe_adjusted).to_flow() + pipe_deserialized = openml.flows.SKLearnConverter.from_flow( flow, initialize_with_defaults=True) # we want to compare pipe_deserialized and pipe_orig. We use the flow # equals function for this - assert_flows_equal(openml.flows.sklearn_to_flow(pipe_orig), - openml.flows.sklearn_to_flow(pipe_deserialized)) + assert_flows_equal(SKLearnConverter(pipe_orig).to_flow(), + SKLearnConverter(pipe_deserialized).to_flow()) def test_deserialize_adaboost_with_defaults(self): # used the 'initialize_with_defaults' flag of the deserialization @@ -830,14 +835,14 @@ def test_deserialize_adaboost_with_defaults(self): 'OneHotEncoder__sparse': False, 'Estimator__n_estimators': 10} pipe_adjusted.set_params(**params) - flow = openml.flows.sklearn_to_flow(pipe_adjusted) - pipe_deserialized = openml.flows.flow_to_sklearn( + flow = SKLearnConverter(pipe_adjusted).to_flow() + pipe_deserialized = SKLearnConverter.from_flow( flow, initialize_with_defaults=True) - # we want to compare pipe_deserialized and pipe_orig. We use the flow - # equals function for this - assert_flows_equal(openml.flows.sklearn_to_flow(pipe_orig), - openml.flows.sklearn_to_flow(pipe_deserialized)) + # we want to compare pipe_deserialized and pipe_orig. We use the flow equals function for this + assert_flows_equal( + SKLearnConverter(pipe_orig).to_flow(), + SKLearnConverter(pipe_deserialized).to_flow()) def test_deserialize_complex_with_defaults(self): # used the 'initialize_with_defaults' flag of the deserialization @@ -859,10 +864,10 @@ def test_deserialize_complex_with_defaults(self): 'Estimator__base_estimator__base_estimator__learning_rate': 0.1, 'Estimator__base_estimator__base_estimator__loss__n_neighbors': 13} pipe_adjusted.set_params(**params) - flow = openml.flows.sklearn_to_flow(pipe_adjusted) - pipe_deserialized = openml.flows.flow_to_sklearn(flow, initialize_with_defaults=True) + flow = SKLearnConverter(pipe_adjusted).to_flow() + pipe_deserialized = SKLearnConverter.from_flow(flow, initialize_with_defaults=True) # we want to compare pipe_deserialized and pipe_orig. We use the flow # equals function for this - assert_flows_equal(openml.flows.sklearn_to_flow(pipe_orig), - openml.flows.sklearn_to_flow(pipe_deserialized)) + assert_flows_equal(SKLearnConverter(pipe_orig).to_flow(), + SKLearnConverter(pipe_deserialized).to_flow())