From 2256f20c6c593c10f9a1d66cc4325ccdd3452e50 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Thu, 21 Feb 2019 20:50:21 +0200 Subject: [PATCH 01/35] PEP8. No unused import. # always follows by space. --- doc/conf.py | 100 ++++++++++++++++++++++++++-------------------------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index d4f88c273..149d1fb69 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -15,7 +15,6 @@ import os import sys import sphinx_bootstrap_theme -from sphinx_gallery.sorting import ExplicitOrder, FileNameSortKey import openml @@ -27,7 +26,7 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')# ) +# sys.path.insert(0, os.path.abspath('.')# ) sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) @@ -35,11 +34,11 @@ # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.autosummary', @@ -64,7 +63,7 @@ source_suffix = '.rst' # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' @@ -87,13 +86,13 @@ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -101,27 +100,27 @@ # The reST default role (used for this markup: `text`) to use for all # documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False +# keep_warnings = False # -- Options for HTML output ---------------------------------------------- @@ -135,7 +134,7 @@ 'navbar_title': "OpenML", # Tab name for entire site. (Default: "Site") - #'navbar_site_name': "Site", + # 'navbar_site_name': "Site", # A list of tuples containting pages to link to. The value should # be in the form [(name, page), ..] @@ -198,19 +197,19 @@ # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, @@ -220,48 +219,48 @@ # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. -#html_extra_path = [] +# html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. html_sidebars = {'**': ['localtoc.html']} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'OpenMLdoc' @@ -271,13 +270,13 @@ latex_elements = { # The paper size ('letterpaper' or 'a4paper'). - #'papersize': 'letterpaper', + # 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). - #'pointsize': '10pt', + # 'pointsize': '10pt', # Additional stuff for the LaTeX preamble. - #'preamble': '', + # 'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples @@ -288,23 +287,23 @@ # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output --------------------------------------- @@ -317,7 +316,7 @@ ] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------- @@ -332,19 +331,20 @@ ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False +# texinfo_no_detailmenu = False -# prefix each section label with the name of the document it is in, in order to avoid -# ambiguity when there are multiple same section labels in different documents. +# prefix each section label with the name of the document it is in, +# in order to avoid ambiguity when there are multiple same section +# labels in different documents. autosectionlabel_prefix_document = True # Sphinx-gallery configuration. sphinx_gallery_conf = { @@ -356,5 +356,5 @@ 'gallery_dirs': 'examples', # compile execute examples in the examples dir 'filename_pattern': '.*example.py$|.*tutorial.py$', - #TODO: fix back/forward references for the examples. + # TODO: fix back/forward references for the examples. } From 59af53c574330afe1d9817614f49233979f79b7f Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Thu, 21 Feb 2019 20:52:13 +0200 Subject: [PATCH 02/35] PEP8 --- examples/datasets_tutorial.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/datasets_tutorial.py b/examples/datasets_tutorial.py index db92a3401..7f85de24d 100644 --- a/examples/datasets_tutorial.py +++ b/examples/datasets_tutorial.py @@ -54,8 +54,8 @@ ############################################################################ # Get the actual data. -# -# Returned as numpy array, with meta-info (e.g. target feature, feature names,...) +# Returned as numpy array, with meta-info +# (e.g. target feature, feature names, ...) X, y, attribute_names = dataset.get_data( target=dataset.default_target_attribute, return_attribute_names=True, @@ -77,4 +77,4 @@ hist_kwds={'bins': 20}, alpha=.8, cmap='plasma' -) \ No newline at end of file +) From 065ee57a520b7d832111fce7fee5fb7a8ee030e2 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Thu, 21 Feb 2019 20:53:37 +0200 Subject: [PATCH 03/35] Edit a blank line back in for file formatting. --- examples/datasets_tutorial.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/datasets_tutorial.py b/examples/datasets_tutorial.py index 7f85de24d..63cc8e29c 100644 --- a/examples/datasets_tutorial.py +++ b/examples/datasets_tutorial.py @@ -54,6 +54,7 @@ ############################################################################ # Get the actual data. +# # Returned as numpy array, with meta-info # (e.g. target feature, feature names, ...) X, y, attribute_names = dataset.get_data( From 0d40cae5a00f9e06ef1cfe1d3b8d3274c04208f3 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Thu, 21 Feb 2019 20:57:35 +0200 Subject: [PATCH 04/35] PEP8. Fix grammar. --- examples/flows_and_runs_tutorial.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/examples/flows_and_runs_tutorial.py b/examples/flows_and_runs_tutorial.py index 78f36195d..0267af02a 100644 --- a/examples/flows_and_runs_tutorial.py +++ b/examples/flows_and_runs_tutorial.py @@ -6,8 +6,6 @@ """ import openml -import pandas as pd -import seaborn as sns from pprint import pprint from sklearn import ensemble, neighbors, preprocessing, pipeline, tree @@ -60,7 +58,7 @@ ############################################################################ # Share the run on the OpenML server # -# So far the run is only available locally. By calling the publish function, the run is send to the OpenML server: +# So far the run is only available locally. By calling the publish function, the run is sent to the OpenML server: myrun = run.publish() # For this tutorial, our configuration publishes to the test server From f75ef71e6c79814ce4416f09f991ff715e0c2af6 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Thu, 21 Feb 2019 21:07:36 +0200 Subject: [PATCH 05/35] PEP8 --- openml/config.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/openml/config.py b/openml/config.py index b5819c282..586654e83 100644 --- a/openml/config.py +++ b/openml/config.py @@ -1,5 +1,5 @@ """ -Stores module level information like the API key, cache directory and the server. +Store module level information like the API key, cache directory and the server """ import logging import os @@ -60,8 +60,12 @@ def _setup(): config = _parse_config() apikey = config.get('FAKE_SECTION', 'apikey') server = config.get('FAKE_SECTION', 'server') - cache_directory = os.path.expanduser(config.get('FAKE_SECTION', 'cachedir')) - avoid_duplicate_runs = config.getboolean('FAKE_SECTION', 'avoid_duplicate_runs') + + short_cache_dir = config.get('FAKE_SECTION', 'cachedir') + cache_directory = os.path.expanduser(short_cache_dir) + + avoid_duplicate_runs = config.getboolean('FAKE_SECTION', + 'avoid_duplicate_runs') connection_n_retries = config.get('FAKE_SECTION', 'connection_n_retries') if connection_n_retries > 20: raise ValueError( From 29a1afee0ac7213c61d25180cc336f9b5c18c34b Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Thu, 21 Feb 2019 21:18:04 +0200 Subject: [PATCH 06/35] PEP8. --- openml/exceptions.py | 5 +++-- openml/testing.py | 17 ++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/openml/exceptions.py b/openml/exceptions.py index d38fdca91..f66feb741 100644 --- a/openml/exceptions.py +++ b/openml/exceptions.py @@ -30,6 +30,7 @@ def __str__(self): self.url, self.code, self.message, ) + class OpenMLServerNoResult(OpenMLServerException): """exception for when the result of the server is empty. """ pass @@ -47,6 +48,6 @@ class OpenMLHashException(PyOpenMLError): class PrivateDatasetError(PyOpenMLError): - "Exception thrown when the user has no rights to access the dataset" + """ Exception thrown when the user has no rights to access the dataset. """ def __init__(self, message): - super(PrivateDatasetError, self).__init__(message) \ No newline at end of file + super(PrivateDatasetError, self).__init__(message) diff --git a/openml/testing.py b/openml/testing.py index c31f1158e..e29fe45d9 100644 --- a/openml/testing.py +++ b/openml/testing.py @@ -29,8 +29,10 @@ def setUp(self): # cache self.maxDiff = None self.static_cache_dir = None - static_cache_dir = os.path.dirname(os.path.abspath(inspect.getfile(self.__class__))) - static_cache_dir = os.path.abspath(os.path.join(static_cache_dir, '..')) + abspath_this_file = os.path.abspath(inspect.getfile(self.__class__)) + static_cache_dir = os.path.dirname(abspath_this_file) + static_cache_dir = os.path.abspath(os.path.join(static_cache_dir, + '..')) content = os.listdir(static_cache_dir) if 'files' in content: self.static_cache_dir = os.path.join(static_cache_dir, 'files') @@ -42,10 +44,7 @@ def setUp(self): workdir = os.path.dirname(os.path.abspath(__file__)) tmp_dir_name = self.id() self.workdir = os.path.join(workdir, tmp_dir_name) - try: - shutil.rmtree(self.workdir) - except: - pass + shutil.rmtree(self.workdir, ignore_errors=True) os.mkdir(self.workdir) os.chdir(self.workdir) @@ -88,9 +87,9 @@ def tearDown(self): def _get_sentinel(self, sentinel=None): if sentinel is None: - # Create a unique prefix for the flow. Necessary because the flow is - # identified by its name and external version online. Having a unique - # name allows us to publish the same flow in each test run + # Create a unique prefix for the flow. Necessary because the flow + # is identified by its name and external version online. Having a + # unique name allows us to publish the same flow in each test run. md5 = hashlib.md5() md5.update(str(time.time()).encode('utf-8')) md5.update(str(os.getpid()).encode('utf-8')) From a0519c2753a73eff2654a40b8edfa48c6f9eecb0 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Thu, 21 Feb 2019 21:26:16 +0200 Subject: [PATCH 07/35] PEP8 --- openml/utils.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/openml/utils.py b/openml/utils.py index 2a9461dbb..65c979bb3 100644 --- a/openml/utils.py +++ b/openml/utils.py @@ -72,15 +72,15 @@ def _tag_entity(entity_type, entity_id, tag, untag=False): """ legal_entities = {'data', 'task', 'flow', 'setup', 'run'} if entity_type not in legal_entities: - raise ValueError('Can\'t tag a %s' %entity_type) + raise ValueError('Can\'t tag a %s' % entity_type) - uri = '%s/tag' %entity_type - main_tag = 'oml:%s_tag' %entity_type + uri = '%s/tag' % entity_type + main_tag = 'oml:%s_tag' % entity_type if untag: - uri = '%s/untag' %entity_type - main_tag = 'oml:%s_untag' %entity_type + uri = '%s/untag' % entity_type + main_tag = 'oml:%s_untag' % entity_type - post_variables = {'%s_id'%entity_type: entity_id, 'tag': tag} + post_variables = {'%s_id' % entity_type: entity_id, 'tag': tag} result_xml = openml._api_calls._perform_api_call(uri, post_variables) result = xmltodict.parse(result_xml, force_list={'oml:tag'})[main_tag] @@ -115,12 +115,14 @@ def _list_all(listing_call, *args, **filters): """ # eliminate filters that have a None value - active_filters = {key: value for key, value in filters.items() if value is not None} + active_filters = {key: value for key, value in filters.items() + if value is not None} page = 0 result = {} - # default batch size per paging. This one can be set in filters (batch_size), - # but should not be changed afterwards. the derived batch_size can be changed. + # Default batch size per paging. + # This one can be set in filters (batch_size), but should not be + # changed afterwards. The derived batch_size can be changed. BATCH_SIZE_ORIG = 10000 if 'batch_size' in active_filters: BATCH_SIZE_ORIG = active_filters['batch_size'] @@ -132,7 +134,8 @@ def _list_all(listing_call, *args, **filters): if 'size' in active_filters: LIMIT = active_filters['size'] del active_filters['size'] - # check if the batch size is greater than the number of results that need to be returned. + # check if the batch size is greater than + # the number of results that need to be returned. if LIMIT is not None: if BATCH_SIZE_ORIG > LIMIT: BATCH_SIZE_ORIG = min(LIMIT, BATCH_SIZE_ORIG) @@ -158,7 +161,8 @@ def _list_all(listing_call, *args, **filters): page += 1 if LIMIT is not None: # check if the number of required results has been achieved - # always do a 'bigger than' check, in case of bugs to prevent infinite loops + # always do a 'bigger than' check, + # in case of bugs to prevent infinite loops if len(result) >= LIMIT: break # check if there are enough results to fulfill a batch @@ -173,7 +177,7 @@ def _create_cache_directory(key): cache_dir = os.path.join(cache, key) try: os.makedirs(cache_dir) - except: + except OSError: pass return cache_dir @@ -233,6 +237,6 @@ def _create_lockfiles_dir(): dir = os.path.join(config.get_cache_directory(), 'locks') try: os.makedirs(dir) - except: + except OSError: pass return dir From b20abfdabb2de5895f8a388e3bee721b241fe05f Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Thu, 21 Feb 2019 21:27:23 +0200 Subject: [PATCH 08/35] Refactor. --- openml/utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/openml/utils.py b/openml/utils.py index 65c979bb3..75a5369a8 100644 --- a/openml/utils.py +++ b/openml/utils.py @@ -134,14 +134,14 @@ def _list_all(listing_call, *args, **filters): if 'size' in active_filters: LIMIT = active_filters['size'] del active_filters['size'] - # check if the batch size is greater than - # the number of results that need to be returned. - if LIMIT is not None: - if BATCH_SIZE_ORIG > LIMIT: - BATCH_SIZE_ORIG = min(LIMIT, BATCH_SIZE_ORIG) + + if LIMIT is not None and BATCH_SIZE_ORIG > LIMIT: + BATCH_SIZE_ORIG = LIMIT + if 'offset' in active_filters: offset = active_filters['offset'] del active_filters['offset'] + batch_size = BATCH_SIZE_ORIG while True: try: From 59bc0269fcb537436bc92e05a94e3b90ff236e32 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Thu, 21 Feb 2019 21:38:42 +0200 Subject: [PATCH 09/35] PEP8, small refactor. --- openml/_api_calls.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/openml/_api_calls.py b/openml/_api_calls.py index 707516651..5a433064c 100644 --- a/openml/_api_calls.py +++ b/openml/_api_calls.py @@ -74,7 +74,8 @@ def _read_url_files(url, data=None, file_elements=None): raise _parse_server_exception(response, url=url) if 'Content-Encoding' not in response.headers or \ response.headers['Content-Encoding'] != 'gzip': - warnings.warn('Received uncompressed content from OpenML for %s.' % url) + warnings.warn('Received uncompressed content from OpenML for {}.' + .format(url)) return response.text @@ -100,7 +101,8 @@ def _read_url(url, data=None): raise _parse_server_exception(response, url=url) if 'Content-Encoding' not in response.headers or \ response.headers['Content-Encoding'] != 'gzip': - warnings.warn('Received uncompressed content from OpenML for %s.' % url) + warnings.warn('Received uncompressed content from OpenML for {}.' + .format(url)) return response.text @@ -137,27 +139,26 @@ def send_request( def _parse_server_exception(response, url=None): - # OpenML has a sopisticated error system + # OpenML has a sophisticated error system # where information about failures is provided. try to parse this try: server_exception = xmltodict.parse(response.text) except Exception: - raise OpenMLServerError(('Unexpected server error. Please ' - 'contact the developers!\nStatus code: ' - '%d\n' % response.status_code) + response.text) - - code = int(server_exception['oml:error']['oml:code']) - message = server_exception['oml:error']['oml:message'] - additional = None - if 'oml:additional_information' in server_exception['oml:error']: - additional = server_exception['oml:error']['oml:additional_information'] + raise OpenMLServerError( + 'Unexpected server error. Please contact the developers!\n' + 'Status code: {}\n{}'.format(response.status_code, response.text)) + + server_error = server_exception['oml:error'] + code = int(server_error['oml:code']) + message = server_error['oml:message'] + additional_information = server_error.get('oml:additional_information') if code in [372, 512, 500, 482, 542, 674]: # 512 for runs, 372 for datasets, 500 for flows # 482 for tasks, 542 for evaluations, 674 for setups - return OpenMLServerNoResult(code, message, additional) + return OpenMLServerNoResult(code, message, additional_information) return OpenMLServerException( code=code, message=message, - additional=additional, + additional=additional_information, url=url ) From 9fecfc48d1d185b41819ee7772434754cf3e5f11 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 22 Feb 2019 11:26:59 +0200 Subject: [PATCH 10/35] OpenMLStudy available through import * --- openml/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/openml/__init__.py b/openml/__init__.py index d34f1bab6..d817fbd3c 100644 --- a/openml/__init__.py +++ b/openml/__init__.py @@ -9,7 +9,7 @@ * analyze experiments (uploaded by you and other collaborators) and conduct meta studies -In particular, this module implemts a python interface for the +In particular, this module implements a python interface for the `OpenML REST API `_ (`REST on wikipedia `_). @@ -24,11 +24,12 @@ from . import setups from . import study from . import evaluations -from . import utils + from .runs import OpenMLRun from .tasks import OpenMLTask, OpenMLSplit from .flows import OpenMLFlow from .evaluations import OpenMLEvaluation +from .study import OpenMLStudy from .__version__ import __version__ @@ -71,5 +72,5 @@ def populate_cache(task_ids=None, dataset_ids=None, flow_ids=None, __all__ = ['OpenMLDataset', 'OpenMLDataFeature', 'OpenMLRun', 'OpenMLSplit', 'OpenMLEvaluation', 'OpenMLSetup', - 'OpenMLTask', 'OpenMLFlow', 'datasets', 'evaluations', - 'config', 'runs', 'flows', 'tasks', 'setups'] + 'OpenMLTask', 'OpenMLFlow', 'OpenMLStudy', 'datasets', + 'evaluations', 'config', 'runs', 'flows', 'tasks', 'setups'] From 97a4ed9300bdfc9d48dbbe5032a9f56192e84bed Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 22 Feb 2019 11:58:30 +0200 Subject: [PATCH 11/35] PEP8. --- openml/__init__.py | 1 - openml/datasets/dataset.py | 89 +++++++++++++++++++++++--------------- 2 files changed, 54 insertions(+), 36 deletions(-) diff --git a/openml/__init__.py b/openml/__init__.py index d817fbd3c..a707374eb 100644 --- a/openml/__init__.py +++ b/openml/__init__.py @@ -22,7 +22,6 @@ from . import runs from . import flows from . import setups -from . import study from . import evaluations from .runs import OpenMLRun diff --git a/openml/datasets/dataset.py b/openml/datasets/dataset.py index 68c1cdaf6..24c09a209 100644 --- a/openml/datasets/dataset.py +++ b/openml/datasets/dataset.py @@ -35,7 +35,8 @@ class OpenMLDataset(object): dataset_id : int, optional Id autogenerated by the server. version : int, optional - Version of this dataset. '1' for original version. Auto-incremented by server. + Version of this dataset. '1' for original version. + Auto-incremented by server. creator : str, optional The person who created the dataset. contributor : str, optional @@ -50,15 +51,20 @@ class OpenMLDataset(object): licence : str, optional License of the data. url : str, optional - Valid URL, points to actual data file, on the OpenML server or another dataset repository. + Valid URL, points to actual data file. + The file can be on the OpenML server or another dataset repository. default_target_attribute : str, optional - The default target attribute, if it exists. Can have multiple values, comma separated. + The default target attribute, if it exists. + Can have multiple values, comma separated. row_id_attribute : str, optional - The attribute that represents the row-id column, if present in the dataset. + The attribute that represents the row-id column, + if present in the dataset. ignore_attribute : str | list, optional - Attributes that should be excluded in modelling, such as identifiers and indexes. + Attributes that should be excluded in modelling, + such as identifiers and indexes. version_label : str, optional - Version label provided by user, can be a date, hash, or some other type of id. + Version label provided by user. + Can be a date, hash, or some other type of id. citation : str, optional Reference(s) that should be cited when building on this data. tag : str, optional @@ -80,9 +86,11 @@ class OpenMLDataset(object): data_file : str, optional Path to where the dataset is located. features : dict, optional - A dictionary of dataset features which maps a feature index to a OpenMLDataFeature. + A dictionary of dataset features, + which maps a feature index to a OpenMLDataFeature. qualities : dict, optional - A dictionary of dataset qualities which maps a quality name to a quality value. + A dictionary of dataset qualities, + which maps a quality name to a quality value. dataset: string, optional Serialized arff dataset string. """ @@ -128,7 +136,8 @@ def __init__(self, name, description, format=None, elif ignore_attribute is None: pass else: - raise ValueError('wrong data type for ignore_attribute. Should be list. ') + raise ValueError('Wrong data type for ignore_attribute. ' + 'Should be list.') self.version_label = version_label self.citation = citation self.tag = tag @@ -144,14 +153,17 @@ def __init__(self, name, description, format=None, if features is not None: self.features = {} + # todo add nominal values (currently not in database) for idx, xmlfeature in enumerate(features['oml:feature']): + nr_missing = xmlfeature.get('oml:number_of_missing_values', 0) feature = OpenMLDataFeature(int(xmlfeature['oml:index']), xmlfeature['oml:name'], xmlfeature['oml:data_type'], - None, # todo add nominal values (currently not in database) - int(xmlfeature.get('oml:number_of_missing_values', 0))) + None, + int(nr_missing)) if idx != feature.index: - raise ValueError('Data features not provided in right order') + raise ValueError('Data features not provided ' + 'in right order') self.features[feature.index] = feature self.qualities = _check_qualities(qualities) @@ -166,19 +178,21 @@ def __init__(self, name, description, format=None, try: data = self._get_arff(self.format) except OSError as e: - logger.critical("Please check that the data file %s is there " - "and can be read.", self.data_file) + logger.critical("Please check that the data file " + "{}* is there and can be read." + .format(self.data_file)) raise e categorical = [False if type(type_) != list else True for name, type_ in data['attributes']] - attribute_names = [name for name, type_ in data['attributes']] + attribute_names = [name for name, _ in data['attributes']] if self.format.lower() == 'sparse_arff': X = data['data'] X_shape = (max(X[1]) + 1, max(X[2]) + 1) X = scipy.sparse.coo_matrix( - (X[0], (X[1], X[2])), shape=X_shape, dtype=np.float32) + (X[0], (X[1], X[2])), + shape=X_shape, dtype=np.float32) X = X.tocsr() elif self.format.lower() == 'arff': X = np.array(data['data'], dtype=np.float32) @@ -187,8 +201,10 @@ def __init__(self, name, description, format=None, with open(self.data_pickle_file, "wb") as fh: pickle.dump((X, categorical, attribute_names), fh, -1) - logger.debug("Saved dataset %d: %s to file %s" % - (int(self.dataset_id or -1), self.name, self.data_pickle_file)) + logger.debug("Saved dataset {}: {} to file {}" + .format(int(self.dataset_id or -1), + self.name, + self.data_pickle_file)) def push_tag(self, tag): """Annotates this data set with a tag on the server. @@ -254,16 +270,17 @@ def _get_arff(self, format): # TODO: add a partial read method which only returns the attribute # headers of the corresponding .arff file! - - # A random number after which we consider a file for too large on a - # 32 bit system...currently 120mb (just a little bit more than covtype) import struct if not self._data_features_supported(): - raise PyOpenMLError('Dataset not compatible, PyOpenML cannot handle string features') + raise PyOpenMLError('Dataset not compatible, ' + 'PyOpenML cannot handle string features') filename = self.data_file bits = (8 * struct.calcsize("P")) + # Files can be considered too large on a 32-bit system, + # if it exceeds 120mb (slightly more than covtype dataset size) + # This number is somewhat arbitrary. if bits != 64 and os.path.getsize(filename) > 120000000: return NotImplementedError("File too big") @@ -290,8 +307,7 @@ def get_data(self, target=None, include_row_id=False, include_ignore_attributes=False, return_categorical_indicator=False, - return_attribute_names=False - ): + return_attribute_names=False): """Returns dataset content as numpy arrays / sparse matrices. Parameters @@ -443,8 +459,8 @@ def get_features_by_type(self, data_type, exclude=None, exclude_ignore_attributes=True, exclude_row_id_attribute=True): """ - Returns indices of features of a given type, e.g., all nominal features. - Can use additional parameters to exclude various features by index or ontology. + Return indices of features of a given type, e.g. all nominal features. + Optional parameters to exclude various features by index or ontology. Parameters ---------- @@ -476,7 +492,8 @@ def get_features_by_type(self, data_type, exclude=None, if exclude is not None: if not isinstance(exclude, list): raise TypeError("Exclude should be a list") - # assert all(isinstance(elem, str) for elem in exclude), "Exclude should be a list of strings" + # assert all(isinstance(elem, str) for elem in exclude), + # "Exclude should be a list of strings" to_exclude = [] if exclude is not None: to_exclude.extend(exclude) @@ -487,14 +504,15 @@ def get_features_by_type(self, data_type, exclude=None, result = [] offset = 0 - # this function assumes that everything in to_exclude will be 'excluded' from the dataset (hence the offset) + # this function assumes that everything in to_exclude will + # be 'excluded' from the dataset (hence the offset) for idx in self.features: name = self.features[idx].name if name in to_exclude: offset += 1 else: if self.features[idx].data_type == data_type: - result.append(idx-offset) + result.append(idx - offset) return result def publish(self): @@ -523,23 +541,24 @@ def publish(self): with io.open(path, encoding='utf8') as fh: decoder.decode(fh, encode_nominal=True) except arff.ArffException: - raise ValueError("The file you have provided is not a valid arff file") + raise ValueError("The file you have provided is not " + "a valid arff file.") file_elements['dataset'] = open(path, 'rb') else: if self.url is None: - raise ValueError("No path/url to the dataset file was given") + raise ValueError("No url/path to the data file was given") return_value = openml._api_calls._perform_api_call( "data/", file_elements=file_elements, ) - self.dataset_id = int(xmltodict.parse(return_value)['oml:upload_data_set']['oml:id']) + response = xmltodict.parse(return_value) + self.dataset_id = int(response['oml:upload_data_set']['oml:id']) return self.dataset_id - def _to_xml(self): - """Serialize object to xml for upload + """ Serialize object to xml for upload Returns ------- @@ -551,7 +570,7 @@ def _to_xml(self): 'licence', 'url', 'default_target_attribute', 'row_id_attribute', 'ignore_attribute', 'version_label', 'citation', 'tag', 'visibility', 'original_data_url', - 'paper_url', 'update_comment', 'md5_checksum'] # , 'data_file'] + 'paper_url', 'update_comment', 'md5_checksum'] data_container = OrderedDict() data_dict = OrderedDict([('@xmlns:oml', 'http://openml.org/openml')]) From f93cc271cf20466a150927e766e7a95a644eefd8 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 22 Feb 2019 12:26:29 +0200 Subject: [PATCH 12/35] PEP8. --- openml/datasets/functions.py | 89 ++++++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 35 deletions(-) diff --git a/openml/datasets/functions.py b/openml/datasets/functions.py index 949315ca7..5984692d9 100644 --- a/openml/datasets/functions.py +++ b/openml/datasets/functions.py @@ -36,14 +36,12 @@ DATASETS_CACHE_DIR_NAME = 'datasets' - - ############################################################################ # Local getters/accessors to the cache directory def _list_cached_datasets(): - """Return list with ids of all cached datasets + """ Return list with ids of all cached datasets. Returns ------- @@ -69,8 +67,8 @@ def _list_cached_datasets(): directory_name) dataset_directory_content = os.listdir(directory_name) - if ("dataset.arff" in dataset_directory_content and - "description.xml" in dataset_directory_content): + if ("dataset.arff" in dataset_directory_content + and "description.xml" in dataset_directory_content): if dataset_id not in datasets: datasets.append(dataset_id) @@ -102,7 +100,10 @@ def _get_cached_dataset(dataset_id): arff_file = _get_cached_dataset_arff(dataset_id) features = _get_cached_dataset_features(dataset_id) qualities = _get_cached_dataset_qualities(dataset_id) - dataset = _create_dataset_from_description(description, features, qualities, arff_file) + dataset = _create_dataset_from_description(description, + features, + qualities, + arff_file) return dataset @@ -144,7 +145,8 @@ def _get_cached_dataset_qualities(dataset_id): try: with io.open(qualities_file, encoding='utf8') as fh: qualities_xml = fh.read() - return xmltodict.parse(qualities_xml)["oml:data_qualities"]['oml:quality'] + qualities_dict = xmltodict.parse(qualities_xml) + return qualities_dict["oml:data_qualities"]['oml:quality'] except (IOError, OSError): raise OpenMLCacheException("Dataset qualities for dataset id %d not " "cached" % dataset_id) @@ -168,7 +170,8 @@ def _get_cached_dataset_arff(dataset_id): def list_datasets(offset=None, size=None, status=None, tag=None, **kwargs): """ - Return a list of all dataset which are on OpenML. (Supports large amount of results) + Return a list of all dataset which are on OpenML. + Supports large amount of results. Parameters ---------- @@ -202,7 +205,12 @@ def list_datasets(offset=None, size=None, status=None, tag=None, **kwargs): these are also returned. """ - return openml.utils._list_all(_list_datasets, offset=offset, size=size, status=status, tag=tag, **kwargs) + return openml.utils._list_all(_list_datasets, + offset=offset, + size=size, + status=status, + tag=tag, + **kwargs) def _list_datasets(**kwargs): @@ -214,7 +222,7 @@ def _list_datasets(**kwargs): ---------- kwargs : dict, optional Legal filter operators (keys in the dict): - {tag, status, limit, offset, data_name, data_version, number_instances, + tag, status, limit, offset, data_name, data_version, number_instances, number_features, number_classes, number_missing_values. Returns @@ -282,8 +290,8 @@ def check_datasets_active(dataset_ids): for did in dataset_ids: if did not in active: - raise ValueError('Could not find dataset %d in OpenML dataset list.' - % did) + raise ValueError('Could not find dataset {} in ' + 'OpenML dataset list.'.format(did)) active = {did: active[did] for did in dataset_ids} @@ -329,7 +337,7 @@ def get_dataset(dataset_id): The downloaded dataset.""" try: dataset_id = int(dataset_id) - except: + except (ValueError, TypeError): raise ValueError("Dataset ID is neither an Integer nor can be " "cast to an Integer.") @@ -349,14 +357,16 @@ def get_dataset(dataset_id): qualities = _get_dataset_qualities(did_cache_dir, dataset_id) remove_dataset_cache = False except OpenMLServerException as e: - # if there was an exception, check if the user had access to the dataset + # if there was an exception, + # check if the user had access to the dataset if e.code == 112: raise PrivateDatasetError(e.message) from None else: raise e finally: if remove_dataset_cache: - _remove_cache_dir_for_id(DATASETS_CACHE_DIR_NAME, did_cache_dir) + _remove_cache_dir_for_id(DATASETS_CACHE_DIR_NAME, + did_cache_dir) dataset = _create_dataset_from_description( description, features, qualities, arff_file @@ -365,7 +375,7 @@ def get_dataset(dataset_id): def attributes_arff_from_df(df): - """Create the attributes as specified by the ARFF format using a dataframe. + """ Describe attributes of the dataframe according to ARFF specification. Parameters ---------- @@ -390,7 +400,7 @@ def attributes_arff_from_df(df): if column_dtype == 'categorical': # for categorical feature, arff expects a list string. However, a - # categorical column can contain mixed type and we should therefore + # categorical column can contain mixed type and should therefore # raise an error asking to convert all entries to string. categories = df[column_name].cat.categories categories_dtype = pd.api.types.infer_dtype(categories) @@ -463,11 +473,13 @@ def create_dataset(name, description, creator, contributor, The default target attribute, if it exists. Can have multiple values, comma separated. ignore_attribute : str | list - Attributes that should be excluded in modelling, such as identifiers and indexes. + Attributes that should be excluded in modelling, + such as identifiers and indexes. citation : str Reference(s) that should be cited when building on this data. version_label : str, optional - Version label provided by user, can be a date, hash, or some other type of id. + Version label provided by user. + Can be a date, hash, or some other type of id. row_id_attribute : str, optional The attribute that represents the row-id column, if present in the dataset. If ``data`` is a dataframe and ``row_id_attribute`` is not @@ -492,14 +504,14 @@ def create_dataset(name, description, creator, contributor, # infer the row id from the index of the dataset if row_id_attribute is None: row_id_attribute = data.index.name - # When calling data.values, the index will be skipped. We need to reset - # the index such that it is part of the data. + # When calling data.values, the index will be skipped. + # We need to reset the index such that it is part of the data. if data.index.name is not None: data = data.reset_index() if attributes == 'auto' or isinstance(attributes, dict): if not hasattr(data, "columns"): - raise ValueError("Automatically inferring the attributes required " + raise ValueError("Automatically inferring attributes requires " "a pandas DataFrame or SparseDataFrame. " "A {!r} was given instead.".format(data)) # infer the type of data for each column of the DataFrame @@ -603,9 +615,9 @@ def create_dataset(name, description, creator, contributor, def status_update(data_id, status): """ - Updates the status of a dataset to either 'active' or 'deactivated'. Please - see the OpenML API documentation for a description of the status and all - legal status transitions: + Updates the status of a dataset to either 'active' or 'deactivated'. + Please see the OpenML API documentation for a description of the status + and all legal status transitions: https://docs.openml.org/#dataset-status Parameters @@ -650,8 +662,7 @@ def _get_dataset_description(did_cache_dir, dataset_id): """ - # TODO implement a cache for this that invalidates itself after some - # time + # TODO implement a cache for this that invalidates itself after some time # This can be saved on disk, but cannot be cached properly, because # it contains the information on whether a dataset is active. description_file = os.path.join(did_cache_dir, "description.xml") @@ -659,7 +670,8 @@ def _get_dataset_description(did_cache_dir, dataset_id): try: return _get_cached_dataset_description(dataset_id) except OpenMLCacheException: - dataset_xml = openml._api_calls._perform_api_call("data/%d" % dataset_id) + url_extension = "data/{}".format(dataset_id) + dataset_xml = openml._api_calls._perform_api_call(url_extension) with io.open(description_file, "w", encoding='utf8') as fh: fh.write(dataset_xml) @@ -672,8 +684,8 @@ def _get_dataset_description(did_cache_dir, dataset_id): def _get_dataset_arff(did_cache_dir, description): """Get the filepath to the dataset ARFF - Checks if the file is in the cache, if yes, return the path to the file. If - not, downloads the file and caches it, then returns the file path. + Checks if the file is in the cache, if yes, return the path to the file. + If not, downloads the file and caches it, then returns the file path. This function is NOT thread/multiprocessing safe. @@ -751,12 +763,14 @@ def _get_dataset_features(did_cache_dir, dataset_id): with io.open(features_file, encoding='utf8') as fh: features_xml = fh.read() except (OSError, IOError): - features_xml = openml._api_calls._perform_api_call("data/features/%d" % dataset_id) + url_extension = "data/features/{}".format(dataset_id) + features_xml = openml._api_calls._perform_api_call(url_extension) with io.open(features_file, "w", encoding='utf8') as fh: fh.write(features_xml) - features = xmltodict.parse(features_xml, force_list=('oml:feature',))["oml:data_features"] + xml_as_dict = xmltodict.parse(features_xml, force_list=('oml:feature',)) + features = xml_as_dict["oml:data_features"] return features @@ -787,17 +801,22 @@ def _get_dataset_qualities(did_cache_dir, dataset_id): with io.open(qualities_file, encoding='utf8') as fh: qualities_xml = fh.read() except (OSError, IOError): - qualities_xml = openml._api_calls._perform_api_call("data/qualities/%d" % dataset_id) + url_extension = "data/qualities/{}".format(dataset_id) + qualities_xml = openml._api_calls._perform_api_call(url_extension) with io.open(qualities_file, "w", encoding='utf8') as fh: fh.write(qualities_xml) - qualities = xmltodict.parse(qualities_xml, force_list=('oml:quality',))['oml:data_qualities']['oml:quality'] + xml_as_dict = xmltodict.parse(qualities_xml, force_list=('oml:quality',)) + qualities = xml_as_dict['oml:data_qualities']['oml:quality'] return qualities -def _create_dataset_from_description(description, features, qualities, arff_file): +def _create_dataset_from_description(description, + features, + qualities, + arff_file): """Create a dataset object from a description dict. Parameters From d270d023e007de14e670c5f7bfb0ed6d1d952c7d Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 22 Feb 2019 13:42:30 +0200 Subject: [PATCH 13/35] PEP8. Removed Py2 support. --- openml/evaluations/__init__.py | 2 + openml/evaluations/evaluation.py | 25 +++---- openml/flows/flow.py | 11 +-- openml/flows/functions.py | 19 ++++-- openml/flows/sklearn_converter.py | 108 ++++++++++++++++-------------- 5 files changed, 91 insertions(+), 74 deletions(-) diff --git a/openml/evaluations/__init__.py b/openml/evaluations/__init__.py index fb5a21876..650ba3502 100644 --- a/openml/evaluations/__init__.py +++ b/openml/evaluations/__init__.py @@ -1,2 +1,4 @@ from .evaluation import OpenMLEvaluation from .functions import list_evaluations + +__all__ = ['OpenMLEvaluation', 'list_evaluations'] diff --git a/openml/evaluations/evaluation.py b/openml/evaluations/evaluation.py index f297d7054..a22b6598f 100644 --- a/openml/evaluations/evaluation.py +++ b/openml/evaluations/evaluation.py @@ -7,29 +7,30 @@ class OpenMLEvaluation(object): Parameters ---------- run_id : int - + Refers to the run. task_id : int - + Refers to the task. setup_id : int - + Refers to the setup. flow_id : int - + Refers to the flow. flow_name : str - + Name of the referred flow. data_id : int - + Refers to the dataset. data_name : str - the name of the dataset + The name of the dataset. function : str - the evaluation function of this item (e.g., accuracy) + The evaluation metric of this item (e.g., accuracy). upload_time : str - the time of evaluation + The time of evaluation. value : float - the value of this evaluation + The value (score) of this evaluation. values : List[float] - the values per repeat and fold (if requested) + The values (scores) per repeat and fold (if requested) array_data : str - list of information per class (e.g., in case of precision, auroc, recall) + list of information per class. + (e.g., in case of precision, auroc, recall) """ def __init__(self, run_id, task_id, setup_id, flow_id, flow_name, data_id, data_name, function, upload_time, value, values, diff --git a/openml/flows/flow.py b/openml/flows/flow.py index aaa8d75a6..725da846e 100644 --- a/openml/flows/flow.py +++ b/openml/flows/flow.py @@ -333,7 +333,8 @@ def publish(self): "flow/", file_elements=file_elements, ) - flow_id = int(xmltodict.parse(return_value)['oml:upload_flow']['oml:id']) + server_response = xmltodict.parse(return_value) + flow_id = int(server_response['oml:upload_flow']['oml:id']) flow = openml.flows.functions.get_flow(flow_id) _copy_server_fields(flow, self) try: @@ -350,10 +351,10 @@ def publish(self): def get_structure(self, key_item): """ - Returns for each sub-component of the flow the path of identifiers that - should be traversed to reach this component. The resulting dict maps a - key (identifying a flow by either its id, name or fullname) to the - parameter prefix. + Returns for each sub-component of the flow the path of identifiers + that should be traversed to reach this component. The resulting dict + maps a key (identifying a flow by either its id, name or fullname) to + the parameter prefix. Parameters ---------- diff --git a/openml/flows/functions.py b/openml/flows/functions.py index aae87b2c7..5b51275f4 100644 --- a/openml/flows/functions.py +++ b/openml/flows/functions.py @@ -32,8 +32,8 @@ def get_flow(flow_id, reinstantiate=False): flow = OpenMLFlow._from_dict(flow_dict) if reinstantiate: - if not (flow.external_version.startswith('sklearn==') or - ',sklearn==' in flow.external_version): + if not (flow.external_version.startswith('sklearn==') + or ',sklearn==' in flow.external_version): raise ValueError('Only sklearn flows can be reinstantiated') flow.model = openml.flows.flow_to_sklearn(flow) @@ -72,7 +72,11 @@ def list_flows(offset=None, size=None, tag=None, **kwargs): - external version - uploader """ - return openml.utils._list_all(_list_flows, offset=offset, size=size, tag=tag, **kwargs) + return openml.utils._list_all(_list_flows, + offset=offset, + size=size, + tag=tag, + **kwargs) def _list_flows(**kwargs): @@ -236,9 +240,9 @@ def assert_flows_equal(flow1, flow2, if key == 'parameters': if ignore_parameter_values or \ ignore_parameter_values_on_older_children: - parameters_flow_1 = set(flow1.parameters.keys()) - parameters_flow_2 = set(flow2.parameters.keys()) - symmetric_difference = parameters_flow_1 ^ parameters_flow_2 + params_flow_1 = set(flow1.parameters.keys()) + params_flow_2 = set(flow2.parameters.keys()) + symmetric_difference = params_flow_1 ^ params_flow_2 if len(symmetric_difference) > 0: raise ValueError('Flow %s: parameter set of flow ' 'differs from the parameters stored ' @@ -260,4 +264,5 @@ def assert_flows_equal(flow1, flow2, if attr1 != attr2: raise ValueError("Flow %s: values for attribute '%s' differ: " "'%s'\nvs\n'%s'." % - (str(flow1.name), str(key), str(attr1), str(attr2))) + (str(flow1.name), str(key), + str(attr1), str(attr2))) diff --git a/openml/flows/sklearn_converter.py b/openml/flows/sklearn_converter.py index fd312403c..b597243c1 100644 --- a/openml/flows/sklearn_converter.py +++ b/openml/flows/sklearn_converter.py @@ -85,8 +85,9 @@ def sklearn_to_flow(o, parent_model=None): def _is_estimator(o): - return (hasattr(o, 'fit') and hasattr(o, 'get_params') and - hasattr(o, 'set_params')) + return (hasattr(o, 'fit') + and hasattr(o, 'get_params') + and hasattr(o, 'set_params')) def _is_cross_validator(o): @@ -389,24 +390,26 @@ def _serialize_model(model): """ # Get all necessary information about the model objects itself - parameters, parameters_meta_info, sub_components, sub_components_explicit =\ + parameters, parameters_meta_info, subcomponents, subcomponents_explicit =\ _extract_information_from_model(model) # Check that a component does not occur multiple times in a flow as this # is not supported by OpenML - _check_multiple_occurence_of_component_in_flow(model, sub_components) + _check_multiple_occurence_of_component_in_flow(model, subcomponents) - # Create a flow name, which contains all components in brackets, for - # example RandomizedSearchCV(Pipeline(StandardScaler,AdaBoostClassifier(DecisionTreeClassifier)),StandardScaler,AdaBoostClassifier(DecisionTreeClassifier)) + # Create a flow name, which contains all components in brackets, e.g.: + # RandomizedSearchCV(Pipeline(StandardScaler, + # AdaBoostClassifier(DecisionTreeClassifier)), StandardScaler, + # AdaBoostClassifier(DecisionTreeClassifier)) class_name = model.__module__ + "." + model.__class__.__name__ # will be part of the name (in brackets) sub_components_names = "" - for key in sub_components: - if key in sub_components_explicit: - sub_components_names += "," + key + "=" + sub_components[key].name + for key in subcomponents: + if key in subcomponents_explicit: + sub_components_names += "," + key + "=" + subcomponents[key].name else: - sub_components_names += "," + sub_components[key].name + sub_components_names += "," + subcomponents[key].name if sub_components_names: # slice operation on string in order to get rid of leading comma @@ -415,24 +418,24 @@ def _serialize_model(model): name = class_name # Get the external versions of all sub-components - external_version = _get_external_version_string(model, sub_components) + external_version = _get_external_version_string(model, subcomponents) dependencies = [_format_external_version('sklearn', sklearn.__version__), 'numpy>=1.6.1', 'scipy>=0.9'] dependencies = '\n'.join(dependencies) + sklearn_version = _format_external_version('sklearn', sklearn.__version__) + sklearn_version_formatted = sklearn_version.replace('==', '_') flow = OpenMLFlow(name=name, class_name=class_name, description='Automatically created scikit-learn flow.', model=model, - components=sub_components, + components=subcomponents, parameters=parameters, parameters_meta_info=parameters_meta_info, external_version=external_version, tags=['openml-python', 'sklearn', 'scikit-learn', - 'python', - _format_external_version('sklearn', - sklearn.__version__).replace('==', '_'), + 'python', sklearn_version_formatted, # TODO: add more tags based on the scikit-learn # module a flow is in? For example automatically # annotate a class of sklearn.svm.SVC() with the @@ -500,9 +503,10 @@ def _extract_information_from_model(model): for k, v in sorted(model_parameters.items(), key=lambda t: t[0]): rval = sklearn_to_flow(v, model) - if (isinstance(rval, (list, tuple)) and len(rval) > 0 and - isinstance(rval[0], (list, tuple)) and - all([isinstance(rval[i], type(rval[0])) + if (isinstance(rval, (list, tuple)) + and len(rval) > 0 + and isinstance(rval[0], (list, tuple)) + and all([isinstance(rval[i], type(rval[0])) for i in range(len(rval))])): # Steps in a pipeline or feature union, or base classifiers in @@ -526,10 +530,10 @@ def _extract_information_from_model(model): raise TypeError(msg) if identifier in reserved_keywords: - parent_model_name = model.__module__ + "." + \ - model.__class__.__name__ + parent_model = "{}.{}".format(model.__module__, + model.__class__.__name__) msg = 'Found element shadowing official '\ - 'parameter for %s: %s' % (parent_model_name, + 'parameter for %s: %s' % (parent_model, identifier) raise PyOpenMLError(msg) @@ -602,8 +606,9 @@ def _extract_information_from_model(model): def _get_fn_arguments_with_defaults(fn_name): """ - Returns i) a dict with all parameter names (as key) that have a default value (as value) and ii) a set with all - parameter names that do not have a default + Returns: + i) a dict with all parameter names that have a default value, and + ii) a set with all parameter names that do not have a default Parameters ---------- @@ -614,21 +619,18 @@ def _get_fn_arguments_with_defaults(fn_name): ------- params_with_defaults: dict a dict mapping parameter name to the default value - params_without_defaults: dict + params_without_defaults: set a set with all parameters that do not have a default value """ - if sys.version_info[0] >= 3: - signature = inspect.getfullargspec(fn_name) - else: - signature = inspect.getargspec(fn_name) - - # len(signature.defaults) <= len(signature.args). Thus, by definition, the last entrees of signature.args - # actually have defaults. Iterate backwards over both arrays to keep them in sync - len_defaults = len(signature.defaults) if signature.defaults is not None else 0 - params_with_defaults = {signature.args[-1*i]: signature.defaults[-1*i] for i in range(1, len_defaults + 1)} - # retrieve the params without defaults - params_without_defaults = {signature.args[i] for i in range(len(signature.args) - len_defaults)} - return params_with_defaults, params_without_defaults + # parameters with defaults are optional, all others are required. + signature = inspect.getfullargspec(fn_name) + optional_params, required_params = dict(), set() + if signature.defaults: + optional_params =\ + dict(zip(reversed(signature.args), reversed(signature.defaults))) + required_params = {arg for arg in signature.args + if arg not in optional_params} + return optional_params, required_params def _deserialize_model(flow, keep_defaults, recursion_depth): @@ -675,15 +677,18 @@ def _deserialize_model(flow, keep_defaults, recursion_depth): if keep_defaults: # obtain all params with a default - param_defaults, _ = _get_fn_arguments_with_defaults(model_class.__init__) + param_defaults, _ =\ + _get_fn_arguments_with_defaults(model_class.__init__) # delete the params that have a default from the dict, # so they get initialized with their default value # except [...] for param in param_defaults: - # [...] the ones that also have a key in the components dict. As OpenML stores different flows for ensembles - # with different (base-)components, in OpenML terms, these are not considered hyperparameters but rather - # constants (i.e., changing them would result in a different flow) + # [...] the ones that also have a key in the components dict. + # As OpenML stores different flows for ensembles with different + # (base-)components, in OpenML terms, these are not considered + # hyperparameters but rather constants (i.e., changing them would + # result in a different flow) if param not in components.keys(): del parameter_dict[param] return model_class(**parameter_dict) @@ -709,8 +714,8 @@ def _check_dependencies(dependencies): elif operation == '>': check = installed_version > required_version elif operation == '>=': - check = installed_version > required_version or \ - installed_version == required_version + check = (installed_version > required_version + or installed_version == required_version) else: raise NotImplementedError( 'operation \'%s\' is not supported' % operation) @@ -770,7 +775,7 @@ def deserialize_rv_frozen(o): try: rv_class = getattr(importlib.import_module(module_name[0]), module_name[1]) - except: + except AttributeError: warnings.warn('Cannot create model %s for flow.' % dist_name) return None @@ -849,7 +854,7 @@ def _serialize_cross_validator(o): def _check_n_jobs(model): """ Returns True if the parameter settings of model are chosen s.t. the model - will run on a single core (in that case, openml-python can measure runtimes) + will run on a single core (if so, openml-python can measure runtimes) """ def check(param_grid, restricted_parameter_name, legal_values): if isinstance(param_grid, dict): @@ -864,10 +869,10 @@ def check(param_grid, restricted_parameter_name, legal_values): return False return True elif isinstance(param_grid, list): - for sub_grid in param_grid: - if not check(sub_grid, restricted_parameter_name, legal_values): - return False - return True + return all(check(sub_grid, + restricted_parameter_name, + legal_values) + for sub_grid in param_grid) if not (isinstance(model, sklearn.base.BaseEstimator) or isinstance(model, sklearn.model_selection._search.BaseSearchCV)): @@ -884,8 +889,11 @@ def check(param_grid, restricted_parameter_name, legal_values): if hasattr(model, 'param_distributions'): param_distributions = model.param_distributions else: - raise AttributeError('Using subclass BaseSearchCV other than {GridSearchCV, RandomizedSearchCV}. Could not find attribute param_distributions. ') - print('Warning! Using subclass BaseSearchCV other than ' \ + raise AttributeError('Using subclass BaseSearchCV other than ' + '{GridSearchCV, RandomizedSearchCV}. ' + 'Could not find attribute ' + 'param_distributions.') + print('Warning! Using subclass BaseSearchCV other than ' '{GridSearchCV, RandomizedSearchCV}. Should implement param check. ') if not check(param_distributions, 'n_jobs', None): From a1345127ab9947d28e96df12cbe14254aee66327 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 22 Feb 2019 13:44:16 +0200 Subject: [PATCH 14/35] PEP8. --- openml/flows/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openml/flows/__init__.py b/openml/flows/__init__.py index 884d32e98..0c72fd36a 100644 --- a/openml/flows/__init__.py +++ b/openml/flows/__init__.py @@ -5,4 +5,5 @@ from .functions import get_flow, list_flows, flow_exists, assert_flows_equal __all__ = ['OpenMLFlow', 'get_flow', 'list_flows', 'sklearn_to_flow', - 'flow_to_sklearn', 'flow_exists', 'openml_param_name_to_sklearn'] + 'flow_to_sklearn', 'flow_exists', 'openml_param_name_to_sklearn', + 'assert_flows_equal', 'obtain_parameter_values'] From 721a24a437de38b7b5780ffbf6772fb75edb3e42 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 22 Feb 2019 14:53:33 +0200 Subject: [PATCH 15/35] PEP8. Minor refactor. --- openml/flows/sklearn_converter.py | 6 +++-- openml/runs/run.py | 43 +++++++++++++++++-------------- openml/setups/functions.py | 34 ++++++++++++++---------- openml/study/__init__.py | 2 ++ openml/study/study.py | 1 - openml/tasks/functions.py | 4 +-- 6 files changed, 51 insertions(+), 39 deletions(-) diff --git a/openml/flows/sklearn_converter.py b/openml/flows/sklearn_converter.py index b597243c1..e5b775145 100644 --- a/openml/flows/sklearn_converter.py +++ b/openml/flows/sklearn_converter.py @@ -601,7 +601,8 @@ def _extract_information_from_model(model): parameters_meta_info[k] = OrderedDict((('description', None), ('data_type', None))) - return parameters, parameters_meta_info, sub_components, sub_components_explicit + return parameters, parameters_meta_info,\ + sub_components, sub_components_explicit def _get_fn_arguments_with_defaults(fn_name): @@ -894,7 +895,8 @@ def check(param_grid, restricted_parameter_name, legal_values): 'Could not find attribute ' 'param_distributions.') print('Warning! Using subclass BaseSearchCV other than ' - '{GridSearchCV, RandomizedSearchCV}. Should implement param check. ') + '{GridSearchCV, RandomizedSearchCV}. ' + 'Should implement param check. ') if not check(param_distributions, 'n_jobs', None): raise PyOpenMLError('openml-python should not be used to ' diff --git a/openml/runs/run.py b/openml/runs/run.py index 9485b60b9..4abf2eabe 100644 --- a/openml/runs/run.py +++ b/openml/runs/run.py @@ -1,6 +1,5 @@ from collections import OrderedDict import errno -import json import pickle import sys import time @@ -187,27 +186,31 @@ def _generate_arff_dict(self): if self.data_content is None: raise ValueError('Run has not been executed.') - run_environment = (_get_version_information() + - [time.strftime("%c")] + ['Created by run_task()']) + run_environment = (_get_version_information() + + [time.strftime("%c")] + + ['Created by run_task()']) task = get_task(self.task_id) arff_dict = OrderedDict() arff_dict['data'] = self.data_content arff_dict['description'] = "\n".join(run_environment) - arff_dict['relation'] = 'openml_task_' + str(task.task_id) + \ - '_predictions' + arff_dict['relation'] =\ + 'openml_task_{}_predictions'.format(task.task_id) if task.task_type_id == TaskTypeEnum.SUPERVISED_CLASSIFICATION: class_labels = task.class_labels - arff_dict['attributes'] = [('repeat', 'NUMERIC'), + instance_specifications = [('repeat', 'NUMERIC'), ('fold', 'NUMERIC'), ('sample', 'NUMERIC'), # Legacy - ('row_id', 'NUMERIC')] + \ - [('confidence.' + class_labels[i], - 'NUMERIC') for i in - range(len(class_labels))] + \ - [('prediction', class_labels), - ('correct', class_labels)] + ('row_id', 'NUMERIC')] + prediction_confidences = [('confidence.' + class_labels[i], + 'NUMERIC') + for i in range(len(class_labels))] + prediction_and_true = [('prediction', class_labels), + ('correct', class_labels)] + arff_dict['attributes'] = (instance_specifications + + prediction_confidences + + prediction_and_true) elif task.task_type_id == TaskTypeEnum.LEARNING_CURVE: class_labels = task.class_labels @@ -276,9 +279,9 @@ def get_metric_fn(self, sklearn_fn, kwargs={}): task = get_task(self.task_id) attribute_names = [att[0] for att in predictions_arff['attributes']] - if (task.task_type_id == TaskTypeEnum.SUPERVISED_CLASSIFICATION or - task.task_type_id == TaskTypeEnum.LEARNING_CURVE) and \ - 'correct' not in attribute_names: + if ('correct' not in attribute_names and + task.task_type_id in [TaskTypeEnum.SUPERVISED_CLASSIFICATION, + TaskTypeEnum.LEARNING_CURVE]): raise ValueError('Attribute "correct" should be set for ' 'classification task runs') if task.task_type_id == TaskTypeEnum.SUPERVISED_REGRESSION and \ @@ -305,7 +308,7 @@ def _attribute_list_to_dict(attribute_list): repeat_idx = attribute_dict['repeat'] fold_idx = attribute_dict['fold'] - predicted_idx = attribute_dict['prediction'] # Assume supervised tasks + predicted_idx = attribute_dict['prediction'] # Assume supervised task if task.task_type_id == TaskTypeEnum.SUPERVISED_CLASSIFICATION or \ task.task_type_id == TaskTypeEnum.LEARNING_CURVE: @@ -321,8 +324,8 @@ def _attribute_list_to_dict(attribute_list): predictions_arff['attributes'][correct_idx][1]: pred = predictions_arff['attributes'][predicted_idx][1] corr = predictions_arff['attributes'][correct_idx][1] - raise ValueError('Predicted and Correct do not have equal values: ' - '%s Vs. %s' % (str(pred), str(corr))) + raise ValueError('Predicted and Correct do not have equal values:' + ' %s Vs. %s' % (str(pred), str(corr))) # TODO: these could be cached values_predict = {} @@ -335,8 +338,8 @@ def _attribute_list_to_dict(attribute_list): else: samp = 0 # No learning curve sample, always 0 - if task.task_type_id == TaskTypeEnum.SUPERVISED_CLASSIFICATION or \ - task.task_type_id == TaskTypeEnum.LEARNING_CURVE: + if task.task_type_id in [TaskTypeEnum.SUPERVISED_CLASSIFICATION, + TaskTypeEnum.LEARNING_CURVE]: prediction = predictions_arff['attributes'][predicted_idx][ 1].index(line[predicted_idx]) correct = predictions_arff['attributes'][predicted_idx][1]. \ diff --git a/openml/setups/functions.py b/openml/setups/functions.py index fdb803453..69486f8a8 100644 --- a/openml/setups/functions.py +++ b/openml/setups/functions.py @@ -8,7 +8,7 @@ from .. import config from .setup import OpenMLSetup, OpenMLParameter from openml.flows import flow_exists -from openml.exceptions import OpenMLServerNoResult +import openml.exceptions import openml.utils @@ -67,7 +67,8 @@ def _get_cached_setup(setup_id): return setup except (OSError, IOError): - raise openml.exceptions.OpenMLCacheException("Setup file for setup id %d not cached" % setup_id) + raise openml.exceptions.OpenMLCacheException( + "Setup file for setup id %d not cached" % setup_id) def get_setup(setup_id): @@ -85,7 +86,9 @@ def get_setup(setup_id): OpenMLSetup an initialized openml setup object """ - setup_dir = os.path.join(config.get_cache_directory(), "setups", str(setup_id)) + setup_dir = os.path.join(config.get_cache_directory(), + "setups", + str(setup_id)) setup_file = os.path.join(setup_dir, "description.xml") if not os.path.exists(setup_dir): @@ -94,7 +97,7 @@ def get_setup(setup_id): try: return _get_cached_setup(setup_id) - except (openml.exceptions.OpenMLCacheException): + except openml.exceptions.OpenMLCacheException: setup_xml = openml._api_calls._perform_api_call('/setup/%d' % setup_id) with io.open(setup_file, "w", encoding='utf8') as fh: fh.write(setup_xml) @@ -119,10 +122,10 @@ def list_setups(offset=None, size=None, flow=None, tag=None, setup=None): ------- dict """ - + batch_size = 1000 # batch size for setups is lower return openml.utils._list_all(_list_setups, offset=offset, size=size, - flow=flow, tag=tag, setup=setup, batch_size=1000) #batch size for setups is lower - + flow=flow, tag=tag, + setup=setup, batch_size=batch_size) def _list_setups(setup=None, **kwargs): """ @@ -157,19 +160,20 @@ def __list_setups(api_call): """Helper function to parse API calls which are lists of setups""" xml_string = openml._api_calls._perform_api_call(api_call) setups_dict = xmltodict.parse(xml_string, force_list=('oml:setup',)) + openml_uri = 'http://openml.org/openml' # Minimalistic check if the XML is useful if 'oml:setups' not in setups_dict: - raise ValueError('Error in return XML, does not contain "oml:setups": %s' - % str(setups_dict)) + raise ValueError('Error in return XML, does not contain "oml:setups":' + ' %s' % str(setups_dict)) elif '@xmlns:oml' not in setups_dict['oml:setups']: raise ValueError('Error in return XML, does not contain ' '"oml:setups"/@xmlns:oml: %s' % str(setups_dict)) - elif setups_dict['oml:setups']['@xmlns:oml'] != 'http://openml.org/openml': + elif setups_dict['oml:setups']['@xmlns:oml'] != openml_uri: raise ValueError('Error in return XML, value of ' '"oml:seyups"/@xmlns:oml is not ' - '"http://openml.org/openml": %s' - % str(setups_dict)) + '"%s": %s' + % (openml_uri, str(setups_dict))) assert type(setups_dict['oml:setups']['oml:setup']) == list, \ type(setups_dict['oml:setups']) @@ -246,9 +250,11 @@ def _create_setup_from_xml(result_dict): elif isinstance(xml_parameters, list): for xml_parameter in xml_parameters: id = int(xml_parameter['oml:id']) - parameters[id] = _create_setup_parameter_from_xml(xml_parameter) + parameters[id] = \ + _create_setup_parameter_from_xml(xml_parameter) else: - raise ValueError('Expected None, list or dict, received someting else: %s' %str(type(xml_parameters))) + raise ValueError('Expected None, list or dict, received ' + 'something else: %s' % str(type(xml_parameters))) return OpenMLSetup(setup_id, flow_id, parameters) diff --git a/openml/study/__init__.py b/openml/study/__init__.py index 3d7f12fe5..cc32043f3 100644 --- a/openml/study/__init__.py +++ b/openml/study/__init__.py @@ -1,2 +1,4 @@ from .study import OpenMLStudy from .functions import get_study + +__all__ = ['OpenMLStudy', 'get_study'] diff --git a/openml/study/study.py b/openml/study/study.py index f4a878411..8c2b77bfd 100644 --- a/openml/study/study.py +++ b/openml/study/study.py @@ -48,4 +48,3 @@ def __init__(self, id, name, description, creation_date, creator, self.flows = flows self.setups = setups pass - diff --git a/openml/tasks/functions.py b/openml/tasks/functions.py index 360a5b574..872b297f4 100644 --- a/openml/tasks/functions.py +++ b/openml/tasks/functions.py @@ -409,9 +409,9 @@ def _create_task_from_xml(xml): "oml:estimation_procedure"]["oml:type"] common_kwargs['estimation_parameters'] = estimation_parameters common_kwargs['target_name'] = inputs[ - "source_data"]["oml:data_set"]["oml:target_feature"] + "source_data"]["oml:data_set"]["oml:target_feature"] common_kwargs['data_splits_url'] = inputs["estimation_procedure"][ - "oml:estimation_procedure"]["oml:data_splits_url"] + "oml:estimation_procedure"]["oml:data_splits_url"] cls = { TaskTypeEnum.SUPERVISED_CLASSIFICATION: OpenMLClassificationTask, From 0e5918f599ab088a9005c768c3da341ad43a98e6 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 22 Feb 2019 14:53:55 +0200 Subject: [PATCH 16/35] PEP8. Refactor/bugfix __eq__ --- openml/tasks/split.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/openml/tasks/split.py b/openml/tasks/split.py index 9bab4918e..4479e747c 100644 --- a/openml/tasks/split.py +++ b/openml/tasks/split.py @@ -34,27 +34,27 @@ def __init__(self, name, description, split): self.samples = len(self.split[0][0]) def __eq__(self, other): - if type(self) != type(other): + if (type(self) != type(other) + or self.name != other.name + or self.description != other.description + or self.split.keys() != other.split.keys()): return False - elif self.name != other.name: - return False - elif self.description != other.description: - return False - elif self.split.keys() != other.split.keys(): + + if any(self.split[repetition].keys() != other.split[repetition].keys() + for repetition in self.split): return False - else: - for repetition in self.split: - if self.split[repetition].keys() != other.split[repetition].keys(): - return False - else: - for fold in self.split[repetition]: - for sample in self.split[repetition][fold]: - if np.all(self.split[repetition][fold][sample].test != - other.split[repetition][fold][sample].test)\ - and \ - np.all(self.split[repetition][fold][sample].train - != other.split[repetition][fold][sample].train): - return False + + samples = [(repetition, fold, sample) + for repetition in self.split + for fold in self.split[repetition] + for sample in self.split[repetition][fold]] + + for repetition, fold, sample in samples: + self_train, self_test = self.split[repetition][fold][sample] + other_train, other_test = other.split[repetition][fold][sample] + if not (np.all(self_train == other_train) + and np.all(self_test == other_test)): + return False return True @classmethod From d963df229c39673f98b364a42d827754d5e0f689 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 22 Feb 2019 14:56:04 +0200 Subject: [PATCH 17/35] PEP8. task type are input for list_tasks anyway, so the enum should be exposed. --- openml/tasks/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openml/tasks/__init__.py b/openml/tasks/__init__.py index 3e872c133..7e919dad2 100644 --- a/openml/tasks/__init__.py +++ b/openml/tasks/__init__.py @@ -21,4 +21,5 @@ 'get_tasks', 'list_tasks', 'OpenMLSplit', + 'TaskTypeEnum' ] From 5bc6a641ca13f6823b443658eecd0422d56aba7e Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 22 Feb 2019 15:05:06 +0200 Subject: [PATCH 18/35] PEP8. --- openml/flows/sklearn_converter.py | 4 ++-- openml/setups/functions.py | 1 + openml/study/functions.py | 2 +- openml/tasks/split.py | 11 +++++++---- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/openml/flows/sklearn_converter.py b/openml/flows/sklearn_converter.py index e5b775145..510b03f51 100644 --- a/openml/flows/sklearn_converter.py +++ b/openml/flows/sklearn_converter.py @@ -601,8 +601,8 @@ def _extract_information_from_model(model): parameters_meta_info[k] = OrderedDict((('description', None), ('data_type', None))) - return parameters, parameters_meta_info,\ - sub_components, sub_components_explicit + return (parameters, parameters_meta_info, + sub_components, sub_components_explicit) def _get_fn_arguments_with_defaults(fn_name): diff --git a/openml/setups/functions.py b/openml/setups/functions.py index 69486f8a8..1cbdf505d 100644 --- a/openml/setups/functions.py +++ b/openml/setups/functions.py @@ -127,6 +127,7 @@ def list_setups(offset=None, size=None, flow=None, tag=None, setup=None): flow=flow, tag=tag, setup=setup, batch_size=batch_size) + def _list_setups(setup=None, **kwargs): """ Perform API call `/setup/list/{filters}` diff --git a/openml/study/functions.py b/openml/study/functions.py index cce4ca4b0..6bf2f7b57 100644 --- a/openml/study/functions.py +++ b/openml/study/functions.py @@ -19,7 +19,7 @@ def get_study(study_id, type=None): Note that some of the (data, tasks, flows, setups) fields can be empty (depending on information on the server) ''' - call_suffix = "study/%s" %str(study_id) + call_suffix = "study/%s" % str(study_id) if type is not None: call_suffix += "/" + type xml_string = openml._api_calls._perform_api_call(call_suffix) diff --git a/openml/tasks/split.py b/openml/tasks/split.py index 4479e747c..c83873cc8 100644 --- a/openml/tasks/split.py +++ b/openml/tasks/split.py @@ -106,12 +106,13 @@ def _from_arff_file(cls, filename): repetitions[repetition][fold] = OrderedDict() if sample not in repetitions[repetition][fold]: repetitions[repetition][fold][sample] = ([], []) + split = repetitions[repetition][fold][sample] type_ = line[type_idx].decode('utf-8') if type_ == 'TRAIN': - repetitions[repetition][fold][sample][0].append(line[rowid_idx]) + split[0].append(line[rowid_idx]) elif type_ == 'TEST': - repetitions[repetition][fold][sample][1].append(line[rowid_idx]) + split[1].append(line[rowid_idx]) else: raise ValueError(type_) @@ -119,8 +120,10 @@ def _from_arff_file(cls, filename): for fold in repetitions[repetition]: for sample in repetitions[repetition][fold]: repetitions[repetition][fold][sample] = Split( - np.array(repetitions[repetition][fold][sample][0], dtype=np.int32), - np.array(repetitions[repetition][fold][sample][1], dtype=np.int32)) + np.array(repetitions[repetition][fold][sample][0], + dtype=np.int32), + np.array(repetitions[repetition][fold][sample][1], + dtype=np.int32)) with open(pkl_filename, "wb") as fh: pickle.dump({"name": name, "repetitions": repetitions}, fh, From 469da356c4ce98bd5d6a7c92be2b946fde890ff6 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 22 Feb 2019 15:38:57 +0200 Subject: [PATCH 19/35] undo linebreaks --- openml/flows/sklearn_converter.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openml/flows/sklearn_converter.py b/openml/flows/sklearn_converter.py index 510b03f51..b7e2371ea 100644 --- a/openml/flows/sklearn_converter.py +++ b/openml/flows/sklearn_converter.py @@ -398,9 +398,7 @@ def _serialize_model(model): _check_multiple_occurence_of_component_in_flow(model, subcomponents) # Create a flow name, which contains all components in brackets, e.g.: - # RandomizedSearchCV(Pipeline(StandardScaler, - # AdaBoostClassifier(DecisionTreeClassifier)), StandardScaler, - # AdaBoostClassifier(DecisionTreeClassifier)) + # RandomizedSearchCV(Pipeline(StandardScaler,AdaBoostClassifier(DecisionTreeClassifier)),StandardScaler,AdaBoostClassifier(DecisionTreeClassifier)) class_name = model.__module__ + "." + model.__class__.__name__ # will be part of the name (in brackets) From 486675fcf628e1b6ef7590ee27cdb66c194f4112 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 22 Feb 2019 15:55:43 +0200 Subject: [PATCH 20/35] Update for updated error message. --- tests/test_datasets/test_dataset_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py index f8c77be11..4b1df8cd3 100644 --- a/tests/test_datasets/test_dataset_functions.py +++ b/tests/test_datasets/test_dataset_functions.py @@ -951,7 +951,7 @@ def test_create_dataset_attributes_auto_without_df(self): citation = 'None' original_data_url = 'http://openml.github.io/openml-python' paper_url = 'http://openml.github.io/openml-python' - err_msg = "Automatically inferring the attributes required a pandas" + err_msg = "Automatically inferring attributes required a pandas" with pytest.raises(ValueError, match=err_msg): openml.datasets.functions.create_dataset( name=name, From a6386407296b73b1ea6e83decef18bf36c375f12 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 22 Feb 2019 16:47:05 +0200 Subject: [PATCH 21/35] Update for updated error message. --- tests/test_datasets/test_dataset_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py index 4b1df8cd3..3bd0f5dfd 100644 --- a/tests/test_datasets/test_dataset_functions.py +++ b/tests/test_datasets/test_dataset_functions.py @@ -951,7 +951,7 @@ def test_create_dataset_attributes_auto_without_df(self): citation = 'None' original_data_url = 'http://openml.github.io/openml-python' paper_url = 'http://openml.github.io/openml-python' - err_msg = "Automatically inferring attributes required a pandas" + err_msg = "Automatically inferring attributes requires a pandas" with pytest.raises(ValueError, match=err_msg): openml.datasets.functions.create_dataset( name=name, From da0cdd2e19b435d0e68d8eb396df5c436849dc51 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 22 Feb 2019 17:41:28 +0200 Subject: [PATCH 22/35] Undo refactor. --- openml/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openml/utils.py b/openml/utils.py index 75a5369a8..187aac916 100644 --- a/openml/utils.py +++ b/openml/utils.py @@ -135,8 +135,9 @@ def _list_all(listing_call, *args, **filters): LIMIT = active_filters['size'] del active_filters['size'] - if LIMIT is not None and BATCH_SIZE_ORIG > LIMIT: - BATCH_SIZE_ORIG = LIMIT + if LIMIT is not None: + if BATCH_SIZE_ORIG > LIMIT: + BATCH_SIZE_ORIG = min(LIMIT, BATCH_SIZE_ORIG) if 'offset' in active_filters: offset = active_filters['offset'] From af6d5c6980ba0c5d2fdeacbaea61b2dd97b783ce Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 22 Feb 2019 19:14:59 +0200 Subject: [PATCH 23/35] Redo refactor. --- openml/utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/openml/utils.py b/openml/utils.py index 187aac916..75a5369a8 100644 --- a/openml/utils.py +++ b/openml/utils.py @@ -135,9 +135,8 @@ def _list_all(listing_call, *args, **filters): LIMIT = active_filters['size'] del active_filters['size'] - if LIMIT is not None: - if BATCH_SIZE_ORIG > LIMIT: - BATCH_SIZE_ORIG = min(LIMIT, BATCH_SIZE_ORIG) + if LIMIT is not None and BATCH_SIZE_ORIG > LIMIT: + BATCH_SIZE_ORIG = LIMIT if 'offset' in active_filters: offset = active_filters['offset'] From b196a97431adeb3a3c04133e25ccbaa07e873829 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Fri, 22 Feb 2019 20:53:55 +0100 Subject: [PATCH 24/35] Fix syntax error due to online merging --- openml/study/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openml/study/__init__.py b/openml/study/__init__.py index aefebf5ee..f99b0d638 100644 --- a/openml/study/__init__.py +++ b/openml/study/__init__.py @@ -5,4 +5,5 @@ __all__ = [ 'OpenMLStudy', 'attach_to_study', 'create_benchmark_suite', 'create_study', - 'delete_study', 'detach_from_study', 'get_study', 'status_update' + 'delete_study', 'detach_from_study', 'get_study', 'status_update', +] From 7450082a50a089cd9007719488bd901c02d91a0e Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Fri, 22 Feb 2019 20:55:56 +0100 Subject: [PATCH 25/35] Fix merge error due to online merging --- openml/datasets/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openml/datasets/functions.py b/openml/datasets/functions.py index 8809f57a8..3bb0f9ec7 100644 --- a/openml/datasets/functions.py +++ b/openml/datasets/functions.py @@ -672,7 +672,7 @@ def _get_dataset_description(did_cache_dir, dataset_id): return _get_cached_dataset_description(dataset_id) except OpenMLCacheException: url_extension = "data/{}".format(dataset_id) - dataset_xml = openml._api_calls._perform_api_call(url_extension, 'get' + dataset_xml = openml._api_calls._perform_api_call(url_extension, 'get') with io.open(description_file, "w", encoding='utf8') as fh: fh.write(dataset_xml) From 5d83c94ffbb2b90736d9555c30fdfe4181881c74 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Fri, 22 Feb 2019 21:07:15 +0100 Subject: [PATCH 26/35] Fix bug due to online merge error --- openml/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openml/utils.py b/openml/utils.py index 7092c2bbc..e324d375c 100644 --- a/openml/utils.py +++ b/openml/utils.py @@ -82,7 +82,6 @@ def _tag_entity(entity_type, entity_id, tag, untag=False): main_tag = 'oml:%s_untag' % entity_type post_variables = {'%s_id' % entity_type: entity_id, 'tag': tag} - result_xml = openml._api_calls._perform_api_call(uri, post_variables) result_xml = openml._api_calls._perform_api_call(uri, 'post', post_variables) From 3a98e9faf15fba98a91c75f4aa9278aacbdf0eab Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Sat, 23 Feb 2019 11:55:26 +0200 Subject: [PATCH 27/35] Change flake scope and arguments. --- ci_scripts/flake8_diff.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci_scripts/flake8_diff.sh b/ci_scripts/flake8_diff.sh index 0c4667176..8e4c56225 100755 --- a/ci_scripts/flake8_diff.sh +++ b/ci_scripts/flake8_diff.sh @@ -140,7 +140,8 @@ check_files() { if [ -n "$files" ]; then # Conservative approach: diff without context (--unified=0) so that code # that was not changed does not create failures - git diff --no-ext-diff --unified=0 $COMMIT_RANGE -- $files | flake8 --ignore E402 --diff --show-source $options + # git diff --no-ext-diff --unified=0 $COMMIT_RANGE -- $files | flake8 --ignore E402 --diff --show-source $options + flake8 --ignore E402,W503 --show-source --max-line-length 100 $options fi } From ecce029d162d4e827e463ee513c8c27e5bfb3b81 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Sat, 23 Feb 2019 12:56:09 +0200 Subject: [PATCH 28/35] Final PEP8 changes. --- openml/__init__.py | 2 +- openml/flows/sklearn_converter.py | 4 ++-- openml/runs/run.py | 16 ++++++++-------- openml/study/functions.py | 2 +- openml/utils.py | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/openml/__init__.py b/openml/__init__.py index a707374eb..fc67ee6b2 100644 --- a/openml/__init__.py +++ b/openml/__init__.py @@ -30,7 +30,7 @@ from .evaluations import OpenMLEvaluation from .study import OpenMLStudy -from .__version__ import __version__ +from .__version__ import __version__ # noqa: F401 def populate_cache(task_ids=None, dataset_ids=None, flow_ids=None, diff --git a/openml/flows/sklearn_converter.py b/openml/flows/sklearn_converter.py index b7e2371ea..755e0f1dd 100644 --- a/openml/flows/sklearn_converter.py +++ b/openml/flows/sklearn_converter.py @@ -873,8 +873,8 @@ def check(param_grid, restricted_parameter_name, legal_values): legal_values) for sub_grid in param_grid) - if not (isinstance(model, sklearn.base.BaseEstimator) or - isinstance(model, sklearn.model_selection._search.BaseSearchCV)): + if not (isinstance(model, sklearn.base.BaseEstimator) + or isinstance(model, sklearn.model_selection._search.BaseSearchCV)): raise ValueError('model should be BaseEstimator or BaseSearchCV') # make sure that n_jobs is not in the parameter grid of optimization diff --git a/openml/runs/run.py b/openml/runs/run.py index 474a5848d..ac4308b1c 100644 --- a/openml/runs/run.py +++ b/openml/runs/run.py @@ -280,17 +280,17 @@ def get_metric_fn(self, sklearn_fn, kwargs={}): task = get_task(self.task_id) attribute_names = [att[0] for att in predictions_arff['attributes']] - if ('correct' not in attribute_names and - task.task_type_id in [TaskTypeEnum.SUPERVISED_CLASSIFICATION, - TaskTypeEnum.LEARNING_CURVE]): + if (task.task_type_id in [TaskTypeEnum.SUPERVISED_CLASSIFICATION, + TaskTypeEnum.LEARNING_CURVE] + and 'correct' not in attribute_names): raise ValueError('Attribute "correct" should be set for ' 'classification task runs') - if task.task_type_id == TaskTypeEnum.SUPERVISED_REGRESSION and \ - 'truth' not in attribute_names: + if (task.task_type_id == TaskTypeEnum.SUPERVISED_REGRESSION + and 'truth' not in attribute_names): raise ValueError('Attribute "truth" should be set for ' 'regression task runs') - if task.task_type_id != TaskTypeEnum.CLUSTERING and \ - 'prediction' not in attribute_names: + if (task.task_type_id != TaskTypeEnum.CLUSTERING + and 'prediction' not in attribute_names): raise ValueError('Attribute "predict" should be set for ' 'supervised task runs') @@ -511,7 +511,7 @@ def _to_dict(taskid, flow_id, setup_string, error_message, parameter_settings, Returns ------- result : an array with version information of the above packages - """ + """ # noqa: W605 description = OrderedDict() description['oml:run'] = OrderedDict() description['oml:run']['@xmlns:oml'] = 'http://openml.org/openml' diff --git a/openml/study/functions.py b/openml/study/functions.py index e526ee246..a2600e4a0 100644 --- a/openml/study/functions.py +++ b/openml/study/functions.py @@ -178,7 +178,7 @@ def create_benchmark_suite(alias, name, description, task_ids): def status_update(study_id, status): """ - Updates the status of a study to either 'active' or 'deactivated'. + Updates the status of a study to either 'active' or 'deactivated'. Parameters ---------- diff --git a/openml/utils.py b/openml/utils.py index e324d375c..a95e1c96b 100644 --- a/openml/utils.py +++ b/openml/utils.py @@ -185,7 +185,7 @@ def _list_all(listing_call, *args, **filters): if 'offset' in active_filters: offset = active_filters['offset'] del active_filters['offset'] - + batch_size = BATCH_SIZE_ORIG while True: try: From a4817ba535055c8ed59440c9697d73fa56c12f77 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Sat, 23 Feb 2019 13:01:33 +0100 Subject: [PATCH 29/35] PEP8 for several test files --- tests/__init__.py | 2 +- tests/test_examples/test_OpenMLDemo.py | 4 +- tests/test_flows/test_flow.py | 124 +++++++++++++++++-------- tests/test_flows/test_sklearn.py | 98 +++++++++++++------ 4 files changed, 156 insertions(+), 72 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index d6b0c7b1a..dc5287024 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,3 +1,3 @@ # Dummy to allow mock classes in the test files to have a version number for # their parent module -__version__ = '0.1' \ No newline at end of file +__version__ = '0.1' diff --git a/tests/test_examples/test_OpenMLDemo.py b/tests/test_examples/test_OpenMLDemo.py index 676138c3f..64c710873 100644 --- a/tests/test_examples/test_OpenMLDemo.py +++ b/tests/test_examples/test_OpenMLDemo.py @@ -30,12 +30,12 @@ def setUp(self): try: shutil.rmtree(self.notebook_output_directory) - except: + except OSError: pass try: os.makedirs(self.notebook_output_directory) - except: + except OSError: pass def _tst_notebook(self, notebook_name): diff --git a/tests/test_flows/test_flow.py b/tests/test_flows/test_flow.py index 4b784e790..d1b67d686 100644 --- a/tests/test_flows/test_flow.py +++ b/tests/test_flows/test_flow.py @@ -110,7 +110,8 @@ def test_tagging(self): def test_from_xml_to_xml(self): # Get the raw xml thing - # TODO maybe get this via get_flow(), which would have to be refactored to allow getting only the xml dictionary + # TODO maybe get this via get_flow(), which would have to be refactored + # to allow getting only the xml dictionary # TODO: no sklearn flows. for flow_id in [3, 5, 7, 9, ]: flow_xml = _perform_api_call("flow/%d" % flow_id, @@ -120,9 +121,15 @@ def test_from_xml_to_xml(self): flow = openml.OpenMLFlow._from_dict(flow_dict) new_xml = flow._to_xml() - flow_xml = flow_xml.replace(' ', '').replace('\t', '').strip().replace('\n\n', '\n').replace('"', '"') + flow_xml = ( + flow_xml.replace(' ', '').replace('\t', ''). + strip().replace('\n\n', '\n').replace('"', '"') + ) flow_xml = re.sub(r'^$', '', flow_xml) - new_xml = new_xml.replace(' ', '').replace('\t', '').strip().replace('\n\n', '\n').replace('"', '"') + new_xml = ( + new_xml.replace(' ', '').replace('\t', ''). + strip().replace('\n\n', '\n').replace('"', '"') + ) new_xml = re.sub(r'^$', '', new_xml) self.assertEqual(new_xml, flow_xml) @@ -169,8 +176,11 @@ def test_publish_existing_flow(self): flow = openml.flows.sklearn_to_flow(clf) flow, _ = self._add_sentinel_to_flow_name(flow, None) flow.publish() - self.assertRaisesRegex(openml.exceptions.OpenMLServerException, - 'flow already exists', flow.publish) + self.assertRaisesRegex( + openml.exceptions.OpenMLServerException, + 'flow already exists', + flow.publish, + ) def test_publish_flow_with_similar_components(self): clf = sklearn.ensemble.VotingClassifier([ @@ -219,8 +229,8 @@ def test_publish_flow_with_similar_components(self): def test_semi_legal_flow(self): # TODO: Test if parameters are set correctly! - # should not throw error as it contains two differentiable forms of Bagging - # i.e., Bagging(Bagging(J48)) and Bagging(J48) + # should not throw error as it contains two differentiable forms of + # Bagging i.e., Bagging(Bagging(J48)) and Bagging(J48) semi_legal = sklearn.ensemble.BaggingClassifier( base_estimator=sklearn.ensemble.BaggingClassifier( base_estimator=sklearn.tree.DecisionTreeClassifier())) @@ -250,12 +260,15 @@ def test_publish_error(self, api_call_mock, get_flow_mock): with self.assertRaises(ValueError) as context_manager: flow.publish() - fixture = "Flow was not stored correctly on the server. " \ - "New flow ID is 1. Please check manually and remove " \ - "the flow if necessary! Error is:\n" \ - "'Flow sklearn.ensemble.forest.RandomForestClassifier: values for attribute 'name' differ: " \ - "'sklearn.ensemble.forest.RandomForestClassifier'" \ - "\nvs\n'sklearn.ensemble.forest.RandomForestClassifie'.'" + fixture = ( + "Flow was not stored correctly on the server. " + "New flow ID is 1. Please check manually and remove " + "the flow if necessary! Error is:\n" + "'Flow sklearn.ensemble.forest.RandomForestClassifier: " + "values for attribute 'name' differ: " + "'sklearn.ensemble.forest.RandomForestClassifier'" + "\nvs\n'sklearn.ensemble.forest.RandomForestClassifie'.'" + ) self.assertEqual(context_manager.exception.args[0], fixture) self.assertEqual(api_call_mock.call_count, 2) @@ -263,16 +276,20 @@ def test_publish_error(self, api_call_mock, get_flow_mock): def test_illegal_flow(self): # should throw error as it contains two imputers - illegal = sklearn.pipeline.Pipeline(steps=[('imputer1', Imputer()), - ('imputer2', Imputer()), - ('classif', sklearn.tree.DecisionTreeClassifier())]) + illegal = sklearn.pipeline.Pipeline( + steps=[ + ('imputer1', Imputer()), + ('imputer2', Imputer()), + ('classif', sklearn.tree.DecisionTreeClassifier()) + ] + ) self.assertRaises(ValueError, openml.flows.sklearn_to_flow, illegal) def test_nonexisting_flow_exists(self): def get_sentinel(): - # Create a unique prefix for the flow. Necessary because the flow is - # identified by its name and external version online. Having a unique - # name allows us to publish the same flow in each test run + # Create a unique prefix for the flow. Necessary because the flow + # is identified by its name and external version online. Having a + # unique name allows us to publish the same flow in each test run md5 = hashlib.md5() md5.update(str(time.time()).encode('utf-8')) sentinel = md5.hexdigest()[:10] @@ -292,10 +309,15 @@ def test_existing_flow_exists(self): ohe_params = {'sparse': False, 'handle_unknown': 'ignore'} if LooseVersion(sklearn.__version__) >= '0.20': ohe_params['categories'] = 'auto' - steps = [('imputation', Imputer(strategy='median')), - ('hotencoding', sklearn.preprocessing.OneHotEncoder(**ohe_params)), - ('variencethreshold', sklearn.feature_selection.VarianceThreshold()), - ('classifier', sklearn.tree.DecisionTreeClassifier())] + steps = [ + ('imputation', Imputer(strategy='median')), + ('hotencoding', sklearn.preprocessing.OneHotEncoder(**ohe_params)), + ( + 'variencethreshold', + sklearn.feature_selection.VarianceThreshold(), + ), + ('classifier', sklearn.tree.DecisionTreeClassifier()) + ] complicated = sklearn.pipeline.Pipeline(steps=steps) for classifier in [nb, complicated]: @@ -308,7 +330,10 @@ def test_existing_flow_exists(self): # check if flow exists can find it flow = openml.flows.get_flow(flow.flow_id) - downloaded_flow_id = openml.flows.flow_exists(flow.name, flow.external_version) + downloaded_flow_id = openml.flows.flow_exists( + flow.name, + flow.external_version, + ) self.assertEqual(downloaded_flow_id, flow.flow_id) def test_sklearn_to_upload_to_flow(self): @@ -329,11 +354,19 @@ def test_sklearn_to_upload_to_flow(self): ('pca', pca), ('fs', fs)]) boosting = sklearn.ensemble.AdaBoostClassifier( base_estimator=sklearn.tree.DecisionTreeClassifier()) - model = sklearn.pipeline.Pipeline(steps=[('ohe', ohe), ('scaler', scaler), - ('fu', fu), ('boosting', boosting)]) - parameter_grid = {'boosting__n_estimators': [1, 5, 10, 100], - 'boosting__learning_rate': scipy.stats.uniform(0.01, 0.99), - 'boosting__base_estimator__max_depth': scipy.stats.randint(1, 10)} + model = sklearn.pipeline.Pipeline( + steps=[ + ('ohe', ohe), + ('scaler', scaler), + ('fu', fu), + ('boosting', boosting), + ] + ) + parameter_grid = { + 'boosting__n_estimators': [1, 5, 10, 100], + 'boosting__learning_rate': scipy.stats.uniform(0.01, 0.99), + 'boosting__base_estimator__max_depth': scipy.stats.randint(1, 10), + } cv = sklearn.model_selection.StratifiedKFold(n_splits=5, shuffle=True) rs = sklearn.model_selection.RandomizedSearchCV( estimator=model, param_distributions=parameter_grid, cv=cv) @@ -364,10 +397,16 @@ def test_sklearn_to_upload_to_flow(self): for i in range(10): # Make sure that we replace all occurences of two newlines local_xml = local_xml.replace(sentinel, '') - local_xml = local_xml.replace(' ', '').replace('\t', '').strip().replace('\n\n', '\n').replace('"', '"') + local_xml = ( + local_xml.replace(' ', '').replace('\t', ''). + strip().replace('\n\n', '\n').replace('"', '"') + ) local_xml = re.sub(r'(^$)', '', local_xml) server_xml = server_xml.replace(sentinel, '') - server_xml = server_xml.replace(' ', '').replace('\t', '').strip().replace('\n\n', '\n').replace('"', '"') + server_xml = ( + server_xml.replace(' ', '').replace('\t', ''). + strip().replace('\n\n', '\n').replace('"', '"') + ) server_xml = re.sub(r'^$', '', server_xml) self.assertEqual(server_xml, local_xml) @@ -380,16 +419,19 @@ def test_sklearn_to_upload_to_flow(self): module_name_encoder = ('_encoders' if LooseVersion(sklearn.__version__) >= "0.20" else 'data') - fixture_name = '%ssklearn.model_selection._search.RandomizedSearchCV(' \ - 'estimator=sklearn.pipeline.Pipeline(' \ - 'ohe=sklearn.preprocessing.%s.OneHotEncoder,' \ - 'scaler=sklearn.preprocessing.data.StandardScaler,' \ - 'fu=sklearn.pipeline.FeatureUnion(' \ - 'pca=sklearn.decomposition.truncated_svd.TruncatedSVD,' \ - 'fs=sklearn.feature_selection.univariate_selection.SelectPercentile),' \ - 'boosting=sklearn.ensemble.weight_boosting.AdaBoostClassifier(' \ - 'base_estimator=sklearn.tree.tree.DecisionTreeClassifier)))' \ - % (sentinel, module_name_encoder) + fixture_name = ( + '%ssklearn.model_selection._search.RandomizedSearchCV(' + 'estimator=sklearn.pipeline.Pipeline(' + 'ohe=sklearn.preprocessing.%s.OneHotEncoder,' + 'scaler=sklearn.preprocessing.data.StandardScaler,' + 'fu=sklearn.pipeline.FeatureUnion(' + 'pca=sklearn.decomposition.truncated_svd.TruncatedSVD,' + 'fs=' + 'sklearn.feature_selection.univariate_selection.SelectPercentile),' + 'boosting=sklearn.ensemble.weight_boosting.AdaBoostClassifier(' + 'base_estimator=sklearn.tree.tree.DecisionTreeClassifier)))' + % (sentinel, module_name_encoder) + ) self.assertEqual(new_flow.name, fixture_name) new_flow.model.fit(X, y) diff --git a/tests/test_flows/test_sklearn.py b/tests/test_flows/test_sklearn.py index 90f8545be..bd13a4408 100644 --- a/tests/test_flows/test_sklearn.py +++ b/tests/test_flows/test_sklearn.py @@ -288,11 +288,21 @@ def test_serialize_pipeline(self): self.assertEqual(len(serialization.parameters), 2) # Hard to compare two representations of a dict due to possibly # different sorting. Making a json makes it easier - self.assertEqual(json.loads(serialization.parameters['steps']), - [{'oml-python:serialized_object': - 'component_reference', 'value': {'key': 'scaler', 'step_name': 'scaler'}}, - {'oml-python:serialized_object': - 'component_reference', 'value': {'key': 'dummy', 'step_name': 'dummy'}}]) + self.assertEqual( + json.loads(serialization.parameters['steps']), + [ + { + 'oml-python:serialized_object': + 'component_reference', + 'value': {'key': 'scaler', 'step_name': 'scaler'} + }, + { + 'oml-python:serialized_object': + 'component_reference', + 'value': {'key': 'dummy', 'step_name': 'dummy'} + } + ] + ) # Checking the sub-component self.assertEqual(len(serialization.components), 2) @@ -301,7 +311,6 @@ def test_serialize_pipeline(self): self.assertIsInstance(serialization.components['dummy'], OpenMLFlow) - #del serialization.model new_model = flow_to_sklearn(serialization) # compares string representations of the dict, as it potentially # contains complex objects that can not be compared with == op @@ -363,11 +372,19 @@ def test_serialize_pipeline_clustering(self): self.assertEqual(len(serialization.parameters), 2) # Hard to compare two representations of a dict due to possibly # different sorting. Making a json makes it easier - self.assertEqual(json.loads(serialization.parameters['steps']), - [{'oml-python:serialized_object': - 'component_reference', 'value': {'key': 'scaler', 'step_name': 'scaler'}}, - {'oml-python:serialized_object': - 'component_reference', 'value': {'key': 'clusterer', 'step_name': 'clusterer'}}]) + self.assertEqual( + json.loads(serialization.parameters['steps']), + [ + { + 'oml-python:serialized_object': 'component_reference', + 'value': {'key': 'scaler', 'step_name': 'scaler'} + }, + { + 'oml-python:serialized_object': 'component_reference', + 'value': {'key': 'clusterer', 'step_name': 'clusterer'} + }, + ] + ) # Checking the sub-component self.assertEqual(len(serialization.components), 2) @@ -684,21 +701,33 @@ def test_serialize_rvs(self): supported_rv.__dict__) def test_serialize_function(self): - serialized = sklearn_to_flow(sklearn.feature_selection.chi2) + serialized = sklearn_to_flow(sklearn.feature_selection.chi2) deserialized = flow_to_sklearn(serialized) self.assertEqual(deserialized, sklearn.feature_selection.chi2) def test_serialize_cvobject(self): methods = [sklearn.model_selection.KFold(3), sklearn.model_selection.LeaveOneOut()] - fixtures = [OrderedDict([('oml-python:serialized_object', 'cv_object'), - ('value', OrderedDict([('name', 'sklearn.model_selection._split.KFold'), - ('parameters', OrderedDict([('n_splits', '3'), - ('random_state', 'null'), - ('shuffle', 'false')]))]))]), - OrderedDict([('oml-python:serialized_object', 'cv_object'), - ('value', OrderedDict([('name', 'sklearn.model_selection._split.LeaveOneOut'), - ('parameters', OrderedDict())]))])] + fixtures = [ + OrderedDict([ + ('oml-python:serialized_object', 'cv_object'), + ('value', OrderedDict([ + ('name', 'sklearn.model_selection._split.KFold'), + ('parameters', OrderedDict([ + ('n_splits', '3'), + ('random_state', 'null'), + ('shuffle', 'false'), + ])) + ])) + ]), + OrderedDict([ + ('oml-python:serialized_object', 'cv_object'), + ('value', OrderedDict([ + ('name', 'sklearn.model_selection._split.LeaveOneOut'), + ('parameters', OrderedDict()) + ])) + ]), + ] for method, fixture in zip(methods, fixtures): m = sklearn_to_flow(method) self.assertEqual(m, fixture) @@ -794,7 +823,7 @@ def test_serialize_advanced_grid(self): def test_serialize_resampling(self): kfold = sklearn.model_selection.StratifiedKFold( n_splits=4, shuffle=True) - serialized = sklearn_to_flow(kfold) + serialized = sklearn_to_flow(kfold) deserialized = flow_to_sklearn(serialized) # Best approximation to get_params() self.assertEqual(str(deserialized), str(kfold)) @@ -967,7 +996,9 @@ def test__get_fn_arguments_with_defaults(self): ] for fn, num_params_with_defaults in fns: - defaults, defaultless = openml.flows.sklearn_converter._get_fn_arguments_with_defaults(fn) + defaults, defaultless = ( + openml.flows.sklearn_converter._get_fn_arguments_with_defaults(fn) + ) self.assertIsInstance(defaults, dict) self.assertIsInstance(defaultless, set) # check whether we have both defaults and defaultless params @@ -1030,12 +1061,20 @@ def test_deserialize_complex_with_defaults(self): # used the 'initialize_with_defaults' flag of the deserialization # method to return a flow that contains default hyperparameter # settings. - steps = [('Imputer', Imputer()), - ('OneHotEncoder', sklearn.preprocessing.OneHotEncoder()), - ('Estimator', sklearn.ensemble.AdaBoostClassifier( - sklearn.ensemble.BaggingClassifier( + steps = [ + ('Imputer', Imputer()), + ('OneHotEncoder', sklearn.preprocessing.OneHotEncoder()), + ( + 'Estimator', + sklearn.ensemble.AdaBoostClassifier( + sklearn.ensemble.BaggingClassifier( sklearn.ensemble.GradientBoostingClassifier( - sklearn.neighbors.KNeighborsClassifier()))))] + sklearn.neighbors.KNeighborsClassifier() + ) + ) + ) + ), + ] pipe_orig = sklearn.pipeline.Pipeline(steps=steps) pipe_adjusted = sklearn.clone(pipe_orig) @@ -1047,7 +1086,10 @@ def test_deserialize_complex_with_defaults(self): 'Estimator__base_estimator__base_estimator__loss__n_neighbors': 13} pipe_adjusted.set_params(**params) flow = openml.flows.sklearn_to_flow(pipe_adjusted) - pipe_deserialized = openml.flows.flow_to_sklearn(flow, initialize_with_defaults=True) + pipe_deserialized = openml.flows.flow_to_sklearn( + flow, + initialize_with_defaults=True, + ) # we want to compare pipe_deserialized and pipe_orig. We use the flow # equals function for this From 00397386af12df5e50b323d76bfeb8fcb431e64a Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Sat, 23 Feb 2019 15:48:46 +0200 Subject: [PATCH 30/35] PEP8. --- tests/test_datasets/test_dataset_functions.py | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py index 3bd0f5dfd..60ca1c386 100644 --- a/tests/test_datasets/test_dataset_functions.py +++ b/tests/test_datasets/test_dataset_functions.py @@ -54,7 +54,8 @@ def _remove_pickle_files(self): 'dataset.pkl') try: os.remove(pickle_path) - except: + except (OSError, FileNotFoundError): + # Replaced a bare except. Not sure why either of these would be acceptable. pass def _get_empty_param_for_dataset(self): @@ -108,39 +109,38 @@ def test_get_cached_dataset_description(self): def test_get_cached_dataset_description_not_cached(self): openml.config.cache_directory = self.static_cache_dir - self.assertRaisesRegex(OpenMLCacheException, "Dataset description for " - "dataset id 3 not cached", - openml.datasets.functions._get_cached_dataset_description, - 3) + self.assertRaisesRegex(OpenMLCacheException, + "Dataset description for dataset id 3 not cached", + openml.datasets.functions._get_cached_dataset_description, + dataset_id=3) def test_get_cached_dataset_arff(self): openml.config.cache_directory = self.static_cache_dir - description = openml.datasets.functions._get_cached_dataset_arff( - dataset_id=2) + description = openml.datasets.functions._get_cached_dataset_arff(dataset_id=2) self.assertIsInstance(description, str) def test_get_cached_dataset_arff_not_cached(self): openml.config.cache_directory = self.static_cache_dir - self.assertRaisesRegex(OpenMLCacheException, "ARFF file for " - "dataset id 3 not cached", - openml.datasets.functions._get_cached_dataset_arff, - 3) + self.assertRaisesRegex(OpenMLCacheException, + "ARFF file for dataset id 3 not cached", + openml.datasets.functions._get_cached_dataset_arff, + dataset_id=3) def _check_dataset(self, dataset): - self.assertEqual(type(dataset), dict) - self.assertGreaterEqual(len(dataset), 2) - self.assertIn('did', dataset) - self.assertIsInstance(dataset['did'], int) - self.assertIn('status', dataset) - self.assertIsInstance(dataset['status'], str) - self.assertIn(dataset['status'], ['in_preparation', 'active', - 'deactivated']) + self.assertEqual(type(dataset), dict) + self.assertGreaterEqual(len(dataset), 2) + self.assertIn('did', dataset) + self.assertIsInstance(dataset['did'], int) + self.assertIn('status', dataset) + self.assertIsInstance(dataset['status'], str) + self.assertIn(dataset['status'], ['in_preparation', 'active', 'deactivated']) + def _check_datasets(self, datasets): for did in datasets: self._check_dataset(datasets[did]) def test_tag_untag_dataset(self): - tag = 'test_tag_%d' %random.randint(1, 1000000) + tag = 'test_tag_%d' % random.randint(1, 1000000) all_tags = _tag_entity('data', 1, tag) self.assertTrue(tag in all_tags) all_tags = _tag_entity('data', 1, tag, untag=True) @@ -185,7 +185,9 @@ def test_list_datasets_by_number_missing_values(self): self._check_datasets(datasets) def test_list_datasets_combined_filters(self): - datasets = openml.datasets.list_datasets(tag='study_14', number_instances="100..1000", number_missing_values="800..1000") + datasets = openml.datasets.list_datasets(tag='study_14', + number_instances="100..1000", + number_missing_values="800..1000") self.assertGreaterEqual(len(datasets), 1) self._check_datasets(datasets) @@ -257,7 +259,6 @@ def test_get_dataset(self): openml.config.server = self.production_server self.assertRaises(PrivateDatasetError, openml.datasets.get_dataset, 45) - def test_get_dataset_with_string(self): dataset = openml.datasets.get_dataset(101) self.assertRaises(PyOpenMLError, dataset._get_arff, 'arff') @@ -329,8 +330,7 @@ def test_deletion_of_cache_dir(self): @mock.patch('openml.datasets.functions._get_dataset_arff') def test_deletion_of_cache_dir_faulty_download(self, patch): patch.side_effect = Exception('Boom!') - self.assertRaisesRegex(Exception, 'Boom!', openml.datasets.get_dataset, - 1) + self.assertRaisesRegex(Exception, 'Boom!', openml.datasets.get_dataset, dataset_id=1) datasets_cache_dir = os.path.join( self.workdir, 'org', 'openml', 'test', 'datasets' ) From b9482ba0b166cb802d0d2a987155e4047991f18c Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Sat, 23 Feb 2019 16:02:53 +0200 Subject: [PATCH 31/35] PEP8. --- openml/flows/functions.py | 2 +- .../test_evaluation_functions.py | 20 ++++++----- tests/test_flows/test_flow_functions.py | 34 +++++++++---------- 3 files changed, 29 insertions(+), 27 deletions(-) diff --git a/openml/flows/functions.py b/openml/flows/functions.py index 2620dbcdc..ab3e6fd5d 100644 --- a/openml/flows/functions.py +++ b/openml/flows/functions.py @@ -197,7 +197,7 @@ def assert_flows_equal(flow1, flow2, flow2 : OpenMLFlow - ignore_parameter_values_on_older_children : str + ignore_parameter_values_on_older_children : str (optional) If set to ``OpenMLFlow.upload_date``, ignores parameters in a child flow if it's upload date predates the upload date of the parent flow. diff --git a/tests/test_evaluations/test_evaluation_functions.py b/tests/test_evaluations/test_evaluation_functions.py index 0254f2b4d..37e8f710d 100644 --- a/tests/test_evaluations/test_evaluation_functions.py +++ b/tests/test_evaluations/test_evaluation_functions.py @@ -11,7 +11,8 @@ def test_evaluation_list_filter_task(self): task_id = 7312 - evaluations = openml.evaluations.list_evaluations("predictive_accuracy", task=[task_id]) + evaluations = openml.evaluations.list_evaluations("predictive_accuracy", + task=[task_id]) self.assertGreater(len(evaluations), 100) for run_id in evaluations.keys(): @@ -25,8 +26,8 @@ def test_evaluation_list_filter_uploader_ID_16(self): openml.config.server = self.production_server uploader_id = 16 - - evaluations = openml.evaluations.list_evaluations("predictive_accuracy", uploader=[uploader_id]) + evaluations = openml.evaluations.list_evaluations("predictive_accuracy", + uploader=[uploader_id]) self.assertGreater(len(evaluations), 50) @@ -34,8 +35,8 @@ def test_evaluation_list_filter_uploader_ID_10(self): openml.config.server = self.production_server setup_id = 10 - - evaluations = openml.evaluations.list_evaluations("predictive_accuracy", setup=[setup_id]) + evaluations = openml.evaluations.list_evaluations("predictive_accuracy", + setup=[setup_id]) self.assertGreater(len(evaluations), 50) for run_id in evaluations.keys(): @@ -50,7 +51,8 @@ def test_evaluation_list_filter_flow(self): flow_id = 100 - evaluations = openml.evaluations.list_evaluations("predictive_accuracy", flow=[flow_id]) + evaluations = openml.evaluations.list_evaluations("predictive_accuracy", + flow=[flow_id]) self.assertGreater(len(evaluations), 2) for run_id in evaluations.keys(): @@ -65,7 +67,8 @@ def test_evaluation_list_filter_run(self): run_id = 12 - evaluations = openml.evaluations.list_evaluations("predictive_accuracy", id=[run_id]) + evaluations = openml.evaluations.list_evaluations("predictive_accuracy", + id=[run_id]) self.assertEqual(len(evaluations), 1) for run_id in evaluations.keys(): @@ -78,7 +81,8 @@ def test_evaluation_list_filter_run(self): def test_evaluation_list_limit(self): openml.config.server = self.production_server - evaluations = openml.evaluations.list_evaluations("predictive_accuracy", size=100, offset=100) + evaluations = openml.evaluations.list_evaluations("predictive_accuracy", + size=100, offset=100) self.assertEqual(len(evaluations), 100) def test_list_evaluations_empty(self): diff --git a/tests/test_flows/test_flow_functions.py b/tests/test_flows/test_flow_functions.py index e6f567fa0..7e5cf181c 100644 --- a/tests/test_flows/test_flow_functions.py +++ b/tests/test_flows/test_flow_functions.py @@ -15,10 +15,10 @@ def _check_flow(self, flow): self.assertIsInstance(flow['name'], str) self.assertIsInstance(flow['full_name'], str) self.assertIsInstance(flow['version'], str) - # There are some runs on openml.org that can have an empty external - # version - self.assertTrue(isinstance(flow['external_version'], str) - or flow['external_version'] is None) # noqa W503 + # There are some runs on openml.org that can have an empty external version + ext_version_str_or_none = (isinstance(flow['external_version'], str) + or flow['external_version'] is None) + self.assertTrue(ext_version_str_or_none) def test_list_flows(self): openml.config.server = self.production_server @@ -191,6 +191,8 @@ def test_are_flows_equal_ignore_parameter_values(self): def test_are_flows_equal_ignore_if_older(self): paramaters = OrderedDict((('a', 5), ('b', 6))) parameters_meta_info = OrderedDict((('a', None), ('b', None))) + flow_upload_date = '2017-01-31T12-01-01' + assert_flows_equal = openml.flows.functions.assert_flows_equal flow = openml.flows.OpenMLFlow(name='Test', description='Test flow', @@ -204,22 +206,18 @@ def test_are_flows_equal_ignore_if_older(self): dependencies='abc', class_name='Test', custom_name='Test', - upload_date='2017-01-31T12-01-01') + upload_date=flow_upload_date) - openml.flows.functions.assert_flows_equal(flow, flow, - ignore_parameter_values_on_older_children='2017-01-31T12-01-01') - openml.flows.functions.assert_flows_equal(flow, flow, - ignore_parameter_values_on_older_children=None) + assert_flows_equal(flow, flow, ignore_parameter_values_on_older_children=flow_upload_date) + assert_flows_equal(flow, flow, ignore_parameter_values_on_older_children=None) new_flow = copy.deepcopy(flow) new_flow.parameters['a'] = 7 - self.assertRaises(ValueError, openml.flows.functions.assert_flows_equal, - flow, new_flow, ignore_parameter_values_on_older_children='2017-01-31T12-01-01') - self.assertRaises(ValueError, openml.flows.functions.assert_flows_equal, - flow, new_flow, ignore_parameter_values_on_older_children=None) + self.assertRaises(ValueError, assert_flows_equal, flow, new_flow, + ignore_parameter_values_on_older_children=flow_upload_date) + self.assertRaises(ValueError, assert_flows_equal, flow, new_flow, + ignore_parameter_values_on_older_children=None) new_flow.upload_date = '2016-01-31T12-01-01' - self.assertRaises(ValueError, openml.flows.functions.assert_flows_equal, - flow, new_flow, - ignore_parameter_values_on_older_children='2017-01-31T12-01-01') - openml.flows.functions.assert_flows_equal(flow, flow, - ignore_parameter_values_on_older_children=None) + self.assertRaises(ValueError, assert_flows_equal, flow, new_flow, + ignore_parameter_values_on_older_children=flow_upload_date) + assert_flows_equal(flow, flow, gnore_parameter_values_on_older_children=None) From a2ef3f961972a774c46652c30d8739cca45220fd Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Sat, 23 Feb 2019 16:11:51 +0200 Subject: [PATCH 32/35] PEP8. Removed import 'unused' import statements. Have to watch unit tests. --- tests/test_flows/dummy_learn/dummy_forest.py | 2 +- tests/test_runs/test_run_functions.py | 7 +++---- tests/test_setups/__init__.py | 2 +- tests/test_setups/test_setup_functions.py | 5 +---- tests/test_study/test_study_examples.py | 14 ++++++-------- 5 files changed, 12 insertions(+), 18 deletions(-) diff --git a/tests/test_flows/dummy_learn/dummy_forest.py b/tests/test_flows/dummy_learn/dummy_forest.py index b01473cbe..06eaab62e 100644 --- a/tests/test_flows/dummy_learn/dummy_forest.py +++ b/tests/test_flows/dummy_learn/dummy_forest.py @@ -9,4 +9,4 @@ def get_params(self, deep=False): return {} def set_params(self, params): - return self \ No newline at end of file + return self diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index 397c49369..3977c1601 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -19,7 +19,7 @@ from openml.testing import TestBase from openml.runs.functions import _run_task_get_arffcontent, \ _get_seeded_model, _run_exists, _extract_arfftrace, \ - _extract_arfftrace_attributes, _prediction_to_row, _check_n_jobs + _extract_arfftrace_attributes, _prediction_to_row from openml.flows.sklearn_converter import sklearn_to_flow from openml.runs.trace import OpenMLRunTrace from openml.tasks import TaskTypeEnum @@ -911,7 +911,7 @@ def test__run_exists(self): avoid_duplicate_runs=True, ) run.publish() - except openml.exceptions.PyOpenMLError as e: + except openml.exceptions.PyOpenMLError: # run already existed. Great. pass @@ -1400,12 +1400,11 @@ def test_run_on_dataset_with_missing_labels(self): # actual data task = openml.tasks.get_task(2) - class_labels = task.class_labels model = Pipeline(steps=[('Imputer', Imputer(strategy='median')), ('Estimator', DecisionTreeClassifier())]) - data_content, _, _, _ = _run_task_get_arffcontent( + data_content, _, _, _ = _run_task_get_arffcontent( model, task, add_local_measures=True, diff --git a/tests/test_setups/__init__.py b/tests/test_setups/__init__.py index d6b0c7b1a..dc5287024 100644 --- a/tests/test_setups/__init__.py +++ b/tests/test_setups/__init__.py @@ -1,3 +1,3 @@ # Dummy to allow mock classes in the test files to have a version number for # their parent module -__version__ = '0.1' \ No newline at end of file +__version__ = '0.1' diff --git a/tests/test_setups/test_setup_functions.py b/tests/test_setups/test_setup_functions.py index 351960428..fe7267d4b 100644 --- a/tests/test_setups/test_setup_functions.py +++ b/tests/test_setups/test_setup_functions.py @@ -1,4 +1,3 @@ -import sys import hashlib import time @@ -6,9 +5,7 @@ import openml.exceptions from openml.testing import TestBase -from sklearn.ensemble import BaggingClassifier from sklearn.tree import DecisionTreeClassifier -from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.base import BaseEstimator, ClassifierMixin @@ -133,7 +130,7 @@ def test_setup_list_filter_flow(self): setups = openml.setups.list_setups(flow=flow_id) - self.assertGreater(len(setups), 0) # TODO: please adjust 0 + self.assertGreater(len(setups), 0) # TODO: please adjust 0 for setup_id in setups.keys(): self.assertEqual(setups[setup_id].flow_id, flow_id) diff --git a/tests/test_study/test_study_examples.py b/tests/test_study/test_study_examples.py index aa894a9a1..79c5c7cf4 100644 --- a/tests/test_study/test_study_examples.py +++ b/tests/test_study/test_study_examples.py @@ -7,10 +7,9 @@ class TestStudyFunctions(TestBase): def test_Figure1a(self): """Test listing in Figure 1a on a single task and the old OpenML100 study. - - The original listing is pasted into the comment below because it the - actual unit test differs a bit, as for example it does not run for all tasks, - but only a single one. + + The original listing is pasted into the comment below because it the actual unit test + differs a bit, as for example it does not run for all tasks, but only a single one. import openml import sklearn.tree, sklearn.preprocessing @@ -25,9 +24,9 @@ def test_Figure1a(self): print('Data set: %s; Accuracy: %0.2f' % (task.get_dataset().name,score.mean())) run.publish() # publish the experiment on OpenML (optional) print('URL for run: %s/run/%d' %(openml.config.server,run.run_id)) - """ + """ # noqa: E501 import openml - import sklearn.tree, sklearn.preprocessing + import sklearn.preprocessing benchmark_suite = openml.study.get_study( 'OpenML100', 'tasks' ) # obtain the benchmark suite @@ -47,7 +46,6 @@ def test_Figure1a(self): score = run.get_metric_fn( sklearn.metrics.accuracy_score ) # print accuracy score - print('Data set: %s; Accuracy: %0.2f' % ( - task.get_dataset().name, score.mean())) + print('Data set: %s; Accuracy: %0.2f' % (task.get_dataset().name, score.mean())) run.publish() # publish the experiment on OpenML (optional) print('URL for run: %s/run/%d' % (openml.config.server, run.run_id)) From 792adb23f1949de796b2156cfbc1fabc63d6c3a6 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Sat, 23 Feb 2019 15:48:58 +0100 Subject: [PATCH 33/35] Fix bug regarding not existing argument --- tests/test_flows/test_flow_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_flows/test_flow_functions.py b/tests/test_flows/test_flow_functions.py index 7e5cf181c..3e5717b31 100644 --- a/tests/test_flows/test_flow_functions.py +++ b/tests/test_flows/test_flow_functions.py @@ -220,4 +220,4 @@ def test_are_flows_equal_ignore_if_older(self): new_flow.upload_date = '2016-01-31T12-01-01' self.assertRaises(ValueError, assert_flows_equal, flow, new_flow, ignore_parameter_values_on_older_children=flow_upload_date) - assert_flows_equal(flow, flow, gnore_parameter_values_on_older_children=None) + assert_flows_equal(flow, flow, ignore_parameter_values_on_older_children=None) From 9a45d03dc7ee35cde3e43f26d2f799096fb5c3b9 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Sat, 23 Feb 2019 17:05:03 +0200 Subject: [PATCH 34/35] PEP8. --- tests/test_flows/test_flow_functions.py | 2 +- tests/test_study/test_study_functions.py | 3 ++- tests/test_tasks/test_split.py | 9 +++++---- tests/test_tasks/test_task_functions.py | 12 +++++++----- tests/test_utils/test_utils.py | 2 +- 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/tests/test_flows/test_flow_functions.py b/tests/test_flows/test_flow_functions.py index 7e5cf181c..3e5717b31 100644 --- a/tests/test_flows/test_flow_functions.py +++ b/tests/test_flows/test_flow_functions.py @@ -220,4 +220,4 @@ def test_are_flows_equal_ignore_if_older(self): new_flow.upload_date = '2016-01-31T12-01-01' self.assertRaises(ValueError, assert_flows_equal, flow, new_flow, ignore_parameter_values_on_older_children=flow_upload_date) - assert_flows_equal(flow, flow, gnore_parameter_values_on_older_children=None) + assert_flows_equal(flow, flow, ignore_parameter_values_on_older_children=None) diff --git a/tests/test_study/test_study_functions.py b/tests/test_study/test_study_functions.py index 10f6ec725..cb9af5e7b 100644 --- a/tests/test_study/test_study_functions.py +++ b/tests/test_study/test_study_functions.py @@ -2,6 +2,7 @@ import openml.study from openml.testing import TestBase + class TestStudyFunctions(TestBase): _multiprocess_can_split_ = True @@ -127,6 +128,6 @@ def test_publish_study(self): openml.study.status_update(study_id, 'deactivated') study_downloaded = openml.study.get_study(study_id) self.assertEqual(study_downloaded.status, 'deactivated') - + res = openml.study.delete_study(study_id) self.assertTrue(res) diff --git a/tests/test_tasks/test_split.py b/tests/test_tasks/test_split.py index 3cd4c90b3..46c6564a1 100644 --- a/tests/test_tasks/test_split.py +++ b/tests/test_tasks/test_split.py @@ -1,6 +1,5 @@ import inspect import os -import unittest import numpy as np @@ -26,7 +25,8 @@ def setUp(self): def tearDown(self): try: os.remove(self.pd_filename) - except: + except (OSError, FileNotFoundError): + # Replaced bare except. Not sure why these exceptions are acceptable. pass def test_eq(self): @@ -64,8 +64,9 @@ def test_from_arff_file(self): for j in range(10): self.assertGreaterEqual(split.split[i][j][0].train.shape[0], 808) self.assertGreaterEqual(split.split[i][j][0].test.shape[0], 89) - self.assertEqual(split.split[i][j][0].train.shape[0] + - split.split[i][j][0].test.shape[0], 898) + self.assertEqual(split.split[i][j][0].train.shape[0] + + split.split[i][j][0].test.shape[0], + 898) def test_get_split(self): split = OpenMLSplit._from_arff_file(self.arff_filename) diff --git a/tests/test_tasks/test_task_functions.py b/tests/test_tasks/test_task_functions.py index 4befc6193..867c14d1b 100644 --- a/tests/test_tasks/test_task_functions.py +++ b/tests/test_tasks/test_task_functions.py @@ -56,8 +56,8 @@ def _check_task(self, task): ['in_preparation', 'active', 'deactivated']) def test_list_tasks_by_type(self): - num_curves_tasks = 200 # number is flexible, check server if fails - ttid=3 + num_curves_tasks = 200 # number is flexible, check server if fails + ttid = 3 tasks = openml.tasks.list_tasks(task_type_id=ttid) self.assertGreaterEqual(len(tasks), num_curves_tasks) for tid in tasks: @@ -72,7 +72,7 @@ def test_list_tasks_empty(self): self.assertIsInstance(tasks, dict) def test_list_tasks_by_tag(self): - num_basic_tasks = 100 # number is flexible, check server if fails + num_basic_tasks = 100 # number is flexible, check server if fails tasks = openml.tasks.list_tasks(tag='study_14') self.assertGreaterEqual(len(tasks), num_basic_tasks) for tid in tasks: @@ -97,7 +97,7 @@ def test_list_tasks_per_type_paginate(self): size = 10 max = 100 task_types = 4 - for j in range(1,task_types): + for j in range(1, task_types): for i in range(0, max, size): tasks = openml.tasks.list_tasks(task_type_id=j, offset=i, size=size) self.assertGreaterEqual(size, len(tasks)) @@ -109,7 +109,7 @@ def test__get_task(self): openml.config.cache_directory = self.static_cache_dir openml.tasks.get_task(1882) - @unittest.skip("Please await outcome of discussion: https://github.com/openml/OpenML/issues/776") + @unittest.skip("Please await outcome of discussion: https://github.com/openml/OpenML/issues/776") # noqa: E501 def test__get_task_live(self): # Test the following task as it used to throw an Unicode Error. # https://github.com/openml/openml-python/issues/378 @@ -133,10 +133,12 @@ def test_get_task(self): def test_removal_upon_download_failure(self, get_dataset): class WeirdException(Exception): pass + def assert_and_raise(*args, **kwargs): # Make sure that the file was created! assert os.path.join(os.getcwd(), "tasks", "1", "tasks.xml") raise WeirdException() + get_dataset.side_effect = assert_and_raise try: openml.tasks.get_task(1) diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index d12a07471..a50ac5cb0 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -21,7 +21,7 @@ def mocked_perform_api_call(call, request_method): def test_list_all(self): openml.utils._list_all(openml.tasks.functions._list_tasks) - @mock.patch('openml._api_calls._perform_api_call', + @mock.patch('openml._api_calls._perform_api_call', side_effect=mocked_perform_api_call) def test_list_all_few_results_available(self, _perform_api_call): # we want to make sure that the number of api calls is only 1. From 4e384757c8b854b9a426ed25092c87fe63b1dee6 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Sat, 23 Feb 2019 17:22:35 +0200 Subject: [PATCH 35/35] Flake8 ignore directive. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index ce34960fe..51a2a6cea 100644 --- a/setup.py +++ b/setup.py @@ -9,13 +9,13 @@ dependency_links = [] try: - import numpy + import numpy # noqa: F401 except ImportError: print('numpy is required during installation') sys.exit(1) try: - import scipy + import scipy # noqa: F401 except ImportError: print('scipy is required during installation') sys.exit(1)