From 5681e451bc9e11a0f9d0213c3138871729992959 Mon Sep 17 00:00:00 2001 From: sahithyaravi1493 Date: Thu, 17 Oct 2019 10:07:07 +0200 Subject: [PATCH 1/7] fix list_evaluations_setups --- doc/progress.rst | 1 + openml/evaluations/functions.py | 6 ++++-- .../test_evaluation_functions.py | 18 +++++++++++++----- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/doc/progress.rst b/doc/progress.rst index 4b227cd2f..355e4f058 100644 --- a/doc/progress.rst +++ b/doc/progress.rst @@ -8,6 +8,7 @@ Changelog 0.10.0 ~~~~~~ +* FIX #838: Fix list_evaluations_setups to work when evaluations are not a 100 multiple. * ADD #737: Add list_evaluations_setups to return hyperparameters along with list of evaluations. * FIX #261: Test server is cleared of all files uploaded during unit testing. * FIX #447: All files created by unit tests no longer persist in local. diff --git a/openml/evaluations/functions.py b/openml/evaluations/functions.py index 044f49370..95a8de0c0 100644 --- a/openml/evaluations/functions.py +++ b/openml/evaluations/functions.py @@ -324,14 +324,15 @@ def list_evaluations_setups( evals = list_evaluations(function=function, offset=offset, size=size, run=run, task=task, setup=setup, flow=flow, uploader=uploader, tag=tag, per_fold=per_fold, sort_order=sort_order, output_format='dataframe') - + print("evals done") # List setups # Split setups in evals into chunks of N setups as list_setups does not support large size df = pd.DataFrame() if len(evals) != 0: N = 100 - setup_chunks = np.split(evals['setup_id'].unique(), + setup_chunks = np.array_split(evals['setup_id'].unique(), ((len(evals['setup_id'].unique()) - 1) // N) + 1) + print( len(evals['setup_id'].unique()),((len(evals['setup_id'].unique()) - 1) // N) + 1) setups = pd.DataFrame() for setup in setup_chunks: result = pd.DataFrame(openml.setups.list_setups(setup=setup, output_format='dataframe')) @@ -353,6 +354,7 @@ def list_evaluations_setups( if parameters_in_separate_columns: df = pd.concat([df.drop('parameters', axis=1), df['parameters'].apply(pd.Series)], axis=1) + print("done") if output_format == 'dataframe': return df diff --git a/tests/test_evaluations/test_evaluation_functions.py b/tests/test_evaluations/test_evaluation_functions.py index 7dac00891..9afc64c7e 100644 --- a/tests/test_evaluations/test_evaluation_functions.py +++ b/tests/test_evaluations/test_evaluation_functions.py @@ -6,18 +6,20 @@ class TestEvaluationFunctions(TestBase): _multiprocess_can_split_ = True - def _check_list_evaluation_setups(self, size, **kwargs): + def _check_list_evaluation_setups(self, **kwargs): evals_setups = openml.evaluations.list_evaluations_setups("predictive_accuracy", - **kwargs, size=size, + **kwargs, sort_order='desc', output_format='dataframe') evals = openml.evaluations.list_evaluations("predictive_accuracy", - **kwargs, size=size, + **kwargs, sort_order='desc', output_format='dataframe') # Check if list is non-empty self.assertGreater(len(evals_setups), 0) + # Check if length is accurate + self.assertEqual(len(evals_setups), len(evals)) # Check if output from sort is sorted in the right order self.assertSequenceEqual(sorted(evals_setups['value'].tolist(), reverse=True), evals_setups['value'].tolist()) @@ -176,7 +178,7 @@ def test_list_evaluations_setups_filter_flow(self): openml.config.server = self.production_server flow_id = [405] size = 100 - evals = self._check_list_evaluation_setups(size, flow=flow_id) + evals = self._check_list_evaluation_setups(flow=flow_id, size=size) # check if parameters in separate columns works evals_cols = openml.evaluations.list_evaluations_setups("predictive_accuracy", flow=flow_id, size=size, @@ -192,4 +194,10 @@ def test_list_evaluations_setups_filter_task(self): openml.config.server = self.production_server task_id = [6] size = 100 - self._check_list_evaluation_setups(size, task=task_id) + self._check_list_evaluation_setups(task=task_id, size=size) + + def test_list_evaluations_setups_not_100_multiple(self): + openml.config.server = self.production_server + task_id = [37] + flow_id = [5891] + self._check_list_evaluation_setups(task=task_id, flow=flow_id) From 5d2f1df5eef71a4cfe3e241d7afe0abff366a5e0 Mon Sep 17 00:00:00 2001 From: sahithyaravi1493 Date: Thu, 17 Oct 2019 10:22:48 +0200 Subject: [PATCH 2/7] edit existing test --- tests/test_evaluations/test_evaluation_functions.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/test_evaluations/test_evaluation_functions.py b/tests/test_evaluations/test_evaluation_functions.py index 9afc64c7e..7015d5857 100644 --- a/tests/test_evaluations/test_evaluation_functions.py +++ b/tests/test_evaluations/test_evaluation_functions.py @@ -193,11 +193,6 @@ def test_list_evaluations_setups_filter_flow(self): def test_list_evaluations_setups_filter_task(self): openml.config.server = self.production_server task_id = [6] - size = 100 + size = 121 self._check_list_evaluation_setups(task=task_id, size=size) - def test_list_evaluations_setups_not_100_multiple(self): - openml.config.server = self.production_server - task_id = [37] - flow_id = [5891] - self._check_list_evaluation_setups(task=task_id, flow=flow_id) From 968d985bb0b203575f56d69dad3e8bdd8e333748 Mon Sep 17 00:00:00 2001 From: sahithyaravi1493 Date: Thu, 17 Oct 2019 10:27:25 +0200 Subject: [PATCH 3/7] remove prints --- openml/evaluations/functions.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openml/evaluations/functions.py b/openml/evaluations/functions.py index 95a8de0c0..34f7ca33a 100644 --- a/openml/evaluations/functions.py +++ b/openml/evaluations/functions.py @@ -331,8 +331,7 @@ def list_evaluations_setups( if len(evals) != 0: N = 100 setup_chunks = np.array_split(evals['setup_id'].unique(), - ((len(evals['setup_id'].unique()) - 1) // N) + 1) - print( len(evals['setup_id'].unique()),((len(evals['setup_id'].unique()) - 1) // N) + 1) + ((len(evals['setup_id'].unique()) - 1) // N) + 1) setups = pd.DataFrame() for setup in setup_chunks: result = pd.DataFrame(openml.setups.list_setups(setup=setup, output_format='dataframe')) @@ -354,7 +353,6 @@ def list_evaluations_setups( if parameters_in_separate_columns: df = pd.concat([df.drop('parameters', axis=1), df['parameters'].apply(pd.Series)], axis=1) - print("done") if output_format == 'dataframe': return df From c269d7d7bc59f4ee922606ad05fbcb95e07660cb Mon Sep 17 00:00:00 2001 From: sahithyaravi1493 Date: Thu, 17 Oct 2019 10:37:04 +0200 Subject: [PATCH 4/7] remove print --- openml/evaluations/functions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openml/evaluations/functions.py b/openml/evaluations/functions.py index 34f7ca33a..21bdc4b5e 100644 --- a/openml/evaluations/functions.py +++ b/openml/evaluations/functions.py @@ -324,7 +324,6 @@ def list_evaluations_setups( evals = list_evaluations(function=function, offset=offset, size=size, run=run, task=task, setup=setup, flow=flow, uploader=uploader, tag=tag, per_fold=per_fold, sort_order=sort_order, output_format='dataframe') - print("evals done") # List setups # Split setups in evals into chunks of N setups as list_setups does not support large size df = pd.DataFrame() From 112ef38d3988f069dee7a533acaa0203fd26e548 Mon Sep 17 00:00:00 2001 From: sahithyaravi1493 Date: Thu, 17 Oct 2019 10:39:14 +0200 Subject: [PATCH 5/7] remove blank line --- tests/test_evaluations/test_evaluation_functions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_evaluations/test_evaluation_functions.py b/tests/test_evaluations/test_evaluation_functions.py index 7015d5857..21e61c471 100644 --- a/tests/test_evaluations/test_evaluation_functions.py +++ b/tests/test_evaluations/test_evaluation_functions.py @@ -195,4 +195,3 @@ def test_list_evaluations_setups_filter_task(self): task_id = [6] size = 121 self._check_list_evaluation_setups(task=task_id, size=size) - From 634178616fd9a2f00e03185f0dd429b9eab3606c Mon Sep 17 00:00:00 2001 From: sahithyaravi1493 Date: Thu, 17 Oct 2019 11:24:34 +0200 Subject: [PATCH 6/7] add comments --- openml/evaluations/functions.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/openml/evaluations/functions.py b/openml/evaluations/functions.py index 21bdc4b5e..a20de5784 100644 --- a/openml/evaluations/functions.py +++ b/openml/evaluations/functions.py @@ -325,12 +325,16 @@ def list_evaluations_setups( setup=setup, flow=flow, uploader=uploader, tag=tag, per_fold=per_fold, sort_order=sort_order, output_format='dataframe') # List setups - # Split setups in evals into chunks of N setups as list_setups does not support large size + # list_setups by setup id does not support large sizes (exceeds URL length limit) + # Hence we split the list of unique setup ids returned by list_evaluations into chunks of size N df = pd.DataFrame() if len(evals) != 0: - N = 100 - setup_chunks = np.array_split(evals['setup_id'].unique(), - ((len(evals['setup_id'].unique()) - 1) // N) + 1) + N = 100 # size of section + length = len(evals['setup_id'].unique()) # length of the array we want to split + # array_split - allows indices_or_sections to not equally divide the array + # array_split -length % N sub-arrays of size length//N + 1 and the rest of size length//N. + setup_chunks = np.array_split(ary=evals['setup_id'].unique(), + indices_or_sections=((length - 1) // N) + 1) setups = pd.DataFrame() for setup in setup_chunks: result = pd.DataFrame(openml.setups.list_setups(setup=setup, output_format='dataframe')) From ac0404b74b566eb37db50e127801e571c431bd9a Mon Sep 17 00:00:00 2001 From: sahithyaravi1493 Date: Thu, 17 Oct 2019 12:13:27 +0200 Subject: [PATCH 7/7] add space comment --- openml/evaluations/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openml/evaluations/functions.py b/openml/evaluations/functions.py index a20de5784..8de69ebc1 100644 --- a/openml/evaluations/functions.py +++ b/openml/evaluations/functions.py @@ -329,7 +329,7 @@ def list_evaluations_setups( # Hence we split the list of unique setup ids returned by list_evaluations into chunks of size N df = pd.DataFrame() if len(evals) != 0: - N = 100 # size of section + N = 100 # size of section length = len(evals['setup_id'].unique()) # length of the array we want to split # array_split - allows indices_or_sections to not equally divide the array # array_split -length % N sub-arrays of size length//N + 1 and the rest of size length//N.