From 2f7b8cdd2deca61710049a1313a122fc347c76ec Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Tue, 15 Oct 2019 22:55:41 +0200 Subject: [PATCH 1/3] add new example regarding svm hyperparameter plotting --- .../plot_svm_hyperparameters_tutorial.py | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 examples/30_extended/plot_svm_hyperparameters_tutorial.py diff --git a/examples/30_extended/plot_svm_hyperparameters_tutorial.py b/examples/30_extended/plot_svm_hyperparameters_tutorial.py new file mode 100644 index 000000000..f10497106 --- /dev/null +++ b/examples/30_extended/plot_svm_hyperparameters_tutorial.py @@ -0,0 +1,74 @@ +""" +================================ +Plotting hyperparameter surfaces +================================ +""" +import openml +import numpy as np + +# Choose an SVM flow, for example 8353, and a task. +df = openml.evaluations.list_evaluations_setups( + function='predictive_accuracy', + flow=[8353], + task=[6], + output_format='dataframe', + parameters_in_separate_columns=True, +) +hyperparameters = ['sklearn.svm.classes.SVC(16)_C', 'sklearn.svm.classes.SVC(16)_gamma'] +df[hyperparameters] = df[hyperparameters].astype(float).apply(np.log) + +#################################################################################################### +# Option 1 - plotting via the pandas helper functions +# =================================================== +# +df.plot.hexbin( + x='sklearn.svm.classes.SVC(16)_C', + y='sklearn.svm.classes.SVC(16)_gamma', + C='value', reduce_C_function=np.mean, gridsize=25, +) + +#################################################################################################### +# Option 2 - plotting via matplotlib +# ================================== +# +import matplotlib.pyplot as plt + +fig, ax = plt.subplots() + +C = df['sklearn.svm.classes.SVC(16)_C'] +gamma = df['sklearn.svm.classes.SVC(16)_gamma'] +score = df['value'] + +# Plotting all evaluations: +ax.plot(C, gamma, 'ko', ms=1) +# Create a contour plot +cntr = ax.tricontourf(C, gamma, score, levels=12, cmap="RdBu_r") +# Adjusting the colorbar +fig.colorbar(cntr, ax=ax, label="accuracy") +# Adjusting the axis limits +ax.set( + xlim=[min(C),max(C)], + ylim=[min(gamma),max(gamma)], + xlabel="C (log10)", + ylabel="gamma (log10)", +) + +#################################################################################################### +# Option 3 - exact code example from the OpenML-Python paper +# ========================================================== +# + +import openml +import numpy as np +import matplotlib.pyplot as plt +df = openml.evaluations.list_evaluations_setups( + 'predictive_accuracy', flow=[8353], task=[6], + output_format='dataframe', parameters_in_separate_columns=True, +) # Choose an SVM flow, for example 8353, and a task. +hp_names = ['sklearn.svm.classes.SVC(16)_C','sklearn.svm.classes.SVC(16)_gamma'] +df[hp_names] = df[hp_names].astype(float).apply(np.log) +C, gamma, score = df[hp_names[0]], df[hp_names[1]], df['value'] +cntr = plt.tricontourf(C, gamma, score, levels=12, cmap="RdBu_r") +plt.colorbar(cntr, label="accuracy") +plt.xlim((min(C), max(C))); plt.ylim((min(gamma), max(gamma))) +plt.xlabel("C (log10)"); plt.ylabel("gamma (log10)") From 69be8c6e4f6c51c83b1633ca18bf9ad6090cf5b7 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Wed, 16 Oct 2019 13:56:54 +0200 Subject: [PATCH 2/3] implement Neeratyoy's suggestions --- .../plot_svm_hyperparameters_tutorial.py | 42 +++++++++---------- .../test_evaluations_example.py | 31 ++++++++++++++ 2 files changed, 52 insertions(+), 21 deletions(-) create mode 100644 tests/test_evaluations/test_evaluations_example.py diff --git a/examples/30_extended/plot_svm_hyperparameters_tutorial.py b/examples/30_extended/plot_svm_hyperparameters_tutorial.py index f10497106..da2fbd35b 100644 --- a/examples/30_extended/plot_svm_hyperparameters_tutorial.py +++ b/examples/30_extended/plot_svm_hyperparameters_tutorial.py @@ -6,14 +6,34 @@ import openml import numpy as np -# Choose an SVM flow, for example 8353, and a task. +#################################################################################################### +# First step - obtaining the data +# =============================== +# First, we nood to choose an SVM flow, for example 8353, and a task. Finding the IDs of them are +# not part of this tutorial, this could for example be done via the website. +# +# For this we use the function ``list_evaluations_setup`` which can automatically join +# evaluations conducted by the server with the hyperparameter settings extracted from the +# uploaded runs (called *setup*). df = openml.evaluations.list_evaluations_setups( function='predictive_accuracy', flow=[8353], task=[6], output_format='dataframe', + # Using this flag incorporates the hyperparameters into the returned dataframe. Otherwise, + # the dataframe would contain a field ``paramaters`` containing an unparsed dictionary. parameters_in_separate_columns=True, ) +print(df.head(n=10)) + +#################################################################################################### +# We can see all the hyperparameter names in the columns of the dataframe: +for name in df.columns: + print(name) + +#################################################################################################### +# Next, we cast and transform the hyperparameters of interest (``C`` and ``gamma``) so that we +# can nicely plot them. hyperparameters = ['sklearn.svm.classes.SVC(16)_C', 'sklearn.svm.classes.SVC(16)_gamma'] df[hyperparameters] = df[hyperparameters].astype(float).apply(np.log) @@ -52,23 +72,3 @@ xlabel="C (log10)", ylabel="gamma (log10)", ) - -#################################################################################################### -# Option 3 - exact code example from the OpenML-Python paper -# ========================================================== -# - -import openml -import numpy as np -import matplotlib.pyplot as plt -df = openml.evaluations.list_evaluations_setups( - 'predictive_accuracy', flow=[8353], task=[6], - output_format='dataframe', parameters_in_separate_columns=True, -) # Choose an SVM flow, for example 8353, and a task. -hp_names = ['sklearn.svm.classes.SVC(16)_C','sklearn.svm.classes.SVC(16)_gamma'] -df[hp_names] = df[hp_names].astype(float).apply(np.log) -C, gamma, score = df[hp_names[0]], df[hp_names[1]], df['value'] -cntr = plt.tricontourf(C, gamma, score, levels=12, cmap="RdBu_r") -plt.colorbar(cntr, label="accuracy") -plt.xlim((min(C), max(C))); plt.ylim((min(gamma), max(gamma))) -plt.xlabel("C (log10)"); plt.ylabel("gamma (log10)") diff --git a/tests/test_evaluations/test_evaluations_example.py b/tests/test_evaluations/test_evaluations_example.py new file mode 100644 index 000000000..bb222608b --- /dev/null +++ b/tests/test_evaluations/test_evaluations_example.py @@ -0,0 +1,31 @@ +import unittest + + +class TestEvaluationsExample(unittest.TestCase): + + def test_example_python_paper(self): + # Example script which will appear in the upcoming OpenML-Python paper + # This test ensures that the example will keep running! + + import openml + import numpy as np + import matplotlib.pyplot as plt + + df = openml.evaluations.list_evaluations_setups( + 'predictive_accuracy', + flow=[8353], + task=[6], + output_format='dataframe', + parameters_in_separate_columns=True, + ) # Choose an SVM flow, for example 8353, and a task. + + hp_names = ['sklearn.svm.classes.SVC(16)_C', 'sklearn.svm.classes.SVC(16)_gamma'] + df[hp_names] = df[hp_names].astype(float).apply(np.log) + C, gamma, score = df[hp_names[0]], df[hp_names[1]], df['value'] + + cntr = plt.tricontourf(C, gamma, score, levels=12, cmap="RdBu_r") + plt.colorbar(cntr, label="accuracy") + plt.xlim((min(C), max(C))) + plt.ylim((min(gamma), max(gamma))) + plt.xlabel("C (log10)") + plt.ylabel("gamma (log10)") From 1b96f37a20464d00f63128139ff64cfdea6210f4 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Wed, 16 Oct 2019 15:19:21 +0200 Subject: [PATCH 3/3] add title & fix pep8 --- .../30_extended/plot_svm_hyperparameters_tutorial.py | 10 +++++++--- tests/test_evaluations/test_evaluations_example.py | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/examples/30_extended/plot_svm_hyperparameters_tutorial.py b/examples/30_extended/plot_svm_hyperparameters_tutorial.py index da2fbd35b..714e64221 100644 --- a/examples/30_extended/plot_svm_hyperparameters_tutorial.py +++ b/examples/30_extended/plot_svm_hyperparameters_tutorial.py @@ -44,7 +44,10 @@ df.plot.hexbin( x='sklearn.svm.classes.SVC(16)_C', y='sklearn.svm.classes.SVC(16)_gamma', - C='value', reduce_C_function=np.mean, gridsize=25, + C='value', + reduce_C_function=np.mean, + gridsize=25, + title='SVM performance landscape', ) #################################################################################################### @@ -67,8 +70,9 @@ fig.colorbar(cntr, ax=ax, label="accuracy") # Adjusting the axis limits ax.set( - xlim=[min(C),max(C)], - ylim=[min(gamma),max(gamma)], + xlim=(min(C), max(C)), + ylim=(min(gamma), max(gamma)), xlabel="C (log10)", ylabel="gamma (log10)", ) +ax.set_title('SVM performance landscape') diff --git a/tests/test_evaluations/test_evaluations_example.py b/tests/test_evaluations/test_evaluations_example.py index bb222608b..0d75c928e 100644 --- a/tests/test_evaluations/test_evaluations_example.py +++ b/tests/test_evaluations/test_evaluations_example.py @@ -22,7 +22,7 @@ def test_example_python_paper(self): hp_names = ['sklearn.svm.classes.SVC(16)_C', 'sklearn.svm.classes.SVC(16)_gamma'] df[hp_names] = df[hp_names].astype(float).apply(np.log) C, gamma, score = df[hp_names[0]], df[hp_names[1]], df['value'] - + cntr = plt.tricontourf(C, gamma, score, levels=12, cmap="RdBu_r") plt.colorbar(cntr, label="accuracy") plt.xlim((min(C), max(C)))