From 99d7bc0556998d8c4f14d3134fbf648be50ee7a3 Mon Sep 17 00:00:00 2001 From: Vishaal Kapoor Date: Mon, 1 Oct 2018 11:04:27 -0700 Subject: [PATCH 1/4] [#12345] Enabling two tests in the Straight Dope Nightly. Two straight dope notebook tests were disabled due to a timeout so they were disabled. I've updated one of the notebooks (rnn-gluon) to use the gpu instead of the cpu so it takes ~ 5 minutes on a p3.2xl, and verified the other notebook takes a minute and was a false alarm (visual-qa). The PR in the Straight Dope is: https://github.com/zackchase/mxnet-the-straight-dope/pull/540 --- tests/nightly/straight_dope/test_notebooks_single_gpu.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/nightly/straight_dope/test_notebooks_single_gpu.py b/tests/nightly/straight_dope/test_notebooks_single_gpu.py index 5eeb52f516e1..fc3a4f240ddc 100644 --- a/tests/nightly/straight_dope/test_notebooks_single_gpu.py +++ b/tests/nightly/straight_dope/test_notebooks_single_gpu.py @@ -35,13 +35,11 @@ 'chapter02_supervised-learning/environment', 'chapter03_deep-neural-networks/kaggle-gluon-kfold', 'chapter04_convolutional-neural-networks/deep-cnns-alexnet', # > 10 mins. - 'chapter05_recurrent-neural-networks/rnns-gluon', # > 10 mins. 'chapter06_optimization/gd-sgd-scratch', # Overflow warning is intended. 'chapter06_optimization/gd-sgd-gluon', # Overflow warning is intended. 'chapter07_distributed-learning/multiple-gpus-scratch', 'chapter07_distributed-learning/multiple-gpus-gluon', 'chapter07_distributed-learning/training-with-multiple-machines', - 'chapter08_computer-vision/visual-question-answer', # > 10 mins. 'chapter11_recommender-systems/intro-recommender-systems', # Early draft, non-working. 'chapter12_time-series/intro-forecasting-gluon', 'chapter12_time-series/intro-forecasting-2-gluon', @@ -178,6 +176,9 @@ def test_lstm_scratch(self): def test_gru_scratch(self): assert _test_notebook('chapter05_recurrent-neural-networks/gru-scratch') + def test_rnn_gluon(self): + assert _test_notebook('chapter05_recurrent-neural-networks/rnns-gluon') + # Chapter 6 def test_optimization_intro(self): @@ -227,6 +228,10 @@ def test_object_detection(self): def test_fine_tuning(self): assert _test_notebook('chapter08_computer-vision/fine-tuning') + def test_visual_qa(self): + assert _test_notebook('chapter08_computer-vision/visual-question-answer') + + # Chapter 9 def test_tree_lstm(self): From 665ef57d034c2ee7f8b22e7d9f1cd38f72c35335 Mon Sep 17 00:00:00 2001 From: Vishaal Kapoor Date: Fri, 5 Oct 2018 11:47:13 -0700 Subject: [PATCH 2/4] Add dependency for IPython update. --- ci/docker/install/ubuntu_nightly_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/docker/install/ubuntu_nightly_tests.sh b/ci/docker/install/ubuntu_nightly_tests.sh index 68358908bdc9..406985ea3a4a 100755 --- a/ci/docker/install/ubuntu_nightly_tests.sh +++ b/ci/docker/install/ubuntu_nightly_tests.sh @@ -32,5 +32,5 @@ apt-get -y install time apt-get install -y subversion maven -y #>/dev/null # Packages needed for the Straight Dope Nightly tests. -pip2 install pandas scikit-image -pip3 install pandas scikit-image +pip2 install pandas scikit-image prompt_toolkit +pip3 install pandas scikit-image prompt_toolkit From deb3130c46e2e0bc733fe761a5f39388bec80956 Mon Sep 17 00:00:00 2001 From: Vishaal Kapoor Date: Fri, 5 Oct 2018 14:43:02 -0700 Subject: [PATCH 3/4] Detect errors in notebook execution failure. --- tests/utils/notebook_test/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/utils/notebook_test/__init__.py b/tests/utils/notebook_test/__init__.py index 25e96ab0fc55..572119debc64 100644 --- a/tests/utils/notebook_test/__init__.py +++ b/tests/utils/notebook_test/__init__.py @@ -80,11 +80,13 @@ def run_notebook(notebook, notebook_dir, kernel=None, no_cache=False, temp_dir=' else: eprocessor = ExecutePreprocessor(timeout=TIME_OUT) + success = False # There is a low (< 1%) chance that starting a notebook executor will fail due to the kernel # taking to long to start, or a port collision, etc. for i in range(RETRIES): try: nb, _ = eprocessor.preprocess(notebook, {'metadata': {'path': working_dir}}) + success = True except RuntimeError as rte: # We check if the exception has to do with the Jupyter kernel failing to start. If # not, we rethrow to prevent the notebook from erring RETRIES times. It is not ideal @@ -97,6 +99,8 @@ def run_notebook(notebook, notebook_dir, kernel=None, no_cache=False, temp_dir=' time.sleep(1) continue break + if not success: + errors.append('Error: Notebook failed to run.') except Exception as err: err_msg = str(err) errors.append(err_msg) From 8056706e16b033f3b6b7a7689a0d53fa4441c487 Mon Sep 17 00:00:00 2001 From: Vishaal Kapoor Date: Fri, 5 Oct 2018 15:17:25 -0700 Subject: [PATCH 4/4] Clean up of naming in retry code. --- tests/utils/notebook_test/__init__.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/utils/notebook_test/__init__.py b/tests/utils/notebook_test/__init__.py index 572119debc64..a32c5269b812 100644 --- a/tests/utils/notebook_test/__init__.py +++ b/tests/utils/notebook_test/__init__.py @@ -32,7 +32,7 @@ IPYTHON_VERSION = 4 # Pin to ipython version 4. TIME_OUT = 10*60 # Maximum 10 mins/test. Reaching timeout causes test failure. -RETRIES = 8 +ATTEMPTS = 8 KERNEL_ERROR_MSG = 'Kernel died before replying to kernel_info' @@ -83,24 +83,25 @@ def run_notebook(notebook, notebook_dir, kernel=None, no_cache=False, temp_dir=' success = False # There is a low (< 1%) chance that starting a notebook executor will fail due to the kernel # taking to long to start, or a port collision, etc. - for i in range(RETRIES): + for i in range(ATTEMPTS): try: nb, _ = eprocessor.preprocess(notebook, {'metadata': {'path': working_dir}}) success = True except RuntimeError as rte: # We check if the exception has to do with the Jupyter kernel failing to start. If - # not, we rethrow to prevent the notebook from erring RETRIES times. It is not ideal - # to inspect the exception message, but necessary for retry logic, as Jupyter client - # throws the generic RuntimeError that can be confused with other Runtime errors. + # not, we rethrow to prevent the notebook from erring ATTEMPTS times. It is not + # ideal to inspect the exception message, but necessary for retry logic, as Jupyter + # client throws the generic RuntimeError that can be confused with other Runtime + # errors. if str(rte) != KERNEL_ERROR_MSG: raise rte - logging.info("Error starting preprocessor: {}. Attempt {}/{}".format(str(rte), i+1, RETRIES)) + logging.info("Error starting preprocessor: {}. Attempt {}/{}".format(str(rte), i+1, ATTEMPTS)) time.sleep(1) continue break if not success: - errors.append('Error: Notebook failed to run.') + errors.append("Error: Notebook failed to run after {} attempts.".format(ATTEMPTS)) except Exception as err: err_msg = str(err) errors.append(err_msg)