This is an automated email from the ASF dual-hosted git repository.
zhasheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new 6372037 [MXAPPS-581] Fixes for broken Straight Dope tests. (#11923)
6372037 is described below
commit 6372037ec4eb56b609a184712e5ec993a21faad0
Author: vishaalkapoor <[email protected]>
AuthorDate: Sun Jul 29 12:40:32 2018 -0700
[MXAPPS-581] Fixes for broken Straight Dope tests. (#11923)
* Update relative paths pointing to the data directory to point to the
correct place in the testing temporary folder.
* Enable the notebooks that were previously broken because of relative
file paths not pointing to the correct place.
* Move some notebooks we do not plan to test to the whitelist. These
notebooks are not published in the Straight Dope book.
* Clean-up: Convert print statements to info/warn/error logging
statements. Add some logging statements for better status.
---
.../straight_dope/straight_dope_test_utils.py | 39 +++++++--
.../straight_dope/test_notebooks_single_gpu.py | 94 ++++++----------------
tests/utils/notebook_test/__init__.py | 7 +-
3 files changed, 61 insertions(+), 79 deletions(-)
diff --git a/tests/nightly/straight_dope/straight_dope_test_utils.py
b/tests/nightly/straight_dope/straight_dope_test_utils.py
index bb64f37..ee499a5 100644
--- a/tests/nightly/straight_dope/straight_dope_test_utils.py
+++ b/tests/nightly/straight_dope/straight_dope_test_utils.py
@@ -24,6 +24,7 @@
the notebook. e.g: `export MXNET_TEST_KERNEL=python2`
"""
import io
+import logging
import os
import re
import shutil
@@ -40,6 +41,7 @@ GIT_PATH = '/usr/bin/git'
GIT_REPO = 'https://github.com/zackchase/mxnet-the-straight-dope'
KERNEL = os.getenv('MXNET_TEST_KERNEL', None)
NOTEBOOKS_DIR = os.path.join(os.path.dirname(__file__), 'tmp_notebook')
+RELATIVE_DATA_PATH_REGEX = r'\.\.\/data\/' # Regular expression to match the
relative data path.
def _test_notebook(notebook, override_epochs=True):
"""Run Jupyter notebook to catch any execution error.
@@ -47,13 +49,18 @@ def _test_notebook(notebook, override_epochs=True):
Args:
notebook : string
notebook name in folder/notebook format
- epochs : boolean
+ override_epochs : boolean
whether or not to override the number of epochs to 1
+
Returns:
True if the notebook runs without warning or error.
"""
+ # Some notebooks will fail to run without error if we do not override the
data path.
+ _override_data_path(notebook)
+
if override_epochs:
_override_epochs(notebook)
+
return run_notebook(notebook, NOTEBOOKS_DIR, kernel=KERNEL,
temp_dir=NOTEBOOKS_DIR)
@@ -63,15 +70,14 @@ def _override_epochs(notebook):
Args:
notebook : string
notebook name in folder/notebook format
-
"""
notebook_path = os.path.join(*([NOTEBOOKS_DIR] + notebook.split('/'))) +
".ipynb"
- # Read the notebook and set epochs to num_epochs
+ # Read the notebook and set epochs to num_epochs.
with io.open(notebook_path, 'r', encoding='utf-8') as f:
notebook = f.read()
- # Set number of epochs to 1
+ # Set number of epochs to 1.
modified_notebook = re.sub(EPOCHS_REGEX, 'epochs = 1', notebook)
# Replace the original notebook with the modified one.
@@ -79,13 +85,34 @@ def _override_epochs(notebook):
f.write(modified_notebook)
+def _override_data_path(notebook):
+ """Overrides the relative path for the data directory to point to the
right place. This is
+ required as we run the notebooks in a different directory hierarchy more
suitable for testing.
+
+ Args:
+ notebook : string
+ notebook name in folder/notebook format
+ """
+ notebook_path = os.path.join(*([NOTEBOOKS_DIR] + notebook.split('/'))) +
".ipynb"
+
+ # Read the notebook and set epochs to num_epochs.
+ with io.open(notebook_path, 'r', encoding='utf-8') as f:
+ notebook = f.read()
+
+ # Update the location for the data directory.
+ modified_notebook = re.sub(RELATIVE_DATA_PATH_REGEX, NOTEBOOKS_DIR +
'/data/', notebook)
+
+ # Replace the original notebook with the modified one.
+ with io.open(notebook_path, 'w', encoding='utf-8') as f:
+ f.write(modified_notebook)
+
def _download_straight_dope_notebooks():
"""Downloads the Straight Dope Notebooks.
Returns:
True if it succeeds in downloading the notebooks without error.
"""
- print('Cleaning and setting up notebooks directory
"{}"'.format(NOTEBOOKS_DIR))
+ logging.info('Cleaning and setting up notebooks directory
"{}"'.format(NOTEBOOKS_DIR))
shutil.rmtree(NOTEBOOKS_DIR, ignore_errors=True)
cmd = [GIT_PATH,
@@ -98,7 +125,7 @@ def _download_straight_dope_notebooks():
if proc.returncode != 0:
err_msg = 'Error downloading Straight Dope notebooks.\n'
err_msg += msg
- print(err_msg)
+ logging.error(err_msg)
return False
return True
diff --git a/tests/nightly/straight_dope/test_notebooks_single_gpu.py
b/tests/nightly/straight_dope/test_notebooks_single_gpu.py
index b87d16c..ee7c94c 100644
--- a/tests/nightly/straight_dope/test_notebooks_single_gpu.py
+++ b/tests/nightly/straight_dope/test_notebooks_single_gpu.py
@@ -32,9 +32,19 @@ NOTEBOOKS_WHITELIST = [
'chapter01_crashcourse/introduction',
'chapter01_crashcourse/chapter-one-problem-set',
'chapter02_supervised-learning/environment',
+ 'chapter03_deep-neural-networks/kaggle-gluon-kfold',
'chapter07_distributed-learning/multiple-gpus-scratch',
'chapter07_distributed-learning/multiple-gpus-gluon',
- 'chapter07_distributed-learning/training-with-multiple-machines'
+ 'chapter07_distributed-learning/training-with-multiple-machines',
+ 'chapter12_time-series/intro-forecasting-gluon',
+ 'chapter12_time-series/intro-forecasting-2-gluon',
+ 'chapter13_unsupervised-learning/vae-gluon',
+ 'chapter18_variational-methods-and-uncertainty/bayes-by-backprop-rnn',
+ 'chapter17_deep-reinforcement-learning/DQN',
+ 'chapter17_deep-reinforcement-learning/DDQN',
+ 'chapter19_graph-neural-networks/Graph-Neural-Networks',
+ 'chapter16_tensor_methods/tensor_basics',
+ 'cheatsheets/kaggle-gluon-kfold'
]
@@ -91,10 +101,8 @@ class StraightDopeSingleGpuTests(unittest.TestCase):
def test_linear_regression_gluon(self):
assert
_test_notebook('chapter02_supervised-learning/linear-regression-gluon')
- # TODO(vishaalk): There is a relative file path needs to be fixed so that
the
- # python code can be run from another directory.
- #def test_logistic_regression_gluon(self):
- # assert
_test_notebook('chapter02_supervised-learning/logistic-regression-gluon')
+ def test_logistic_regression_gluon(self):
+ assert
_test_notebook('chapter02_supervised-learning/logistic-regression-gluon')
def test_softmax_regression_scratch(self):
assert
_test_notebook('chapter02_supervised-learning/softmax-regression-scratch')
@@ -132,9 +140,6 @@ class StraightDopeSingleGpuTests(unittest.TestCase):
def test_custom_layer(self):
assert _test_notebook('chapter03_deep-neural-networks/custom-layer')
- #def test_kaggle_gluon_kfold(self):
- # assert
_test_notebook('chapter03_deep-neural-networks/kaggle-gluon-kfold')
-
# TODO(vishaalk): Load params and Save params are deprecated warning.
#def test_serialization(self):
# assert _test_notebook('chapter03_deep-neural-networks/serialization')
@@ -162,20 +167,14 @@ class StraightDopeSingleGpuTests(unittest.TestCase):
# Chapter 5
- # TODO(vishaalk): There is a relative file path needs to be fixed so that
the
- # python code can be run from another directory.
- #def test_simple_rnn(self):
- # assert
_test_notebook('chapter05_recurrent-neural-networks/simple-rnn')
+ def test_simple_rnn(self):
+ assert _test_notebook('chapter05_recurrent-neural-networks/simple-rnn')
- # TODO(vishaalk): There is a relative file path needs to be fixed so that
the
- # python code can be run from another directory.
- #def test_lstm_scratch(self):
- # assert
_test_notebook('chapter05_recurrent-neural-networks/lstm-scratch')
+ def test_lstm_scratch(self):
+ assert
_test_notebook('chapter05_recurrent-neural-networks/lstm-scratch')
- # TODO(vishaalk): There is a relative file path needs to be fixed so that
the
- # python code can be run from another directory.
- #def test_gru_scratch(self):
- # assert
_test_notebook('chapter05_recurrent-neural-networks/gru-scratch')
+ def test_gru_scratch(self):
+ assert
_test_notebook('chapter05_recurrent-neural-networks/gru-scratch')
#def test_rnns_gluon(self):
# assert
_test_notebook('chapter05_recurrent-neural-networks/rnns-gluon')
@@ -263,19 +262,6 @@ class StraightDopeSingleGpuTests(unittest.TestCase):
#def test_issm_scratch(self):
# assert _test_notebook('chapter12_time-series/issm-scratch')
- # TODO(vishaalk): Error: sequential1_batchnorm0_running_mean' has not been
initialized
- # def test_intro_forecasting_gluon(self):
- # assert _test_notebook('chapter12_time-series/intro-forecasting-gluon')
-
- #def test_intro_forecasting_2_gluon(self):
- # assert
_test_notebook('chapter12_time-series/intro-forecasting-2-gluon')
-
- # Chapter 13
-
- # TODO(vishaalk): Load params and Save params are deprecated warning.
- #def test_vae_gluon(self):
- # assert _test_notebook('chapter13_unsupervised-learning/vae-gluon')
-
# Chapter 14
def test_igan_intro(self):
@@ -287,46 +273,14 @@ class StraightDopeSingleGpuTests(unittest.TestCase):
def test_generative_adversarial_networks(self):
assert
_test_notebook('chapter14_generative-adversarial-networks/conditional')
- # Chapter 16
-
- # TODO(vishaalk): Checked failed oshape.Size() != dshape.Size()
- #def test_tensor_basics(self):
- # assert _test_notebook('chapter16_tensor_methods/tensor_basics')
-
# TODO(vishaalk): Notebook does not appear to be valid JSON.
#def test_pixel2pixel(self):
# assert
_test_notebook('chapter14_generative-adversarial-networks/pixel2pixel')
- # Chapter 17
-
- # TODO(vishaalk): Requires OpenAI Gym. Also uses deprecated load_params.
- #def test_dqn(self):
-# assert _test_notebook('chapter17_deep-reinforcement-learning/DQN')
-
-#def test_ddqn(self):
-# assert _test_notebook('chapter17_deep-reinforcement-learning/DDQN')
-
-# Chapter 18
-
-#def test_bayes_by_backprop(self):
-# assert
_test_notebook('chapter18_variational-methods-and-uncertainty/bayes-by-backprop')
-
-#def test_bayes_by_backprop_gluon(self):
-# assert
_test_notebook('chapter18_variational-methods-and-uncertainty/bayes-by-backprop-gluon')
-
-# TODO(vishaalk): AttributeError: 'list' object has no attribute 'keys'
-#def test_bayes_by_backprop_rnn(self):
-# assert
_test_notebook('chapter18_variational-methods-and-uncertainty/bayes-by-backprop-rnn')
-
-# Chapter 19
-
-# TODO(vishaalk): Requires deepchem
-#def test_graph_neural_networks(self):
-# assert
_test_notebook('chapter19_graph-neural-networks/Graph-Neural-Networks')
+ # Chapter 18
-# Cheatsheets
+ #def test_bayes_by_backprop(self):
+ # assert
_test_notebook('chapter18_variational-methods-and-uncertainty/bayes-by-backprop')
-# TODO(vishaalk): There is a relative file path needs to be fixed so that the
-# python code can be run from another directory.
-#def test_kaggle_gluon_kfold(self):
-# assert _test_notebook('cheatsheets/kaggle-gluon-kfold')
+ #def test_bayes_by_backprop_gluon(self):
+ # assert
_test_notebook('chapter18_variational-methods-and-uncertainty/bayes-by-backprop-gluon')
diff --git a/tests/utils/notebook_test/__init__.py
b/tests/utils/notebook_test/__init__.py
index cb5282f..2cdb613 100644
--- a/tests/utils/notebook_test/__init__.py
+++ b/tests/utils/notebook_test/__init__.py
@@ -21,6 +21,7 @@
warning or exception.
"""
import io
+import logging
import os
import shutil
import time
@@ -57,12 +58,12 @@ def run_notebook(notebook, notebook_dir, kernel=None,
no_cache=False, temp_dir='
-------
Returns true if the workbook runs with no warning or exception.
"""
-
+ logging.info("Running notebook '{}'".format(notebook))
notebook_path = os.path.join(*([notebook_dir] + notebook.split('/')))
working_dir = os.path.join(*([temp_dir] + notebook.split('/')))
if no_cache == '1':
- print("Cleaning and setting up temp directory
'{}'".format(working_dir))
+ logging.info("Cleaning and setting up temp directory
'{}'".format(working_dir))
shutil.rmtree(temp_dir, ignore_errors=True)
errors = []
@@ -92,6 +93,6 @@ def run_notebook(notebook, notebook_dir, kernel=None,
no_cache=False, temp_dir='
if "Warning:" in line:
errors.append("Warning:\n" + line)
if len(errors) > 0:
- print('\n'.join(errors))
+ logging.error('\n'.join(errors))
return False
return True
