szha closed pull request #11923: [MXAPPS-581] Fixes for broken Straight Dope
tests.
URL: https://github.com/apache/incubator-mxnet/pull/11923
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/tests/nightly/straight_dope/straight_dope_test_utils.py
b/tests/nightly/straight_dope/straight_dope_test_utils.py
index bb64f37fe5f..ee499a56fbd 100644
--- a/tests/nightly/straight_dope/straight_dope_test_utils.py
+++ b/tests/nightly/straight_dope/straight_dope_test_utils.py
@@ -24,6 +24,7 @@
the notebook. e.g: `export MXNET_TEST_KERNEL=python2`
"""
import io
+import logging
import os
import re
import shutil
@@ -40,6 +41,7 @@
GIT_REPO = 'https://github.com/zackchase/mxnet-the-straight-dope'
KERNEL = os.getenv('MXNET_TEST_KERNEL', None)
NOTEBOOKS_DIR = os.path.join(os.path.dirname(__file__), 'tmp_notebook')
+RELATIVE_DATA_PATH_REGEX = r'\.\.\/data\/' # Regular expression to match the
relative data path.
def _test_notebook(notebook, override_epochs=True):
"""Run Jupyter notebook to catch any execution error.
@@ -47,13 +49,18 @@ def _test_notebook(notebook, override_epochs=True):
Args:
notebook : string
notebook name in folder/notebook format
- epochs : boolean
+ override_epochs : boolean
whether or not to override the number of epochs to 1
+
Returns:
True if the notebook runs without warning or error.
"""
+ # Some notebooks will fail to run without error if we do not override the
data path.
+ _override_data_path(notebook)
+
if override_epochs:
_override_epochs(notebook)
+
return run_notebook(notebook, NOTEBOOKS_DIR, kernel=KERNEL,
temp_dir=NOTEBOOKS_DIR)
@@ -63,15 +70,14 @@ def _override_epochs(notebook):
Args:
notebook : string
notebook name in folder/notebook format
-
"""
notebook_path = os.path.join(*([NOTEBOOKS_DIR] + notebook.split('/'))) +
".ipynb"
- # Read the notebook and set epochs to num_epochs
+ # Read the notebook and set epochs to num_epochs.
with io.open(notebook_path, 'r', encoding='utf-8') as f:
notebook = f.read()
- # Set number of epochs to 1
+ # Set number of epochs to 1.
modified_notebook = re.sub(EPOCHS_REGEX, 'epochs = 1', notebook)
# Replace the original notebook with the modified one.
@@ -79,13 +85,34 @@ def _override_epochs(notebook):
f.write(modified_notebook)
+def _override_data_path(notebook):
+ """Overrides the relative path for the data directory to point to the
right place. This is
+ required as we run the notebooks in a different directory hierarchy more
suitable for testing.
+
+ Args:
+ notebook : string
+ notebook name in folder/notebook format
+ """
+ notebook_path = os.path.join(*([NOTEBOOKS_DIR] + notebook.split('/'))) +
".ipynb"
+
+ # Read the notebook and set epochs to num_epochs.
+ with io.open(notebook_path, 'r', encoding='utf-8') as f:
+ notebook = f.read()
+
+ # Update the location for the data directory.
+ modified_notebook = re.sub(RELATIVE_DATA_PATH_REGEX, NOTEBOOKS_DIR +
'/data/', notebook)
+
+ # Replace the original notebook with the modified one.
+ with io.open(notebook_path, 'w', encoding='utf-8') as f:
+ f.write(modified_notebook)
+
def _download_straight_dope_notebooks():
"""Downloads the Straight Dope Notebooks.
Returns:
True if it succeeds in downloading the notebooks without error.
"""
- print('Cleaning and setting up notebooks directory
"{}"'.format(NOTEBOOKS_DIR))
+ logging.info('Cleaning and setting up notebooks directory
"{}"'.format(NOTEBOOKS_DIR))
shutil.rmtree(NOTEBOOKS_DIR, ignore_errors=True)
cmd = [GIT_PATH,
@@ -98,7 +125,7 @@ def _download_straight_dope_notebooks():
if proc.returncode != 0:
err_msg = 'Error downloading Straight Dope notebooks.\n'
err_msg += msg
- print(err_msg)
+ logging.error(err_msg)
return False
return True
diff --git a/tests/nightly/straight_dope/test_notebooks_single_gpu.py
b/tests/nightly/straight_dope/test_notebooks_single_gpu.py
index b87d16cb0a0..ee7c94c80af 100644
--- a/tests/nightly/straight_dope/test_notebooks_single_gpu.py
+++ b/tests/nightly/straight_dope/test_notebooks_single_gpu.py
@@ -32,9 +32,19 @@
'chapter01_crashcourse/introduction',
'chapter01_crashcourse/chapter-one-problem-set',
'chapter02_supervised-learning/environment',
+ 'chapter03_deep-neural-networks/kaggle-gluon-kfold',
'chapter07_distributed-learning/multiple-gpus-scratch',
'chapter07_distributed-learning/multiple-gpus-gluon',
- 'chapter07_distributed-learning/training-with-multiple-machines'
+ 'chapter07_distributed-learning/training-with-multiple-machines',
+ 'chapter12_time-series/intro-forecasting-gluon',
+ 'chapter12_time-series/intro-forecasting-2-gluon',
+ 'chapter13_unsupervised-learning/vae-gluon',
+ 'chapter18_variational-methods-and-uncertainty/bayes-by-backprop-rnn',
+ 'chapter17_deep-reinforcement-learning/DQN',
+ 'chapter17_deep-reinforcement-learning/DDQN',
+ 'chapter19_graph-neural-networks/Graph-Neural-Networks',
+ 'chapter16_tensor_methods/tensor_basics',
+ 'cheatsheets/kaggle-gluon-kfold'
]
@@ -91,10 +101,8 @@ def test_linear_regression_scratch(self):
def test_linear_regression_gluon(self):
assert
_test_notebook('chapter02_supervised-learning/linear-regression-gluon')
- # TODO(vishaalk): There is a relative file path needs to be fixed so that
the
- # python code can be run from another directory.
- #def test_logistic_regression_gluon(self):
- # assert
_test_notebook('chapter02_supervised-learning/logistic-regression-gluon')
+ def test_logistic_regression_gluon(self):
+ assert
_test_notebook('chapter02_supervised-learning/logistic-regression-gluon')
def test_softmax_regression_scratch(self):
assert
_test_notebook('chapter02_supervised-learning/softmax-regression-scratch')
@@ -132,9 +140,6 @@ def test_plumbing(self):
def test_custom_layer(self):
assert _test_notebook('chapter03_deep-neural-networks/custom-layer')
- #def test_kaggle_gluon_kfold(self):
- # assert
_test_notebook('chapter03_deep-neural-networks/kaggle-gluon-kfold')
-
# TODO(vishaalk): Load params and Save params are deprecated warning.
#def test_serialization(self):
# assert _test_notebook('chapter03_deep-neural-networks/serialization')
@@ -162,20 +167,14 @@ def test_cnn_batch_norm_gluon(self):
# Chapter 5
- # TODO(vishaalk): There is a relative file path needs to be fixed so that
the
- # python code can be run from another directory.
- #def test_simple_rnn(self):
- # assert
_test_notebook('chapter05_recurrent-neural-networks/simple-rnn')
+ def test_simple_rnn(self):
+ assert _test_notebook('chapter05_recurrent-neural-networks/simple-rnn')
- # TODO(vishaalk): There is a relative file path needs to be fixed so that
the
- # python code can be run from another directory.
- #def test_lstm_scratch(self):
- # assert
_test_notebook('chapter05_recurrent-neural-networks/lstm-scratch')
+ def test_lstm_scratch(self):
+ assert
_test_notebook('chapter05_recurrent-neural-networks/lstm-scratch')
- # TODO(vishaalk): There is a relative file path needs to be fixed so that
the
- # python code can be run from another directory.
- #def test_gru_scratch(self):
- # assert
_test_notebook('chapter05_recurrent-neural-networks/gru-scratch')
+ def test_gru_scratch(self):
+ assert
_test_notebook('chapter05_recurrent-neural-networks/gru-scratch')
#def test_rnns_gluon(self):
# assert
_test_notebook('chapter05_recurrent-neural-networks/rnns-gluon')
@@ -263,19 +262,6 @@ def test_lds_scratch(self):
#def test_issm_scratch(self):
# assert _test_notebook('chapter12_time-series/issm-scratch')
- # TODO(vishaalk): Error: sequential1_batchnorm0_running_mean' has not been
initialized
- # def test_intro_forecasting_gluon(self):
- # assert _test_notebook('chapter12_time-series/intro-forecasting-gluon')
-
- #def test_intro_forecasting_2_gluon(self):
- # assert
_test_notebook('chapter12_time-series/intro-forecasting-2-gluon')
-
- # Chapter 13
-
- # TODO(vishaalk): Load params and Save params are deprecated warning.
- #def test_vae_gluon(self):
- # assert _test_notebook('chapter13_unsupervised-learning/vae-gluon')
-
# Chapter 14
def test_igan_intro(self):
@@ -287,46 +273,14 @@ def test_dcgan(self):
def test_generative_adversarial_networks(self):
assert
_test_notebook('chapter14_generative-adversarial-networks/conditional')
- # Chapter 16
-
- # TODO(vishaalk): Checked failed oshape.Size() != dshape.Size()
- #def test_tensor_basics(self):
- # assert _test_notebook('chapter16_tensor_methods/tensor_basics')
-
# TODO(vishaalk): Notebook does not appear to be valid JSON.
#def test_pixel2pixel(self):
# assert
_test_notebook('chapter14_generative-adversarial-networks/pixel2pixel')
- # Chapter 17
-
- # TODO(vishaalk): Requires OpenAI Gym. Also uses deprecated load_params.
- #def test_dqn(self):
-# assert _test_notebook('chapter17_deep-reinforcement-learning/DQN')
-
-#def test_ddqn(self):
-# assert _test_notebook('chapter17_deep-reinforcement-learning/DDQN')
-
-# Chapter 18
-
-#def test_bayes_by_backprop(self):
-# assert
_test_notebook('chapter18_variational-methods-and-uncertainty/bayes-by-backprop')
-
-#def test_bayes_by_backprop_gluon(self):
-# assert
_test_notebook('chapter18_variational-methods-and-uncertainty/bayes-by-backprop-gluon')
-
-# TODO(vishaalk): AttributeError: 'list' object has no attribute 'keys'
-#def test_bayes_by_backprop_rnn(self):
-# assert
_test_notebook('chapter18_variational-methods-and-uncertainty/bayes-by-backprop-rnn')
-
-# Chapter 19
-
-# TODO(vishaalk): Requires deepchem
-#def test_graph_neural_networks(self):
-# assert
_test_notebook('chapter19_graph-neural-networks/Graph-Neural-Networks')
+ # Chapter 18
-# Cheatsheets
+ #def test_bayes_by_backprop(self):
+ # assert
_test_notebook('chapter18_variational-methods-and-uncertainty/bayes-by-backprop')
-# TODO(vishaalk): There is a relative file path needs to be fixed so that the
-# python code can be run from another directory.
-#def test_kaggle_gluon_kfold(self):
-# assert _test_notebook('cheatsheets/kaggle-gluon-kfold')
+ #def test_bayes_by_backprop_gluon(self):
+ # assert
_test_notebook('chapter18_variational-methods-and-uncertainty/bayes-by-backprop-gluon')
diff --git a/tests/utils/notebook_test/__init__.py
b/tests/utils/notebook_test/__init__.py
index cb5282fb404..2cdb6134a60 100644
--- a/tests/utils/notebook_test/__init__.py
+++ b/tests/utils/notebook_test/__init__.py
@@ -21,6 +21,7 @@
warning or exception.
"""
import io
+import logging
import os
import shutil
import time
@@ -57,12 +58,12 @@ def run_notebook(notebook, notebook_dir, kernel=None,
no_cache=False, temp_dir='
-------
Returns true if the workbook runs with no warning or exception.
"""
-
+ logging.info("Running notebook '{}'".format(notebook))
notebook_path = os.path.join(*([notebook_dir] + notebook.split('/')))
working_dir = os.path.join(*([temp_dir] + notebook.split('/')))
if no_cache == '1':
- print("Cleaning and setting up temp directory
'{}'".format(working_dir))
+ logging.info("Cleaning and setting up temp directory
'{}'".format(working_dir))
shutil.rmtree(temp_dir, ignore_errors=True)
errors = []
@@ -92,6 +93,6 @@ def run_notebook(notebook, notebook_dir, kernel=None,
no_cache=False, temp_dir='
if "Warning:" in line:
errors.append("Warning:\n" + line)
if len(errors) > 0:
- print('\n'.join(errors))
+ logging.error('\n'.join(errors))
return False
return True
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
