ThomasDelteil commented on a change in pull request #10495: [MXNET-307] Add
tutorials to the CI + Fix them
URL: https://github.com/apache/incubator-mxnet/pull/10495#discussion_r180914315
##
File path: tests/nightly/test_tutorial.py
##
@@ -25,87 +25,116 @@
import os
import warnings
import imp
-
+import shutil
+import time
+import argparse
import traceback
import nbformat
from nbconvert.preprocessors import ExecutePreprocessor
+import sys
fail_dict = {}
+TIME_OUT = 1800
-def test_tutorial(file_path):
-"""Run tutorial python script and save any error or warning.
- If no error or warning occurs, run notebook.
-
-Parameters
---
-file_path : str
-path of tutorial markdown file
-"""
-with warnings.catch_warnings(record=True) as w:
-tutorial_name = os.path.basename(file_path)
-print file_path + '.py'
-try:
-imp.load_source('tutorial', file_path + '.py')
-if len(w) > 0:
-err_msg = "%s.py has %d warnings.\n" % (tutorial_name, len(w))
-fail_dict[tutorial_name] = err_msg
-else:
-test_tutorial_nb(file_path)
-except Exception:
-err_msg = "%s.py has error:\n%s" % (tutorial_name,
traceback.format_exc())
-fail_dict[tutorial_name] = err_msg
-
-def test_tutorial_nb(file_path):
+def test_tutorial_nb(file_path, workingdir, kernel=None):
"""Run tutorial jupyter notebook to catch any execution error.
Parameters
--
file_path : str
-path of tutorial markdown file
+path of tutorial .ipynb file
+workingdir: str
+path of the directory to run the tutorial in
+kernel: str
+Default None
+name of the kernel to use, if none, will use first kernel
+in the list
"""
tutorial_name = os.path.basename(file_path)
+sys.stdout.write('Testing {}...'.format(file_path))
+sys.stdout.flush()
+tick = time.time()
notebook = nbformat.read(file_path + '.ipynb', as_version=4)
-eprocessor = ExecutePreprocessor(timeout=1800)
+if kernel:
+eprocessor = ExecutePreprocessor(timeout=TIME_OUT, kernel_name=kernel)
+else:
+eprocessor = ExecutePreprocessor(timeout=TIME_OUT)
+success = True
try:
-eprocessor.preprocess(notebook, {'metadata': {}})
+os.environ['MXNET_STORAGE_FALLBACK_LOG_VERBOSE'] = '0'
+os.environ['MXNET_CUDNN_AUTOTUNE_DEFAULT'] = '0'
+eprocessor.preprocess(notebook, {'metadata': {'path':workingdir}})
except Exception as err:
err_msg = str(err)
fail_dict[tutorial_name] = err_msg
+success = False
finally:
-output_nb = open("output.txt", mode='w')
+output_file = os.path.join(workingdir, "output.txt")
+output_nb = open(output_file, mode='w')
nbformat.write(notebook, output_nb)
output_nb.close()
-output_nb = open("output.txt", mode='r')
+output_nb = open(output_file, mode='r')
for line in output_nb:
if "Warning:" in line:
-fail_dict[tutorial_name] = "%s has warning." % (tutorial_name)
-return
+success = False
+if tutorial_name in fail_dict:
+fail_dict[tutorial_name] += "\n"+line
+else:
+fail_dict[tutorial_name] = "Warning:\n"+line
+sys.stdout.write(' Elapsed time: {0:.2f}s '.format(time.time()-tick ))
+sys.stdout.write(' [{}] \n'.format('Success' if success else 'Failed'))
+sys.stdout.flush()
if __name__ == "__main__":
-tutorial_dir = '../../docs/_build/html/tutorials/'
-with open('test_tutorial_config.txt') as config_file:
-tutorial_list = []
-for line in config_file:
-tutorial_list.append(line.lstrip().rstrip())
-file_dir = tutorial_dir + line.lstrip().rstrip()
-test_tutorial_nb(file_dir)
+tutorial_dir = os.path.join('..','..','docs', '_build', 'html',
'tutorials')
+tick = time.time()
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--tutorial", help="tutorial to test, if not set, read
from test_tutorial_config.txt")
+parser.add_argument("--kernel", help="name of the jupyter kernel to use
for the test")
+parser.add_argument("--no-cache", help="clean the temp directory",
action="store_true", dest="no_cache")
+args = parser.parse_args()
+
+
+tutorial_list = []
+if args.tutorial:
+tutorial_list.append(args.tutorial)
+else:
+with open('test_tutorial_config.txt') as config_file:
+for line in config_file:
+tutorial_list.append(line.lstrip().rstrip())
+
+temp_dir = 'tmp_notebook'
+if args.no_cache:
+print("Cleaning and setting up temp directory '{}'".format(temp_dir))
+shutil.rmtree(temp_dir,