ThomasDelteil commented on a change in pull request #10495: [MXNET-307] Add
tutorials to the CI + Fix them
URL: https://github.com/apache/incubator-mxnet/pull/10495#discussion_r180914315
##########
File path: tests/nightly/test_tutorial.py
##########
@@ -25,87 +25,116 @@
import os
import warnings
import imp
-
+import shutil
+import time
+import argparse
import traceback
import nbformat
from nbconvert.preprocessors import ExecutePreprocessor
+import sys
fail_dict = {}
+TIME_OUT = 1800
-def test_tutorial(file_path):
- """Run tutorial python script and save any error or warning.
- If no error or warning occurs, run notebook.
-
- Parameters
- ----------
- file_path : str
- path of tutorial markdown file
- """
- with warnings.catch_warnings(record=True) as w:
- tutorial_name = os.path.basename(file_path)
- print file_path + '.py'
- try:
- imp.load_source('tutorial', file_path + '.py')
- if len(w) > 0:
- err_msg = "%s.py has %d warnings.\n" % (tutorial_name, len(w))
- fail_dict[tutorial_name] = err_msg
- else:
- test_tutorial_nb(file_path)
- except Exception:
- err_msg = "%s.py has error:\n%s" % (tutorial_name,
traceback.format_exc())
- fail_dict[tutorial_name] = err_msg
-
-def test_tutorial_nb(file_path):
+def test_tutorial_nb(file_path, workingdir, kernel=None):
"""Run tutorial jupyter notebook to catch any execution error.
Parameters
----------
file_path : str
- path of tutorial markdown file
+ path of tutorial .ipynb file
+ workingdir: str
+ path of the directory to run the tutorial in
+ kernel: str
+ Default None
+ name of the kernel to use, if none, will use first kernel
+ in the list
"""
tutorial_name = os.path.basename(file_path)
+ sys.stdout.write('Testing {}...'.format(file_path))
+ sys.stdout.flush()
+ tick = time.time()
notebook = nbformat.read(file_path + '.ipynb', as_version=4)
- eprocessor = ExecutePreprocessor(timeout=1800)
+ if kernel:
+ eprocessor = ExecutePreprocessor(timeout=TIME_OUT, kernel_name=kernel)
+ else:
+ eprocessor = ExecutePreprocessor(timeout=TIME_OUT)
+ success = True
try:
- eprocessor.preprocess(notebook, {'metadata': {}})
+ os.environ['MXNET_STORAGE_FALLBACK_LOG_VERBOSE'] = '0'
+ os.environ['MXNET_CUDNN_AUTOTUNE_DEFAULT'] = '0'
+ eprocessor.preprocess(notebook, {'metadata': {'path':workingdir}})
except Exception as err:
err_msg = str(err)
fail_dict[tutorial_name] = err_msg
+ success = False
finally:
- output_nb = open("output.txt", mode='w')
+ output_file = os.path.join(workingdir, "output.txt")
+ output_nb = open(output_file, mode='w')
nbformat.write(notebook, output_nb)
output_nb.close()
- output_nb = open("output.txt", mode='r')
+ output_nb = open(output_file, mode='r')
for line in output_nb:
if "Warning:" in line:
- fail_dict[tutorial_name] = "%s has warning." % (tutorial_name)
- return
+ success = False
+ if tutorial_name in fail_dict:
+ fail_dict[tutorial_name] += "\n"+line
+ else:
+ fail_dict[tutorial_name] = "Warning:\n"+line
+ sys.stdout.write(' Elapsed time: {0:.2f}s '.format(time.time()-tick ))
+ sys.stdout.write(' [{}] \n'.format('Success' if success else 'Failed'))
+ sys.stdout.flush()
if __name__ == "__main__":
- tutorial_dir = '../../docs/_build/html/tutorials/'
- with open('test_tutorial_config.txt') as config_file:
- tutorial_list = []
- for line in config_file:
- tutorial_list.append(line.lstrip().rstrip())
- file_dir = tutorial_dir + line.lstrip().rstrip()
- test_tutorial_nb(file_dir)
+ tutorial_dir = os.path.join('..','..','docs', '_build', 'html',
'tutorials')
+ tick = time.time()
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--tutorial", help="tutorial to test, if not set, read
from test_tutorial_config.txt")
+ parser.add_argument("--kernel", help="name of the jupyter kernel to use
for the test")
+ parser.add_argument("--no-cache", help="clean the temp directory",
action="store_true", dest="no_cache")
+ args = parser.parse_args()
+
+
+ tutorial_list = []
+ if args.tutorial:
+ tutorial_list.append(args.tutorial)
+ else:
+ with open('test_tutorial_config.txt') as config_file:
+ for line in config_file:
+ tutorial_list.append(line.lstrip().rstrip())
+
+ temp_dir = 'tmp_notebook'
+ if args.no_cache:
+ print("Cleaning and setting up temp directory '{}'".format(temp_dir))
+ shutil.rmtree(temp_dir, ignore_errors=True)
+
+ kernel = args.kernel if args.kernel else None
+
+ for tutorial in tutorial_list:
+ file_dir = os.path.join(*([tutorial_dir]+tutorial.split('/')))
+ working_dir = os.path.join(*([temp_dir]+tutorial.split('/')))
+ if not os.path.isdir(working_dir):
+ os.makedirs(working_dir)
+ test_tutorial_nb(file_dir, working_dir, kernel)
- fail_num = len(fail_dict)
- success_num = len(tutorial_list) - fail_num
- print "Test Summary Start"
- print "%d tutorials tested:" % (len(tutorial_list))
- for tutorial in tutorial_list:
- print tutorial
- print "\n%d tests failed:" % (fail_num)
- for tutorial, msg in fail_dict.items():
- print tutorial + ":"
- print msg
- print "Test Summary End"
- print "Stats start"
- print "[Passed: %d of %d]" % (success_num, len(tutorial_list))
- print "Stats end"
+ fail_num = len(fail_dict)
Review comment:
it is declared at the top of the file, and written to here:
```
+ if tutorial_name in fail_dict:
+ fail_dict[tutorial_name] += "\n"+line
+ else:
+ fail_dict[tutorial_name] = "Warning:\n"+line
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
