This is an automated email from the ASF dual-hosted git repository.
marcoabreu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new 815f36c [MXNET-707] Add unit test for mxnet to coreml converter
(#11952)
815f36c is described below
commit 815f36ce8b4ed16fe27d500f5c8c930cd10cee5c
Author: Lin Yuan <[email protected]>
AuthorDate: Tue Oct 16 10:16:15 2018 -0700
[MXNET-707] Add unit test for mxnet to coreml converter (#11952)
* Add unittest to coreml converter
* Add unittest to coreml converter
* Add docstring and remove unused method
* updated test and removed unittest folder
* remove unittest
* Add coreml test to CI
* fix lint
* install mxnet-to-coreml for testing
* exclude test that takes too long
* linting to 100 max line width
---
.travis.yml | 2 +
ci/travis/install.sh | 2 +-
tools/coreml/test/test_mxnet_converter.py | 69 +++++++++++++++++--------------
tools/coreml/test/test_mxnet_image.py | 45 ++++++++++----------
tools/coreml/test/test_mxnet_models.py | 48 +++++++++++----------
5 files changed, 92 insertions(+), 74 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index d6b38f4..b5c323f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -34,7 +34,9 @@ script:
- export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
- mv make/osx.mk config.mk
- make -j 2
+
# We ignore several tests to avoid possible timeouts on large PRs.
# This lowers our test coverage, but is required for consistent Travis runs.
# These tests will be tested in a variety of environments in Jenkins based
tests.
- python -m nose --with-timer
--exclude-test=test_sparse_operator.test_elemwise_binary_ops
--exclude-test=test_gluon_model_zoo.test_models
--exclude-test=test_random.test_shuffle
--exclude-test=test_operator.test_broadcast_binary_op
--exclude-test=test_operator.test_pick
--exclude-test=test_profiler.test_continuous_profile_and_instant_marker
--exclude-test=test_metric_perf.test_metric_performance
--exclude-test=test_operator.test_order --verbose tests/python/unittest/
+ - python2 -m nose --verbose tools/coreml/test --exclude-test=test_mxnet_image
diff --git a/ci/travis/install.sh b/ci/travis/install.sh
index ae95976..6412849 100644
--- a/ci/travis/install.sh
+++ b/ci/travis/install.sh
@@ -22,5 +22,5 @@ export HOMEBREW_NO_AUTO_UPDATE=1
if [ ${TRAVIS_OS_NAME} == "osx" ]; then
brew install opencv
- python -m pip install --user nose numpy cython scipy requests mock
nose-timer nose-exclude
+ python -m pip install --user nose numpy cython scipy requests mock
nose-timer nose-exclude mxnet-to-coreml
fi
diff --git a/tools/coreml/test/test_mxnet_converter.py
b/tools/coreml/test/test_mxnet_converter.py
index 6020041..5d26c5f 100644
--- a/tools/coreml/test/test_mxnet_converter.py
+++ b/tools/coreml/test/test_mxnet_converter.py
@@ -18,15 +18,12 @@
import unittest
import mxnet as mx
import numpy as np
-import sys
-import os
-current_working_directory = os.getcwd()
-sys.path.append(current_working_directory + "/..")
-sys.path.append(current_working_directory + "/../converter/")
-import _mxnet_converter as mxnet_converter
+
+from converter._mxnet_converter import convert
from collections import namedtuple
from converter import utils
+
def _mxnet_remove_batch(input_data):
for blob in input_data:
input_data[blob] = np.reshape(input_data[blob],
input_data[blob].shape[1:])
@@ -39,7 +36,8 @@ def _get_mxnet_module(net, data_shapes, mode, label_names,
input_names=None):
"""
mx.random.seed(1993)
- mod = utils.create_module(sym=net, data_shapes=data_shapes,
label_shapes=input_names, label_names=label_names)
+ mod = utils.create_module(sym=net, data_shapes=data_shapes,
label_shapes=input_names,
+ label_names=label_names)
if mode == 'random':
mod.init_params(
@@ -62,11 +60,14 @@ def _get_mxnet_module(net, data_shapes, mode, label_names,
input_names=None):
class SingleLayerTest(unittest.TestCase):
"""
Unit test class for testing where converter is able to convert individual
layers or not.
- In order to do so, it converts model and generates preds on both CoreML
and MXNet and check they are the same.
+ In order to do so, it converts model and generates preds on both CoreML
and MXNet and check
+ they are the same.
"""
- def _test_mxnet_model(self, net, input_shape, mode, class_labels=None,
coreml_mode=None, label_names=None, delta=1e-3,
+ def _test_mxnet_model(self, net, input_shape, mode, class_labels=None,
+ coreml_mode=None, label_names=None, delta=1e-2,
pre_processing_args=None, input_name='data'):
- """ Helper method that convert the CoreML model into CoreML and
compares the predictions over random data.
+ """ Helper method that convert the CoreML model into CoreML and
compares the predictions
+ over random data.
Parameters
----------
@@ -89,7 +90,7 @@ class SingleLayerTest(unittest.TestCase):
The name of the input variable to the symbolic graph.
"""
- data_shapes=[(input_name, input_shape)]
+ data_shapes = [(input_name, input_shape)]
mod = _get_mxnet_module(net, data_shapes, mode, label_names)
@@ -100,7 +101,7 @@ class SingleLayerTest(unittest.TestCase):
mxnet_preds = mod.get_outputs()[0].asnumpy().flatten()
# Get predictions from coreml
- coreml_model = mxnet_converter.convert(
+ coreml_model = convert(
model=mod,
class_labels=class_labels,
mode=coreml_mode,
@@ -112,7 +113,7 @@ class SingleLayerTest(unittest.TestCase):
# Check prediction accuracy
self.assertEquals(len(mxnet_preds), len(coreml_preds))
for i in range(len(mxnet_preds)):
- self.assertAlmostEquals(mxnet_preds[i], coreml_preds[i], delta =
delta)
+ self.assertAlmostEquals(mxnet_preds[i], coreml_preds[i],
delta=delta)
def test_tiny_inner_product_zero_input(self):
np.random.seed(1988)
@@ -140,7 +141,7 @@ class SingleLayerTest(unittest.TestCase):
input_shape = (1, 10)
net = mx.sym.Variable('data')
net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=5)
- self._test_mxnet_model(net, input_shape=input_shape, mode='ones')
+ self._test_mxnet_model(net, input_shape=input_shape, mode='ones',
delta=0.05)
def test_tiny_inner_product_random_input(self):
np.random.seed(1988)
@@ -162,7 +163,8 @@ class SingleLayerTest(unittest.TestCase):
net = mx.sym.Variable('data')
net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=5)
net = mx.sym.SoftmaxOutput(net, name='softmax')
- self._test_mxnet_model(net, input_shape=input_shape, mode='random',
label_names=['softmax_label'])
+ self._test_mxnet_model(net, input_shape=input_shape, mode='random',
+ label_names=['softmax_label'])
def test_tiny_relu_activation_random_input(self):
np.random.seed(1988)
@@ -228,7 +230,7 @@ class SingleLayerTest(unittest.TestCase):
pad=pad,
name='conv_1'
)
- self._test_mxnet_model(net, input_shape=input_shape, mode='ones')
+ self._test_mxnet_model(net, input_shape=input_shape, mode='ones',
delta=0.05)
def test_tiny_conv_random_input(self):
np.random.seed(1988)
@@ -481,7 +483,8 @@ class SingleLayerTest(unittest.TestCase):
net = mx.sym.Flatten(data=net, name='flatten1')
net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=5)
net = mx.sym.SoftmaxOutput(net, name='softmax')
- self._test_mxnet_model(net, input_shape=input_shape, mode='random',
label_names=['softmax_label'])
+ self._test_mxnet_model(net, input_shape=input_shape, mode='random',
+ label_names=['softmax_label'])
def test_transpose(self):
np.random.seed(1988)
@@ -516,10 +519,8 @@ class SingleLayerTest(unittest.TestCase):
net = mx.sym.Variable('data')
net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=5)
net = mx.sym.SoftmaxOutput(net, name='softmax')
- mod = _get_mxnet_module(net,
- data_shapes=[('data', input_shape)],
- mode='random',
- label_names=['softmax_label'])
+ mod = _get_mxnet_module(net, data_shapes=[('data', input_shape)],
+ mode='random', label_names=['softmax_label'])
# Generate some dummy data
input_data = np.random.uniform(-0.1, 0.1, input_shape)
@@ -529,14 +530,15 @@ class SingleLayerTest(unittest.TestCase):
kwargs = {'input_shape': {'data': input_shape}}
# Get predictions from coreml
- coreml_model = mxnet_converter.convert(
+ coreml_model = convert(
model=mod,
-
class_labels=['Category1','Category2','Category3','Category4','Category5'],
+ class_labels=['Category1', 'Category2', 'Category3', 'Category4',
'Category5'],
mode='classifier',
**kwargs
)
- prediction = coreml_model.predict(_mxnet_remove_batch({'data':
input_data}))
+ prediction = coreml_model.predict(
+ _mxnet_remove_batch({'data': input_data}))
self.assertEqual(prediction['classLabel'], 'Category3')
def test_really_tiny_deconv_random_input(self):
@@ -579,7 +581,7 @@ class SingleLayerTest(unittest.TestCase):
name='deconv_1'
)
# Test the mxnet model
- self._test_mxnet_model(net, input_shape=input_shape, mode='ones')
+ self._test_mxnet_model(net, input_shape=input_shape, mode='ones',
delta=0.05)
def test_tiny_deconv_random_input(self):
np.random.seed(1988)
@@ -915,7 +917,6 @@ class SingleLayerTest(unittest.TestCase):
def test_batch_norm_no_global_stats(self):
""" This test should throw an exception since converter doesn't support
- conversion of MXNet models that use local batch stats (i.e.
use_global_stats=False). The reason for this is CoreML doesn't
support
local batch stats.
"""
@@ -943,12 +944,17 @@ class SingleLayerTest(unittest.TestCase):
net = mx.sym.Variable('data')
net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=5)
net = mx.sym.SoftmaxOutput(net, name='softmax')
- self._test_mxnet_model(net, input_shape=input_shape, mode='random',
label_names=['softmax_label'],
- pre_processing_args={'red_bias':0,
'blue_bias':0, 'green_bias':0, 'image_scale':1})
+ self._test_mxnet_model(net, input_shape=input_shape, mode='random',
+ label_names=['softmax_label'],
+ pre_processing_args={'red_bias': 0,
+ 'blue_bias': 0,
+ 'green_bias': 0,
+ 'image_scale': 1})
def test_different_input_variables(self):
"""
- Verifying the behavior when input variable name is different than the
standard name - 'data'.
+ Verifying the behavior when input variable name is different than the
+ standard name - 'data'.
"""
np.random.seed(1988)
input_shape = (1, 10)
@@ -958,12 +964,13 @@ class SingleLayerTest(unittest.TestCase):
def test_really_tiny_conv_optional_params(self):
"""
- Verifying the behavior of a convolutional layer when stride and pad
are not provided.
+ Verifying the behavior of a convolutional layer when stride and pad
+ are not provided.
"""
np.random.seed(1988)
input_shape = (1, 1, 10, 10)
num_filter = 1
- kernel = (1 ,1)
+ kernel = (1, 1)
# Define a model
net = mx.sym.Variable('data')
diff --git a/tools/coreml/test/test_mxnet_image.py
b/tools/coreml/test/test_mxnet_image.py
index e754519..b3165f0 100644
--- a/tools/coreml/test/test_mxnet_image.py
+++ b/tools/coreml/test/test_mxnet_image.py
@@ -1,4 +1,4 @@
-from __future__ import print_function
+
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,19 +16,13 @@ from __future__ import print_function
# specific language governing permissions and limitations
# under the License.
+import unittest
import mxnet as mx
import numpy as np
-import unittest
-import sys
-import os
from six.moves import xrange
-
-current_working_directory = os.getcwd()
-sys.path.append(current_working_directory + "/..")
-sys.path.append(current_working_directory + "/../converter/")
-import _mxnet_converter as mxnet_converter
-from converter.utils import load_model
+from converter._mxnet_converter import convert
+from converter import utils
VAL_DATA = 'data/val-5k-256.rec'
URL = 'http://data.mxnet.io/data/val-5k-256.rec'
@@ -44,7 +38,7 @@ def read_image(data_val, label_name):
label_width=1,
preprocess_threads=4,
batch_size=32,
- data_shape=(3,224,224),
+ data_shape=(3, 224, 224),
label_name=label_name,
rand_corp=False,
rand_mirror=False,
@@ -80,11 +74,11 @@ class ImageNetTest(unittest.TestCase):
epoch_num=epoch,
data_shapes=data.provide_data,
label_shapes=data.provide_label,
- label_names=[label_name,]
+ label_names=[label_name, ]
)
input_shape = (1, 3, 224, 224)
- coreml_model = mxnet_converter.convert(mod, input_shape={'data':
input_shape})
+ coreml_model = convert(mod, input_shape={'data': input_shape})
mxnet_acc = []
mxnet_top_5_acc = []
@@ -104,34 +98,43 @@ class ImageNetTest(unittest.TestCase):
mxnet_predict = mxnet_preds[i]
label = label_numpy[i]
mxnet_acc.append(is_correct_top_one(mxnet_predict, label))
- mxnet_top_5_acc.append(is_correct_top_five(mxnet_predict,
label))
+ mxnet_top_5_acc.append(is_correct_top_five(mxnet_predict,
+ label))
coreml_acc.append(is_correct_top_one(coreml_predict, label))
- coreml_top_5_acc.append(is_correct_top_five(coreml_predict,
label))
+ coreml_top_5_acc.append(is_correct_top_five(coreml_predict,
+ label))
num_batch += 1
- if (num_batch == 5): break # we only use a subset of the batches.
+ if (num_batch == 5):
+ break # we only use a subset of the batches.
print("MXNet acc %s" % np.mean(mxnet_acc))
print("Coreml acc %s" % np.mean(coreml_acc))
print("MXNet top 5 acc %s" % np.mean(mxnet_top_5_acc))
print("Coreml top 5 acc %s" % np.mean(coreml_top_5_acc))
self.assertAlmostEqual(np.mean(mxnet_acc), np.mean(coreml_acc),
delta=1e-4)
- self.assertAlmostEqual(np.mean(mxnet_top_5_acc),
np.mean(coreml_top_5_acc), delta=1e-4)
+ self.assertAlmostEqual(np.mean(mxnet_top_5_acc),
+ np.mean(coreml_top_5_acc),
+ delta=1e-4)
def test_squeezenet(self):
print("Testing Image Classification with Squeezenet")
- self._test_image_prediction(model_name='squeezenet_v1.1', epoch=0,
label_name='prob_label')
+ self._test_image_prediction(model_name='squeezenet_v1.1', epoch=0,
+ label_name='prob_label')
def test_inception_with_batch_normalization(self):
print("Testing Image Classification with Inception/BatchNorm")
- self._test_image_prediction(model_name='Inception-BN', epoch=126,
label_name='softmax_label')
+ self._test_image_prediction(model_name='Inception-BN', epoch=126,
+ label_name='softmax_label')
def test_resnet18(self):
print("Testing Image Classification with ResNet18")
- self._test_image_prediction(model_name='resnet-18', epoch=0,
label_name='softmax_label')
+ self._test_image_prediction(model_name='resnet-18', epoch=0,
+ label_name='softmax_label')
def test_vgg16(self):
print("Testing Image Classification with vgg16")
- self._test_image_prediction(model_name='vgg16', epoch=0,
label_name='prob_label')
+ self._test_image_prediction(model_name='vgg16', epoch=0,
+ label_name='prob_label')
if __name__ == '__main__':
diff --git a/tools/coreml/test/test_mxnet_models.py
b/tools/coreml/test/test_mxnet_models.py
index 7c294db..8dd319a 100644
--- a/tools/coreml/test/test_mxnet_models.py
+++ b/tools/coreml/test/test_mxnet_models.py
@@ -19,21 +19,16 @@ from __future__ import print_function
import unittest
import mxnet as mx
import numpy as np
-import sys
-import os
-from collections import namedtuple
+from collections import namedtuple
+from converter._mxnet_converter import convert
from six.moves import xrange
-current_working_directory = os.getcwd()
-sys.path.append(current_working_directory + "/..")
-sys.path.append(current_working_directory + "/../converter/")
-import _mxnet_converter as mxnet_converter
-
def _mxnet_remove_batch(input_data):
for blob in input_data:
- input_data[blob] = np.reshape(input_data[blob],
input_data[blob].shape[1:])
+ input_data[blob] = np.reshape(input_data[blob], input_data[blob]
+ .shape[1:])
return input_data
@@ -53,9 +48,10 @@ def _kl_divergence(distribution1, distribution2):
class ModelsTest(unittest.TestCase):
"""
- Unit test class that tests converter on entire MXNet models .
- In order to test each unit test converts MXNet model into CoreML model
using the converter, generate predictions
- on both MXNet and CoreML and verifies that predictions are same (or
similar).
+ Unit test class that tests converter on entire MXNet models.
+ In order to test each unit test converts MXNet model into CoreML model
using the converter,
+ generate predictions on both MXNet and CoreML and verifies that
predictions are same
+ (or similar).
"""
def _load_model(self, model_name, epoch_num, input_shape):
sym, arg_params, aux_params = mx.model.load_checkpoint(model_name,
epoch_num)
@@ -76,8 +72,10 @@ class ModelsTest(unittest.TestCase):
)
return mod
- def _test_model(self, model_name, epoch_num, input_shape=(1, 3, 224, 224),
files=None):
- """ Tests whether the converted CoreML model's preds are equal to
MXNet preds for a given model or not.
+ def _test_model(self, model_name, epoch_num, input_shape=(1, 3, 224, 224),
+ files=None):
+ """ Tests whether the converted CoreML model's preds are equal to MXNet
+ preds for a given model or not.
Parameters
----------
@@ -88,10 +86,12 @@ class ModelsTest(unittest.TestCase):
Epoch number of model we would like to load.
input_shape: tuple
- The shape of the input data in the form of (batch_size, channels,
height, width)
+ The shape of the input data in the form of (batch_size, channels,
+ height, width)
files: list of strings
- List of URLs pertaining to files that need to be downloaded in
order to use the model.
+ List of URLs pertaining to files that need to be downloaded in
+ order to use the model.
"""
if files is not None:
@@ -106,17 +106,22 @@ class ModelsTest(unittest.TestCase):
input_shape=input_shape
)
- coreml_model = mxnet_converter.convert(module, input_shape={'data':
input_shape})
+ coreml_model = convert(module, input_shape={'data': input_shape})
# Get predictions from MXNet and coreml
- div=[] # For storing KL divergence for each input.
+ div = [] # For storing KL divergence for each input.
for _ in xrange(1):
np.random.seed(1993)
- input_data = {'data': np.random.uniform(0, 1,
input_shape).astype(np.float32)}
+ input_data = {'data': np.random.uniform(0, 1, input_shape)
+ .astype(np.float32)}
Batch = namedtuple('Batch', ['data'])
- module.forward(Batch([mx.nd.array(input_data['data'])]),
is_train=False)
+ module.forward(Batch([mx.nd.array(input_data['data'])]),
+ is_train=False)
mxnet_pred = module.get_outputs()[0].asnumpy().flatten()
- coreml_pred =
coreml_model.predict(_mxnet_remove_batch(input_data)).values()[0].flatten()
+ coreml_pred = coreml_model \
+ .predict(_mxnet_remove_batch(input_data)) \
+ .values()[0] \
+ .flatten()
self.assertEqual(len(mxnet_pred), len(coreml_pred))
div.append(_kl_divergence(mxnet_pred, coreml_pred))
@@ -138,6 +143,7 @@ class ModelsTest(unittest.TestCase):
files=["http://data.mxnet.io/models/imagenet/resnet/50-layers/resnet-50-symbol.json";,
"http://data.mxnet.io/models/imagenet/resnet/50-layers/resnet-50-0000.params";])
+ @unittest.skip("Model is too big for unit test")
def test_pred_vgg16(self):
self._test_model(model_name='vgg16', epoch_num=0,
files=["http://data.mxnet.io/models/imagenet/vgg/vgg16-symbol.json";,
