Justobe opened a new issue #17258: mxnet.base.MXNetError: [12:37:13] src/executor/../common/exec_utils.h:391: InferShape pass cannot decide shapes for the following arguments URL: https://github.com/apache/incubator-mxnet/issues/17258 ## Description I find that MXNET behaves differently from Tensorflow when I try to load one simple lstm model using the following code. I get the exception under the MXNET backend. But the code runs correctly under Tensorflow. ### Error Message ``` Traceback (most recent call last): File "/root/anaconda3/lib/python3.6/site-packages/mxnet/symbol/symbol.py", line 1776, in simple_bind ctypes.byref(exe_handle))) File "/root/anaconda3/lib/python3.6/site-packages/mxnet/base.py", line 278, in check_call raise MXNetError(py_str(_LIB.MXGetLastError())) mxnet.base.MXNetError: [12:29:57] src/executor/../common/exec_utils.h:391: InferShape pass cannot decide shapes for the following arguments (-1 means unknown dimensions). Please consider providing them as inputs: /lstm_1_input2: [-1,50,25], Stack trace: [bt] (0) /root/anaconda3/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x6ccefb) [0x7f0743234efb] [bt] (1) /root/anaconda3/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x392c8c3) [0x7f07464948c3] [bt] (2) /root/anaconda3/lib/python3.6/site-packages/mxnet/libmxnet.so(mxnet::exec::GraphExecutor::Init(nnvm::Symbol, mxnet::Context const&, std::map<std::string, mxnet::Context, std::less<std::string>, std::allocator<std::pair<std::string const, mxnet::Context> > > const&, std::vector<mxnet::Context, std::allocator<mxnet::Context> > const&, std::vector<mxnet::Context, std::allocator<mxnet::Context> > const&, std::vector<mxnet::Context, std::allocator<mxnet::Context> > const&, std::unordered_map<std::string, mxnet::TShape, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, mxnet::TShape> > > const&, std::unordered_map<std::string, int, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, int> > > const&, std::unordered_map<std::string, int, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, int> > > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, std::unordered_set<std::string, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::string> > const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> >*, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> >*, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> >*, std::unordered_map<std::string, mxnet::NDArray, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, mxnet::NDArray> > >*, mxnet::Executor*, std::unordered_map<nnvm::NodeEntry, mxnet::NDArray, nnvm::NodeEntryHash, nnvm::NodeEntryEqual, std::allocator<std::pair<nnvm::NodeEntry const, mxnet::NDArray> > > const&)+0x40f) [0x7f07464c161f] [bt] (3) /root/anaconda3/lib/python3.6/site-packages/mxnet/libmxnet.so(mxnet::Executor::SimpleBind(nnvm::Symbol, mxnet::Context const&, std::map<std::string, mxnet::Context, std::less<std::string>, std::allocator<std::pair<std::string const, mxnet::Context> > > const&, std::vector<mxnet::Context, std::allocator<mxnet::Context> > const&, std::vector<mxnet::Context, std::allocator<mxnet::Context> > const&, std::vector<mxnet::Context, std::allocator<mxnet::Context> > const&, std::unordered_map<std::string, mxnet::TShape, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, mxnet::TShape> > > const&, std::unordered_map<std::string, int, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, int> > > const&, std::unordered_map<std::string, int, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, int> > > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, std::unordered_set<std::string, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::string> > const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> >*, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> >*, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> >*, std::unordered_map<std::string, mxnet::NDArray, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, mxnet::NDArray> > >*, mxnet::Executor*)+0x2a8) [0x7f07464c2198] [bt] (4) /root/anaconda3/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x3887d83) [0x7f07463efd83] [bt] (5) /root/anaconda3/lib/python3.6/lib-dynload/../../libffi.so.6(ffi_call_unix64+0x4c) [0x7f078434aec0] [bt] (6) /root/anaconda3/lib/python3.6/lib-dynload/../../libffi.so.6(ffi_call+0x22d) [0x7f078434a87d] [bt] (7) /root/anaconda3/lib/python3.6/lib-dynload/_ctypes.cpython-36m-x86_64-linux-gnu.so(_ctypes_callproc+0x2ce) [0x7f0784560ede] [bt] (8) /root/anaconda3/lib/python3.6/lib-dynload/_ctypes.cpython-36m-x86_64-linux-gnu.so(+0x13915) [0x7f0784561915] During handling of the above exception, another exception occurred: Traceback (most recent call last): File "crash_checker.py", line 45, in <module> model = keras.models.load_model(file_path,custom_objects=custom_objects()) File "/root/anaconda3/lib/python3.6/site-packages/keras/engine/saving.py", line 496, in load_model model = _deserialize_model(f, custom_objects, compile) File "/root/anaconda3/lib/python3.6/site-packages/keras/engine/saving.py", line 302, in _deserialize_model model = model_from_config(model_config, custom_objects=custom_objects) File "/root/anaconda3/lib/python3.6/site-packages/keras/engine/saving.py", line 535, in model_from_config return deserialize(config, custom_objects=custom_objects) File "/root/anaconda3/lib/python3.6/site-packages/keras/layers/__init__.py", line 55, in deserialize printable_module_name='layer') File "/root/anaconda3/lib/python3.6/site-packages/keras/utils/generic_utils.py", line 145, in deserialize_keras_object list(custom_objects.items()))) File "/root/anaconda3/lib/python3.6/site-packages/keras/engine/sequential.py", line 301, in from_config model.add(layer) File "/root/anaconda3/lib/python3.6/site-packages/keras/engine/sequential.py", line 181, in add output_tensor = layer(self.outputs[0]) File "/root/anaconda3/lib/python3.6/site-packages/keras/engine/base_layer.py", line 480, in __call__ x = K.identity(x) File "/root/anaconda3/lib/python3.6/site-packages/keras/backend/mxnet_backend.py", line 780, in identity x_value = eval(x) File "/root/anaconda3/lib/python3.6/site-packages/keras/backend/mxnet_backend.py", line 595, in eval ret = _forward_pass(x)[0].asnumpy() File "/root/anaconda3/lib/python3.6/site-packages/keras/backend/mxnet_backend.py", line 4640, in _forward_pass executor = x.symbol.simple_bind(mx.cpu(), grad_req='null') File "/root/anaconda3/lib/python3.6/site-packages/mxnet/symbol/symbol.py", line 1782, in simple_bind raise RuntimeError(error_msg) RuntimeError: simple_bind error. Arguments: [12:29:57] src/executor/../common/exec_utils.h:391: InferShape pass cannot decide shapes for the following arguments (-1 means unknown dimensions). Please consider providing them as inputs: /lstm_1_input2: [-1,50,25], Stack trace: [bt] (0) /root/anaconda3/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x6ccefb) [0x7f0743234efb] [bt] (1) /root/anaconda3/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x392c8c3) [0x7f07464948c3] [bt] (2) /root/anaconda3/lib/python3.6/site-packages/mxnet/libmxnet.so(mxnet::exec::GraphExecutor::Init(nnvm::Symbol, mxnet::Context const&, std::map<std::string, mxnet::Context, std::less<std::string>, std::allocator<std::pair<std::string const, mxnet::Context> > > const&, std::vector<mxnet::Context, std::allocator<mxnet::Context> > const&, std::vector<mxnet::Context, std::allocator<mxnet::Context> > const&, std::vector<mxnet::Context, std::allocator<mxnet::Context> > const&, std::unordered_map<std::string, mxnet::TShape, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, mxnet::TShape> > > const&, std::unordered_map<std::string, int, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, int> > > const&, std::unordered_map<std::string, int, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, int> > > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, std::unordered_set<std::string, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::string> > const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> >*, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> >*, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> >*, std::unordered_map<std::string, mxnet::NDArray, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, mxnet::NDArray> > >*, mxnet::Executor*, std::unordered_map<nnvm::NodeEntry, mxnet::NDArray, nnvm::NodeEntryHash, nnvm::NodeEntryEqual, std::allocator<std::pair<nnvm::NodeEntry const, mxnet::NDArray> > > const&)+0x40f) [0x7f07464c161f] [bt] (3) /root/anaconda3/lib/python3.6/site-packages/mxnet/libmxnet.so(mxnet::Executor::SimpleBind(nnvm::Symbol, mxnet::Context const&, std::map<std::string, mxnet::Context, std::less<std::string>, std::allocator<std::pair<std::string const, mxnet::Context> > > const&, std::vector<mxnet::Context, std::allocator<mxnet::Context> > const&, std::vector<mxnet::Context, std::allocator<mxnet::Context> > const&, std::vector<mxnet::Context, std::allocator<mxnet::Context> > const&, std::unordered_map<std::string, mxnet::TShape, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, mxnet::TShape> > > const&, std::unordered_map<std::string, int, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, int> > > const&, std::unordered_map<std::string, int, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, int> > > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, std::unordered_set<std::string, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::string> > const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> >*, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> >*, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> >*, std::unordered_map<std::string, mxnet::NDArray, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, mxnet::NDArray> > >*, mxnet::Executor*)+0x2a8) [0x7f07464c2198] [bt] (4) /root/anaconda3/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x3887d83) [0x7f07463efd83] [bt] (5) /root/anaconda3/lib/python3.6/lib-dynload/../../libffi.so.6(ffi_call_unix64+0x4c) [0x7f078434aec0] [bt] (6) /root/anaconda3/lib/python3.6/lib-dynload/../../libffi.so.6(ffi_call+0x22d) [0x7f078434a87d] [bt] (7) /root/anaconda3/lib/python3.6/lib-dynload/_ctypes.cpython-36m-x86_64-linux-gnu.so(_ctypes_callproc+0x2ce) [0x7f0784560ede] [bt] (8) /root/anaconda3/lib/python3.6/lib-dynload/_ctypes.cpython-36m-x86_64-linux-gnu.so(+0x13915) [0x7f0784561915] ``` ## To Reproduce script.py ``` import os import sys import argparse parse = argparse.ArgumentParser() parse.add_argument("--backend", type=str, help="name of backends") flags, unparsed = parse.parse_known_args(sys.argv[1:]) bk = flags.backend os.environ['KERAS_BACKEND'] = bk os.environ["TF_CPP_MIN_LOG_LEVEL"] = '2' os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = "" import keras new_model = keras.models.load_model("lstm1-PM-data_origin.h5") new_model.summary() print("Done") ``` The structure of model is shown below  ### Steps to reproduce 1. download the model [lstm1-PM-data_origin.zip](https://github.com/apache/incubator-mxnet/files/4040429/lstm1-PM-data_origin.zip) 2. You can run it like this: > python -u script.py --backend mxnet ## Environment - numpy 1.16.1 - tensorflow 1.14.0 - mxnet-cu101_1.6.0b20191122 Thanks in advance!
---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
