leleamol commented on a change in pull request #13294: [MXNET-1083] Add the example to demonstrate the inference workflow using C++ API URL: https://github.com/apache/incubator-mxnet/pull/13294#discussion_r239275908
########## File path: cpp-package/example/inference/inception_inference.cpp ########## @@ -0,0 +1,414 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * This example demonstrates image classification workflow with pre-trained models using MXNet C++ API. + * The example performs following tasks. + * 1. Load the pre-trained model, + * 2. Load the parameters of pre-trained model, + * 3. Load the image to be classified in to NDArray. + * 4. Normalize the image using the mean of images that were used for training. + * 5. Run the forward pass and predict the input image. + */ + +#include <sys/stat.h> +#include <iostream> +#include <fstream> +#include <map> +#include <string> +#include <vector> +#include "mxnet-cpp/MxNetCpp.h" +#include <opencv2/opencv.hpp> + +using namespace mxnet::cpp; + + +/* + * class Predictor + * + * This class encapsulates the functionality to load the model, process input image and run the forward pass. + */ + +class Predictor { + public: + Predictor() {} + Predictor(const std::string& model_json, + const std::string& model_params, + const Shape& input_shape, + bool gpu_context_type = false, + const std::string& synset_file = "", + const std::string& mean_image_file = ""); + void PredictImage(const std::string& image_file); + ~Predictor(); + + private: + void LoadModel(const std::string& model_json_file); + void LoadParameters(const std::string& model_parameters_file); + void LoadSynset(const std::string& synset_file); + NDArray LoadInputImage(const std::string& image_file); + void LoadMeanImageData(); + void NormalizeInput(const std::string& mean_image_file); + inline bool FileExists(const std::string& name) { + struct stat buffer; + return (stat(name.c_str(), &buffer) == 0); + } + NDArray mean_img; + std::map<std::string, NDArray> args_map; + std::map<std::string, NDArray> aux_map; + std::vector<std::string> output_labels; + Symbol net; + Executor *executor; + Shape input_shape; + NDArray mean_image_data; + Context global_ctx = Context::cpu(); + std::string mean_image_file; +}; + + +/* + * The constructor takes following parameters as input: + * 1. model_json: The model in json formatted file. + * 2. model_params: File containing model parameters + * 3. synset_file: File containing the list of image labels + * 4. input_shape: Shape of input data to the model. Since this class will be running one inference at a time, + * the input shape is required to be in format Shape(1, number_of_channels, height, width) + * The input image will be resized to (height x width) size before running the inference. + * The constructor will: + * 1. Load the model and parameter files. + * 2. Load the synset file. + * 3. Invoke the SimpleBind to bind the input argument to the model and create an executor. + * + * The SimpleBind is expected to be invoked only once. + */ +Predictor::Predictor(const std::string& model_json, + const std::string& model_params, + const Shape& input_shape, + bool gpu_context_type, + const std::string& synset_file, + const std::string& mean_image_file): + input_shape(input_shape), + mean_image_file(mean_image_file) { + if (gpu_context_type) { + global_ctx = Context::gpu(); + } + // Load the model + LoadModel(model_json); + + // Load the model parameters. + LoadParameters(model_params); + + /* + * The data will be used to output the exact label that matches highest output of the model. + */ + LoadSynset(synset_file); + + /* + * Load the mean image data if specified. + */ + if (!mean_image_file.empty()) { + LoadMeanImageData(); + } else { + LG << "Mean image file for normalizing the input is not provide." + << " It may affect the accuracy of the prediction."; + } + + // Create an executor after binding the model to input parameters. + args_map["data"] = NDArray(input_shape, global_ctx, false); + executor = net.SimpleBind(global_ctx, args_map, std::map<std::string, NDArray>(), + std::map<std::string, OpReqType>(), aux_map); +} + +/* + * The following function loads the model from json file. + */ +void Predictor::LoadModel(const std::string& model_json_file) { + if (!FileExists(model_json_file)) { + LG << "Model file " << model_json_file << " does not exist"; + throw std::runtime_error("Model file does not exist"); + } + LG << "Loading the model from " << model_json_file << std::endl; + net = Symbol::Load(model_json_file); +} + + +/* + * The following function loads the model parameters. + */ +void Predictor::LoadParameters(const std::string& model_parameters_file) { + if (!FileExists(model_parameters_file)) { + LG << "Parameter file " << model_parameters_file << " does not exist"; + throw std::runtime_error("Model parameters does not exist"); + } + LG << "Loading the model parameters from " << model_parameters_file << std::endl; + std::map<std::string, NDArray> paramters; + NDArray::Load(model_parameters_file, 0, ¶mters); + for (const auto &k : paramters) { + if (k.first.substr(0, 4) == "aux:") { + auto name = k.first.substr(4, k.first.size() - 4); + aux_map[name] = k.second.Copy(global_ctx); + } + if (k.first.substr(0, 4) == "arg:") { + auto name = k.first.substr(4, k.first.size() - 4); + args_map[name] = k.second.Copy(global_ctx); + } + } + /*WaitAll is need when we copy data between GPU and the main memory*/ + NDArray::WaitAll(); +} + + +/* + * The following function loads the synset file. + * This information will be used later to report the label of input image. + */ +void Predictor::LoadSynset(const std::string& synset_file) { + if (!FileExists(synset_file)) { + LG << "Synset file " << synset_file << " does not exist"; + throw std::runtime_error("Synset file does not exist"); + } + LG << "Loading the synset file."; + std::ifstream fi(synset_file.c_str()); + if (!fi.is_open()) { + std::cerr << "Error opening synset file " << synset_file << std::endl; + assert(false); + } + std::string synset, lemma; + while (fi >> synset) { + getline(fi, lemma); + output_labels.push_back(lemma); + } + fi.close(); +} + + +/* + * The following function loads the mean data from mean image file. + * This data will be used for normalizing the image before running the forward + * pass. + * + */ +void Predictor::LoadMeanImageData() { + LG << "Load the mean image data that will be used to normalize " + << "the image before running forward pass."; + mean_image_data = NDArray(input_shape, global_ctx, false); + mean_image_data.SyncCopyFromCPU( + NDArray::LoadToMap(mean_image_file)["mean_img"].GetData(), + input_shape.Size()); + NDArray::WaitAll(); +} + + +/* + * The following function loads the input image. + */ +NDArray Predictor::LoadInputImage(const std::string& image_file) { + if (!FileExists(image_file)) { + LG << "Image file " << image_file << " does not exist"; + throw std::runtime_error("Image file does not exist"); + } + LG << "Loading the image " << image_file << std::endl; + std::vector<float> array; + cv::Mat mat = cv::imread(image_file); + /*resize pictures to (224, 224) according to the pretrained model*/ + int height = input_shape[2]; + int width = input_shape[3]; + int channels = input_shape[1]; + cv::resize(mat, mat, cv::Size(height, width)); + for (int c = 0; c < channels; ++c) { + for (int i = 0; i < height; ++i) { + for (int j = 0; j < width; ++j) { + array.push_back(static_cast<float>(mat.data[(i * height + j) * 3 + c])); + } + } + } + NDArray image_data = NDArray(input_shape, global_ctx, false); + image_data.SyncCopyFromCPU(array.data(), input_shape.Size()); + NDArray::WaitAll(); + return image_data; +} + + +/* + * The following function runs the forward pass on the model. + * The executor is created in the constructor. + * + */ +void Predictor::PredictImage(const std::string& image_file) { + // Load the input image + NDArray image_data = LoadInputImage(image_file); + + // Normalize the image + if (!mean_image_file.empty()) { + image_data.Slice(0, 1) -= mean_image_data; + } + + LG << "Running the forward pass on model to predict the image"; + /* + * The executor->arg_arrays represent the arguments to the model. + * + * Copying the image_data that contains the NDArray of input image + * to the arg map of the executor. The input is stored with the key "data" in the map. + * + */ + image_data.CopyTo(&(executor->arg_dict()["data"])); + NDArray::WaitAll(); + + // Run the forward pass. + executor->Forward(false); + + // The output is available in executor->outputs. + auto array = executor->outputs[0].Copy(global_ctx); + NDArray::WaitAll(); + + float best_accuracy = 0.0f; + std::size_t best_idx = 0; + + // Find out the maximum accuracy and the index associated with that accuracy. + for (std::size_t i = 0; i < array.Size(); ++i) { Review comment: Done. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
