SINGA-213 Implement Encoder and Decoder for CSV Rename text_encoder and text_decoder to csv_encoder and csv_decoder. Rename jpg2proto and proto2jpg to JPGEncoder and JPGDecoder. Rename the corresponding files.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/49319223 Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/49319223 Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/49319223 Branch: refs/heads/dev Commit: 49319223316bbaffc9082f0cb42e931111dbf6f1 Parents: 7444f0a Author: Wei Wang <[email protected]> Authored: Mon Jul 11 14:57:19 2016 +0800 Committer: Wei Wang <[email protected]> Committed: Mon Jul 11 14:57:19 2016 +0800 ---------------------------------------------------------------------- include/singa/io/decoder.h | 10 ++-- include/singa/io/encoder.h | 8 +-- src/io/csv_decoder.cc | 55 +++++++++++++++++++++ src/io/csv_encoder.cc | 43 ++++++++++++++++ src/io/jpg2proto_encoder.cc | 83 ------------------------------- src/io/jpg_decoder.cc | 75 ++++++++++++++++++++++++++++ src/io/jpg_encoder.cc | 83 +++++++++++++++++++++++++++++++ src/io/proto2jpg_decoder.cc | 75 ---------------------------- src/io/textfile_decoder.cc | 54 -------------------- src/io/textfile_encoder.cc | 43 ---------------- test/singa/test_csv.cc | 60 +++++++++++++++++++++++ test/singa/test_decoder.cc | 84 -------------------------------- test/singa/test_jpg.cc | 84 ++++++++++++++++++++++++++++++++ test/singa/test_textfile_decoder.cc | 60 ----------------------- 14 files changed, 409 insertions(+), 408 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/include/singa/io/decoder.h ---------------------------------------------------------------------- diff --git a/include/singa/io/decoder.h b/include/singa/io/decoder.h index 7d47d6a..bf9a1bc 100644 --- a/include/singa/io/decoder.h +++ b/include/singa/io/decoder.h @@ -55,9 +55,9 @@ class JPGDecoder : public Decoder { }; #endif -/// Decode the string and convert it into a text -/// tensor (dtype is kFloat32) and a label tensor (dtype is kInt). -class TextDecoder : public Decoder { +/// Decode the string of csv formated data into data tensor +/// (dtype is kFloat32) and optionally a label tensor (dtype is kInt). +class CSVDecoder : public Decoder { public: void Setup(const DecoderConf& conf) override { has_label_ = conf.has_label(); @@ -67,8 +67,8 @@ class TextDecoder : public Decoder { const bool has_label() const { return has_label_; } private: - /// Indicate the dimension order for the output image tensor. - bool has_label_ = true; + /// if ture the first value is the label + bool has_label_ = false; }; } // namespace singa #endif // SINGA_IO_DECODER_H_ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/include/singa/io/encoder.h ---------------------------------------------------------------------- diff --git a/include/singa/io/encoder.h b/include/singa/io/encoder.h index 90cea0e..5ca7b21 100644 --- a/include/singa/io/encoder.h +++ b/include/singa/io/encoder.h @@ -60,12 +60,12 @@ class JPGEncoder : public Encoder { }; #endif // USE_OPENCV -/// Convert a set of tensors parsed from csv file into strings -class TextEncoder : public Encoder { +/// Convert values from tensors into a csv formated string. +class CSVEncoder : public Encoder { public: void Setup(const EncoderConf& conf) override {} - /// 'data' has two tesors, one for the text vector (1D) and one for the - /// label. The text tensor's data type is kFloat. + /// 'data' has two tesors, one for the data vector (1D) and one for the + /// label. The data tensor's data type is kFloat. /// The label tensor's data type is kInt. std::string Encode(vector<Tensor>& data) override; }; http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/src/io/csv_decoder.cc ---------------------------------------------------------------------- diff --git a/src/io/csv_decoder.cc b/src/io/csv_decoder.cc new file mode 100644 index 0000000..0c11028 --- /dev/null +++ b/src/io/csv_decoder.cc @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "singa/io/decoder.h" +#include <string> +#include <sstream> + +const int kMaxCSVBufSize = 40960; + +namespace singa { + +std::vector<Tensor> CSVDecoder::Decode(std::string value) { + std::vector<Tensor> output; + std::stringstream ss; + ss.str(value); + int l = 0; + if (has_label_ == true) + ss >> l; + std::string str; + float d[kMaxCSVBufSize]; + size_t size = 0; + while (std::getline(ss, str, ',')) { + float temp; + if (std::stringstream(str) >> temp) { + CHECK_LE(size, kMaxCSVBufSize - 1); + d[size++] = temp; + } + } + + Tensor data(Shape {size}, kFloat32); + data.CopyDataFromHostPtr(d, size); + output.push_back(data); + if (has_label_ == true) { + Tensor label(Shape {1}, kInt); + label.CopyDataFromHostPtr(&l, 1); + output.push_back(label); + } + return output; +} +} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/src/io/csv_encoder.cc ---------------------------------------------------------------------- diff --git a/src/io/csv_encoder.cc b/src/io/csv_encoder.cc new file mode 100644 index 0000000..1b797a9 --- /dev/null +++ b/src/io/csv_encoder.cc @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "singa/io/encoder.h" +#include <sstream> + +namespace singa { + +std::string CSVEncoder::Encode(vector<Tensor>& data) { + CHECK_GE(data.size(), 1); + size_t size = data[0].Size(); + const float* value = data[0].data<float>(); + std::string des = ""; + if (data.size() == 2) { + const float label = (const float)data[1].data<int>()[0]; + std::ostringstream buff; + buff << label; + des += buff.str() + ','; + } + for (size_t i = 0; i < size; i++) { + std::ostringstream buff; + buff << value[i]; + if (i == size - 1) des += buff.str(); + else des += buff.str() + ','; + } + return des; +} +} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/src/io/jpg2proto_encoder.cc ---------------------------------------------------------------------- diff --git a/src/io/jpg2proto_encoder.cc b/src/io/jpg2proto_encoder.cc deleted file mode 100644 index c5d81b8..0000000 --- a/src/io/jpg2proto_encoder.cc +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "singa/io/encoder.h" - -#ifdef USE_OPENCV -#include <opencv2/highgui/highgui.hpp> -#include <opencv2/imgproc/imgproc.hpp> - -namespace singa { - -std::string JPG2ProtoEncoder::Encode(vector<Tensor>& data) { - // suppose image: image, data[1]: label - CHECK_LE(data.size(), 2u); - const Tensor& image = data.at(0); - CHECK_EQ(image.nDim(), 3u); - CHECK_EQ(image.data_type(), kUChar) << "Data type " << image.data_type() - << " is invalid for an raw image"; - const auto* raw = image.data<unsigned char>(); - cv::Mat mat; - if (image_dim_order_ == "HWC") { - size_t height = image.shape(0), width = image.shape(1), - channel = image.shape(2); - mat = cv::Mat(height, width, CV_8UC3, cv::Scalar(0, 0, 0)); - for (size_t i = 0; i < height; i++) - for (size_t j = 0; j < width; j++) - for (size_t k = 0; k < channel; k++) - mat.at<cv::Vec3b>(i, j)[k] = - raw[i * width * channel + j * channel + k]; - } else if (image_dim_order_ == "CHW") { - size_t channel = image.shape(0), height = image.shape(1), - width = image.shape(2); - mat = cv::Mat(height, width, CV_8UC3, cv::Scalar(0, 0, 0)); - for (size_t i = 0; i < height; i++) - for (size_t j = 0; j < width; j++) - for (size_t k = 0; k < channel; k++) - mat.at<cv::Vec3b>(i, j)[k] = raw[k * height * width + i * width + j]; - } else { - LOG(FATAL) << "Unknow dimension order for images " << image_dim_order_ - << " Only support 'HWC' and 'CHW'"; - } - - // encode image with jpg format - std::vector<uchar> buff; - std::vector<int> param = std::vector<int>(2); - param[0] = CV_IMWRITE_JPEG_QUALITY; - param[1] = 100; // default is 95 - cv::imencode(".jpg", mat, buff, param); - std::string buf(buff.begin(), buff.end()); - - std::string output; - ImageRecord record; - for (size_t i = 0; i < image.nDim(); i++) - record.add_shape(image.shape(i)); - record.set_pixel(buf); - - // suppose each image is attached with at most one label - if (data.size() == 2) { - const int* label = data[1].data<int>(); - CHECK_EQ(label[0], 2); - record.add_label(label[0]); - } - - record.SerializeToString(&output); - return output; -} -} // namespace singa -#endif // USE_OPENCV http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/src/io/jpg_decoder.cc ---------------------------------------------------------------------- diff --git a/src/io/jpg_decoder.cc b/src/io/jpg_decoder.cc new file mode 100644 index 0000000..f885537 --- /dev/null +++ b/src/io/jpg_decoder.cc @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "singa/io/decoder.h" + +#ifdef USE_OPENCV + +#include <opencv2/highgui/highgui.hpp> +#include <opencv2/imgproc/imgproc.hpp> + +namespace singa { + +std::vector<Tensor> JPGDecoder::Decode(std::string value) { + std::vector<Tensor> output; + + ImageRecord record; + record.ParseFromString(value); + std::vector<uchar> pixel(record.pixel().begin(), record.pixel().end()); + + // decode image + cv::Mat mat = cv::imdecode(cv::Mat(pixel), CV_LOAD_IMAGE_COLOR); + size_t height = mat.rows, width = mat.cols, channel = mat.channels(); + Shape shape(record.shape().begin(), record.shape().end()); + CHECK_EQ(shape[0], height); + CHECK_EQ(shape[1], width); + CHECK_EQ(shape[2], channel); + Tensor image(shape); + + float* data = new float[image.Size()]; + if (image_dim_order_ == "CHW") { + for (size_t i = 0; i < height; i++) + for (size_t j = 0; j < width; j++) + for (size_t k = 0; k < channel; k++) + data[k * height * width + i * width + j] = static_cast<float>( + static_cast<int>(mat.at<cv::Vec3b>(i, j)[k])); + } else if (image_dim_order_ == "HWC") { + + for (size_t i = 0; i < height; i++) + for (size_t j = 0; j < width; j++) + for (size_t k = 0; k < channel; k++) + data[i * width * channel + j * channel + k] = + static_cast<float>(static_cast<int>(mat.at<cv::Vec3b>(i, j)[k])); + } else { + LOG(FATAL) << "Unknow dimension order for images " << image_dim_order_ + << " Only support 'HWC' and 'CHW'"; + } + image.CopyDataFromHostPtr<float>(data, image.Size()); + output.push_back(image); + delete data; + + if (record.label_size()) { + Tensor label(Shape{1}, &defaultDevice, kInt); + int labelid = record.label(0); + label.CopyDataFromHostPtr(&labelid, 1); + output.push_back(label); + } + return output; +} +} // namespace singa +#endif http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/src/io/jpg_encoder.cc ---------------------------------------------------------------------- diff --git a/src/io/jpg_encoder.cc b/src/io/jpg_encoder.cc new file mode 100644 index 0000000..9db799d --- /dev/null +++ b/src/io/jpg_encoder.cc @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "singa/io/encoder.h" + +#ifdef USE_OPENCV +#include <opencv2/highgui/highgui.hpp> +#include <opencv2/imgproc/imgproc.hpp> + +namespace singa { + +std::string JPGEncoder::Encode(vector<Tensor>& data) { + // suppose image: image, data[1]: label + CHECK_LE(data.size(), 2u); + const Tensor& image = data.at(0); + CHECK_EQ(image.nDim(), 3u); + CHECK_EQ(image.data_type(), kUChar) << "Data type " << image.data_type() + << " is invalid for an raw image"; + const auto* raw = image.data<unsigned char>(); + cv::Mat mat; + if (image_dim_order_ == "HWC") { + size_t height = image.shape(0), width = image.shape(1), + channel = image.shape(2); + mat = cv::Mat(height, width, CV_8UC3, cv::Scalar(0, 0, 0)); + for (size_t i = 0; i < height; i++) + for (size_t j = 0; j < width; j++) + for (size_t k = 0; k < channel; k++) + mat.at<cv::Vec3b>(i, j)[k] = + raw[i * width * channel + j * channel + k]; + } else if (image_dim_order_ == "CHW") { + size_t channel = image.shape(0), height = image.shape(1), + width = image.shape(2); + mat = cv::Mat(height, width, CV_8UC3, cv::Scalar(0, 0, 0)); + for (size_t i = 0; i < height; i++) + for (size_t j = 0; j < width; j++) + for (size_t k = 0; k < channel; k++) + mat.at<cv::Vec3b>(i, j)[k] = raw[k * height * width + i * width + j]; + } else { + LOG(FATAL) << "Unknow dimension order for images " << image_dim_order_ + << " Only support 'HWC' and 'CHW'"; + } + + // encode image with jpg format + std::vector<uchar> buff; + std::vector<int> param = std::vector<int>(2); + param[0] = CV_IMWRITE_JPEG_QUALITY; + param[1] = 100; // default is 95 + cv::imencode(".jpg", mat, buff, param); + std::string buf(buff.begin(), buff.end()); + + std::string output; + ImageRecord record; + for (size_t i = 0; i < image.nDim(); i++) + record.add_shape(image.shape(i)); + record.set_pixel(buf); + + // suppose each image is attached with at most one label + if (data.size() == 2) { + const int* label = data[1].data<int>(); + CHECK_EQ(label[0], 2); + record.add_label(label[0]); + } + + record.SerializeToString(&output); + return output; +} +} // namespace singa +#endif // USE_OPENCV http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/src/io/proto2jpg_decoder.cc ---------------------------------------------------------------------- diff --git a/src/io/proto2jpg_decoder.cc b/src/io/proto2jpg_decoder.cc deleted file mode 100644 index 0125ea3..0000000 --- a/src/io/proto2jpg_decoder.cc +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "singa/io/decoder.h" - -#ifdef USE_OPENCV - -#include <opencv2/highgui/highgui.hpp> -#include <opencv2/imgproc/imgproc.hpp> - -namespace singa { - -std::vector<Tensor> Proto2JPGDecoder::Decode(std::string value) { - std::vector<Tensor> output; - - ImageRecord record; - record.ParseFromString(value); - std::vector<uchar> pixel(record.pixel().begin(), record.pixel().end()); - - // decode image - cv::Mat mat = cv::imdecode(cv::Mat(pixel), CV_LOAD_IMAGE_COLOR); - size_t height = mat.rows, width = mat.cols, channel = mat.channels(); - Shape shape(record.shape().begin(), record.shape().end()); - CHECK_EQ(shape[0], height); - CHECK_EQ(shape[1], width); - CHECK_EQ(shape[2], channel); - Tensor image(shape); - - float* data = new float[image.Size()]; - if (image_dim_order_ == "CHW") { - for (size_t i = 0; i < height; i++) - for (size_t j = 0; j < width; j++) - for (size_t k = 0; k < channel; k++) - data[k * height * width + i * width + j] = static_cast<float>( - static_cast<int>(mat.at<cv::Vec3b>(i, j)[k])); - } else if (image_dim_order_ == "HWC") { - - for (size_t i = 0; i < height; i++) - for (size_t j = 0; j < width; j++) - for (size_t k = 0; k < channel; k++) - data[i * width * channel + j * channel + k] = - static_cast<float>(static_cast<int>(mat.at<cv::Vec3b>(i, j)[k])); - } else { - LOG(FATAL) << "Unknow dimension order for images " << image_dim_order_ - << " Only support 'HWC' and 'CHW'"; - } - image.CopyDataFromHostPtr<float>(data, image.Size()); - output.push_back(image); - delete data; - - if (record.label_size()) { - Tensor label(Shape{1}, &defaultDevice, kInt); - int labelid = record.label(0); - label.CopyDataFromHostPtr(&labelid, 1); - output.push_back(label); - } - return output; -} -} // namespace singa -#endif http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/src/io/textfile_decoder.cc ---------------------------------------------------------------------- diff --git a/src/io/textfile_decoder.cc b/src/io/textfile_decoder.cc deleted file mode 100644 index 221fb9f..0000000 --- a/src/io/textfile_decoder.cc +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "singa/io/decoder.h" -#include <string> -#include <sstream> - -#define MAXSIZE 4096 - -namespace singa { - -std::vector<Tensor> TextDecoder::Decode(std::string value) { - std::vector<Tensor> output; - std::stringstream ss; - ss.str(value); - int l = 0; - if (has_label_ == true) ss >> l; - std::string str; - float* d = new float[MAXSIZE]; - size_t size = 0; - while(std::getline(ss, str, ',')) { - float temp; - if (std::stringstream(str) >> temp) { - CHECK_LE(size, MAXSIZE-1); - d[size++] = temp; - } - } - - Tensor data(Shape{size}, kFloat32); - data.CopyDataFromHostPtr(d, size); - output.push_back(data); - if (has_label_ == true) { - Tensor label(Shape{1}, kInt); - label.CopyDataFromHostPtr(&l, 1); - output.push_back(label); - } - return output; -} -} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/src/io/textfile_encoder.cc ---------------------------------------------------------------------- diff --git a/src/io/textfile_encoder.cc b/src/io/textfile_encoder.cc deleted file mode 100644 index 72401ee..0000000 --- a/src/io/textfile_encoder.cc +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "singa/io/encoder.h" -#include <sstream> - -namespace singa { - -std::string TextEncoder::Encode(vector<Tensor>& data) { - CHECK_GE(data.size(), 1); - size_t size = data[0].Size(); - const float* value = data[0].data<float>(); - std::string des = ""; - if (data.size() == 2) { - const float label = (const float)data[1].data<int>()[0]; - std::ostringstream buff; - buff << label; - des += buff.str() + ','; - } - for (size_t i = 0; i < size; i++) { - std::ostringstream buff; - buff << value[i]; - if (i == size - 1) des += buff.str(); - else des += buff.str() + ','; - } - return des; -} -} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/test/singa/test_csv.cc ---------------------------------------------------------------------- diff --git a/test/singa/test_csv.cc b/test/singa/test_csv.cc new file mode 100644 index 0000000..d8dbe69 --- /dev/null +++ b/test/singa/test_csv.cc @@ -0,0 +1,60 @@ +/************************************************************ +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*************************************************************/ + +#include "singa/io/encoder.h" +#include "singa/io/decoder.h" +#include "gtest/gtest.h" +#include <sstream> +#include <algorithm> + +using singa::Shape; +using singa::Tensor; +TEST(CSV, EncoderDecode) { + singa::CSVEncoder encoder; + singa::CSVDecoder decoder; + + singa::DecoderConf decoder_conf; + decoder_conf.set_has_label(true); + decoder.Setup(decoder_conf); + EXPECT_EQ(true, decoder.has_label()); + + float in_data[] = {1.23, 4.5, 5.1, 3.33, 0.44}; + std::string in_str = "2, 1.23, 4.5, 5.1, 3.33, 0.44"; + int in_label = 2; + size_t size = 5; + + std::vector<Tensor> input; + Tensor data(Shape{size}, singa::kFloat32), label(Shape{1}, singa::kInt); + data.CopyDataFromHostPtr<float>(in_data, size); + label.CopyDataFromHostPtr<int>(&in_label, 1); + input.push_back(data); + input.push_back(label); + + std::string value = encoder.Encode(input); + in_str.erase(std::remove(in_str.begin(), in_str.end(), ' '), in_str.end()); + EXPECT_EQ(in_str, value); + + std::vector<Tensor> output = decoder.Decode(value); + const auto* out_data = output.at(0).data<float>(); + const auto* out_label = output.at(1).data<int>(); + for (size_t i = 0; i < size; i++) EXPECT_EQ(in_data[i], out_data[i]); + EXPECT_EQ(in_label, out_label[0]); +} http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/test/singa/test_decoder.cc ---------------------------------------------------------------------- diff --git a/test/singa/test_decoder.cc b/test/singa/test_decoder.cc deleted file mode 100644 index fa8683f..0000000 --- a/test/singa/test_decoder.cc +++ /dev/null @@ -1,84 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#include "singa/io/encoder.h" -#include "singa/io/decoder.h" -#include "gtest/gtest.h" -#include <time.h> - -#ifdef USE_OPENCV -#include <opencv2/highgui/highgui.hpp> -#include <opencv2/imgproc/imgproc.hpp> -using singa::Shape; -using singa::Tensor; -TEST(Decoder, Decode) { - singa::JPG2ProtoEncoder encoder; - singa::Proto2JPGDecoder decoder; - - // initial random seed - srand(time(NULL)); - - size_t height = 40, width = 30; - size_t nheight = 256, nwidth = 256, channel = 3; - size_t total = nheight * nwidth * channel; - cv::Mat image(height, width, CV_8UC3); - for (size_t i = 0; i < height; i++) - for (size_t j = 0; j < width; j++) - for (size_t k = 0; k < channel; k++) - image.at<cv::Vec3b>(i, j)[k] = static_cast<uchar>(rand() % 256); - - cv::Mat transformed; - cv::Size size(nwidth, nheight); - cv::resize(image, transformed, size); - EXPECT_EQ(static_cast<int>(nwidth), transformed.cols); - EXPECT_EQ(static_cast<int>(nheight), transformed.rows); - EXPECT_EQ(static_cast<int>(channel), transformed.channels()); - - unsigned char* buff = transformed.data; - Shape shape{nheight, nwidth, channel}; - Tensor pixel(shape, singa::kUChar), label(Shape{1}, singa::kInt); - pixel.CopyDataFromHostPtr<unsigned char>(buff, total); - int raw_label = 2; - label.CopyDataFromHostPtr<int>(&raw_label, 1); - - std::vector<Tensor> input; - input.push_back(pixel); - input.push_back(label); - const auto* in_pixel = input[0].data<unsigned char>(); - for (size_t i = 0; i < total; i++) EXPECT_EQ(buff[i], in_pixel[i]); - const int* in_label = input[1].data<int>(); - EXPECT_EQ(2, in_label[0]); - EXPECT_EQ(2u, input.size()); - - std::string tmp = encoder.Encode(input); - std::vector<Tensor> output = decoder.Decode(tmp); - EXPECT_EQ(2u, output.size()); - EXPECT_EQ(singa::kFloat32, output[0].data_type()); - Shape out_shape = output[0].shape(); - for (size_t i = 0; i < shape.size(); i++) EXPECT_EQ(shape[i], out_shape[i]); - const int* out_label = output[1].data<int>(); - EXPECT_EQ(raw_label, out_label[0]); - // opencv imencode will have some information loss - // const float* out_pixel = output[0].data<const float*>(); - // for(size_t i = 0; i < total; i++) - // EXPECT_LE(fabs(in_pixel[i]-out_pixel[i]), 10.f); -} -#endif http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/test/singa/test_jpg.cc ---------------------------------------------------------------------- diff --git a/test/singa/test_jpg.cc b/test/singa/test_jpg.cc new file mode 100644 index 0000000..f60546a --- /dev/null +++ b/test/singa/test_jpg.cc @@ -0,0 +1,84 @@ +/************************************************************ +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*************************************************************/ + +#include "singa/io/encoder.h" +#include "singa/io/decoder.h" +#include "gtest/gtest.h" +#include <time.h> + +#ifdef USE_OPENCV +#include <opencv2/highgui/highgui.hpp> +#include <opencv2/imgproc/imgproc.hpp> +using singa::Shape; +using singa::Tensor; +TEST(Decoder, Decode) { + singa::JPGEncoder encoder; + singa::JPGDecoder decoder; + + // initial random seed + srand(time(NULL)); + + size_t height = 40, width = 30; + size_t nheight = 256, nwidth = 256, channel = 3; + size_t total = nheight * nwidth * channel; + cv::Mat image(height, width, CV_8UC3); + for (size_t i = 0; i < height; i++) + for (size_t j = 0; j < width; j++) + for (size_t k = 0; k < channel; k++) + image.at<cv::Vec3b>(i, j)[k] = static_cast<uchar>(rand() % 256); + + cv::Mat transformed; + cv::Size size(nwidth, nheight); + cv::resize(image, transformed, size); + EXPECT_EQ(static_cast<int>(nwidth), transformed.cols); + EXPECT_EQ(static_cast<int>(nheight), transformed.rows); + EXPECT_EQ(static_cast<int>(channel), transformed.channels()); + + unsigned char* buff = transformed.data; + Shape shape{nheight, nwidth, channel}; + Tensor pixel(shape, singa::kUChar), label(Shape{1}, singa::kInt); + pixel.CopyDataFromHostPtr<unsigned char>(buff, total); + int raw_label = 2; + label.CopyDataFromHostPtr<int>(&raw_label, 1); + + std::vector<Tensor> input; + input.push_back(pixel); + input.push_back(label); + const auto* in_pixel = input[0].data<unsigned char>(); + for (size_t i = 0; i < total; i++) EXPECT_EQ(buff[i], in_pixel[i]); + const int* in_label = input[1].data<int>(); + EXPECT_EQ(2, in_label[0]); + EXPECT_EQ(2u, input.size()); + + std::string tmp = encoder.Encode(input); + std::vector<Tensor> output = decoder.Decode(tmp); + EXPECT_EQ(2u, output.size()); + EXPECT_EQ(singa::kFloat32, output[0].data_type()); + Shape out_shape = output[0].shape(); + for (size_t i = 0; i < shape.size(); i++) EXPECT_EQ(shape[i], out_shape[i]); + const int* out_label = output[1].data<int>(); + EXPECT_EQ(raw_label, out_label[0]); + // opencv imencode will have some information loss + // const float* out_pixel = output[0].data<const float*>(); + // for(size_t i = 0; i < total; i++) + // EXPECT_LE(fabs(in_pixel[i]-out_pixel[i]), 10.f); +} +#endif http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/test/singa/test_textfile_decoder.cc ---------------------------------------------------------------------- diff --git a/test/singa/test_textfile_decoder.cc b/test/singa/test_textfile_decoder.cc deleted file mode 100644 index bb31b88..0000000 --- a/test/singa/test_textfile_decoder.cc +++ /dev/null @@ -1,60 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#include "singa/io/encoder.h" -#include "singa/io/decoder.h" -#include "gtest/gtest.h" -#include <sstream> -#include <algorithm> - -using singa::Shape; -using singa::Tensor; -TEST(TextDecoder, Decode) { - singa::TextEncoder encoder; - singa::TextDecoder decoder; - - singa::DecoderConf decoder_conf; - decoder_conf.set_has_label(true); - decoder.Setup(decoder_conf); - EXPECT_EQ(true, decoder.has_label()); - - float in_data[] = {1.23, 4.5, 5.1, 3.33, 0.44}; - std::string in_str = "2, 1.23, 4.5, 5.1, 3.33, 0.44"; - int in_label = 2; - size_t size = 5; - - std::vector<Tensor> input; - Tensor data(Shape{size}, singa::kFloat32), label(Shape{1}, singa::kInt); - data.CopyDataFromHostPtr<float>(in_data, size); - label.CopyDataFromHostPtr<int>(&in_label, 1); - input.push_back(data); - input.push_back(label); - - std::string value = encoder.Encode(input); - in_str.erase(std::remove(in_str.begin(), in_str.end(), ' '), in_str.end()); - EXPECT_EQ(in_str, value); - - std::vector<Tensor> output = decoder.Decode(value); - const auto* out_data = output.at(0).data<float>(); - const auto* out_label = output.at(1).data<int>(); - for (size_t i = 0; i < size; i++) EXPECT_EQ(in_data[i], out_data[i]); - EXPECT_EQ(in_label, out_label[0]); -}
