SINGA-84 Header Files Rearrange Move deprecated input layers into input/deprecated.h and input/deprecated.cc Add "deprecated" tag in the comments. Remove the DataShard class in utils/ folder, use it as an alias of io::KVFile.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/e469503d Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/e469503d Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/e469503d Branch: refs/heads/master Commit: e469503de96ef3184c983f568b25a2d9710edd96 Parents: 239ed21 Author: Wei Wang <[email protected]> Authored: Thu Oct 8 17:53:08 2015 +0800 Committer: Wei Wang <[email protected]> Committed: Thu Oct 8 17:53:08 2015 +0800 ---------------------------------------------------------------------- Makefile.am | 9 +- include/singa/io/kvfile.h | 4 +- .../singa/neuralnet/connection_layer/bridge.h | 4 - .../singa/neuralnet/connection_layer/concate.h | 4 - .../singa/neuralnet/connection_layer/slice.h | 4 - .../singa/neuralnet/connection_layer/split.h | 4 - .../singa/neuralnet/input_layer/csv_record.h | 17 +- include/singa/neuralnet/input_layer/data.h | 76 ---- .../singa/neuralnet/input_layer/deprecated.h | 172 +++++++++ .../neuralnet/input_layer/image_preprocess.h | 17 - include/singa/neuralnet/input_layer/label.h | 59 --- include/singa/neuralnet/input_layer/lmdb_data.h | 75 ---- include/singa/neuralnet/input_layer/mnist.h | 62 ---- include/singa/neuralnet/input_layer/parser.h | 65 ---- include/singa/neuralnet/input_layer/prefetch.h | 22 +- .../singa/neuralnet/input_layer/proto_record.h | 15 - include/singa/neuralnet/input_layer/rgb_image.h | 66 ---- .../singa/neuralnet/input_layer/shard_data.h | 65 ---- .../singa/neuralnet/input_layer/store_input.h | 14 - include/singa/neuralnet/loss_layer/euclidean.h | 8 +- include/singa/neuralnet/loss_layer/softmax.h | 6 +- .../singa/neuralnet/neuron_layer/convolution.h | 4 - include/singa/neuralnet/neuron_layer/dropout.h | 5 - .../neuralnet/neuron_layer/inner_product.h | 6 - include/singa/neuralnet/neuron_layer/lrn.h | 4 - include/singa/neuralnet/neuron_layer/pooling.h | 4 - include/singa/neuralnet/neuron_layer/rbm.h | 4 - include/singa/neuralnet/neuron_layer/relu.h | 5 - include/singa/neuralnet/neuron_layer/sigmoid.h | 4 - include/singa/neuralnet/neuron_layer/stanh.h | 4 - include/singa/utils/data_shard.h | 171 --------- src/driver.cc | 9 +- src/neuralnet/input_layer/deprecated.cc | 360 +++++++++++++++++++ src/neuralnet/input_layer/label.cc | 56 --- src/neuralnet/input_layer/lmdb_data.cc | 143 -------- src/neuralnet/input_layer/mnist.cc | 86 ----- src/neuralnet/input_layer/parser.cc | 35 -- src/neuralnet/input_layer/prefetch.cc | 2 - src/neuralnet/input_layer/rgb_image.cc | 131 ------- src/neuralnet/input_layer/shard_data.cc | 80 ----- src/neuralnet/neuron_layer/convolution.cc | 1 - src/neuralnet/neuron_layer/dropout.cc | 1 - src/utils/data_shard.cc | 214 ----------- 43 files changed, 544 insertions(+), 1553 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/Makefile.am ---------------------------------------------------------------------- diff --git a/Makefile.am b/Makefile.am index 5c341a3..6419980 100644 --- a/Makefile.am +++ b/Makefile.am @@ -33,7 +33,6 @@ SINGA_SRCS := src/driver.cc \ src/utils/common.cc \ src/utils/param.cc \ src/utils/updater.cc \ - src/utils/data_shard.cc \ src/utils/blob.cc \ src/utils/image_transform.cc \ src/server.cc \ @@ -44,16 +43,11 @@ SINGA_SRCS := src/driver.cc \ src/neuralnet/connection_layer/concate.cc \ src/neuralnet/connection_layer/slice.cc \ src/neuralnet/connection_layer/split.cc \ - src/neuralnet/input_layer/parser.cc \ src/neuralnet/input_layer/csv_record.cc \ src/neuralnet/input_layer/image_preprocess.cc \ - src/neuralnet/input_layer/label.cc \ - src/neuralnet/input_layer/lmdb_data.cc \ - src/neuralnet/input_layer/mnist.cc \ src/neuralnet/input_layer/prefetch.cc \ src/neuralnet/input_layer/proto_record.cc \ - src/neuralnet/input_layer/rgb_image.cc \ - src/neuralnet/input_layer/shard_data.cc \ + src/neuralnet/input_layer/deprecated.cc \ src/neuralnet/input_layer/store_input.cc \ src/neuralnet/loss_layer/euclidean.cc \ src/neuralnet/loss_layer/softmax.cc \ @@ -80,7 +74,6 @@ SINGA_HDRS := include/singa/singa.h \ include/singa/utils/param.h \ include/singa/utils/common.h \ include/singa/utils/factory.h \ - include/singa/utils/data_shard.h \ include/singa/utils/singleton.h \ include/singa/utils/graph.h \ include/singa/utils/blob.h \ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/io/kvfile.h ---------------------------------------------------------------------- diff --git a/include/singa/io/kvfile.h b/include/singa/io/kvfile.h index 27dd35e..6d9a709 100644 --- a/include/singa/io/kvfile.h +++ b/include/singa/io/kvfile.h @@ -175,8 +175,8 @@ class KVFile { /** * @deprecated {ShardData is deprecated! Use KVFile}. -using ShardData = KVFile; -*/ + */ +using DataShard = io::KVFile; } // namespace singa #endif // SINGA_IO_KVFILE_H_ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/connection_layer/bridge.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/connection_layer/bridge.h b/include/singa/neuralnet/connection_layer/bridge.h index 333d6c9..5f224b3 100644 --- a/include/singa/neuralnet/connection_layer/bridge.h +++ b/include/singa/neuralnet/connection_layer/bridge.h @@ -25,10 +25,6 @@ #include <vector> #include "singa/neuralnet/layer.h" -/** - * \file this file includes the declarations of layers that inherit the - * base ConnectionLayer. - */ namespace singa { class BridgeLayer : virtual public ConnectionLayer { public: http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/connection_layer/concate.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/connection_layer/concate.h b/include/singa/neuralnet/connection_layer/concate.h index 9c5180b..5875835 100644 --- a/include/singa/neuralnet/connection_layer/concate.h +++ b/include/singa/neuralnet/connection_layer/concate.h @@ -25,10 +25,6 @@ #include <vector> #include "singa/neuralnet/layer.h" -/** - * \file this file includes the declarations of layers that inherit the - * base ConnectionLayer. - */ namespace singa { /** * Connect multiple (src) layers with a single (dst) layer. http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/connection_layer/slice.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/connection_layer/slice.h b/include/singa/neuralnet/connection_layer/slice.h index 87b1754..615685b 100644 --- a/include/singa/neuralnet/connection_layer/slice.h +++ b/include/singa/neuralnet/connection_layer/slice.h @@ -25,10 +25,6 @@ #include <vector> #include "singa/neuralnet/layer.h" -/** - * \file this file includes the declarations of layers that inherit the - * base ConnectionLayer. - */ namespace singa { /** * Connect a single (src) layer with multiple (dst) layers. http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/connection_layer/split.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/connection_layer/split.h b/include/singa/neuralnet/connection_layer/split.h index 391473a..7557766 100644 --- a/include/singa/neuralnet/connection_layer/split.h +++ b/include/singa/neuralnet/connection_layer/split.h @@ -25,10 +25,6 @@ #include <vector> #include "singa/neuralnet/layer.h" -/** - * \file this file includes the declarations of layers that inherit the - * base ConnectionLayer. - */ namespace singa { /** * Connect a single (src) layer with multiple dst layers. http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/input_layer/csv_record.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/input_layer/csv_record.h b/include/singa/neuralnet/input_layer/csv_record.h index 1c50ceb..2653b63 100644 --- a/include/singa/neuralnet/input_layer/csv_record.h +++ b/include/singa/neuralnet/input_layer/csv_record.h @@ -24,23 +24,8 @@ #include <string> #include <vector> -#include "singa/io/store.h" -#include "singa/neuralnet/layer.h" #include "singa/neuralnet/input_layer/store_input.h" -#include "singa/utils/data_shard.h" -/** - * \file this file includes the declarations of input layers that inherit the - * base InputLayer to load input features. - * - * The feature loading phase can be implemented using a single layer or - * separated into DataLayer (for loading features as records) and ParserLayer - * (for parsing features from records). SINGA has provided some subclasses of - * DataLayer and ParserLayer. - * - * Data prefetching can be implemented as a sub-class of InputLayer. - * SINGA provides a built-in PrefetchLayer which embeds DataLayer and - * ParserLayer. - */ + namespace singa { using std::string; using std::vector; http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/input_layer/data.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/input_layer/data.h b/include/singa/neuralnet/input_layer/data.h deleted file mode 100644 index 46ffbf7..0000000 --- a/include/singa/neuralnet/input_layer/data.h +++ /dev/null @@ -1,76 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#ifndef SINGA_NEURALNET_INPUT_LAYER_DATA_H_ -#define SINGA_NEURALNET_INPUT_LAYER_DATA_H_ - -#include <string> -#include <vector> -#include "singa/io/store.h" -#include "singa/neuralnet/layer.h" -#include "singa/utils/data_shard.h" -/** - * \file this file includes the declarations of input layers that inherit the - * base InputLayer to load input features. - * - * The feature loading phase can be implemented using a single layer or - * separated into DataLayer (for loading features as records) and ParserLayer - * (for parsing features from records). SINGA has provided some subclasses of - * DataLayer and ParserLayer. - * - * Data prefetching can be implemented as a sub-class of InputLayer. - * SINGA provides a built-in PrefetchLayer which embeds DataLayer and - * ParserLayer. - */ -namespace singa { -using std::string; -using std::vector; -/** - * Base layer for reading ::Record from local Shard, HDFS, lmdb, etc. - */ -class DataLayer: virtual public InputLayer { - public: - Blob<float>* mutable_data(const Layer* layer) override { return nullptr; } - ConnectionType dst_layer_connection() const override { - return kOneToMany; - } - - inline int batchsize() const { return batchsize_; } - virtual const Record& sample() const { - return sample_; - } - /** - * @return the loaded records - */ - virtual const std::vector<Record>& records() const { - return records_; - } - - protected: - int random_skip_; - int batchsize_; - Record sample_; - std::vector<Record> records_; -}; - -} // namespace singa - -#endif // SINGA_NEURALNET_INPUT_LAYER_DATA_H_ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/input_layer/deprecated.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/input_layer/deprecated.h b/include/singa/neuralnet/input_layer/deprecated.h new file mode 100644 index 0000000..bd2f60c --- /dev/null +++ b/include/singa/neuralnet/input_layer/deprecated.h @@ -0,0 +1,172 @@ +/************************************************************ +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*************************************************************/ + +#ifndef SINGA_NEURALNET_INPUT_LAYER_DEPRECATED_H_ +#define SINGA_NEURALNET_INPUT_LAYER_DEPRECATED_H_ + +#include <string> +#include <vector> +#include "singa/neuralnet/layer.h" +#include "singa/io/kvfile.h" +namespace singa { +using std::string; +using std::vector; +/** + * @deprecated please use the StoreInputLayer. + * + * Base layer for reading ::Record from local Shard, HDFS, lmdb, etc. + */ +class DataLayer: virtual public InputLayer { + public: + Blob<float>* mutable_data(const Layer* layer) override { return nullptr; } + ConnectionType dst_layer_connection() const override { + return kOneToMany; + } + + inline int batchsize() const { return batchsize_; } + virtual const Record& sample() const { + return sample_; + } + /** + * @return the loaded records + */ + virtual const std::vector<Record>& records() const { + return records_; + } + + protected: + int random_skip_; + int batchsize_; + Record sample_; + std::vector<Record> records_; +}; +/** + * @deprecated Please use the subclasses of StoreInputLayer. + * + * Layer for loading Record from DataShard. + */ +class ShardDataLayer : public DataLayer { + public: + ~ShardDataLayer(); + + void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override; + void ComputeFeature(int flag, const vector<Layer*>& srclayers) override; + + private: + DataShard* shard_; +}; +/** + * @deprecated please use the subclasses of StoreInputLayer. + * + * Layer for loading Record from LMDB. + */ +#ifdef USE_LMDB +#include <lmdb.h> +class LMDBDataLayer : public DataLayer { + public: + ~LMDBDataLayer(); + + void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override; + void OpenLMDB(const std::string& path); + void ComputeFeature(int flag, const vector<Layer*>& srclayers) override; + void ConvertCaffeDatumToRecord(const CaffeDatum& datum, + SingleLabelImageRecord* record); + + private: + MDB_env* mdb_env_; + MDB_dbi mdb_dbi_; + MDB_txn* mdb_txn_; + MDB_cursor* mdb_cursor_; + MDB_val mdb_key_, mdb_value_; +}; +#endif + +/******************Parser layers***************/ +/** + * @deprecated Please use the subclasses of StoreInputLayer which load and parse + * data in a single layer. + * + * Base layer for parsing the input records into Blobs. + */ +class ParserLayer : public InputLayer { + public: + void ComputeFeature(int flag, const vector<Layer*>& srclayers) override; + void ComputeGradient(int flag, const vector<Layer*>& srclayers) override {} + ConnectionType dst_layer_connection() const override { + return kOneToMany; + } + /** + * Parse records from DataLayer into blob. + */ + virtual void ParseRecords(int flag, const std::vector<Record>& records, + Blob<float>* blob) = 0; +}; +/** + * + * @deprecated Please use the SingleLabelRecordLayer which parses both feature + * and label for each record. Its aux_data() function returns the parsed labels. + * + * Derived from ParserLayer to parse label in SingaleLabelImageRecord loaded by + * ShardDataLayer. + */ +class LabelLayer : public ParserLayer { + public: + void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override; + void ParseRecords(int flag, const std::vector<Record>& records, + Blob<float>* blob) override; +}; + +/** + * @deprecated Please use the subclasses of StoreInputLayer. + * + * Derived from ParserLayer to parse MNIST feature from SingaleLabelImageRecord. + */ +class MnistLayer : public ParserLayer { + public: + void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override; + void ParseRecords(int flag, const std::vector<Record>& records, + Blob<float>* blob) override; + + protected: + float norm_a_, norm_b_; +}; +/** + * @deprecated please use the ImagePreprocessLayer which preprocess image + * feature from data Blob of source layers. + * + * Derived from ParserLayer to parse RGB image feature from + * SingaleLabelImageRecord. + */ +class RGBImageLayer : public ParserLayer { + public: + void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override; + void ParseRecords(int flag, const std::vector<Record>& records, + Blob<float>* blob) override; + + private: + float scale_; + int cropsize_; + bool mirror_; + Blob<float> mean_; +}; +} // namespace singa + +#endif // SINGA_NEURALNET_INPUT_LAYER_DEPRECATED_H_ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/input_layer/image_preprocess.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/input_layer/image_preprocess.h b/include/singa/neuralnet/input_layer/image_preprocess.h index 2407990..f3bf90a 100644 --- a/include/singa/neuralnet/input_layer/image_preprocess.h +++ b/include/singa/neuralnet/input_layer/image_preprocess.h @@ -22,26 +22,9 @@ #ifndef SINGA_NEURALNET_INPUT_LAYER_IMAGE_PREPROCESS_H_ #define SINGA_NEURALNET_INPUT_LAYER_IMAGE_PREPROCESS_H_ -#include <string> #include <vector> -#include "singa/io/store.h" #include "singa/neuralnet/layer.h" -#include "singa/utils/data_shard.h" -/** - * \file this file includes the declarations of input layers that inherit the - * base InputLayer to load input features. - * - * The feature loading phase can be implemented using a single layer or - * separated into DataLayer (for loading features as records) and ParserLayer - * (for parsing features from records). SINGA has provided some subclasses of - * DataLayer and ParserLayer. - * - * Data prefetching can be implemented as a sub-class of InputLayer. - * SINGA provides a built-in PrefetchLayer which embeds DataLayer and - * ParserLayer. - */ namespace singa { -using std::string; using std::vector; /** * Do preprocessing for images, including cropping, mirroring, resizing. http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/input_layer/label.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/input_layer/label.h b/include/singa/neuralnet/input_layer/label.h deleted file mode 100644 index 1c40275..0000000 --- a/include/singa/neuralnet/input_layer/label.h +++ /dev/null @@ -1,59 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#ifndef SINGA_NEURALNET_INPUT_LAYER_LABEL_H_ -#define SINGA_NEURALNET_INPUT_LAYER_LABEL_H_ - -#include <string> -#include <vector> -#include "singa/io/store.h" -#include "singa/neuralnet/layer.h" -#include "singa/neuralnet/input_layer/parser.h" -#include "singa/utils/data_shard.h" -/** - * \file this file includes the declarations of input layers that inherit the - * base InputLayer to load input features. - * - * The feature loading phase can be implemented using a single layer or - * separated into DataLayer (for loading features as records) and ParserLayer - * (for parsing features from records). SINGA has provided some subclasses of - * DataLayer and ParserLayer. - * - * Data prefetching can be implemented as a sub-class of InputLayer. - * SINGA provides a built-in PrefetchLayer which embeds DataLayer and - * ParserLayer. - */ -namespace singa { -using std::string; -using std::vector; -/** - * Derived from ParserLayer to parse label from SingaleLabelImageRecord. - */ -class LabelLayer : public ParserLayer { - public: - void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override; - void ParseRecords(int flag, const std::vector<Record>& records, - Blob<float>* blob) override; -}; - -} // namespace singa - -#endif // SINGA_NEURALNET_INPUT_LAYER_LABEL_H_ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/input_layer/lmdb_data.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/input_layer/lmdb_data.h b/include/singa/neuralnet/input_layer/lmdb_data.h deleted file mode 100644 index bc26ab0..0000000 --- a/include/singa/neuralnet/input_layer/lmdb_data.h +++ /dev/null @@ -1,75 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#ifndef SINGA_NEURALNET_INPUT_LAYER_LMDB_DATA_H_ -#define SINGA_NEURALNET_INPUT_LAYER_LMDB_DATA_H_ - -#include <string> -#include <vector> -#include "singa/io/store.h" -#include "singa/neuralnet/layer.h" -#include "singa/neuralnet/input_layer/data.h" -#include "singa/utils/data_shard.h" -/** - * \file this file includes the declarations of input layers that inherit the - * base InputLayer to load input features. - * - * The feature loading phase can be implemented using a single layer or - * separated into DataLayer (for loading features as records) and ParserLayer - * (for parsing features from records). SINGA has provided some subclasses of - * DataLayer and ParserLayer. - * - * Data prefetching can be implemented as a sub-class of InputLayer. - * SINGA provides a built-in PrefetchLayer which embeds DataLayer and - * ParserLayer. - */ -namespace singa { -using std::string; -using std::vector; -/** - * Layer for loading Record from LMDB. - * - * It is derived from DataLayer. - */ -#ifdef USE_LMDB -#include <lmdb.h> -class LMDBDataLayer : public DataLayer { - public: - ~LMDBDataLayer(); - - void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override; - void OpenLMDB(const std::string& path); - void ComputeFeature(int flag, const vector<Layer*>& srclayers) override; - void ConvertCaffeDatumToRecord(const CaffeDatum& datum, - SingleLabelImageRecord* record); - - private: - MDB_env* mdb_env_; - MDB_dbi mdb_dbi_; - MDB_txn* mdb_txn_; - MDB_cursor* mdb_cursor_; - MDB_val mdb_key_, mdb_value_; -}; -#endif - -} // namespace singa - -#endif // SINGA_NEURALNET_INPUT_LAYER_LMDB_DATA_H_ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/input_layer/mnist.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/input_layer/mnist.h b/include/singa/neuralnet/input_layer/mnist.h deleted file mode 100644 index 7c91173..0000000 --- a/include/singa/neuralnet/input_layer/mnist.h +++ /dev/null @@ -1,62 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#ifndef SINGA_NEURALNET_INPUT_LAYER_MNIST_H_ -#define SINGA_NEURALNET_INPUT_LAYER_MNIST_H_ - -#include <string> -#include <vector> -#include "singa/io/store.h" -#include "singa/neuralnet/layer.h" -#include "singa/neuralnet/input_layer/parser.h" -#include "singa/utils/data_shard.h" -/** - * \file this file includes the declarations of input layers that inherit the - * base InputLayer to load input features. - * - * The feature loading phase can be implemented using a single layer or - * separated into DataLayer (for loading features as records) and ParserLayer - * (for parsing features from records). SINGA has provided some subclasses of - * DataLayer and ParserLayer. - * - * Data prefetching can be implemented as a sub-class of InputLayer. - * SINGA provides a built-in PrefetchLayer which embeds DataLayer and - * ParserLayer. - */ -namespace singa { -using std::string; -using std::vector; -/** - * Derived from ParserLayer to parse MNIST feature from SingaleLabelImageRecord. - */ -class MnistLayer : public ParserLayer { - public: - void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override; - void ParseRecords(int flag, const std::vector<Record>& records, - Blob<float>* blob) override; - - protected: - float norm_a_, norm_b_; -}; - -} // namespace singa - -#endif // SINGA_NEURALNET_INPUT_LAYER_MNIST_H_ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/input_layer/parser.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/input_layer/parser.h b/include/singa/neuralnet/input_layer/parser.h deleted file mode 100644 index de52b0a..0000000 --- a/include/singa/neuralnet/input_layer/parser.h +++ /dev/null @@ -1,65 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#ifndef SINGA_NEURALNET_INPUT_LAYER_PARSER_H_ -#define SINGA_NEURALNET_INPUT_LAYER_PARSER_H_ - -#include <string> -#include <vector> -#include "singa/io/store.h" -#include "singa/neuralnet/layer.h" -#include "singa/utils/data_shard.h" -/** - * \file this file includes the declarations of input layers that inherit the - * base InputLayer to load input features. - * - * The feature loading phase can be implemented using a single layer or - * separated into DataLayer (for loading features as records) and ParserLayer - * (for parsing features from records). SINGA has provided some subclasses of - * DataLayer and ParserLayer. - * - * Data prefetching can be implemented as a sub-class of InputLayer. - * SINGA provides a built-in PrefetchLayer which embeds DataLayer and - * ParserLayer. - */ -namespace singa { -using std::string; -using std::vector; -/** - * Base layer for parsing the input records into Blobs. - */ -class ParserLayer : public InputLayer { - public: - void ComputeFeature(int flag, const vector<Layer*>& srclayers) override; - void ComputeGradient(int flag, const vector<Layer*>& srclayers) override {} - ConnectionType dst_layer_connection() const override { - return kOneToMany; - } - /** - * Parse records from DataLayer into blob. - */ - virtual void ParseRecords(int flag, const std::vector<Record>& records, - Blob<float>* blob) = 0; -}; - -} // namespace singa - -#endif // SINGA_NEURALNET_INPUT_LAYER_PARSER_H_ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/input_layer/prefetch.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/input_layer/prefetch.h b/include/singa/neuralnet/input_layer/prefetch.h index 038f7e2..d30521d 100644 --- a/include/singa/neuralnet/input_layer/prefetch.h +++ b/include/singa/neuralnet/input_layer/prefetch.h @@ -24,30 +24,14 @@ #include <string> #include <vector> -#include "singa/io/store.h" +#include <thread> #include "singa/neuralnet/layer.h" -#include "singa/utils/data_shard.h" -/** - * \file this file includes the declarations of input layers that inherit the - * base InputLayer to load input features. - * - * The feature loading phase can be implemented using a single layer or - * separated into DataLayer (for loading features as records) and ParserLayer - * (for parsing features from records). SINGA has provided some subclasses of - * DataLayer and ParserLayer. - * - * Data prefetching can be implemented as a sub-class of InputLayer. - * SINGA provides a built-in PrefetchLayer which embeds DataLayer and - * ParserLayer. - */ namespace singa { -using std::string; using std::vector; /** - * Layer for prefetching data records and parsing them. + * TODO(wangwei) Layer for prefetching data records and parsing them. * - * The data loading and parsing work is done by internal DataLayer and - * ParserLayer respectively. This layer controls the prefetching thread, i.e., + * This layer controls the prefetching thread, i.e., * creating and joining the prefetching thread. */ class PrefetchLayer : public Layer { http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/input_layer/proto_record.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/input_layer/proto_record.h b/include/singa/neuralnet/input_layer/proto_record.h index bfc9934..6a4ad0c 100644 --- a/include/singa/neuralnet/input_layer/proto_record.h +++ b/include/singa/neuralnet/input_layer/proto_record.h @@ -25,22 +25,7 @@ #include <string> #include <vector> #include "singa/io/store.h" -#include "singa/neuralnet/layer.h" #include "singa/neuralnet/input_layer/store_input.h" -#include "singa/utils/data_shard.h" -/** - * \file this file includes the declarations of input layers that inherit the - * base InputLayer to load input features. - * - * The feature loading phase can be implemented using a single layer or - * separated into DataLayer (for loading features as records) and ParserLayer - * (for parsing features from records). SINGA has provided some subclasses of - * DataLayer and ParserLayer. - * - * Data prefetching can be implemented as a sub-class of InputLayer. - * SINGA provides a built-in PrefetchLayer which embeds DataLayer and - * ParserLayer. - */ namespace singa { using std::string; using std::vector; http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/input_layer/rgb_image.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/input_layer/rgb_image.h b/include/singa/neuralnet/input_layer/rgb_image.h deleted file mode 100644 index c4d6f18..0000000 --- a/include/singa/neuralnet/input_layer/rgb_image.h +++ /dev/null @@ -1,66 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#ifndef SINGA_NEURALNET_INPUT_LAYER_RGB_IMAGE_H_ -#define SINGA_NEURALNET_INPUT_LAYER_RGB_IMAGE_H_ - -#include <string> -#include <vector> -#include "singa/io/store.h" -#include "singa/neuralnet/layer.h" -#include "singa/neuralnet/input_layer/parser.h" -#include "singa/utils/data_shard.h" -/** - * \file this file includes the declarations of input layers that inherit the - * base InputLayer to load input features. - * - * The feature loading phase can be implemented using a single layer or - * separated into DataLayer (for loading features as records) and ParserLayer - * (for parsing features from records). SINGA has provided some subclasses of - * DataLayer and ParserLayer. - * - * Data prefetching can be implemented as a sub-class of InputLayer. - * SINGA provides a built-in PrefetchLayer which embeds DataLayer and - * ParserLayer. - */ -namespace singa { -using std::string; -using std::vector; -/** - * Derived from ParserLayer to parse RGB image feature from - * SingaleLabelImageRecord. - */ -class RGBImageLayer : public ParserLayer { - public: - void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override; - void ParseRecords(int flag, const std::vector<Record>& records, - Blob<float>* blob) override; - - private: - float scale_; - int cropsize_; - bool mirror_; - Blob<float> mean_; -}; - -} // namespace singa - -#endif // SINGA_NEURALNET_INPUT_LAYER_RGB_IMAGE_H_ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/input_layer/shard_data.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/input_layer/shard_data.h b/include/singa/neuralnet/input_layer/shard_data.h deleted file mode 100644 index 13ea9d8..0000000 --- a/include/singa/neuralnet/input_layer/shard_data.h +++ /dev/null @@ -1,65 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#ifndef SINGA_NEURALNET_INPUT_LAYER_SHARD_DATA_H_ -#define SINGA_NEURALNET_INPUT_LAYER_SHARD_DATA_H_ - -#include <string> -#include <vector> -#include "singa/io/store.h" -#include "singa/neuralnet/layer.h" -#include "singa/neuralnet/input_layer/data.h" -#include "singa/utils/data_shard.h" -/** - * \file this file includes the declarations of input layers that inherit the - * base InputLayer to load input features. - * - * The feature loading phase can be implemented using a single layer or - * separated into DataLayer (for loading features as records) and ParserLayer - * (for parsing features from records). SINGA has provided some subclasses of - * DataLayer and ParserLayer. - * - * Data prefetching can be implemented as a sub-class of InputLayer. - * SINGA provides a built-in PrefetchLayer which embeds DataLayer and - * ParserLayer. - */ -namespace singa { -using std::string; -using std::vector; -/** - * Layer for loading Record from DataShard. - * - * It is derived from DataLayer. - */ -class ShardDataLayer : public DataLayer { - public: - ~ShardDataLayer(); - - void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override; - void ComputeFeature(int flag, const vector<Layer*>& srclayers) override; - - private: - DataShard* shard_; -}; - -} // namespace singa - -#endif http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/input_layer/store_input.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/input_layer/store_input.h b/include/singa/neuralnet/input_layer/store_input.h index dd63be6..86fbaf2 100644 --- a/include/singa/neuralnet/input_layer/store_input.h +++ b/include/singa/neuralnet/input_layer/store_input.h @@ -26,20 +26,6 @@ #include <vector> #include "singa/io/store.h" #include "singa/neuralnet/layer.h" -#include "singa/utils/data_shard.h" -/** - * \file this file includes the declarations of input layers that inherit the - * base InputLayer to load input features. - * - * The feature loading phase can be implemented using a single layer or - * separated into DataLayer (for loading features as records) and ParserLayer - * (for parsing features from records). SINGA has provided some subclasses of - * DataLayer and ParserLayer. - * - * Data prefetching can be implemented as a sub-class of InputLayer. - * SINGA provides a built-in PrefetchLayer which embeds DataLayer and - * ParserLayer. - */ namespace singa { using std::string; using std::vector; http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/loss_layer/euclidean.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/loss_layer/euclidean.h b/include/singa/neuralnet/loss_layer/euclidean.h index 57703e0..4d50dc5 100644 --- a/include/singa/neuralnet/loss_layer/euclidean.h +++ b/include/singa/neuralnet/loss_layer/euclidean.h @@ -25,15 +25,11 @@ #include <vector> #include "singa/neuralnet/layer.h" -/** - * @file this file includes the declarations of layers that inherit the base - * LossLayer for measuring the objective training loss. - */ namespace singa { using std::vector; /** - * Squared Euclidean loss as @f$0.5 ||p - t||^2@f$, where p is for prediction - * t is for ground truth. + * Squared Euclidean loss as @f$0.5 ||p - t||^2@f$, where p is prediction + * result, t is the ground truth. */ class EuclideanLossLayer : public LossLayer { public: http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/loss_layer/softmax.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/loss_layer/softmax.h b/include/singa/neuralnet/loss_layer/softmax.h index 7a57517..c497377 100644 --- a/include/singa/neuralnet/loss_layer/softmax.h +++ b/include/singa/neuralnet/loss_layer/softmax.h @@ -25,10 +25,6 @@ #include <vector> #include "singa/neuralnet/layer.h" -/** - * @file this file includes the declarations of layers that inherit the base - * LossLayer for measuring the objective training loss. - */ namespace singa { using std::vector; /** @@ -43,7 +39,7 @@ class SoftmaxLossLayer : public LossLayer { void ComputeGradient(int flag, const vector<Layer*>& srclayers) override; /** - * softmax is not recommendeded for partition because it requires the whole + * This layer is not recommendeded for partition because it requires the whole * src layer for normalization. */ ConnectionType src_neuron_connection(int k) const override { http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/neuron_layer/convolution.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/neuron_layer/convolution.h b/include/singa/neuralnet/neuron_layer/convolution.h index a73131d..2ffbf46 100644 --- a/include/singa/neuralnet/neuron_layer/convolution.h +++ b/include/singa/neuralnet/neuron_layer/convolution.h @@ -26,10 +26,6 @@ #include "singa/neuralnet/layer.h" #include "singa/proto/job.pb.h" -/** - * \file this file includes the declarations neuron layer classes that conduct - * the transformation of features. - */ namespace singa { /** * Convolution layer. http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/neuron_layer/dropout.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/neuron_layer/dropout.h b/include/singa/neuralnet/neuron_layer/dropout.h index 1fe189e..8c1883c 100644 --- a/include/singa/neuralnet/neuron_layer/dropout.h +++ b/include/singa/neuralnet/neuron_layer/dropout.h @@ -24,12 +24,7 @@ #include <vector> #include "singa/neuralnet/layer.h" -#include "singa/proto/job.pb.h" -/** - * \file this file includes the declarations neuron layer classes that conduct - * the transformation of features. - */ namespace singa { class DropoutLayer : public NeuronLayer { http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/neuron_layer/inner_product.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/neuron_layer/inner_product.h b/include/singa/neuralnet/neuron_layer/inner_product.h index e109bb3..bf3fdee 100644 --- a/include/singa/neuralnet/neuron_layer/inner_product.h +++ b/include/singa/neuralnet/neuron_layer/inner_product.h @@ -24,12 +24,6 @@ #include <vector> #include "singa/neuralnet/layer.h" -#include "singa/proto/job.pb.h" - -/** - * \file this file includes the declarations neuron layer classes that conduct - * the transformation of features. - */ namespace singa { class InnerProductLayer : public NeuronLayer { http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/neuron_layer/lrn.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/neuron_layer/lrn.h b/include/singa/neuralnet/neuron_layer/lrn.h index ba983a4..42b10ed 100644 --- a/include/singa/neuralnet/neuron_layer/lrn.h +++ b/include/singa/neuralnet/neuron_layer/lrn.h @@ -26,10 +26,6 @@ #include "singa/neuralnet/layer.h" #include "singa/proto/job.pb.h" -/** - * \file this file includes the declarations neuron layer classes that conduct - * the transformation of features. - */ namespace singa { /** * Local Response Normalization edge http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/neuron_layer/pooling.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/neuron_layer/pooling.h b/include/singa/neuralnet/neuron_layer/pooling.h index d8f0f68..a66a4ef 100644 --- a/include/singa/neuralnet/neuron_layer/pooling.h +++ b/include/singa/neuralnet/neuron_layer/pooling.h @@ -26,10 +26,6 @@ #include "singa/neuralnet/layer.h" #include "singa/proto/job.pb.h" -/** - * \file this file includes the declarations neuron layer classes that conduct - * the transformation of features. - */ namespace singa { class PoolingLayer : public NeuronLayer { http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/neuron_layer/rbm.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/neuron_layer/rbm.h b/include/singa/neuralnet/neuron_layer/rbm.h index c2e1140..7c6f81a 100644 --- a/include/singa/neuralnet/neuron_layer/rbm.h +++ b/include/singa/neuralnet/neuron_layer/rbm.h @@ -26,10 +26,6 @@ #include "singa/neuralnet/layer.h" #include "singa/proto/job.pb.h" -/** - * \file this file includes the declarations neuron layer classes that conduct - * the transformation of features. - */ namespace singa { /** * Base layer for RBM models. http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/neuron_layer/relu.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/neuron_layer/relu.h b/include/singa/neuralnet/neuron_layer/relu.h index 735be7c..48a40a8 100644 --- a/include/singa/neuralnet/neuron_layer/relu.h +++ b/include/singa/neuralnet/neuron_layer/relu.h @@ -25,11 +25,6 @@ #include <vector> #include "singa/neuralnet/layer.h" #include "singa/proto/job.pb.h" - -/** - * \file this file includes the declarations neuron layer classes that conduct - * the transformation of features. - */ namespace singa { class ReLULayer : public NeuronLayer { http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/neuron_layer/sigmoid.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/neuron_layer/sigmoid.h b/include/singa/neuralnet/neuron_layer/sigmoid.h index 2d32206..88f15e9 100644 --- a/include/singa/neuralnet/neuron_layer/sigmoid.h +++ b/include/singa/neuralnet/neuron_layer/sigmoid.h @@ -26,10 +26,6 @@ #include "singa/neuralnet/layer.h" #include "singa/proto/job.pb.h" -/** - * \file this file includes the declarations neuron layer classes that conduct - * the transformation of features. - */ namespace singa { /** * This layer apply Sigmoid function to neuron activations. http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/neuron_layer/stanh.h ---------------------------------------------------------------------- diff --git a/include/singa/neuralnet/neuron_layer/stanh.h b/include/singa/neuralnet/neuron_layer/stanh.h index e25f44c..08edb7d 100644 --- a/include/singa/neuralnet/neuron_layer/stanh.h +++ b/include/singa/neuralnet/neuron_layer/stanh.h @@ -26,10 +26,6 @@ #include "singa/neuralnet/layer.h" #include "singa/proto/job.pb.h" -/** - * \file this file includes the declarations neuron layer classes that conduct - * the transformation of features. - */ namespace singa { /** * This layer apply scaled Tanh function to neuron activations. http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/utils/data_shard.h ---------------------------------------------------------------------- diff --git a/include/singa/utils/data_shard.h b/include/singa/utils/data_shard.h deleted file mode 100644 index 7d69ae5..0000000 --- a/include/singa/utils/data_shard.h +++ /dev/null @@ -1,171 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#ifndef SINGA_UTILS_DATA_SHARD_H_ -#define SINGA_UTILS_DATA_SHARD_H_ - -#include <google/protobuf/message.h> -#include <fstream> -#include <string> -#include <unordered_set> - -namespace singa { - -/** - * Data shard stores training/validation/test tuples. - * Every worker node should have a training shard (validation/test shard - * is optional). The shard file for training is - * singa::Cluster::workspace()/train/shard.dat; The shard file for validation - * is singa::Cluster::workspace()/train/shard.dat; Similar path for test. - * - * shard.dat consists of a set of unordered tuples. Each tuple is - * encoded as [key_len key record_len val] (key_len and record_len are of type - * uint32, which indicate the bytes of key and record respectively. - * - * When Shard obj is created, it will remove the last key if the record size - * and key size do not match because the last write of tuple crashed. - * - * TODO - * 1. split one shard into multiple shards. - * 2. add threading to prefetch and parse records - * - */ -class DataShard { - public: - enum { - // read only mode used in training - kRead = 0, - // write mode used in creating shard (will overwrite previous one) - kCreate = 1, - // append mode, e.g. used when previous creating crashes - kAppend = 2 - }; - - /** - * Init the shard obj. - * - * @param folder Shard folder (path excluding shard.dat) on worker node - * @param mode Shard open mode, Shard::kRead, Shard::kWrite or Shard::kAppend - * @param bufsize Batch bufsize bytes data for every disk op (read or write), - * default is 100MB - */ - DataShard(const std::string& folder, int mode); - DataShard(const std::string& folder, int mode, int capacity); - ~DataShard(); - - /** - * read next tuple from the shard. - * - * @param key Tuple key - * @param val Record of type Message - * @return false if read unsuccess, e.g., the tuple was not inserted - * completely. - */ - bool Next(std::string* key, google::protobuf::Message* val); - /** - * read next tuple from the shard. - * - * @param key Tuple key - * @param val Record of type string - * @return false if read unsuccess, e.g., the tuple was not inserted - * completely. - */ - bool Next(std::string* key, std::string* val); - /** - * Append one tuple to the shard. - * - * @param key e.g., image path - * @param val - * @return false if unsucess, e.g., inserted before - */ - bool Insert(const std::string& key, const google::protobuf::Message& tuple); - /** - * Append one tuple to the shard. - * - * @param key e.g., image path - * @param val - * @return false if unsucess, e.g., inserted before - */ - bool Insert(const std::string& key, const std::string& tuple); - /** - * Move the read pointer to the head of the shard file. - * Used for repeated reading. - */ - void SeekToFirst(); - /** - * Flush buffered data to disk. - * Used only for kCreate or kAppend. - */ - void Flush(); - /** - * Iterate through all tuples to get the num of all tuples. - * - * @return num of tuples - */ - int Count(); - /** - * @return path to shard file - */ - inline std::string path() { return path_; } - - protected: - /** - * Read the next key and prepare buffer for reading value. - * - * @param key - * @return length (i.e., bytes) of value field. - */ - int Next(std::string* key); - /** - * Setup the disk pointer to the right position for append in case that - * the pervious write crashes. - * - * @param path shard path. - * @return offset (end pos) of the last success written record. - */ - int PrepareForAppend(const std::string& path); - /** - * Read data from disk if the current data in the buffer is not a full field. - * - * @param size size of the next field. - */ - bool PrepareNextField(int size); - - private: - char mode_ = 0; - std::string path_ = ""; - // either ifstream or ofstream - std::fstream fdat_; - // to avoid replicated record - std::unordered_set<std::string> keys_; - // internal buffer - char* buf_ = nullptr; - // offset inside the buf_ - int offset_ = 0; - // allocated bytes for the buf_ - int capacity_ = 0; - // bytes in buf_, used in reading - int bufsize_ = 0; -}; - -} // namespace singa - -#endif // SINGA_UTILS_DATA_SHARD_H_ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/src/driver.cc ---------------------------------------------------------------------- diff --git a/src/driver.cc b/src/driver.cc index 5f81325..90d7bde 100644 --- a/src/driver.cc +++ b/src/driver.cc @@ -37,17 +37,11 @@ #include "singa/neuralnet/connection_layer/concate.h" #include "singa/neuralnet/connection_layer/slice.h" #include "singa/neuralnet/connection_layer/split.h" -#include "singa/neuralnet/input_layer/parser.h" +#include "singa/neuralnet/input_layer/deprecated.h" #include "singa/neuralnet/input_layer/csv_record.h" -#include "singa/neuralnet/input_layer/data.h" #include "singa/neuralnet/input_layer/image_preprocess.h" -#include "singa/neuralnet/input_layer/label.h" -#include "singa/neuralnet/input_layer/lmdb_data.h" -#include "singa/neuralnet/input_layer/mnist.h" #include "singa/neuralnet/input_layer/prefetch.h" #include "singa/neuralnet/input_layer/proto_record.h" -#include "singa/neuralnet/input_layer/rgb_image.h" -#include "singa/neuralnet/input_layer/shard_data.h" #include "singa/neuralnet/input_layer/store_input.h" #include "singa/neuralnet/loss_layer/euclidean.h" #include "singa/neuralnet/loss_layer/softmax.h" @@ -62,6 +56,7 @@ #include "singa/neuralnet/neuron_layer/stanh.h" #include "singa/neuralnet/output_layer/output_layer.h" + extern "C" void openblas_set_num_threads(int num); namespace singa { http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/src/neuralnet/input_layer/deprecated.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/input_layer/deprecated.cc b/src/neuralnet/input_layer/deprecated.cc new file mode 100644 index 0000000..b0baf1d --- /dev/null +++ b/src/neuralnet/input_layer/deprecated.cc @@ -0,0 +1,360 @@ +/************************************************************ +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*************************************************************/ + +#include "singa/neuralnet/input_layer/deprecated.h" +#include "mshadow/tensor.h" +namespace singa { + +using namespace mshadow; +using mshadow::cpu; +using mshadow::Shape4; +using mshadow::Tensor; + +using std::string; +using std::vector; + +ShardDataLayer::~ShardDataLayer() { + if (shard_ != nullptr) + delete shard_; + shard_ = nullptr; +} + +void ShardDataLayer::Setup(const LayerProto& proto, + const vector<Layer*>& srclayers) { + Layer::Setup(proto, srclayers); + shard_ = new DataShard(proto.sharddata_conf().path(), DataShard::kRead); + string key; + shard_->Next(&key, &sample_); + delete shard_; + shard_ = nullptr; + batchsize_ = proto.sharddata_conf().batchsize(); + if (partition_dim() == 0) + batchsize_ /= proto.num_partitions(); + records_.resize(batchsize_); + random_skip_ = proto.sharddata_conf().random_skip(); +} + +void ShardDataLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) { + if (shard_ == nullptr) + shard_ = new DataShard(layer_conf_.sharddata_conf().path(), + DataShard::kRead); + if (random_skip_) { + int nskip = rand() % random_skip_; + LOG(INFO) << "Random Skip " << nskip << " records, there are " + << shard_->Count() << " records in total"; + string key; + for (int i = 0; i < nskip; i++) { + shard_->Next(&key, &sample_); + } + random_skip_ = 0; + } + for (auto& record : records_) { + string key; + if (!shard_->Next(&key, &record)) { + shard_->SeekToFirst(); + CHECK(shard_->Next(&key, &record)); + } + } +} + +/*****************LMDB data layer*******************/ +#ifdef USE_LMDB +LMDBDataLayer::~LMDBDataLayer() { + mdb_cursor_close(mdb_cursor_); + mdb_txn_abort(mdb_txn_); + mdb_cursor_ = nullptr; +} + +void LMDBDataLayer::Setup(const LayerProto& proto, + const vector<Layer*>& srclayers) { + Layer::Setup(proto, srclayers); + OpenLMDB(proto.lmdbdata_conf().path()); + CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_NEXT), + MDB_SUCCESS); + mdb_cursor_close(mdb_cursor_); + mdb_txn_abort(mdb_txn_); + mdb_cursor_ = nullptr; + CaffeDatum datum; + datum.ParseFromArray(mdb_value_.mv_data, mdb_value_.mv_size); + SingleLabelImageRecord* record = sample_.mutable_image(); + ConvertCaffeDatumToRecord(datum, record); + batchsize_ = proto.lmdbdata_conf().batchsize(); + if (partition_dim() == 0) + batchsize_ /= proto.num_partitions(); + records_.resize(batchsize_); + random_skip_ = proto.lmdbdata_conf().random_skip(); +} + +void LMDBDataLayer::OpenLMDB(const std::string& path) { + CHECK_EQ(mdb_env_create(&mdb_env_), MDB_SUCCESS) << "mdb_env_create failed"; + CHECK_EQ(mdb_env_set_mapsize(mdb_env_, 1099511627776), MDB_SUCCESS); // 1TB + CHECK_EQ(mdb_env_open(mdb_env_, path.c_str(), + MDB_RDONLY, 0664), MDB_SUCCESS) << "cannot open lmdb " << path; + CHECK_EQ(mdb_txn_begin(mdb_env_, NULL, MDB_RDONLY, &mdb_txn_), MDB_SUCCESS) + << "mdb_txn_begin failed"; + CHECK_EQ(mdb_open(mdb_txn_, NULL, 0, &mdb_dbi_), MDB_SUCCESS) + << "mdb_open failed"; + CHECK_EQ(mdb_cursor_open(mdb_txn_, mdb_dbi_, &mdb_cursor_), MDB_SUCCESS) + << "mdb_cursor_open failed"; + LOG(INFO) << "Opening lmdb " << path; + CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_FIRST), + MDB_SUCCESS) << "mdb_cursor_get failed"; +} + +void LMDBDataLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) { + if (mdb_cursor_ == nullptr) + OpenLMDB(layer_conf_.lmdbdata_conf().path()); + if (random_skip_) { + int nskip = rand() % random_skip_; + int n = 0; + CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, + &mdb_value_, MDB_FIRST), MDB_SUCCESS); + while (mdb_cursor_get(mdb_cursor_, &mdb_key_, + &mdb_value_, MDB_NEXT) == MDB_SUCCESS) + n++; + LOG(INFO) << "Random Skip " << nskip << " records of total " + << n << "records"; + // We have reached the end. Restart from the first. + CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, + &mdb_value_, MDB_FIRST), MDB_SUCCESS); + for (int i = 0; i < nskip; i++) { + if (mdb_cursor_get(mdb_cursor_, &mdb_key_, + &mdb_value_, MDB_NEXT) != MDB_SUCCESS) { + // We have reached the end. Restart from the first. + DLOG(INFO) << "Restarting data prefetching from start."; + CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, + &mdb_value_, MDB_FIRST), MDB_SUCCESS); + } + } + random_skip_ = 0; + } + CaffeDatum datum; + for (auto& record : records_) { + SingleLabelImageRecord* image = record.mutable_image(); + CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, + &mdb_value_, MDB_GET_CURRENT), MDB_SUCCESS); + datum.ParseFromArray(mdb_value_.mv_data, mdb_value_.mv_size); + ConvertCaffeDatumToRecord(datum, image); + if (mdb_cursor_get(mdb_cursor_, &mdb_key_, + &mdb_value_, MDB_NEXT) != MDB_SUCCESS) { + // We have reached the end. Restart from the first. + DLOG(INFO) << "Restarting data prefetching from start."; + CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, + &mdb_value_, MDB_FIRST), MDB_SUCCESS); + } + } +} + +void LMDBDataLayer::ConvertCaffeDatumToRecord(const CaffeDatum& datum, + SingleLabelImageRecord* record) { + record->set_label(datum.label()); + record->clear_shape(); + if (datum.has_channels()) + record->add_shape(datum.channels()); + if (datum.has_height()) + record->add_shape(datum.height()); + if (datum.has_width()) + record->add_shape(datum.width()); + if (datum.has_data()) + record->set_pixel(datum.data()); + if (datum.float_data_size()) { + record->clear_data(); + for (float x : datum.float_data()) + record->add_data(x); + } +} +#endif + +/***************Parser layer*******************/ +void ParserLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) { + CHECK_EQ(srclayers.size(), 1); + auto datalayer = dynamic_cast<DataLayer*>(*srclayers.begin()); + ParseRecords(flag, datalayer->records(), &data_); +} + +/**********Mnist Layer************/ +void MnistLayer::ParseRecords(int flag, const vector<Record>& records, + Blob<float>* blob) { + LOG_IF(ERROR, records.size() == 0) << "Empty records to parse"; + int ndim = records.at(0).image().shape_size(); + int inputsize = records.at(0).image().shape(ndim-1); + CHECK_EQ(inputsize, blob->shape()[2]); + + float* dptr = blob->mutable_cpu_data(); + for (const Record& record : records) { + const SingleLabelImageRecord& imagerecord = record.image(); + if (imagerecord.pixel().size()) { + string pixel = imagerecord.pixel(); + for (int i = 0, k = 0; i < inputsize; i++) { + for (int j = 0; j < inputsize; j++) { + // NOTE!!! must cast pixel to uint8_t then to float!!! waste a lot of + // time to debug this + float x = static_cast<float>(static_cast<uint8_t>(pixel[k++])); + x = x / norm_a_-norm_b_; + *dptr = x; + dptr++; + } + } + } else { + for (int i = 0, k = 0; i < inputsize; i++) { + for (int j = 0; j < inputsize; j++) { + *dptr = imagerecord.data(k++) / norm_a_ - norm_b_; + dptr++; + } + } + } + } + CHECK_EQ(dptr, blob->mutable_cpu_data() + blob->count()); +} + +void MnistLayer::Setup(const LayerProto& proto, + const vector<Layer*>& srclayers) { + Layer::Setup(proto, srclayers); + CHECK_EQ(srclayers.size(), 1); + int batchsize = dynamic_cast<DataLayer*>(srclayers[0])->batchsize(); + Record sample = dynamic_cast<DataLayer*>(srclayers[0])->sample(); + norm_a_ = proto.mnist_conf().norm_a(); + norm_b_ = proto.mnist_conf().norm_b(); + int ndim = sample.image().shape_size(); + CHECK_GE(ndim, 2); + int s = sample.image().shape(ndim - 1); + CHECK_EQ(s, sample.image().shape(ndim - 2)); + data_.Reshape(vector<int>{batchsize, 1, s, s}); +} + +/**********RGB image layer****************/ +void RGBImageLayer::ParseRecords(int flag, const vector<Record>& records, + Blob<float>* blob) { + const vector<int>& s = blob->shape(); + Tensor<cpu, 4> images(data_.mutable_cpu_data(), + Shape4(s[0], s[1], s[2], s[3])); + const SingleLabelImageRecord& r = records.at(0).image(); + Tensor<cpu, 3> raw_image(Shape3(r.shape(0), r.shape(1), r.shape(2))); + AllocSpace(raw_image); + Tensor<cpu, 3> croped_image(nullptr, Shape3(s[1], s[2], s[3])); + if (cropsize_) + AllocSpace(croped_image); + int rid = 0; + const float* meandptr = mean_.cpu_data(); + for (const Record& record : records) { + auto image = images[rid]; + bool do_crop = cropsize_> 0 && ((flag & kTrain) == kTrain); + bool do_mirror = mirror_ && rand() % 2 && ((flag & kTrain) == kTrain); + float* dptr = nullptr; + if (do_crop || do_mirror) + dptr = raw_image.dptr; + else + dptr = image.dptr; + if (record.image().pixel().size()) { + string pixel = record.image().pixel(); + for (size_t i = 0; i < pixel.size(); i++) + dptr[i] = static_cast<float>(static_cast<uint8_t>(pixel[i])); + } else { + memcpy(dptr, record.image().data().data(), + sizeof(float) * record.image().data_size()); + } + for (int i = 0; i < mean_.count(); i++) + dptr[i] -= meandptr[i]; + if (do_crop) { + int hoff = rand() % (r.shape(1) - cropsize_); + int woff = rand() % (r.shape(2) - cropsize_); + Shape<2> cropshape = Shape2(cropsize_, cropsize_); + if (do_mirror) { + croped_image = expr::crop(raw_image, cropshape, hoff, woff); + image = expr::mirror(croped_image); + } else { + image = expr::crop(raw_image, cropshape, hoff, woff); + } + } else if (do_mirror) { + image = expr::mirror(raw_image); + } + rid++; + } + if (scale_) + images = images * scale_; + FreeSpace(raw_image); + if (cropsize_) + FreeSpace(croped_image); +} + +void RGBImageLayer::Setup(const LayerProto& proto, + const vector<Layer*>& srclayers) { + ParserLayer::Setup(proto, srclayers); + CHECK_EQ(srclayers.size(), 1); + scale_ = proto.rgbimage_conf().scale(); + cropsize_ = proto.rgbimage_conf().cropsize(); + mirror_ = proto.rgbimage_conf().mirror(); + int batchsize = dynamic_cast<DataLayer*>(srclayers[0])->batchsize(); + Record sample = dynamic_cast<DataLayer*>(srclayers[0])->sample(); + vector<int> shape; + shape.push_back(batchsize); + for (int x : sample.image().shape()) { + shape.push_back(x); + } + CHECK_EQ(shape.size(), 4); + if (cropsize_) { + shape[2] = cropsize_; + shape[3] = cropsize_; + } + data_.Reshape(shape); + mean_.Reshape({shape[1], shape[2], shape[3]}); + if (proto.rgbimage_conf().has_meanfile()) { + if (proto.rgbimage_conf().meanfile().find("binaryproto") != string::npos) { + CaffeBlob mean; + ReadProtoFromBinaryFile(proto.rgbimage_conf().meanfile().c_str(), &mean); + CHECK_EQ(mean_.count(), mean.data_size()); + memcpy(mean_.mutable_cpu_data(), mean.data().data(), + sizeof(float)*mean.data_size()); + } else { + SingleLabelImageRecord mean; + ReadProtoFromBinaryFile(proto.rgbimage_conf().meanfile().c_str(), &mean); + CHECK_EQ(mean_.count(), mean.data_size()); + memcpy(mean_.mutable_cpu_data(), mean.data().data(), + sizeof(float)*mean.data_size()); + } + } else { + memset(mean_.mutable_cpu_data(), 0, sizeof(float) * mean_.count()); + } +} + +/*************Label layer *************/ + +void LabelLayer::Setup(const LayerProto& proto, + const vector<Layer*>& srclayers) { + Layer::Setup(proto, srclayers); + CHECK_EQ(srclayers.size(), 1); + int batchsize = dynamic_cast<DataLayer*>(srclayers[0])->batchsize(); + data_.Reshape(vector<int>{batchsize}); +} + +void LabelLayer::ParseRecords(int flag, const vector<Record>& records, + Blob<float>* blob) { + int rid = 0; + float *label = blob->mutable_cpu_data(); + for (const Record& record : records) { + label[rid++] = record.image().label(); + // CHECK_LT(record.image().label(),10); + } + CHECK_EQ(rid, blob->shape()[0]); +} + + +} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/src/neuralnet/input_layer/label.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/input_layer/label.cc b/src/neuralnet/input_layer/label.cc deleted file mode 100644 index 6de9324..0000000 --- a/src/neuralnet/input_layer/label.cc +++ /dev/null @@ -1,56 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#include "singa/neuralnet/input_layer/label.h" -#include "singa/neuralnet/input_layer/data.h" -#include "mshadow/tensor.h" -#include "singa/utils/image_transform.h" -#include "singa/utils/tokenizer.h" -namespace singa { - -using namespace mshadow; -using mshadow::cpu; -using mshadow::Shape4; -using mshadow::Tensor; - -using std::string; -using std::vector; - -void LabelLayer::Setup(const LayerProto& proto, - const vector<Layer*>& srclayers) { - Layer::Setup(proto, srclayers); - CHECK_EQ(srclayers.size(), 1); - int batchsize = dynamic_cast<DataLayer*>(srclayers[0])->batchsize(); - data_.Reshape(vector<int>{batchsize}); -} - -void LabelLayer::ParseRecords(int flag, const vector<Record>& records, - Blob<float>* blob) { - int rid = 0; - float *label = blob->mutable_cpu_data(); - for (const Record& record : records) { - label[rid++] = record.image().label(); - // CHECK_LT(record.image().label(),10); - } - CHECK_EQ(rid, blob->shape()[0]); -} - -} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/src/neuralnet/input_layer/lmdb_data.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/input_layer/lmdb_data.cc b/src/neuralnet/input_layer/lmdb_data.cc deleted file mode 100644 index 37da538..0000000 --- a/src/neuralnet/input_layer/lmdb_data.cc +++ /dev/null @@ -1,143 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#include "singa/neuralnet/input_layer/lmdb_data.h" -#include "mshadow/tensor.h" -#include "singa/utils/image_transform.h" -#include "singa/utils/tokenizer.h" -namespace singa { - -using namespace mshadow; -using mshadow::cpu; -using mshadow::Shape4; -using mshadow::Tensor; - -using std::string; -using std::vector; - -#ifdef USE_LMDB -LMDBDataLayer::~LMDBDataLayer() { - mdb_cursor_close(mdb_cursor_); - mdb_txn_abort(mdb_txn_); - mdb_cursor_ = nullptr; -} - -void LMDBDataLayer::Setup(const LayerProto& proto, - const vector<Layer*>& srclayers) { - Layer::Setup(proto, srclayers); - OpenLMDB(proto.lmdbdata_conf().path()); - CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_NEXT), - MDB_SUCCESS); - mdb_cursor_close(mdb_cursor_); - mdb_txn_abort(mdb_txn_); - mdb_cursor_ = nullptr; - CaffeDatum datum; - datum.ParseFromArray(mdb_value_.mv_data, mdb_value_.mv_size); - SingleLabelImageRecord* record = sample_.mutable_image(); - ConvertCaffeDatumToRecord(datum, record); - batchsize_ = proto.lmdbdata_conf().batchsize(); - if (partition_dim() == 0) - batchsize_ /= proto.num_partitions(); - records_.resize(batchsize_); - random_skip_ = proto.lmdbdata_conf().random_skip(); -} - -void LMDBDataLayer::OpenLMDB(const std::string& path) { - CHECK_EQ(mdb_env_create(&mdb_env_), MDB_SUCCESS) << "mdb_env_create failed"; - CHECK_EQ(mdb_env_set_mapsize(mdb_env_, 1099511627776), MDB_SUCCESS); // 1TB - CHECK_EQ(mdb_env_open(mdb_env_, path.c_str(), - MDB_RDONLY, 0664), MDB_SUCCESS) << "cannot open lmdb " << path; - CHECK_EQ(mdb_txn_begin(mdb_env_, NULL, MDB_RDONLY, &mdb_txn_), MDB_SUCCESS) - << "mdb_txn_begin failed"; - CHECK_EQ(mdb_open(mdb_txn_, NULL, 0, &mdb_dbi_), MDB_SUCCESS) - << "mdb_open failed"; - CHECK_EQ(mdb_cursor_open(mdb_txn_, mdb_dbi_, &mdb_cursor_), MDB_SUCCESS) - << "mdb_cursor_open failed"; - LOG(INFO) << "Opening lmdb " << path; - CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_FIRST), - MDB_SUCCESS) << "mdb_cursor_get failed"; -} - -void LMDBDataLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) { - if (mdb_cursor_ == nullptr) - OpenLMDB(layer_conf_.lmdbdata_conf().path()); - if (random_skip_) { - int nskip = rand() % random_skip_; - int n = 0; - CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, - &mdb_value_, MDB_FIRST), MDB_SUCCESS); - while (mdb_cursor_get(mdb_cursor_, &mdb_key_, - &mdb_value_, MDB_NEXT) == MDB_SUCCESS) - n++; - LOG(INFO) << "Random Skip " << nskip << " records of total " - << n << "records"; - // We have reached the end. Restart from the first. - CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, - &mdb_value_, MDB_FIRST), MDB_SUCCESS); - for (int i = 0; i < nskip; i++) { - if (mdb_cursor_get(mdb_cursor_, &mdb_key_, - &mdb_value_, MDB_NEXT) != MDB_SUCCESS) { - // We have reached the end. Restart from the first. - DLOG(INFO) << "Restarting data prefetching from start."; - CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, - &mdb_value_, MDB_FIRST), MDB_SUCCESS); - } - } - random_skip_ = 0; - } - CaffeDatum datum; - for (auto& record : records_) { - SingleLabelImageRecord* image = record.mutable_image(); - CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, - &mdb_value_, MDB_GET_CURRENT), MDB_SUCCESS); - datum.ParseFromArray(mdb_value_.mv_data, mdb_value_.mv_size); - ConvertCaffeDatumToRecord(datum, image); - if (mdb_cursor_get(mdb_cursor_, &mdb_key_, - &mdb_value_, MDB_NEXT) != MDB_SUCCESS) { - // We have reached the end. Restart from the first. - DLOG(INFO) << "Restarting data prefetching from start."; - CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, - &mdb_value_, MDB_FIRST), MDB_SUCCESS); - } - } -} - -void LMDBDataLayer::ConvertCaffeDatumToRecord(const CaffeDatum& datum, - SingleLabelImageRecord* record) { - record->set_label(datum.label()); - record->clear_shape(); - if (datum.has_channels()) - record->add_shape(datum.channels()); - if (datum.has_height()) - record->add_shape(datum.height()); - if (datum.has_width()) - record->add_shape(datum.width()); - if (datum.has_data()) - record->set_pixel(datum.data()); - if (datum.float_data_size()) { - record->clear_data(); - for (float x : datum.float_data()) - record->add_data(x); - } -} -#endif - -} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/src/neuralnet/input_layer/mnist.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/input_layer/mnist.cc b/src/neuralnet/input_layer/mnist.cc deleted file mode 100644 index 41e4390..0000000 --- a/src/neuralnet/input_layer/mnist.cc +++ /dev/null @@ -1,86 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#include "singa/neuralnet/input_layer/mnist.h" -#include "singa/neuralnet/input_layer/data.h" -#include "mshadow/tensor.h" -#include "singa/utils/image_transform.h" -#include "singa/utils/tokenizer.h" -namespace singa { - -using namespace mshadow; -using mshadow::cpu; -using mshadow::Shape4; -using mshadow::Tensor; - -using std::string; -using std::vector; - -void MnistLayer::ParseRecords(int flag, const vector<Record>& records, - Blob<float>* blob) { - LOG_IF(ERROR, records.size() == 0) << "Empty records to parse"; - int ndim = records.at(0).image().shape_size(); - int inputsize = records.at(0).image().shape(ndim-1); - CHECK_EQ(inputsize, blob->shape()[2]); - - float* dptr = blob->mutable_cpu_data(); - for (const Record& record : records) { - const SingleLabelImageRecord& imagerecord = record.image(); - if (imagerecord.pixel().size()) { - string pixel = imagerecord.pixel(); - for (int i = 0, k = 0; i < inputsize; i++) { - for (int j = 0; j < inputsize; j++) { - // NOTE!!! must cast pixel to uint8_t then to float!!! waste a lot of - // time to debug this - float x = static_cast<float>(static_cast<uint8_t>(pixel[k++])); - x = x / norm_a_-norm_b_; - *dptr = x; - dptr++; - } - } - } else { - for (int i = 0, k = 0; i < inputsize; i++) { - for (int j = 0; j < inputsize; j++) { - *dptr = imagerecord.data(k++) / norm_a_ - norm_b_; - dptr++; - } - } - } - } - CHECK_EQ(dptr, blob->mutable_cpu_data() + blob->count()); -} - -void MnistLayer::Setup(const LayerProto& proto, - const vector<Layer*>& srclayers) { - Layer::Setup(proto, srclayers); - CHECK_EQ(srclayers.size(), 1); - int batchsize = dynamic_cast<DataLayer*>(srclayers[0])->batchsize(); - Record sample = dynamic_cast<DataLayer*>(srclayers[0])->sample(); - norm_a_ = proto.mnist_conf().norm_a(); - norm_b_ = proto.mnist_conf().norm_b(); - int ndim = sample.image().shape_size(); - CHECK_GE(ndim, 2); - int s = sample.image().shape(ndim - 1); - CHECK_EQ(s, sample.image().shape(ndim - 2)); - data_.Reshape(vector<int>{batchsize, 1, s, s}); -} - -} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/src/neuralnet/input_layer/parser.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/input_layer/parser.cc b/src/neuralnet/input_layer/parser.cc deleted file mode 100644 index dfd9d8e..0000000 --- a/src/neuralnet/input_layer/parser.cc +++ /dev/null @@ -1,35 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#include "singa/neuralnet/input_layer/parser.h" -#include "mshadow/tensor.h" -#include "singa/utils/image_transform.h" -#include "singa/utils/tokenizer.h" -#include "singa/neuralnet/input_layer/data.h" -namespace singa { - -void ParserLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) { - CHECK_EQ(srclayers.size(), 1); - auto datalayer = dynamic_cast<DataLayer*>(*srclayers.begin()); - ParseRecords(flag, datalayer->records(), &data_); -} - -} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/src/neuralnet/input_layer/prefetch.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/input_layer/prefetch.cc b/src/neuralnet/input_layer/prefetch.cc index e726838..9c1d7ee 100644 --- a/src/neuralnet/input_layer/prefetch.cc +++ b/src/neuralnet/input_layer/prefetch.cc @@ -21,8 +21,6 @@ #include "singa/neuralnet/input_layer/prefetch.h" #include "mshadow/tensor.h" -#include "singa/utils/image_transform.h" -#include "singa/utils/tokenizer.h" namespace singa { using namespace mshadow; http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/src/neuralnet/input_layer/rgb_image.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/input_layer/rgb_image.cc b/src/neuralnet/input_layer/rgb_image.cc deleted file mode 100644 index 3763932..0000000 --- a/src/neuralnet/input_layer/rgb_image.cc +++ /dev/null @@ -1,131 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#include "singa/neuralnet/input_layer/rgb_image.h" -#include "singa/neuralnet/input_layer/data.h" -#include "mshadow/tensor.h" -#include "singa/utils/image_transform.h" -#include "singa/utils/tokenizer.h" -namespace singa { - -using namespace mshadow; -using mshadow::cpu; -using mshadow::Shape4; -using mshadow::Tensor; - -using std::string; -using std::vector; - -void RGBImageLayer::ParseRecords(int flag, const vector<Record>& records, - Blob<float>* blob) { - const vector<int>& s = blob->shape(); - Tensor<cpu, 4> images(data_.mutable_cpu_data(), - Shape4(s[0], s[1], s[2], s[3])); - const SingleLabelImageRecord& r = records.at(0).image(); - Tensor<cpu, 3> raw_image(Shape3(r.shape(0), r.shape(1), r.shape(2))); - AllocSpace(raw_image); - Tensor<cpu, 3> croped_image(nullptr, Shape3(s[1], s[2], s[3])); - if (cropsize_) - AllocSpace(croped_image); - int rid = 0; - const float* meandptr = mean_.cpu_data(); - for (const Record& record : records) { - auto image = images[rid]; - bool do_crop = cropsize_> 0 && ((flag & kTrain) == kTrain); - bool do_mirror = mirror_ && rand() % 2 && ((flag & kTrain) == kTrain); - float* dptr = nullptr; - if (do_crop || do_mirror) - dptr = raw_image.dptr; - else - dptr = image.dptr; - if (record.image().pixel().size()) { - string pixel = record.image().pixel(); - for (size_t i = 0; i < pixel.size(); i++) - dptr[i] = static_cast<float>(static_cast<uint8_t>(pixel[i])); - } else { - memcpy(dptr, record.image().data().data(), - sizeof(float) * record.image().data_size()); - } - for (int i = 0; i < mean_.count(); i++) - dptr[i] -= meandptr[i]; - if (do_crop) { - int hoff = rand() % (r.shape(1) - cropsize_); - int woff = rand() % (r.shape(2) - cropsize_); - Shape<2> cropshape = Shape2(cropsize_, cropsize_); - if (do_mirror) { - croped_image = expr::crop(raw_image, cropshape, hoff, woff); - image = expr::mirror(croped_image); - } else { - image = expr::crop(raw_image, cropshape, hoff, woff); - } - } else if (do_mirror) { - image = expr::mirror(raw_image); - } - rid++; - } - if (scale_) - images = images * scale_; - FreeSpace(raw_image); - if (cropsize_) - FreeSpace(croped_image); -} - -void RGBImageLayer::Setup(const LayerProto& proto, - const vector<Layer*>& srclayers) { - ParserLayer::Setup(proto, srclayers); - CHECK_EQ(srclayers.size(), 1); - scale_ = proto.rgbimage_conf().scale(); - cropsize_ = proto.rgbimage_conf().cropsize(); - mirror_ = proto.rgbimage_conf().mirror(); - int batchsize = dynamic_cast<DataLayer*>(srclayers[0])->batchsize(); - Record sample = dynamic_cast<DataLayer*>(srclayers[0])->sample(); - vector<int> shape; - shape.push_back(batchsize); - for (int x : sample.image().shape()) { - shape.push_back(x); - } - CHECK_EQ(shape.size(), 4); - if (cropsize_) { - shape[2] = cropsize_; - shape[3] = cropsize_; - } - data_.Reshape(shape); - mean_.Reshape({shape[1], shape[2], shape[3]}); - if (proto.rgbimage_conf().has_meanfile()) { - if (proto.rgbimage_conf().meanfile().find("binaryproto") != string::npos) { - CaffeBlob mean; - ReadProtoFromBinaryFile(proto.rgbimage_conf().meanfile().c_str(), &mean); - CHECK_EQ(mean_.count(), mean.data_size()); - memcpy(mean_.mutable_cpu_data(), mean.data().data(), - sizeof(float)*mean.data_size()); - } else { - SingleLabelImageRecord mean; - ReadProtoFromBinaryFile(proto.rgbimage_conf().meanfile().c_str(), &mean); - CHECK_EQ(mean_.count(), mean.data_size()); - memcpy(mean_.mutable_cpu_data(), mean.data().data(), - sizeof(float)*mean.data_size()); - } - } else { - memset(mean_.mutable_cpu_data(), 0, sizeof(float) * mean_.count()); - } -} - -} // namespace singa
