Repository: incubator-singa Updated Branches: refs/heads/master f1ecd78ac -> 921f9277f
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/6b1e65ee/src/test/dist_test/test_tuple.cc ---------------------------------------------------------------------- diff --git a/src/test/dist_test/test_tuple.cc b/src/test/dist_test/test_tuple.cc deleted file mode 100644 index 727f8e3..0000000 --- a/src/test/dist_test/test_tuple.cc +++ /dev/null @@ -1,258 +0,0 @@ -#include <cstdio> -#include <iostream> -#include <sstream> -#include <string> -#include <vector> - -#include "server.h" -#include "proto/worker.pb.h" -#include "utils/network_service.h" -#include "core/common.h" -#include "core/network_queue.h" -#include "proto/model.pb.h" -#include "proto/common.pb.h" -#include "utils/global_context.h" - -/** - * @file test_tuple.cc - * - * Test performance of TableServer put/get/update operations. - */ -DECLARE_double(sleep_time); - -using namespace lapis; -using namespace std; -using std::vector; - -#define NKEYS 1000 -#define TUPLE_SIZE 50000000 - -#ifndef FLAGS_v - DEFINE_int32(v, 3, "vlog controller"); -#endif - - -#define SIZE 16 -#define THRESHOLD 500000 -int tuple_sizes[SIZE] = {37448736, 16777216, 4096000, 1327104, 884736, 884736, 614400,14112,4096,4096,1000,384,384,256,256,96}; -vector<int> valsizes; -int collect_size; -int num_tuples; - -void Put(int tid, int size, int version) { - RequestBase request; - request.set_table(0); - request.set_source(NetworkService::Get()->id()); - PutRequest *put_req = request.MutableExtension(PutRequest::name); - int shard = tid % GlobalContext::Get()->num_servers(); - put_req->set_shard(shard); - TableData *tuple = put_req->mutable_data(); - - TKey* key = tuple->mutable_key(); - TVal* val = tuple->mutable_value(); - - key->set_id(tid); - key->set_version(version); - - DAryProto *data = val->mutable_data(); - for (int i = 0; i < size; i++){ - data->add_value(0.0f); - } - - // TODO check the msg type - NetworkService::Get()->Send(shard, MTYPE_REQUEST, request); -} - -void Update(int tid, int size, int version) { - RequestBase request; - request.set_table(0); - request.set_source(NetworkService::Get()->id()); - UpdateRequest *update_req = request.MutableExtension(UpdateRequest::name); - int shard = tid % GlobalContext::Get()->num_servers(); - update_req->set_shard(shard); - TableData *tuple = update_req->mutable_data(); - - TKey* key = tuple->mutable_key(); - TVal* val = tuple->mutable_value(); - - key->set_id(tid); - key->set_version(version); - - DAryProto *data = val->mutable_grad(); - for (int i = 0; i < size; i++) - data->add_value(1.0f); - // TODO check the msg type - NetworkService::Get()->Send(shard, MTYPE_REQUEST, request); -} - -void print_result(TableData *data){ - TKey *key = data->mutable_key(); - TVal *val = data->mutable_value(); - int k = key->id(); - VLOG(3) << "key = " << k; - string s; - for (int i=0; i<TUPLE_SIZE; i++) - s.append(to_string(val->mutable_data()->value(i))).append(" "); - VLOG(3) << "val = " <<s; -} - -void AsyncGet(int tid, int version) { - RequestBase request; - request.set_table(0); - request.set_source(GlobalContext::Get()->rank()); //NetworkService::Get()->id()); - GetRequest *get_req = request.MutableExtension(GetRequest::name); - int shard = tid % GlobalContext::Get()->num_servers(); - get_req->set_shard(shard); - - TKey *key = get_req->mutable_key(); - key->set_id(tid); - key->set_version(version); - NetworkService::Get()->Send(shard, MTYPE_REQUEST, request); - -} - -void Collect(){ - int count = collect_size; - double start_collect = Now(); - while (count){ - while (true) { - Message *resp = NetworkService::Get()->Receive(); - if (!resp) - Sleep(FLAGS_sleep_time); - else{ - delete resp; - break; - } - } - count--; - } - double end_collect = Now(); - VLOG(3) << "Collected " << collect_size << " tuples in " << (end_collect-start_collect); -} - -/** - * Workers wait for the barrier, then one of them send SHUTDOWN message - * to all table servers. - */ -void worker_send_shutdown(int id){ - auto gc = lapis::GlobalContext::Get(); - NetworkService *network_service_ = NetworkService::Get().get(); - MPI_Barrier(gc->workergroup_comm()); - if (gc->rank()==id){ - for (int i=0; i<gc->num_procs(); i++){ - if (gc->IsTableServer(i)){ - EmptyMessage msg; - network_service_->Send(i, MTYPE_SHUTDOWN,msg); - } - } - } -} - -/** - * One worker with the specific ID puts, others wait. - */ -void worker_load_data(int id){ - auto gc = lapis::GlobalContext::Get(); - for (int i = 0; i < SIZE; i++) { - int m = tuple_sizes[i]; - if (m < THRESHOLD) - valsizes.push_back(m); - else { - for (int j = 0; j < m / THRESHOLD; j++) - valsizes.push_back(THRESHOLD); - if (m % THRESHOLD) - valsizes.push_back(m%THRESHOLD); - } - } - num_tuples = (int)valsizes.size(); - collect_size = 0; - for (int i=0; i<num_tuples; i++) - if (i%gc->group_size()==gc->worker_id()) - collect_size++; - - if (gc->rank()==id){ - for (size_t i=0; i<valsizes.size(); i++) - Put(i,valsizes[i],0); - VLOG(3) << "Done loading data, num_keys = "<<valsizes.size() << " process " << id; - } - VLOG(3) << "Collect size = " << collect_size; - MPI_Barrier(gc->workergroup_comm()); -} - -void worker_update_data() { - auto gc = lapis::GlobalContext::Get(); - for (int i = 0; i < num_tuples; i++) - if (i%gc->group_size()==gc->worker_id()) - Update(i,valsizes[i],0); - - VLOG(3) << "Done update ... for "<<collect_size << " tuples "; -} - -/* - * Async get. - */ -void worker_get_data(){ - auto gc = lapis::GlobalContext::Get(); - for (int i=0; i<num_tuples; i++) - if (i%gc->group_size()==gc->worker_id()) - AsyncGet(i,0); - Collect(); - VLOG(3) << "Done collect ..."; -} - -void start_network_service_for_worker(){ - NetworkService *network_service_ = NetworkService::Get().get(); - network_service_->Init(GlobalContext::Get()->rank(), Network::Get().get(), new SimpleQueue()); - network_service_->StartNetworkService(); -} - -int main(int argc, char **argv) { - google::InitGoogleLogging(argv[0]); - gflags::ParseCommandLineFlags(&argc, &argv, true); - - int provided; - - - MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); - - - FLAGS_logtostderr = 1; - - - // Init GlobalContext - Cluster cluster; - cluster.set_server_start(0); - cluster.set_server_end(8); - cluster.set_worker_start(8); - cluster.set_worker_end(24); - cluster.set_group_size(8); - cluster.set_data_folder("/data1/wangwei/lapis"); - - auto gc = lapis::GlobalContext::Get(cluster); - - // worker or table server - if (gc->AmITableServer()) { - lapis::TableServer server; - SGDProto sgd; - sgd.set_learning_rate(0.01); - sgd.set_momentum(0.9); - sgd.set_weight_decay(0.1); - sgd.set_gamma(0.5); - sgd.set_learning_rate_change_steps(1); - server.Start(sgd); - } else { - start_network_service_for_worker(); - worker_load_data(cluster.worker_start()); - for (int i=0; i<10; i++){ - worker_update_data(); - worker_get_data(); - } - worker_send_shutdown(cluster.worker_start()); - NetworkService::Get()->Shutdown(); - } - gc->Finalize(); - MPI_Finalize(); - VLOG(3) << "End, process "<< gc->rank(); - return 0; -} - http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/6b1e65ee/src/test/model/test_blob.cc ---------------------------------------------------------------------- diff --git a/src/test/model/test_blob.cc b/src/test/model/test_blob.cc deleted file mode 100644 index 75f1921..0000000 --- a/src/test/model/test_blob.cc +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright © 2014 Wei Wang. All Rights Reserved. -// 2014-07-18 19:44 -#include <gtest/gtest.h> -#include "proto/model.pb.h" -#include "model/lapis.h" - -namespace lapis { -class BlobTest : public ::testing::Test { - public: - BlobTest() : blob1(new Blob()), blob2(new Blob()) {} - ~BlobTest() { - delete blob1; - delete blob2; - } - protected: - Blob *blob1, *blob2; - Blob blob3, blob4; -}; - -TEST_F(BlobTest, Constructor) { - EXPECT_EQ(blob1->length(), 0); - EXPECT_EQ(blob1->width(), 0); - EXPECT_EQ(blob1->height(), 0); - EXPECT_EQ(blob3.length(), 0); - EXPECT_EQ(blob3.width(), 0); - EXPECT_EQ(blob3.height(), 0); - EXPECT_TRUE(blob2->dptr == nullptr); - EXPECT_TRUE(blob4.dptr == nullptr); -} - -TEST_F(BlobTest, TestResize) { - blob1->Resize(10,1,1,1); - EXPECT_EQ(blob1->length(), 10); - EXPECT_EQ(blob1->num(), 10); - EXPECT_EQ(blob1->height(), 1); - EXPECT_EQ(blob1->width(), 1); - EXPECT_TRUE(blob1->dptr != nullptr); - blob2->Resize(4,1,1,3); - EXPECT_EQ(blob2->length(), 12); - EXPECT_EQ(blob2->num(), 4); - EXPECT_EQ(blob2->height(), 1); - EXPECT_EQ(blob2->width(), 3); - EXPECT_TRUE(blob2->dptr != nullptr); - blob3.Resize(5,1,4,3); - EXPECT_EQ(blob3.length(), 60); - EXPECT_EQ(blob3.num(), 5); - EXPECT_EQ(blob3.height(), 4); - EXPECT_EQ(blob3.width(), 3); - EXPECT_TRUE(blob3.dptr != nullptr); - blob4.Resize(6,5,4,3); - EXPECT_EQ(blob4.length(), 360); - EXPECT_EQ(blob4.num(), 6); - EXPECT_EQ(blob4.height(), 4); - EXPECT_EQ(blob4.width(), 3); - EXPECT_TRUE(blob4.dptr != nullptr); -} - -} // namespace lapis http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/6b1e65ee/src/test/model/test_data_layer.cc ---------------------------------------------------------------------- diff --git a/src/test/model/test_data_layer.cc b/src/test/model/test_data_layer.cc deleted file mode 100644 index 49519a5..0000000 --- a/src/test/model/test_data_layer.cc +++ /dev/null @@ -1,178 +0,0 @@ -// Copyright © 2014 Wei Wang. All Rights Reserved. -// 2014-08-01 16:09 - -#include <gtest/gtest.h> -#include <glog/logging.h> -#include <map> -#include <vector> - -#include "model/data_layer.h" -#include "model/trainer.h" -#include "model/sgd_trainer.h" -#include "model/conv_edge.h" -#include "model/relu_layer.h" -#include "proto/model.pb.h" - -#include "utils/proto_helper.h" - -namespace lapis { -class ModelTest : public ::testing::Test { - public: - ModelTest () { - ReadProtoFromTextFile("src/test/data/model.conf", &model_proto); - } - protected: - ModelProto model_proto; -}; -/********************************************************************** - * DataLayer Test - **********************************************************************/ -class DataLayerTest : public ModelTest { - public: - DataLayerTest() { - label_layer.Init(model_proto.net().layer(0)); - img_layer.Init(model_proto.net().layer(1)); - Trainer::InitDataSource(model_proto.trainer().train_data(), &sources); - EXPECT_EQ(2, sources.size()); - sources[0]->LoadData(nullptr); - sources[1]->LoadData(nullptr); - DLOG(INFO)<<"after init datasources"; - label_layer.Setup(2, TrainerProto::kBackPropagation, sources); - DLOG(INFO)<<"after setup label layer"; - img_layer.Setup(2, TrainerProto::kBackPropagation, sources); - DLOG(INFO)<<"after setup img layer"; - } - ~DataLayerTest() { - for(auto& source: sources) - delete source; - } - protected: - DataLayer img_layer, label_layer; - std::vector<DataSource*> sources; -}; - -TEST_F(DataLayerTest, InitSetupForward) { - EXPECT_TRUE(label_layer.HasInput()); - EXPECT_TRUE(img_layer.HasInput()); - EXPECT_STREQ("DataLayer", DataLayer::kType.c_str()); - - EXPECT_EQ(2, label_layer.feature(nullptr).num()); - EXPECT_EQ(1, label_layer.feature(nullptr).channels()); - EXPECT_EQ(1, label_layer.feature(nullptr).height()); - EXPECT_EQ(1, label_layer.feature(nullptr).width()); - - EXPECT_EQ(2, img_layer.feature(nullptr).num()); - EXPECT_EQ(3, img_layer.feature(nullptr).channels()); - EXPECT_EQ(227, img_layer.feature(nullptr).height()); - EXPECT_EQ(227, img_layer.feature(nullptr).width()); - - img_layer.Forward(); -} -// TODO(wangwei) test this after outgoing edges are tested - -/********************************************************************** - * ConvEdge Test - **********************************************************************/ -class ConvEdgeTest : public DataLayerTest { - public: - ConvEdgeTest() { - relu.Init(model_proto.net().layer(2)); - DLOG(INFO)<<"init both layers"; - layer_map["input_img"]=&img_layer; - layer_map["hidden1_relu"]=&relu; - - edge_proto=model_proto.net().edge(0); - convedge.Init(edge_proto, layer_map); - convedge.Setup(true); - } - protected: - std::map<std::string, Layer*> layer_map; - ConvEdge convedge; - EdgeProto edge_proto; - ReLULayer relu; -}; - -TEST_F(ConvEdgeTest, InitSetupForward) { - Layer* dest=layer_map.at("hidden1_relu"); - Blob &b=dest->feature(&convedge); - EXPECT_EQ(0,b.num()); - convedge.SetupTopBlob(&b); - int conv_height = (227 + 2 * edge_proto.pad() - edge_proto.kernel_size()) - / edge_proto.stride() + 1; - int conv_width=conv_height; - CHECK_EQ(2, b.num()); - CHECK_EQ(edge_proto.num_output(), b.channels()); - CHECK_EQ(conv_height, b.height()); - CHECK_EQ(conv_width, b.width()); - DLOG(INFO)<<"after shape check"; - - Layer* src=layer_map["input_img"]; - convedge.Forward(src->feature(&convedge), &b, true); -} - -/********************************************************************** - * ReLULayer Test - **********************************************************************/ -class ReLULayerTest : public ConvEdgeTest { - public: - ReLULayerTest() { - relu.Setup(2, TrainerProto::kBackPropagation, sources); - relu_proto=model_proto.net().layer(3); - } - protected: - LayerProto relu_proto; -}; - -TEST_F(ReLULayerTest, ForwardWithoutDropout) { - EXPECT_EQ(2, relu.feature(&convedge).num()); - EXPECT_EQ(2, relu.gradient(&convedge).num()); - - relu.Forward(); -} -/********************************************************************** - * PoolingEdge Test -class PoolingEdgeTest : public ReLULayerTest { - public: - PoolingEdgeTest() { - linearlayer.Init(model.net().layer(3)); - pooledge.Init(model.net().edge(1)); - } - - protected: - PoolingEdge pooledge; - LinearLayer linearlayer; -} - **********************************************************************/ -/********************************************************************** - * LinearLayer Test - **********************************************************************/ - -/********************************************************************** - * LRNEdge Test - **********************************************************************/ - -/********************************************************************** - * InnerProductEdge Test - **********************************************************************/ - -/********************************************************************** - * SoftmaxLayerLossEdge Test - **********************************************************************/ - - - - -/********************************************************************** - * SGDTrainer Test - **********************************************************************/ -class SGDTrainerTest : public ModelTest { - protected: - SGDTrainer sgd; -}; - -TEST_F(SGDTrainerTest, Init) { - sgd.Init(model_proto.trainer()); - EXPECT_TRUE(Trainer::phase==Phase::kInit); -} - -} // namespace lapis http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/6b1e65ee/src/test/model/test_label_source.cc ---------------------------------------------------------------------- diff --git a/src/test/model/test_label_source.cc b/src/test/model/test_label_source.cc deleted file mode 100644 index 9b25c2a..0000000 --- a/src/test/model/test_label_source.cc +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright © 2014 Wei Wang. All Rights Reserved. -// 2014-07-21 19:40 - -#include <gtest/gtest.h> -#include <glog/logging.h> -#include "proto/model.pb.h" -#include "disk/label_source.h" - -namespace lapis { -class LabelSourceTest : public ::testing::Test { - public: - LabelSourceTest() { - DataSourceProto ds; - ds.set_path("src/test/data/label_source.dat"); - ds.set_size(12); - ds.set_name("label source"); - ls.Init(ds); - } - - protected: - LabelSource ls; -}; - -TEST_F(LabelSourceTest, LoadData) { - auto ptr2names = ls.LoadData(nullptr); - EXPECT_EQ(12, ptr2names->size()); - EXPECT_STREQ("img0.JPEG", ptr2names->at(0).c_str()); - EXPECT_STREQ("img1.JPEG", ptr2names->at(1).c_str()); - EXPECT_STREQ("img5.JPEG", ptr2names->at(5).c_str()); - EXPECT_STREQ("img10.JPEG", ptr2names->at(10).c_str()); - EXPECT_STREQ("img11.JPEG", ptr2names->at(11).c_str()); -} - -TEST_F(LabelSourceTest, GetData) { - ls.LoadData(nullptr); - Blob b; - b.Resize(1, 1, 1, 5); - ls.GetData(&b); - const float *val = b.dptr; - EXPECT_EQ(0.0f, val[0]); - EXPECT_EQ(1.0f, val[1]); - EXPECT_EQ(4.0f, val[2]); - EXPECT_EQ(9.0f, val[3]); - EXPECT_EQ(16.0f, val[4]); - ls.GetData(&b); - EXPECT_EQ(4.0f, val[0]); - EXPECT_EQ(5.0f, val[1]); - EXPECT_EQ(6.0f, val[2]); - EXPECT_EQ(7.0f, val[3]); - EXPECT_EQ(8.0f, val[4]); - ls.GetData(&b); - EXPECT_EQ(1.0f, val[0]); - EXPECT_EQ(2.0f, val[1]); - EXPECT_EQ(0.0f, val[2]); - EXPECT_EQ(1.0f, val[3]); - EXPECT_EQ(4.0f, val[4]); -} - -} // namespace lapis http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/6b1e65ee/src/test/model/test_param.cc ---------------------------------------------------------------------- diff --git a/src/test/model/test_param.cc b/src/test/model/test_param.cc deleted file mode 100644 index 520fbe2..0000000 --- a/src/test/model/test_param.cc +++ /dev/null @@ -1,138 +0,0 @@ -#include <gtest/gtest.h> -#include <glog/logging.h> -#include "proto/model.pb.h" - -#include "utils/param.h" - -using namespace singa; - -class ParamTest : public ::testing::Test { - public: - ParamTest() { - wp.set_name("weight"); - wp.add_shape(3); - wp.add_shape(4); - bp.set_name("bias"); - bp.add_shape(4); - } - protected: - Param w, b; - ParamProto wp, bp; -}; - -TEST_F(ParamTest, ConstantInit) { - bp.set_init_method(ParamProto::kConstant); - bp.set_value(0.5); - b.Init(bp); - const float *val = b.content().dptr; - EXPECT_EQ(0.5f, val[0]); - EXPECT_EQ(0.5f, val[1]); - EXPECT_EQ(0.5f, val[2]); - EXPECT_EQ(0.5f, val[3]); - wp.set_init_method(ParamProto::kConstant); - wp.set_value(1.5); - w.Init(wp); - val = w.content().dptr; - EXPECT_EQ(1.5f, val[0]); - EXPECT_EQ(1.5f, val[3]); - EXPECT_EQ(1.5f, val[4]); - EXPECT_EQ(1.5f, val[11]); -} - -TEST_F(ParamTest, UniformInit) { - bp.set_init_method(ParamProto::kUniform); - bp.set_value(1.0f); - b.Init(bp); - const float *val = b.content().dptr; - EXPECT_TRUE(val[0] >= -1 && val[0] <= 1); - EXPECT_TRUE(val[1] >= -1 && val[2] <= 1); - EXPECT_TRUE(val[2] >= -1 && val[2] <= 1); - EXPECT_TRUE(val[3] >= -1 && val[3] <= 1); - wp.set_init_method(ParamProto::kUniform); - wp.set_value(1.0f); - w.Init(wp); - val = w.content().dptr; - EXPECT_TRUE(val[0] >= -1 && val[0] <= 1); - EXPECT_TRUE(val[3] >= -1 && val[3] <= 1); - EXPECT_TRUE(val[4] >= -1 && val[4] <= 1); - EXPECT_TRUE(val[11] >= -1 && val[11] <= 1); -} - -TEST_F(ParamTest, UniformSqrtFanInInit) { - wp.set_init_method(ParamProto::kUniformSqrtFanIn); - wp.set_value(2.0f); - w.Init(wp); - const float *val = w.content().dptr; - EXPECT_TRUE(val[0] >= -2 && val[0] <= 2); - EXPECT_TRUE(val[3] >= -2 && val[3] <= 2); - EXPECT_TRUE(val[4] >= -2 && val[4] <= 2); - EXPECT_TRUE(val[11] >= -2 && val[11] <= 2); -} - - -TEST_F(ParamTest, UniformSqrtFanInOutInit) { - wp.set_init_method(ParamProto::kUniformSqrtFanInOut); - wp.set_value(1.0f); - float low=1.0f, high=5.0f; - wp.set_low(low); - wp.set_high(high); - w.Init(wp); - const float *val = w.content().dptr; - /* - LOG(INFO) << val[0] << " " << val[1] << " " << val[2] << " " << val[3]; - LOG(INFO) << val[4] << " " << val[5] << " " << val[6] << " " << val[7]; - LOG(INFO) << val[8] << " " << val[9] << " " << val[10] << " " << val[11]; - */ - float factor = wp.value() / sqrt(wp.shape(0) + wp.shape(1)); - low=low*factor; - high=high*factor; - LOG(INFO)<<low<<" "<<high; - EXPECT_TRUE(val[0] >= low && val[0] <= high); - EXPECT_TRUE(val[3] >= low && val[3] <= high); - EXPECT_TRUE(val[4] >= low && val[4] <= high); - EXPECT_TRUE(val[11] >= low && val[11] <= high); -} - -TEST_F(ParamTest, GaussianInit) { - int len=5000, mean=0.0f, std=1.0f; - ParamProto p; - p.set_name("bias"); - p.add_shape(1); - p.add_shape(len); - p.set_init_method(ParamProto::kGaussain); - p.set_value(1.0f); - p.set_mean(mean); - p.set_std(std); - w.Init(p); - - const float *val = w.content().dptr; - float dmean=0.0f; - for(int i=0;i<len;i++) - dmean+=val[i]; - dmean/=len; - float dstd=0.0f; - for(int i=0;i<len;i++) - dstd+=(dmean-val[i])*(dmean-val[i]); - dstd/=len; - EXPECT_TRUE(std::abs(mean-dmean)<0.1); - EXPECT_TRUE(std::abs(std-dstd)<0.1); - /* - LOG(INFO) << val[0] << " " << val[1] << " " << val[2] << " " << val[3]; - LOG(INFO) << val[4] << " " << val[5] << " " << val[6] << " " << val[7]; - LOG(INFO) << val[8] << " " << val[9] << " " << val[10] << " " << val[11]; - */ -} - -TEST_F(ParamTest, GaussianSqrtFanInInit) { - wp.set_init_method(ParamProto::kGaussainSqrtFanIn); - wp.set_value(1.0f); - wp.set_mean(0); - wp.set_std(1.0f); - w.Init(wp); - //const float *val = w.content().dptr; - /* - LOG(INFO) << val[0] << " " << val[1] << " " << val[2] << " " << val[3]; - LOG(INFO) << val[4] << " " << val[5] << " " << val[6] << " " << val[7]; - LOG(INFO) << val[8] << " " << val[9] << " " << val[10] << " " << val[11]; - */ -} http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/6b1e65ee/src/test/model/test_proto.cc ---------------------------------------------------------------------- diff --git a/src/test/model/test_proto.cc b/src/test/model/test_proto.cc deleted file mode 100644 index f6d81fd..0000000 --- a/src/test/model/test_proto.cc +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright © 2014 Wei Wang. All Rights Reserved. -// 2014-07-15 21:54 -#include <glog/logging.h> -#include <gtest/gtest.h> -#include "proto/model.pb.h" -#include "utils/proto_helper.h" -namespace lapis { - -// use const Message& m=..., otherwise may lead to segment fault -TEST(ProtoTest, ReadFromFile) { - ModelProto model; - LOG(INFO)<<"start...."; - lapis::ReadProtoFromTextFile("src/test/data/model.conf", &model); - LOG(INFO)<<"after reading file..."; - EXPECT_STREQ("caffe_config", model.name().c_str()); - - // layer and edge size - const NetProto& net = model.net(); - EXPECT_EQ(15, net.layer().size()); - EXPECT_EQ(14, net.edge().size()); - LOG(INFO)<<"after size check..."; - - // layer config - LayerProto layer1 = net.layer().Get(1); - EXPECT_STREQ("input_img", layer1.name().c_str()); - EXPECT_STREQ("DataLayer", layer1.type().c_str()); - LOG(INFO)<<"after datalayer check..."; - // edge config - EdgeProto edge0 = net.edge().Get(0); - EXPECT_STREQ("input_img-hidden1_relu", edge0.name().c_str()); - EXPECT_STREQ("ConvEdge", edge0.type().c_str()); - EXPECT_EQ(2, edge0.param().size()); - LOG(INFO)<<"after first edge check..."; - // param config - ParamProto param1 = edge0.param().Get(0); - EXPECT_TRUE(ParamProto::kGaussain == param1.init_method()); - EXPECT_EQ(0.0f, param1.mean()); - EXPECT_EQ(0.01f, param1.std()); - EXPECT_EQ(1.0f, param1.learning_rate_multiplier()); - LOG(INFO)<<"after param of first edge check..."; - - ParamProto param2 = edge0.param().Get(1); - EXPECT_TRUE(ParamProto::kConstant == param2.init_method()); - EXPECT_EQ(0.0f, param2.value()); - EXPECT_EQ(0.0f, param2.weight_decay_multiplier()); - LOG(INFO)<<"after param of second edge check..."; - - // trainer config - const TrainerProto& trainer = model.trainer(); - const SGDProto& sgd=trainer.sgd(); - EXPECT_EQ(227, sgd.train_batchsize()); - EXPECT_EQ(0.01f, sgd.base_learning_rate()); - EXPECT_TRUE(SGDProto::kStep== sgd.learning_rate_change()); - LOG(INFO)<<"after sgd check..."; - - // data source config - EXPECT_EQ(2,trainer.train_data().size()); - LOG(INFO)<<"after size check..."; - const DataSourceProto& data=trainer.train_data(0); - LOG(INFO)<<"after get data..."; - EXPECT_STREQ("RGBDirSource", data.type().c_str()); - LOG(INFO)<<"after type check..."; - EXPECT_EQ(50000, data.size()); - EXPECT_EQ(3, data.channels()); - LOG(INFO)<<"after data source check..."; -} -} // namespace lapis http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/6b1e65ee/src/test/model/test_rgb_dir_source.cc ---------------------------------------------------------------------- diff --git a/src/test/model/test_rgb_dir_source.cc b/src/test/model/test_rgb_dir_source.cc deleted file mode 100644 index 36ac21a..0000000 --- a/src/test/model/test_rgb_dir_source.cc +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright © 2014 Wei Wang. All Rights Reserved. -// 2014-07-21 21:52 - -#include <gtest/gtest.h> -#include <glog/logging.h> -#include <algorithm> - -#include "proto/model.pb.h" -#include "disk/rgb_dir_source.h" -#include "disk/label_source.h" - -namespace lapis { -class RGBDirSourceTest : public ::testing::Test { - public: - RGBDirSourceTest() { - DataSourceProto ds; - ds.set_path("src/test/data/rgb_dir"); - ds.set_mean_file("src/test/data/imagenet_mean.binaryproto"); - ds.set_size(3); - ds.set_height(256); - ds.set_width(256); - ds.set_offset(2); - ds.set_name("rgb dir source"); - rgbs.Init(ds); - } - - protected: - RGBDirSource rgbs; -}; - -TEST_F(RGBDirSourceTest, LoadDataNoInputKeys) { - auto &ptr2names = rgbs.LoadData(nullptr); - EXPECT_EQ(3, ptr2names->size()); - sort(ptr2names->begin(), ptr2names->end()); - EXPECT_STREQ("img0.JPEG", ptr2names->at(0).c_str()); - EXPECT_STREQ("img1.JPEG", ptr2names->at(1).c_str()); - EXPECT_STREQ("img2.JPEG", ptr2names->at(2).c_str()); -} - -TEST_F(RGBDirSourceTest, LoadDataWithInputKeys) { - LabelSource ls; - DataSourceProto ds; - ds.set_path("src/test/data/label_source.dat"); - ds.set_name("label source"); - ds.set_size(3); - ls.Init(ds); - auto ptr2names1 = ls.LoadData(nullptr); - auto ptr2names2 = rgbs.LoadData(ptr2names1); - EXPECT_EQ(3, ptr2names2->size()); - for (int i = 0; i < 3; i++) - EXPECT_STREQ(ptr2names1->at(i).c_str(), ptr2names2->at(i).c_str()); -} - -TEST_F(RGBDirSourceTest, GetData) { - Blob b; - b.Resize(256,256,3,2); - rgbs.LoadData(nullptr); - rgbs.GetData(&b); - rgbs.GetData(&b); - rgbs.GetData(&b); -} -} // namespace lapis - http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/6b1e65ee/src/test/test_cluster.cc ---------------------------------------------------------------------- diff --git a/src/test/test_cluster.cc b/src/test/test_cluster.cc index d86463a..fc49e53 100644 --- a/src/test/test_cluster.cc +++ b/src/test/test_cluster.cc @@ -6,7 +6,13 @@ using namespace singa; string folder="src/test/data/"; -/* + +TEST(CluserRuntimeTest, Basic){ + ClusterRuntime* rt = new ZKClusterRT("localhost:2181"); + delete rt; +} + +/** ClusterProto GenClusterProto(){ ClusterProto proto; int nworker=6, nserver=4; @@ -92,4 +98,4 @@ TEST(ClusterTest, MultiServerGroups){ ASSERT_EQ(2, cluster->nserver_groups()); ASSERT_STREQ("awan-0-08-0", cluster->host_addr().c_str()); } -*/ +**/ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/6b1e65ee/src/test/test_communication.cc ---------------------------------------------------------------------- diff --git a/src/test/test_communication.cc b/src/test/test_communication.cc deleted file mode 100644 index c9c035f..0000000 --- a/src/test/test_communication.cc +++ /dev/null @@ -1,158 +0,0 @@ -#include <thread> -#include <vector> -#include "gtest/gtest.h" -#include "communication/msg.h" -#include "communication/socket.h" -using std::vector; -using namespace singa; - -const char* ping="PING",*pong="PONG"; -/** - * Connect dealer with (gid, id, flag) to stub router - */ -void Connect(Dealer* dealer, int gid, int id, int flag){ - dealer->Connect("inproc://router"); - Msg msg; - msg.set_src(gid, id, flag); - msg.set_dst(0,0,2); - msg.set_type(0); - msg.add_frame(ping, 4); - dealer->Send(&msg); -} - -/** - * Dealer thread, ping-pong with the stub router - */ -void DealerPingPong(int id){ - Dealer* dealer=new Dealer(); - Connect(dealer, 0, id, 0); - Msg* msg=dealer->Receive(); - int flag=msg->src_flag(); - ASSERT_EQ(2, flag); - ASSERT_EQ(0, msg->dst_group_id()); - ASSERT_EQ(id, msg->dst_id()); - ASSERT_STREQ(pong, (char*)msg->frame_data()); - delete msg; - delete dealer; -} - -/** - * Worker thread, connect to router and communicate with server thread - */ -void WorkerDealer(int sid, int did){ - Dealer* dealer=new Dealer(); - Connect(dealer, 0, sid, 0); - for(int i=0;i<2;i++){ - { - Msg msg; - msg.set_src(0, sid, 0); - msg.set_dst(0, did, 1); - msg.set_type(3); - msg.set_target(i); - dealer->Send(&msg); - } - { - Msg *msg=dealer->Receive(); - ASSERT_EQ(0, msg->src_group_id()); - ASSERT_EQ(did, msg->src_id()); - ASSERT_EQ(1, msg->src_flag()); - delete msg; - } - } - delete dealer; -} - -/** - * Server thread, connect to router and communicate with worker thread - */ -void ServerDealer(int id, int n){ - Dealer* dealer=new Dealer(); - Connect(dealer, 0, id, 1); - for(int i=0;i<n;i++){ - Msg *msg=dealer->Receive(); - Msg reply; - reply.set_dst(msg->src_group_id(), msg->src_id(), msg->src_flag()); - reply.set_src(0, id, 1); - dealer->Send(&reply); - delete msg; - } - delete dealer; -} - -TEST(CommunicationTest, DealerRouterPingPong){ - int n=2; - vector<std::thread> threads; - for(int i=0;i<n;i++) - threads.push_back(std::thread(DealerPingPong, i)); - Router* router=new Router(); - router->Bind(""); - for(int k=0;k<n;k++){ - Msg* msg=router->Receive(); - ASSERT_EQ(0, msg->src_group_id()); - ASSERT_EQ(2, msg->dst_flag()); - ASSERT_STREQ(ping, (char*)msg->frame_data()); - - Msg reply; - reply.set_src(0,0,2); - reply.set_dst(msg->src_group_id(), msg->src_id(), msg->src_flag()); - reply.add_frame(pong, 4); - router->Send(&reply); - delete msg; - } - - delete router; - for(auto& thread:threads) - thread.join(); -} - -TEST(CommunicationTest, nWorkers1Server){ - int nworker=2; - vector<std::thread> threads; - for(int i=0;i<nworker;i++) - threads.push_back(std::thread(WorkerDealer, i, 0)); - //threads.push_back(std::thread(ServerDealer, 0, 4)); - Router* router=new Router(); - router->Bind(""); - int nmsg=4*nworker; - int k=0; - while(nmsg>0){ - Msg* msg=router->Receive(); - if(2== msg->dst_flag()){ - ASSERT_STREQ(ping, (char*)msg->frame_data()); - k++; - if(k==nworker) - threads.push_back(std::thread(ServerDealer, 0, 2*nworker)); - }else{ - nmsg--; - router->Send(msg); - } - delete msg; - } - delete router; - for(auto& thread:threads) - thread.join(); -} - -TEST(CommunicationTest, 2Workers2Server){ - vector<std::thread> threads; - threads.push_back(std::thread(WorkerDealer, 0, 0)); - threads.push_back(std::thread(WorkerDealer, 1, 1)); - threads.push_back(std::thread(ServerDealer, 0, 2)); - threads.push_back(std::thread(ServerDealer, 1, 2)); - Router* router=new Router(); - router->Bind(""); - int n=8; - while(n>0){ - Msg* msg=router->Receive(); - if(2== msg->dst_flag()){ - ASSERT_STREQ(ping, (char*)msg->frame_data()); - }else{ - n--; - router->Send(msg); - } - delete msg; - } - delete router; - for(auto& thread:threads) - thread.join(); -} http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/6b1e65ee/src/test/test_shard.cc ---------------------------------------------------------------------- diff --git a/src/test/test_shard.cc b/src/test/test_shard.cc deleted file mode 100644 index c96d876..0000000 --- a/src/test/test_shard.cc +++ /dev/null @@ -1,56 +0,0 @@ -#include <gtest/gtest.h> -#include <sys/stat.h> - -#include "utils/data_shard.h" - -std::string key[]={"firstkey","secondkey","3key", "key4", "key5"}; -std::string tuple[]={"firsttuple","2th-tuple","thridtuple", "tuple4", "tuple5"}; - -using namespace singa; - -TEST(DataShardTest, CreateDataShard){ - std::string path="src/test/data/shard_test"; - mkdir(path.c_str(), 0755); - DataShard shard(path, DataShard::kCreate, 50); - shard.Insert(key[0], tuple[0]); - shard.Insert(key[1], tuple[1]); - shard.Insert(key[2], tuple[2]); - shard.Flush(); -} - -TEST(DataShardTest, AppendDataShard){ - std::string path="src/test/data/shard_test"; - DataShard shard(path, DataShard::kAppend, 50); - shard.Insert(key[3], tuple[3]); - shard.Insert(key[4], tuple[4]); - shard.Flush(); -} -TEST(DataShardTest, CountDataShard){ - std::string path="src/test/data/shard_test"; - DataShard shard(path, DataShard::kRead, 50); - int count=shard.Count(); - ASSERT_EQ(5, count); -} - -TEST(DataShardTest, ReadDataShard){ - std::string path="src/test/data/shard_test"; - DataShard shard(path, DataShard::kRead, 50); - std::string k, t; - ASSERT_TRUE(shard.Next(&k, &t)); - ASSERT_STREQ(key[0].c_str(), k.c_str()); - ASSERT_STREQ(tuple[0].c_str(), t.c_str()); - ASSERT_TRUE(shard.Next(&k, &t)); - ASSERT_STREQ(key[1].c_str(), k.c_str()); - ASSERT_STREQ(tuple[1].c_str(), t.c_str()); - ASSERT_TRUE(shard.Next(&k, &t)); - ASSERT_TRUE(shard.Next(&k, &t)); - ASSERT_TRUE(shard.Next(&k, &t)); - ASSERT_STREQ(key[4].c_str(), k.c_str()); - ASSERT_STREQ(tuple[4].c_str(), t.c_str()); - - ASSERT_FALSE(shard.Next(&k, &t)); - shard.SeekToFirst(); - ASSERT_TRUE(shard.Next(&k, &t)); - ASSERT_STREQ(key[0].c_str(), k.c_str()); - ASSERT_STREQ(tuple[0].c_str(), t.c_str()); -} http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/6b1e65ee/src/utils/cluster_rt.cc ---------------------------------------------------------------------- diff --git a/src/utils/cluster_rt.cc b/src/utils/cluster_rt.cc new file mode 100644 index 0000000..f99d4f5 --- /dev/null +++ b/src/utils/cluster_rt.cc @@ -0,0 +1,32 @@ +#include "utils/cluster_rt.h" + +namespace singa { + +/********* Implementation for ZKClusterRT **************/ + + +ZKClusterRT::ZKClusterRT(string host){ + //fprintf(stderr, "Create ZKClusterRT"); +} + +ZKClusterRT::~ZKClusterRT(){ + //fprintf(stderr, "Destroy ZKClusterRT"); +} + +bool ZKClusterRT::Init(){ + return false; +} + +bool ZKClusterRT::sWatchSGroup(int gid, int sid){ + return false; +} + +bool ZKClusterRT::wJoinSGroup(int gid, int wid, int s_group){ + return false; +} + +bool ZKClusterRT::wLeaveSGroup(int gid, int wid, int s_group){ + return false; +} + +} // namespace singa
