refactor iterator implementations, they were broken
Project: http://git-wip-us.apache.org/repos/asf/marmotta/repo Commit: http://git-wip-us.apache.org/repos/asf/marmotta/commit/2e32da5d Tree: http://git-wip-us.apache.org/repos/asf/marmotta/tree/2e32da5d Diff: http://git-wip-us.apache.org/repos/asf/marmotta/diff/2e32da5d Branch: refs/heads/MARMOTTA-584 Commit: 2e32da5d9cc298272edce2a64e22088335107f21 Parents: 27889eb Author: Sebastian Schaffert <[email protected]> Authored: Sat Dec 19 16:15:46 2015 +0100 Committer: Sebastian Schaffert <[email protected]> Committed: Sat Dec 19 16:15:46 2015 +0100 ---------------------------------------------------------------------- libraries/ostrich/backend/CMakeLists.txt | 4 +- libraries/ostrich/backend/client/client.cc | 27 +- .../ostrich/backend/model/rdf_operators.cc | 8 +- .../ostrich/backend/persistence/CMakeLists.txt | 22 +- .../backend/persistence/leveldb_persistence.cc | 63 +- .../backend/persistence/leveldb_service.cc | 22 +- .../backend/persistence/leveldb_sparql.cc | 25 +- .../backend/persistence/marmotta_updatedb.cc | 225 + .../backend/serializer/serializer_base.h | 4 +- .../ostrich/backend/sparql/rasqal_adapter.cc | 13 +- libraries/ostrich/backend/test/CMakeLists.txt | 17 +- .../ostrich/backend/test/PersistenceTest.cc | 76 + libraries/ostrich/backend/test/SparqlTest.cc | 29 +- libraries/ostrich/backend/test/StatementTest.cc | 2 +- .../ostrich/backend/test/gmock-gtest-all.cc | 12243 ++++++++++ libraries/ostrich/backend/test/gmock/gmock.h | 14978 ++++++++++++ libraries/ostrich/backend/test/gtest-all.cc | 9592 -------- libraries/ostrich/backend/test/gtest.h | 20061 ---------------- libraries/ostrich/backend/test/gtest/gtest.h | 21197 +++++++++++++++++ libraries/ostrich/backend/test/main.cc | 2 +- libraries/ostrich/backend/util/iterator.h | 173 +- 21 files changed, 48949 insertions(+), 29834 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/CMakeLists.txt b/libraries/ostrich/backend/CMakeLists.txt index 6a96f63..e05c28b 100644 --- a/libraries/ostrich/backend/CMakeLists.txt +++ b/libraries/ostrich/backend/CMakeLists.txt @@ -14,10 +14,10 @@ find_package (Protobuf REQUIRED) find_package (GRPC REQUIRED) find_package (LevelDB REQUIRED) find_package (GLog REQUIRED) -find_package (Boost 1.54.0 COMPONENTS iostreams) +find_package (Boost 1.54.0 COMPONENTS iostreams filesystem system) find_package (Tcmalloc) -add_definitions(-DNDEBUG) +#add_definitions(-DNDEBUG) if (Boost_IOSTREAMS_FOUND) message(STATUS "Enabling gzip/bzip2 support (Boost iostreams found)") http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/client/client.cc ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/client/client.cc b/libraries/ostrich/backend/client/client.cc index da875a4..396389e 100644 --- a/libraries/ostrich/backend/client/client.cc +++ b/libraries/ostrich/backend/client/client.cc @@ -69,28 +69,24 @@ class ClientReaderIterator : public util::CloseableIterator<T> { public: ClientReaderIterator() : finished(true) { } - ClientReaderIterator(ClientReader<Proto>* r) : reader(r), finished(false) { - // Immediately move to first element. - operator++(); + ClientReaderIterator(ClientReader<Proto>* r) : reader(r) { + finished = !reader->Read(&buffer); } - ClientReaderIterator& operator++() override { + const T& next() override { + current_ = T(buffer); + if (!finished) { finished = !reader->Read(&buffer); - current = T(buffer); if (finished) { reader->Finish(); } } - return *this; - } - - T& operator*() override { - return current; + return current_; } - T* operator->() override { - return ¤t; + const T& current() const override { + return current_; } bool hasNext() override { @@ -100,7 +96,7 @@ class ClientReaderIterator : public util::CloseableIterator<T> { private: ClientReader<Proto>* reader; Proto buffer; - T current; + T current_; bool finished; }; @@ -208,8 +204,9 @@ class MarmottaClient { stub_->GetNamespaces(&context, pattern)); NamespaceReader it(reader.get()); - for (; it.hasNext(); ++it) { - out << (*it).getPrefix() << " = " << (*it).getUri() << std::endl; + while (it.hasNext()) { + const auto& ns = it.next(); + out << ns.getPrefix() << " = " << ns.getUri() << std::endl; } } http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/model/rdf_operators.cc ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/model/rdf_operators.cc b/libraries/ostrich/backend/model/rdf_operators.cc index b9e49ad..b2bc2a3 100644 --- a/libraries/ostrich/backend/model/rdf_operators.cc +++ b/libraries/ostrich/backend/model/rdf_operators.cc @@ -28,14 +28,18 @@ bool operator==(const Value &lhs, const Value &rhs) { } else if (lhs.resource().has_bnode() && rhs.resource().has_bnode()) { return lhs.resource().bnode() == rhs.resource().bnode(); } + return (lhs.resource().has_uri() == rhs.resource().has_uri()) && + (lhs.resource().has_bnode() == rhs.resource().has_bnode()); } else if(lhs.has_literal() && rhs.has_literal()) { if (lhs.literal().has_stringliteral() && rhs.literal().has_stringliteral()) { return lhs.literal().stringliteral() == rhs.literal().stringliteral(); } else if (lhs.literal().has_dataliteral() && rhs.literal().has_dataliteral()) { return lhs.literal().dataliteral() == rhs.literal().dataliteral(); } + return (lhs.literal().has_stringliteral() == rhs.literal().has_stringliteral()) && + (lhs.literal().has_dataliteral() == rhs.literal().has_dataliteral()); } - return false; + return (lhs.has_resource() == rhs.has_resource()) && (lhs.has_literal() == rhs.has_literal()); } bool operator==(const Resource &lhs, const Resource &rhs) { @@ -44,7 +48,7 @@ bool operator==(const Resource &lhs, const Resource &rhs) { } else if (lhs.has_bnode() && rhs.has_bnode()) { return lhs.bnode() == rhs.bnode(); } - return false; + return (lhs.has_uri() == rhs.has_uri()) && (lhs.has_bnode() == rhs.has_bnode()); } bool operator==(const Statement &lhs, const Statement &rhs) { http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/persistence/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/persistence/CMakeLists.txt b/libraries/ostrich/backend/persistence/CMakeLists.txt index d4a274a..8392c61 100644 --- a/libraries/ostrich/backend/persistence/CMakeLists.txt +++ b/libraries/ostrich/backend/persistence/CMakeLists.txt @@ -1,11 +1,21 @@ include_directories(.. ${CMAKE_CURRENT_BINARY_DIR}/.. ${CMAKE_CURRENT_BINARY_DIR}/../model ${RAPTOR_INCLUDE_DIR}/raptor2) +# Shared Marmotta Ostrich persistence implementation +add_library(marmotta_leveldb + leveldb_persistence.cc leveldb_persistence.h leveldb_sparql.cc leveldb_sparql.h) +target_link_libraries(marmotta_leveldb + marmotta_model marmotta_util marmotta_sparql marmotta_service + ${LevelDB_LIBRARY} ${GLOG_LIBRARY} ${PROTOBUF_LIBRARIES}) + +# Server binary add_executable(marmotta_persistence - leveldb_persistence.cc leveldb_persistence.h leveldb_service.cc leveldb_service.h - leveldb_server.cc leveldb_sparql.cc leveldb_sparql.h) -target_link_libraries(marmotta_persistence - marmotta_model marmotta_service marmotta_util marmotta_sparql - ${LevelDB_LIBRARY} ${GFLAGS_LIBRARY} ${GLOG_LIBRARY} - ${CMAKE_THREAD_LIBS_INIT} ${PROTOBUF_LIBRARIES} ${GRPC_LIBRARIES} ${Tcmalloc_LIBRARIES}) + leveldb_service.cc leveldb_service.h leveldb_server.cc ) +target_link_libraries(marmotta_persistence marmotta_service marmotta_leveldb + ${GFLAGS_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${GRPC_LIBRARIES} ${Tcmalloc_LIBRARIES}) install(TARGETS marmotta_persistence DESTINATION bin) +# Command line admin tool +add_executable(marmotta_updatedb marmotta_updatedb.cc) +target_link_libraries(marmotta_updatedb marmotta_leveldb marmotta_parser marmotta_serializer + ${GFLAGS_LIBRARY} ${Boost_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${Tcmalloc_LIBRARIES}) +install(TARGETS marmotta_updatedb DESTINATION bin) http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/persistence/leveldb_persistence.cc ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/persistence/leveldb_persistence.cc b/libraries/ostrich/backend/persistence/leveldb_persistence.cc index 5f18eff..dd9eef9 100644 --- a/libraries/ostrich/backend/persistence/leveldb_persistence.cc +++ b/libraries/ostrich/backend/persistence/leveldb_persistence.cc @@ -187,7 +187,7 @@ class LevelDBIterator : public util::CloseableIterator<T> { public: LevelDBIterator(leveldb::Iterator *it) - : it(it), parsed(false) { + : it(it) { it->SeekToFirst(); } @@ -195,35 +195,24 @@ class LevelDBIterator : public util::CloseableIterator<T> { delete it; }; - util::CloseableIterator<T> &operator++() override { + const T& next() override { + // Parse current position, then iterate to next position for next call. + proto.ParseFromString(it->value().ToString()); it->Next(); - parsed = false; - return *this; - }; - - T &operator*() override { - if (!parsed) - proto.ParseFromString(it->value().ToString()); return proto; }; - T *operator->() override { - if (!parsed) - proto.ParseFromString(it->value().ToString()); - return &proto; + const T& current() const override { + return proto; }; virtual bool hasNext() override { return it->Valid(); } - - protected: leveldb::Iterator* it; - T proto; - bool parsed; }; @@ -313,6 +302,11 @@ LevelDBPersistence::LevelDBPersistence(const std::string &path, int64_t cacheSiz t.join(); } + CHECK_NOTNULL(db_spoc.get()); + CHECK_NOTNULL(db_cspo.get()); + CHECK_NOTNULL(db_opsc.get()); + CHECK_NOTNULL(db_pcos.get()); + LOG(INFO) << "LevelDB Database initialised."; } @@ -323,8 +317,8 @@ int64_t LevelDBPersistence::AddNamespaces(NamespaceIterator& it) { leveldb::WriteBatch batch_prefix, batch_url; - for (; it.hasNext(); ++it) { - AddNamespace(*it, batch_prefix, batch_url); + while (it.hasNext()) { + AddNamespace(it.next(), batch_prefix, batch_url); count++; } CHECK_STATUS(db_ns_prefix->Write(leveldb::WriteOptions(), &batch_prefix)); @@ -374,8 +368,8 @@ void LevelDBPersistence::GetNamespaces( int64_t count = 0; bool cbsuccess = true; - for(auto it = GetNamespaces(pattern); cbsuccess && it->hasNext(); ++(*it)) { - cbsuccess = callback(**it); + for(auto it = GetNamespaces(pattern); cbsuccess && it->hasNext();) { + cbsuccess = callback(it->next()); count++; } @@ -389,8 +383,8 @@ int64_t LevelDBPersistence::AddStatements(StatementIterator& it) { int64_t count = 0; leveldb::WriteBatch batch_spoc, batch_cspo, batch_opsc, batch_pcos; - for (; it.hasNext(); ++it) { - AddStatement(*it, batch_spoc, batch_cspo, batch_opsc, batch_pcos); + while (it.hasNext()) { + AddStatement(it.next(), batch_spoc, batch_cspo, batch_opsc, batch_pcos); count++; } @@ -473,8 +467,8 @@ void LevelDBPersistence::GetStatements( int64_t count = 0; bool cbsuccess = true; - for(auto it = GetStatements(pattern); cbsuccess && it->hasNext(); ++(*it)) { - cbsuccess = callback(**it); + for(auto it = GetStatements(pattern); cbsuccess && it->hasNext(); ) { + cbsuccess = callback(it->next()); count++; } @@ -528,18 +522,19 @@ UpdateStatistics LevelDBPersistence::Update(LevelDBPersistence::UpdateIterator & UpdateStatistics stats; WriteBatch b_spoc, b_cspo, b_opsc, b_pcos, b_prefix, b_url; - for (; it.hasNext(); ++it) { - if (it->has_stmt_added()) { - AddStatement(it->stmt_added(), b_spoc, b_cspo, b_opsc, b_pcos); + while (it.hasNext()) { + auto next = it.next(); + if (next.has_stmt_added()) { + AddStatement(next.stmt_added(), b_spoc, b_cspo, b_opsc, b_pcos); stats.added_stmts++; - } else if (it->has_stmt_removed()) { + } else if (next.has_stmt_removed()) { stats.removed_stmts += - RemoveStatements(it->stmt_removed(), b_spoc, b_cspo, b_opsc, b_pcos); - } else if(it->has_ns_added()) { - AddNamespace(it->ns_added(), b_prefix, b_url); + RemoveStatements(next.stmt_removed(), b_spoc, b_cspo, b_opsc, b_pcos); + } else if(next.has_ns_added()) { + AddNamespace(next.ns_added(), b_prefix, b_url); stats.added_ns++; - } else if(it->has_ns_removed()) { - RemoveNamespace(it->ns_removed(), b_prefix, b_url); + } else if(next.has_ns_removed()) { + RemoveNamespace(next.ns_removed(), b_prefix, b_url); } } std::vector<std::thread> writers; http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/persistence/leveldb_service.cc ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/persistence/leveldb_service.cc b/libraries/ostrich/backend/persistence/leveldb_service.cc index e31af2d..9ab9fd8 100644 --- a/libraries/ostrich/backend/persistence/leveldb_service.cc +++ b/libraries/ostrich/backend/persistence/leveldb_service.cc @@ -48,24 +48,21 @@ template <class Proto> class ReaderIterator : public util::CloseableIterator<Proto> { public: - ReaderIterator(grpc::ServerReader<Proto>* r) : reader(r), finished(false) { + ReaderIterator(grpc::ServerReader<Proto>* r) : reader(r) { // Immediately move to first element. - operator++(); + finished = !reader->Read(&next_); } - util::CloseableIterator<Proto>& operator++() override { + const Proto& next() override { + current_.Swap(&next_); if (!finished) { - finished = !reader->Read(&buffer); + finished = !reader->Read(&next_); } - return *this; + return current_; } - Proto& operator*() override { - return buffer; - } - - Proto* operator->() override { - return &buffer; + const Proto& current() const override { + return current_; } bool hasNext() override { @@ -74,7 +71,8 @@ class ReaderIterator : public util::CloseableIterator<Proto> { private: grpc::ServerReader<Proto>* reader; - Proto buffer; + Proto current_; + Proto next_; bool finished; }; http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/persistence/leveldb_sparql.cc ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/persistence/leveldb_sparql.cc b/libraries/ostrich/backend/persistence/leveldb_sparql.cc index 5d44db6..274c247 100644 --- a/libraries/ostrich/backend/persistence/leveldb_sparql.cc +++ b/libraries/ostrich/backend/persistence/leveldb_sparql.cc @@ -29,26 +29,13 @@ class WrapProtoStatementIterator : public StatementIterator { WrapProtoStatementIterator(std::unique_ptr<persistence::LevelDBPersistence::StatementIterator> it) : it(std::move(it)) { } - util::CloseableIterator<rdf::Statement> &operator++() override { - ++(*it); - parsed = false; - return *this; + const rdf::Statement& next() override { + current_ = std::move(it->next()); + return current_; }; - rdf::Statement &operator*() override { - if (!parsed) { - current = std::move(**it); - parsed = true; - } - return current; - }; - - rdf::Statement *operator->() override { - if (!parsed) { - current = std::move(**it); - parsed = true; - } - return ¤t; + const rdf::Statement& current() const override { + return current_; }; bool hasNext() override { @@ -57,7 +44,7 @@ class WrapProtoStatementIterator : public StatementIterator { private: std::unique_ptr<persistence::LevelDBPersistence::StatementIterator> it; - rdf::Statement current; + rdf::Statement current_; bool parsed; }; http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/persistence/marmotta_updatedb.cc ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/persistence/marmotta_updatedb.cc b/libraries/ostrich/backend/persistence/marmotta_updatedb.cc new file mode 100644 index 0000000..b26b019 --- /dev/null +++ b/libraries/ostrich/backend/persistence/marmotta_updatedb.cc @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <fstream> + +#ifdef HAVE_IOSTREAMS +// support b/gzipped files +#include <boost/iostreams/filtering_streambuf.hpp> +#include <boost/iostreams/copy.hpp> +#include <boost/iostreams/filter/gzip.hpp> +#include <boost/iostreams/filter/bzip2.hpp> +#endif + +#include <google/protobuf/text_format.h> +#include <google/protobuf/empty.pb.h> +#include <google/protobuf/wrappers.pb.h> + +#include <gflags/gflags.h> +#include <thread> +#include <glog/logging.h> +#include <sys/stat.h> + +#include "model/rdf_model.h" +#include "parser/rdf_parser.h" +#include "serializer/serializer.h" +#include "persistence/leveldb_persistence.h" + +using namespace marmotta; +using google::protobuf::TextFormat; + +#ifdef HAVE_IOSTREAMS +using namespace boost::iostreams; +#endif + +class MarmottaClient { + public: + MarmottaClient(marmotta::persistence::LevelDBPersistence* db) + : db(db){ } + + void importDataset(std::istream& in, parser::Format format) { + auto start = std::chrono::steady_clock::now(); + int64_t count = 0; + + parser::Parser p("http://www.example.com";, format); + util::ProducerConsumerIterator<rdf::proto::Statement> stmtit; + util::ProducerConsumerIterator<rdf::proto::Namespace> nsit; + p.setStatementHandler([&stmtit](const rdf::Statement& stmt) { + stmtit.add(stmt.getMessage()); + }); + p.setNamespaceHandler([&nsit](const rdf::Namespace& ns) { + nsit.add(ns.getMessage()); + }); + + std::thread([&p, &in, &stmtit, &nsit]() { + p.parse(in); + stmtit.finish(); + nsit.finish(); + }); + + db->AddStatements(stmtit); + db->AddNamespaces(nsit); + } + + + void patternQuery(const rdf::Statement &pattern, std::ostream &out, serializer::Format format) { + } + + void patternDelete(const rdf::Statement &pattern) { + db->RemoveStatements(pattern.getMessage()); + } + + void tupleQuery(const std::string& query, std::ostream &out) { + /* + ClientContext context; + spq::SparqlRequest request; + request.set_query(query); + + std::unique_ptr<ClientReader<spq::SparqlResponse>> reader( + sparql_->TupleQuery(&context, request)); + + auto out_ = new google::protobuf::io::OstreamOutputStream(&out); + spq::SparqlResponse result; + while (reader->Read(&result)) { + TextFormat::Print(result, dynamic_cast<google::protobuf::io::ZeroCopyOutputStream*>(out_)); + } + delete out_; + */ + } + + void listNamespaces(std::ostream &out) { + /* + ClientContext context; + + google::protobuf::Empty pattern; + + std::unique_ptr<ClientReader<rdf::proto::Namespace> > reader( + stub_->GetNamespaces(&context, pattern)); + + NamespaceReader it(reader.get()); + for (; it.hasNext(); ++it) { + out << (*it).getPrefix() << " = " << (*it).getUri() << std::endl; + } + */ + } + + int64_t size() { + /* + ClientContext context; + google::protobuf::Int64Value result; + + Status status = stub_->Size(&context, r, &result); + if (status.ok()) { + return result.value(); + } else { + return -1; + } + */ + } + private: + marmotta::persistence::LevelDBPersistence* db; +}; + + +DEFINE_string(format, "rdfxml", "RDF format to use for parsing/serializing."); +DEFINE_string(output, "", "File to write result to."); +DEFINE_string(db, "/tmp/testdb", "Path to database. Will be created if non-existant."); +DEFINE_int64(cache_size, 100 * 1048576, "Cache size used by the database (in bytes)."); + +#ifdef HAVE_IOSTREAMS +DEFINE_bool(gzip, false, "Input files are gzip compressed."); +DEFINE_bool(bzip2, false, "Input files are bzip2 compressed."); +#endif + +int main(int argc, char** argv) { + GOOGLE_PROTOBUF_VERIFY_VERSION; + + // Initialize Google's logging library. + google::InitGoogleLogging(argv[0]); + google::ParseCommandLineFlags(&argc, &argv, true); + + mkdir(FLAGS_db.c_str(), 0700); + marmotta::persistence::LevelDBPersistence persistence(FLAGS_db, FLAGS_cache_size); + + MarmottaClient client(&persistence); + + if ("import" == std::string(argv[1])) { +#ifdef HAVE_IOSTREAMS + std::ifstream file(argv[2]); + filtering_streambuf<input> cin; + if (FLAGS_bzip2) { + cin.push(bzip2_decompressor()); + } + if (FLAGS_gzip) { + cin.push(gzip_decompressor()); + } + cin.push(file); + + std::istream in(&cin); +#else + std::ifstream in(argv[2]); +#endif + std::cout << "Importing " << argv[2] << " ... " << std::endl; + client.importDataset(in, parser::FormatFromString(FLAGS_format)); + std::cout << "Finished!" << std::endl; + } + + if ("select" == std::string(argv[1])) { + rdf::proto::Statement query; + TextFormat::ParseFromString(argv[2], &query); + if (FLAGS_output != "") { + std::ofstream out(FLAGS_output); + client.patternQuery(rdf::Statement(query), out, serializer::FormatFromString(FLAGS_format)); + } else { + client.patternQuery(rdf::Statement(query), std::cout, serializer::FormatFromString(FLAGS_format)); + } + } + + if ("sparql" == std::string(argv[1])) { + std::string query = argv[2]; + if (FLAGS_output != "") { + std::ofstream out(FLAGS_output); + client.tupleQuery(query, out); + } else { + client.tupleQuery(query, std::cout); + } + } + + if ("delete" == std::string(argv[1])) { + rdf::proto::Statement query; + TextFormat::ParseFromString(argv[2], &query); + client.patternDelete(rdf::Statement(query)); + } + + if ("size" == std::string(argv[1])) { + std::cout << "Size: " << client.size() << std::endl; + } + + + if ("namespaces" == std::string(argv[1])) { + if (FLAGS_output != "") { + std::ofstream out(FLAGS_output); + client.listNamespaces(out); + } else { + client.listNamespaces(std::cout); + } + } + + google::protobuf::ShutdownProtobufLibrary(); + + return 0; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/serializer/serializer_base.h ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/serializer/serializer_base.h b/libraries/ostrich/backend/serializer/serializer_base.h index 24a64f9..168da98 100644 --- a/libraries/ostrich/backend/serializer/serializer_base.h +++ b/libraries/ostrich/backend/serializer/serializer_base.h @@ -65,8 +65,8 @@ class SerializerBase { void serialize(StatementIterator &it, std::ostream &out) { prepare(out); - for (; it.hasNext(); ++it) { - serialize(*it); + while (it.hasNext()) { + serialize(it.next()); } close(); }; http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/sparql/rasqal_adapter.cc ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/sparql/rasqal_adapter.cc b/libraries/ostrich/backend/sparql/rasqal_adapter.cc index 7fc3cad..c9c89a6 100644 --- a/libraries/ostrich/backend/sparql/rasqal_adapter.cc +++ b/libraries/ostrich/backend/sparql/rasqal_adapter.cc @@ -65,26 +65,27 @@ rasqal_triple_parts bind_match( struct rasqal_triples_match_s *rtm, void *user_data, rasqal_variable *bindings[4], rasqal_triple_parts parts) { StatementIterator *it = (StatementIterator *) rtm->user_data; + const rdf::Statement& s = it->next(); int r = 0; #ifndef NDEBUG - DLOG(INFO) << "Binding variables " << formatVariables(bindings) << " for statement " << (*it)->as_turtle(); + DLOG(INFO) << "Binding variables " << formatVariables(bindings) << " for statement " << s.as_turtle(); #endif if ((parts & RASQAL_TRIPLE_SUBJECT) != 0) { - rasqal_variable_set_value(bindings[0], rasqal::AsLiteral(rtm->world, (*it)->getSubject())); + rasqal_variable_set_value(bindings[0], rasqal::AsLiteral(rtm->world, s.getSubject())); r |= RASQAL_TRIPLE_SUBJECT; } if ((parts & RASQAL_TRIPLE_PREDICATE) != 0) { - rasqal_variable_set_value(bindings[1], rasqal::AsLiteral(rtm->world, (*it)->getPredicate())); + rasqal_variable_set_value(bindings[1], rasqal::AsLiteral(rtm->world, s.getPredicate())); r |= RASQAL_TRIPLE_PREDICATE; } if ((parts & RASQAL_TRIPLE_OBJECT) != 0) { - rasqal_variable_set_value(bindings[2], rasqal::AsLiteral(rtm->world, (*it)->getObject())); + rasqal_variable_set_value(bindings[2], rasqal::AsLiteral(rtm->world, s.getObject())); r |= RASQAL_TRIPLE_OBJECT; } if ((parts & RASQAL_TRIPLE_ORIGIN) != 0) { - rasqal_variable_set_value(bindings[3], rasqal::AsLiteral(rtm->world, (*it)->getContext())); + rasqal_variable_set_value(bindings[3], rasqal::AsLiteral(rtm->world, s.getContext())); r |= RASQAL_TRIPLE_ORIGIN; } @@ -94,8 +95,6 @@ rasqal_triple_parts bind_match( // Increment the iterator contained in the triple match user data. void next_match(struct rasqal_triples_match_s *rtm, void *user_data) { DLOG(INFO) << "Next result"; - StatementIterator *it = (StatementIterator *) rtm->user_data; - ++(*it); } // Return true in case the iterator has no next element. http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/test/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/test/CMakeLists.txt b/libraries/ostrich/backend/test/CMakeLists.txt index 841b982..660f40f 100644 --- a/libraries/ostrich/backend/test/CMakeLists.txt +++ b/libraries/ostrich/backend/test/CMakeLists.txt @@ -1,6 +1,11 @@ -include_directories(.. ${CMAKE_CURRENT_BINARY_DIR}/.. ${RAPTOR_INCLUDE_DIR}/raptor2) +enable_testing() +include_directories(${GTEST_INCLUDE_DIRS}) +include_directories(..) +include_directories(${CMAKE_CURRENT_BINARY_DIR}/..) +include_directories(${CMAKE_CURRENT_BINARY_DIR}/../model) +include_directories(${RAPTOR_INCLUDE_DIR}/raptor2) -add_library(gtest STATIC gtest.h gtest-all.cc) +add_library(gtest STATIC gtest/gtest.h gmock/gmock.h gmock-gtest-all.cc) add_executable(model_tests StatementTest.cc main.cc) target_link_libraries(model_tests gtest marmotta_model ${GLOG_LIBRARY}) @@ -8,5 +13,9 @@ target_link_libraries(model_tests gtest marmotta_model ${GLOG_LIBRARY}) add_executable(sparql_tests SparqlTest.cc main.cc) target_link_libraries(sparql_tests gtest marmotta_model marmotta_sparql ${GLOG_LIBRARY}) -add_test(NAME ModelTest - COMMAND model_tests) +add_executable(persistence_tests main.cc PersistenceTest.cc) +target_link_libraries(persistence_tests gtest marmotta_leveldb ${GLOG_LIBRARY} ${Boost_LIBRARIES}) + +add_test(NAME ModelTest COMMAND model_tests) +add_test(NAME SparqlTest COMMAND sparql_tests) +add_test(NAME PersistenceTest COMMAND persistence_tests) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/test/PersistenceTest.cc ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/test/PersistenceTest.cc b/libraries/ostrich/backend/test/PersistenceTest.cc new file mode 100644 index 0000000..9878cd6 --- /dev/null +++ b/libraries/ostrich/backend/test/PersistenceTest.cc @@ -0,0 +1,76 @@ +// +// Created by wastl on 19.12.15. +// +#include <cstdlib> +#include <vector> + +#include <glog/logging.h> + +#include "gtest/gtest.h" +#include "gmock/gmock.h" +#include "boost/filesystem.hpp" + +#include "util/iterator.h" +#include "model/rdf_operators.h" +#include "persistence/leveldb_persistence.h" + +using namespace boost::filesystem; + +namespace marmotta { +namespace rdf { +namespace proto { +std::ostream& operator<<(std::ostream& out, const Statement& stmt) { + out << rdf::Statement(stmt).as_turtle(); + return out; +} +} +} + +namespace persistence { +namespace { + + +class PersistenceTest : public ::testing::Test { + protected: + PersistenceTest() { + testdir = temp_directory_path()/unique_path(); + create_directory(testdir); + + LOG(INFO) << "Test DB Path: " << testdir.string(); + + db = new LevelDBPersistence(testdir.string(), 10 * 1048576); + } + + ~PersistenceTest() { + LOG(INFO) << "Destroying Test DB"; + delete db; + remove_all(testdir); + } + + LevelDBPersistence* db; + path testdir; +}; + +TEST_F(PersistenceTest, TestAddStatements) { + std::vector<rdf::proto::Statement> stmts = { + rdf::Statement(rdf::URI("http://example.com/s1";), rdf::URI("http://example.com/p1";), + rdf::URI("http://example.com/o1";)).getMessage(), + rdf::Statement(rdf::URI("http://example.com/s2";), rdf::URI("http://example.com/p2";), + rdf::URI("http://example.com/o2";)).getMessage() + }; + + util::CollectionIterator<rdf::proto::Statement> it(stmts); + db->AddStatements(it); + + EXPECT_EQ(2, db->Size()); + for (const auto& stmt : stmts) { + auto it = db->GetStatements(stmt); + ASSERT_TRUE(it->hasNext()); + EXPECT_EQ(stmt, it->next()); + EXPECT_FALSE(it->hasNext()); + } +} + +} +} +} http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/test/SparqlTest.cc ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/test/SparqlTest.cc b/libraries/ostrich/backend/test/SparqlTest.cc index 7e56eba..356e43f 100644 --- a/libraries/ostrich/backend/test/SparqlTest.cc +++ b/libraries/ostrich/backend/test/SparqlTest.cc @@ -2,7 +2,7 @@ // Created by wastl on 09.12.15. // #include <glog/logging.h> -#include "gtest.h" +#include "gtest/gtest.h" #include "sparql/rasqal_adapter.h" #include "model/rdf_operators.h" @@ -10,33 +10,8 @@ namespace marmotta { namespace sparql { namespace { -class MockStatementIterator : public StatementIterator { - public: - MockStatementIterator(std::vector<rdf::Statement> statements) - : statements(statements), index(0) { - } - - StatementIterator& operator++() override { - index++; - return *this; - }; - - rdf::Statement& operator*() override { - return statements[index]; - }; - - rdf::Statement* operator->() override { - return &statements[index]; - }; - bool hasNext() override { - return index < statements.size(); - }; - - private: - std::vector<rdf::Statement> statements; - int index; -}; +using MockStatementIterator = util::CollectionIterator<rdf::Statement>; class MockTripleSource : public TripleSource { http://git-wip-us.apache.org/repos/asf/marmotta/blob/2e32da5d/libraries/ostrich/backend/test/StatementTest.cc ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/test/StatementTest.cc b/libraries/ostrich/backend/test/StatementTest.cc index 56fb11e..458f037 100644 --- a/libraries/ostrich/backend/test/StatementTest.cc +++ b/libraries/ostrich/backend/test/StatementTest.cc @@ -2,7 +2,7 @@ // Created by wastl on 18.04.15. // -#include "gtest.h" +#include "gtest/gtest.h" #include "model/rdf_model.h" #include "model/rdf_operators.h"
