- support SPARQL graph queries in ostrich - cleanups
Project: http://git-wip-us.apache.org/repos/asf/marmotta/repo Commit: http://git-wip-us.apache.org/repos/asf/marmotta/commit/e764c2bd Tree: http://git-wip-us.apache.org/repos/asf/marmotta/tree/e764c2bd Diff: http://git-wip-us.apache.org/repos/asf/marmotta/diff/e764c2bd Branch: refs/heads/MARMOTTA-584 Commit: e764c2bd0edf0f1778f888e016c2cf1711565c3f Parents: 0b07a02 Author: Sebastian Schaffert <[email protected]> Authored: Sat Feb 13 14:38:11 2016 +0100 Committer: Sebastian Schaffert <[email protected]> Committed: Sat Feb 13 14:38:11 2016 +0100 ---------------------------------------------------------------------- libraries/ostrich/backend/CMakeLists.txt | 2 +- libraries/ostrich/backend/client/client.cc | 32 ++++++ libraries/ostrich/backend/parser/CMakeLists.txt | 4 +- libraries/ostrich/backend/parser/rdf_parser.cc | 73 +----------- .../backend/persistence/leveldb_service.cc | 37 +++++- .../backend/persistence/leveldb_service.h | 4 + .../backend/serializer/serializer_raptor.cc | 2 +- libraries/ostrich/backend/service/sparql.proto | 4 + libraries/ostrich/backend/sparql/CMakeLists.txt | 3 +- .../ostrich/backend/sparql/rasqal_adapter.cc | 63 ++++++++-- .../ostrich/backend/sparql/rasqal_adapter.h | 15 ++- libraries/ostrich/backend/sparql/rasqal_model.h | 1 - libraries/ostrich/backend/test/SparqlTest.cc | 50 ++++++-- libraries/ostrich/backend/util/CMakeLists.txt | 5 +- libraries/ostrich/backend/util/raptor_util.cc | 115 +++++++++++++++++++ libraries/ostrich/backend/util/raptor_util.h | 74 ++++++++++++ libraries/ostrich/backend/util/time_logger.cc | 17 +++ libraries/ostrich/backend/util/time_logger.h | 32 ++++++ 18 files changed, 433 insertions(+), 100 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/marmotta/blob/e764c2bd/libraries/ostrich/backend/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/CMakeLists.txt b/libraries/ostrich/backend/CMakeLists.txt index 61156a5..5a8f110 100644 --- a/libraries/ostrich/backend/CMakeLists.txt +++ b/libraries/ostrich/backend/CMakeLists.txt @@ -17,7 +17,7 @@ find_package (GLog REQUIRED) find_package (Boost 1.54.0 COMPONENTS iostreams filesystem system) find_package (Tcmalloc) -add_definitions(-DNDEBUG) +#add_definitions(-DNDEBUG) if (Boost_IOSTREAMS_FOUND) message(STATUS "Enabling gzip/bzip2 support (Boost iostreams found)") http://git-wip-us.apache.org/repos/asf/marmotta/blob/e764c2bd/libraries/ostrich/backend/client/client.cc ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/client/client.cc b/libraries/ostrich/backend/client/client.cc index 396389e..b4a5178 100644 --- a/libraries/ostrich/backend/client/client.cc +++ b/libraries/ostrich/backend/client/client.cc @@ -37,6 +37,7 @@ #include <google/protobuf/wrappers.pb.h> #include <gflags/gflags.h> +#include <glog/logging.h> #include "model/rdf_model.h" #include "parser/rdf_parser.h" @@ -195,6 +196,25 @@ class MarmottaClient { delete out_; } + void graphQuery(const std::string& query, std::ostream &out, serializer::Format format) { + ClientContext context; + spq::SparqlRequest request; + request.set_query(query); + + std::unique_ptr<ClientReader<rdf::proto::Statement>> reader( + sparql_->GraphQuery(&context, request)); + + StatementReader it(reader.get()); + + try { + serializer::Serializer serializer("http://www.example.com";, format); + serializer.serialize(it, out); + } catch(serializer::SerializationError e) { + LOG(FATAL) << "Serialization error: " << e.getMessage(); + } + } + + void listNamespaces(std::ostream &out) { ClientContext context; @@ -240,6 +260,8 @@ DEFINE_bool(bzip2, false, "Input files are bzip2 compressed."); int main(int argc, char** argv) { GOOGLE_PROTOBUF_VERIFY_VERSION; + // Initialize Google's logging library. + google::InitGoogleLogging(argv[0]); google::ParseCommandLineFlags(&argc, &argv, true); MarmottaClient client(FLAGS_host + ":" + FLAGS_port); @@ -286,6 +308,16 @@ int main(int argc, char** argv) { } } + if ("construct" == std::string(argv[1])) { + std::string query = argv[2]; + if (FLAGS_output != "") { + std::ofstream out(FLAGS_output); + client.graphQuery(query, out, serializer::FormatFromString(FLAGS_format)); + } else { + client.graphQuery(query, std::cout, serializer::FormatFromString(FLAGS_format)); + } + } + if ("delete" == std::string(argv[1])) { rdf::proto::Statement query; TextFormat::ParseFromString(argv[2], &query); http://git-wip-us.apache.org/repos/asf/marmotta/blob/e764c2bd/libraries/ostrich/backend/parser/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/parser/CMakeLists.txt b/libraries/ostrich/backend/parser/CMakeLists.txt index 3ed5634..e698690 100644 --- a/libraries/ostrich/backend/parser/CMakeLists.txt +++ b/libraries/ostrich/backend/parser/CMakeLists.txt @@ -1,4 +1,6 @@ include_directories(.. ${CMAKE_CURRENT_BINARY_DIR}/..) add_library(marmotta_parser rdf_parser.h rdf_parser.cc) -target_link_libraries(marmotta_parser marmotta_model ${CMAKE_THREAD_LIBS_INIT} ${RAPTOR_LIBRARY}) \ No newline at end of file +target_link_libraries( + marmotta_parser marmotta_model marmotta_raptor_util + ${CMAKE_THREAD_LIBS_INIT} ${RAPTOR_LIBRARY} ${GLOG_LIBRARY}) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/marmotta/blob/e764c2bd/libraries/ostrich/backend/parser/rdf_parser.cc ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/parser/rdf_parser.cc b/libraries/ostrich/backend/parser/rdf_parser.cc index 4706e8b..cf2dc4d 100644 --- a/libraries/ostrich/backend/parser/rdf_parser.cc +++ b/libraries/ostrich/backend/parser/rdf_parser.cc @@ -15,8 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include <raptor2/raptor2.h> #include "rdf_parser.h" +#include <raptor2/raptor2.h> +#include <util/raptor_util.h> namespace marmotta { namespace parser { @@ -67,75 +68,7 @@ Parser::~Parser() { void Parser::raptor_stmt_handler(void *user_data, raptor_statement *statement) { Parser* p = static_cast<Parser*>(user_data); - - rdf::Resource subject; rdf::URI predicate; rdf::Value object; rdf::Resource context; - switch (statement->subject->type) { - case RAPTOR_TERM_TYPE_URI: - subject = rdf::URI((const char*)raptor_uri_as_string(statement->subject->value.uri)); - break; - case RAPTOR_TERM_TYPE_BLANK: - subject = rdf::BNode(std::string((const char*)statement->subject->value.blank.string, statement->subject->value.blank.string_len)); - break; - default: - raptor_parser_parse_abort(p->parser); - throw ParseError("invalid subject term type"); - } - - switch (statement->predicate->type) { - case RAPTOR_TERM_TYPE_URI: - predicate = rdf::URI((const char*)raptor_uri_as_string(statement->predicate->value.uri)); - break; - default: - raptor_parser_parse_abort(p->parser); - throw ParseError("invalid predicate term type"); - } - - switch (statement->object->type) { - case RAPTOR_TERM_TYPE_URI: - object = rdf::URI((const char*)raptor_uri_as_string(statement->object->value.uri)); - break; - case RAPTOR_TERM_TYPE_BLANK: - object = rdf::BNode(std::string((const char*)statement->object->value.blank.string, statement->object->value.blank.string_len)); - break; - case RAPTOR_TERM_TYPE_LITERAL: - if(statement->object->value.literal.language != NULL) { - object = rdf::StringLiteral( - std::string((const char*)statement->object->value.literal.string, statement->object->value.literal.string_len), - std::string((const char*)statement->object->value.literal.language, statement->object->value.literal.language_len) - ); - } else if(statement->object->value.literal.datatype != NULL) { - object = rdf::DatatypeLiteral( - std::string((const char*)statement->object->value.literal.string, statement->object->value.literal.string_len), - rdf::URI((const char*)raptor_uri_as_string(statement->object->value.literal.datatype)) - ); - } else { - object = rdf::StringLiteral( - std::string((const char*)statement->object->value.literal.string, statement->object->value.literal.string_len) - ); - } - break; - default: - raptor_parser_parse_abort(p->parser); - throw ParseError("invalid object term type"); - } - - if (statement->graph != NULL) { - switch (statement->graph->type) { - case RAPTOR_TERM_TYPE_URI: - context = rdf::URI((const char*)raptor_uri_as_string(statement->graph->value.uri)); - break; - case RAPTOR_TERM_TYPE_BLANK: - context = rdf::BNode(std::string((const char*)statement->graph->value.blank.string, statement->graph->value.blank.string_len)); - break; - default: - raptor_parser_parse_abort(p->parser); - throw ParseError("invalid graph term type"); - } - } else { - context = rdf::URI(); - } - - p->stmt_handler(rdf::Statement(subject, predicate, object, context)); + p->stmt_handler(util::raptor::ConvertStatement(statement)); } http://git-wip-us.apache.org/repos/asf/marmotta/blob/e764c2bd/libraries/ostrich/backend/persistence/leveldb_service.cc ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/persistence/leveldb_service.cc b/libraries/ostrich/backend/persistence/leveldb_service.cc index c11b002..5ea08a1 100644 --- a/libraries/ostrich/backend/persistence/leveldb_service.cc +++ b/libraries/ostrich/backend/persistence/leveldb_service.cc @@ -20,8 +20,8 @@ #include <unordered_set> #include <model/rdf_operators.h> -#include <util/iterator.h> #include <util/unique.h> +#include <util/time_logger.h> using grpc::Status; using grpc::StatusCode; @@ -119,6 +119,7 @@ grpc::Status LevelDBService::GetNamespaces( Status LevelDBService::AddStatements( ServerContext* context, ServerReader<Statement>* reader, Int64Value* result) { + util::TimeLogger timeLogger("Adding statements"); auto it = StatementIterator(reader); int64_t count = persistence->AddStatements(it); @@ -130,6 +131,7 @@ Status LevelDBService::AddStatements( Status LevelDBService::GetStatements( ServerContext* context, const Statement* pattern, ServerWriter<Statement>* result) { + util::TimeLogger timeLogger("Retrieving statements"); persistence->GetStatements(*pattern, [&result](const Statement& stmt) -> bool { return result->Write(stmt); @@ -140,6 +142,7 @@ Status LevelDBService::GetStatements( Status LevelDBService::RemoveStatements( ServerContext* context, const Statement* pattern, Int64Value* result) { + util::TimeLogger timeLogger("Removing statements"); int64_t count = persistence->RemoveStatements(*pattern); result->set_value(count); @@ -149,7 +152,7 @@ Status LevelDBService::RemoveStatements( Status LevelDBService::Clear( ServerContext* context, const ContextRequest* contexts, Int64Value* result) { - + util::TimeLogger timeLogger("Clearing contexts"); int64_t count = 0; @@ -169,6 +172,7 @@ Status LevelDBService::Clear( Status LevelDBService::Size( ServerContext* context, const ContextRequest* contexts, Int64Value* result) { + util::TimeLogger timeLogger("Computing context size"); int64_t count = 0; @@ -194,6 +198,8 @@ Status LevelDBService::Size( grpc::Status LevelDBService::GetContexts( ServerContext *context, const Empty *ignored, ServerWriter<Resource> *result) { + util::TimeLogger timeLogger("Retrieving contexts"); + // Currently we need to iterate over all statements and collect the results. Statement pattern; std::unordered_set<Resource> contexts; @@ -214,6 +220,7 @@ grpc::Status LevelDBService::GetContexts( grpc::Status LevelDBService::Update(grpc::ServerContext *context, grpc::ServerReader<service::proto::UpdateRequest> *reader, service::proto::UpdateResponse *result) { + util::TimeLogger timeLogger("Updating database"); auto it = UpdateIterator(reader); persistence::UpdateStatistics stats = persistence->Update(it); @@ -233,19 +240,39 @@ grpc::Status LevelDBSparqlService::TupleQuery( SparqlService svc(util::make_unique<LevelDBTripleSource>(persistence)); - svc.TupleQuery(query->query(), [&result](const SparqlService::RowType& row) { + rdf::URI base_uri = query->base_uri(); + + svc.TupleQuery(query->query(), base_uri, + [&result](const SparqlService::RowType& row) { spq::SparqlResponse response; for (auto it = row.cbegin(); it != row.cend(); it++) { auto b = response.add_binding(); b->set_variable(it->first); *b->mutable_value() = it->second.getMessage(); } - result->Write(response); - return true; + return result->Write(response); }); return Status::OK; } + +grpc::Status LevelDBSparqlService::GraphQuery(grpc::ServerContext* context, + const spq::SparqlRequest* query, + grpc::ServerWriter<rdf::proto::Statement>* result) { + + SparqlService svc(util::make_unique<LevelDBTripleSource>(persistence)); + + rdf::URI base_uri = query->base_uri(); + + svc.GraphQuery(query->query(), base_uri, + [&result](const rdf::Statement& triple) { + return result->Write(triple.getMessage()); + }); + + return Status::OK; +} + + } // namespace service } // namespace marmotta \ No newline at end of file http://git-wip-us.apache.org/repos/asf/marmotta/blob/e764c2bd/libraries/ostrich/backend/persistence/leveldb_service.h ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/persistence/leveldb_service.h b/libraries/ostrich/backend/persistence/leveldb_service.h index 0cf4df9..44056a3 100644 --- a/libraries/ostrich/backend/persistence/leveldb_service.h +++ b/libraries/ostrich/backend/persistence/leveldb_service.h @@ -110,6 +110,10 @@ class LevelDBSparqlService : public spq::SparqlService::Service { grpc::Status TupleQuery(grpc::ServerContext* context, const spq::SparqlRequest* pattern, grpc::ServerWriter<spq::SparqlResponse>* result) override; + + grpc::Status GraphQuery(grpc::ServerContext* context, + const spq::SparqlRequest* pattern, + grpc::ServerWriter<rdf::proto::Statement>* result) override; private: persistence::LevelDBPersistence* persistence; }; http://git-wip-us.apache.org/repos/asf/marmotta/blob/e764c2bd/libraries/ostrich/backend/serializer/serializer_raptor.cc ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/serializer/serializer_raptor.cc b/libraries/ostrich/backend/serializer/serializer_raptor.cc index 551ecf2..ff37691 100644 --- a/libraries/ostrich/backend/serializer/serializer_raptor.cc +++ b/libraries/ostrich/backend/serializer/serializer_raptor.cc @@ -249,7 +249,7 @@ void RaptorSerializer::serialize(const rdf::Statement &stmt) { triple->graph = raptor_new_term_from_blank( world, (unsigned char const *) stmt.getMessage().context().bnode().id().c_str()); } else { - throw SerializationError("invalid context type"); + triple->graph = nullptr; } raptor_serializer_serialize_statement(serializer, triple); http://git-wip-us.apache.org/repos/asf/marmotta/blob/e764c2bd/libraries/ostrich/backend/service/sparql.proto ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/service/sparql.proto b/libraries/ostrich/backend/service/sparql.proto index 892539b..6d75a5f 100644 --- a/libraries/ostrich/backend/service/sparql.proto +++ b/libraries/ostrich/backend/service/sparql.proto @@ -26,6 +26,7 @@ import "model.proto"; // SPARQL request consisting of a single query string. message SparqlRequest { string query = 1; + marmotta.rdf.proto.URI base_uri = 2; } // SPARQL response row, containing a set of bindings. @@ -42,4 +43,7 @@ message SparqlResponse { service SparqlService { // Execute a SPARQL 1.1 tuple query and stream back the results. rpc TupleQuery(SparqlRequest) returns (stream SparqlResponse); + + // Execute a SPARQL 1.1 graph query and stream back the triples. + rpc GraphQuery(SparqlRequest) returns (stream marmotta.rdf.proto.Statement); } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/marmotta/blob/e764c2bd/libraries/ostrich/backend/sparql/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/sparql/CMakeLists.txt b/libraries/ostrich/backend/sparql/CMakeLists.txt index 9bb00ef..89eb0d0 100644 --- a/libraries/ostrich/backend/sparql/CMakeLists.txt +++ b/libraries/ostrich/backend/sparql/CMakeLists.txt @@ -2,6 +2,7 @@ include_directories(.. ${CMAKE_CURRENT_BINARY_DIR}/.. ${RAPTOR_INCLUDE_DIR}/rapt add_library(marmotta_sparql rasqal_model.cc rasqal_model.h rasqal_adapter.cc rasqal_adapter.h) -target_link_libraries(marmotta_sparql marmotta_model ${CMAKE_THREAD_LIBS_INIT} +target_link_libraries(marmotta_sparql marmotta_model marmotta_util marmotta_raptor_util + ${CMAKE_THREAD_LIBS_INIT} ${PROTOBUF_LIBRARIES} ${GRPC_LIBRARIES} ${RASQAL_LIBRARIES} ${RAPTOR_LIBRARY} ${GLOG_LIBRARY}) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/marmotta/blob/e764c2bd/libraries/ostrich/backend/sparql/rasqal_adapter.cc ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/sparql/rasqal_adapter.cc b/libraries/ostrich/backend/sparql/rasqal_adapter.cc index c9c89a6..09e7bd6 100644 --- a/libraries/ostrich/backend/sparql/rasqal_adapter.cc +++ b/libraries/ostrich/backend/sparql/rasqal_adapter.cc @@ -24,6 +24,11 @@ #include "sparql/rasqal_adapter.h" #include "sparql/rasqal_model.h" +#include "util/raptor_util.h" +#include "util/time_logger.h" + +// Rasqal notoriously uses unsigned strings, macro to convert C++ strings. +#define STR(s) (const unsigned char*)s.c_str() namespace marmotta { namespace sparql { @@ -247,13 +252,14 @@ SparqlService::~SparqlService() { rasqal_free_world(world); } -void SparqlService::TupleQuery(const std::string& query, std::function<bool(const RowType&)> row_handler) { - auto start = std::chrono::steady_clock::now(); - LOG(INFO) << "Starting SPARQL tuple query."; +void SparqlService::TupleQuery(const std::string& query, const rdf::URI& base_uri, + std::function<bool(const RowType&)> row_handler) { + util::TimeLogger timeLogger("SPARQL tuple query"); auto q = rasqal_new_query(world, "sparql11-query", nullptr); - auto base = raptor_new_uri(rasqal_world_get_raptor(world), (const unsigned char*)"http://example.com";); - if (rasqal_query_prepare(q, (const unsigned char*)query.c_str(), base) != 0) { + auto base = raptor_new_uri(rasqal_world_get_raptor(world), + STR(base_uri.getUri())); + if (rasqal_query_prepare(q, STR(query), base) != 0) { raptor_free_uri(base); rasqal_free_query(q); throw SparqlException("Query preparation failed", query); @@ -267,6 +273,13 @@ void SparqlService::TupleQuery(const std::string& query, std::function<bool(cons throw SparqlException("Query execution failed", query); } + if (!rasqal_query_results_is_bindings(r)) { + rasqal_free_query_results(r); + rasqal_free_query(q); + raptor_free_uri(base); + throw SparqlException("Query is not a tuple query", query); + } + int rowcount = 0; while (next && rasqal_query_results_finished(r) == 0) { RowType row; @@ -287,12 +300,48 @@ void SparqlService::TupleQuery(const std::string& query, std::function<bool(cons rasqal_free_query_results(r); rasqal_free_query(q); raptor_free_uri(base); +} + + +void SparqlService::GraphQuery(const std::string& query, const rdf::URI& base_uri, + std::function<bool(const rdf::Statement&)> stmt_handler) { + util::TimeLogger timeLogger("SPARQL graph query"); - LOG(INFO) << "SPARQL query finished (time=" << std::chrono::duration <double, std::milli> ( - std::chrono::steady_clock::now() - start).count() << "ms)."; + auto q = rasqal_new_query(world, "sparql11-query", nullptr); + auto base = raptor_new_uri(rasqal_world_get_raptor(world), + STR(base_uri.getUri())); + if (rasqal_query_prepare(q, STR(query), base) != 0) { + raptor_free_uri(base); + rasqal_free_query(q); + throw SparqlException("Query preparation failed", query); + } + bool next = true; + auto r = rasqal_query_execute(q); + if (r == nullptr) { + raptor_free_uri(base); + rasqal_free_query(q); + throw SparqlException("Query execution failed", query); + } + + if (!rasqal_query_results_is_graph(r)) { + rasqal_free_query_results(r); + rasqal_free_query(q); + raptor_free_uri(base); + throw SparqlException("Query is not a graph query", query); + } + + while (next) { + next = stmt_handler(util::raptor::ConvertStatement(rasqal_query_results_get_triple(r))) + && rasqal_query_results_next_triple(r) == 0; + } + + rasqal_free_query_results(r); + rasqal_free_query(q); + raptor_free_uri(base); } + } // namespace sparql } // namespace marmotta http://git-wip-us.apache.org/repos/asf/marmotta/blob/e764c2bd/libraries/ostrich/backend/sparql/rasqal_adapter.h ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/sparql/rasqal_adapter.h b/libraries/ostrich/backend/sparql/rasqal_adapter.h index 48ce127..32e82d2 100644 --- a/libraries/ostrich/backend/sparql/rasqal_adapter.h +++ b/libraries/ostrich/backend/sparql/rasqal_adapter.h @@ -35,7 +35,6 @@ using StatementIterator = util::CloseableIterator<rdf::Statement>; */ class TripleSource { public: - /** * Check for presence of a complete statement. * @@ -85,7 +84,19 @@ class SparqlService { */ ~SparqlService(); - void TupleQuery(const std::string& query, std::function<bool(const RowType&)> row_handler); + /** + * Execute a tuple (SELECT) query, calling the row handler for each set of + * variable bindings. + */ + void TupleQuery(const std::string& query, const rdf::URI& base_uri, + std::function<bool(const RowType&)> row_handler); + + /** + * Execute a graph (CONSTRUCT) query, calling the statement handler for + * each triple. + */ + void GraphQuery(const std::string& query, const rdf::URI& base_uri, + std::function<bool(const rdf::Statement&)> stmt_handler); /** * Return a reference to the triple source managed by this service. http://git-wip-us.apache.org/repos/asf/marmotta/blob/e764c2bd/libraries/ostrich/backend/sparql/rasqal_model.h ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/sparql/rasqal_model.h b/libraries/ostrich/backend/sparql/rasqal_model.h index 549b3c4..6547c19 100644 --- a/libraries/ostrich/backend/sparql/rasqal_model.h +++ b/libraries/ostrich/backend/sparql/rasqal_model.h @@ -66,7 +66,6 @@ rasqal_literal* AsLiteral(rasqal_world* world, const rdf::Value& v); */ rasqal_literal* AsLiteral(rasqal_world* world, const rdf::URI& u); - } // namespace rasqal } // namespace sparql } // namespace marmotta http://git-wip-us.apache.org/repos/asf/marmotta/blob/e764c2bd/libraries/ostrich/backend/test/SparqlTest.cc ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/test/SparqlTest.cc b/libraries/ostrich/backend/test/SparqlTest.cc index 356e43f..0f663f5 100644 --- a/libraries/ostrich/backend/test/SparqlTest.cc +++ b/libraries/ostrich/backend/test/SparqlTest.cc @@ -11,14 +11,16 @@ namespace sparql { namespace { +const rdf::URI base_uri("http://example.com/";); + + using MockStatementIterator = util::CollectionIterator<rdf::Statement>; class MockTripleSource : public TripleSource { public: - MockTripleSource(std::vector<rdf::Statement> statements) : statements(statements) { - - } + MockTripleSource(std::vector<rdf::Statement> statements) + : statements(statements) { } bool HasStatement(const rdf::Resource *s, const rdf::URI *p, const rdf::Value *o, const rdf::Resource *c) override { for (const auto& stmt : statements) { @@ -68,6 +70,7 @@ class MockTripleSource : public TripleSource { private: std::vector<rdf::Statement> statements; + }; } // namespace @@ -81,7 +84,8 @@ TEST(SPARQLTest, Simple) { int count = 0; rdf::Value s, p, o; - svc.TupleQuery("SELECT * WHERE {?s ?p ?o}", [&](const SparqlService::RowType& row) { + svc.TupleQuery("SELECT * WHERE {?s ?p ?o}", base_uri, + [&](const SparqlService::RowType& row) { count++; s = row.at("s"); p = row.at("p"); @@ -106,7 +110,8 @@ TEST(SPARQLTest, SubjectPattern) { int count = 0; rdf::Value p, o; - svc.TupleQuery("SELECT * WHERE {<http://example.com/s1> ?p ?o}", [&](const SparqlService::RowType& row) { + svc.TupleQuery("SELECT * WHERE {<http://example.com/s1> ?p ?o}", base_uri, + [&](const SparqlService::RowType& row) { count++; p = row.at("p"); o = row.at("o"); @@ -129,7 +134,8 @@ TEST(SPARQLTest, PredicatePattern) { int count = 0; rdf::Value s, o; - svc.TupleQuery("SELECT * WHERE {?s <http://example.com/p1> ?o}", [&](const SparqlService::RowType& row) { + svc.TupleQuery("SELECT * WHERE {?s <http://example.com/p1> ?o}", base_uri, + [&](const SparqlService::RowType& row) { count++; s = row.at("s"); o = row.at("o"); @@ -152,7 +158,8 @@ TEST(SPARQLTest, ObjectPattern) { int count = 0; rdf::Value s, p; - svc.TupleQuery("SELECT * WHERE {?s ?p <http://example.com/o1>}", [&](const SparqlService::RowType& row) { + svc.TupleQuery("SELECT * WHERE {?s ?p <http://example.com/o1>}", base_uri, + [&](const SparqlService::RowType& row) { count++; s = row.at("s"); p = row.at("p"); @@ -175,7 +182,8 @@ TEST(SPARQLTest, BNode) { int count = 0; rdf::Value s, p; - svc.TupleQuery("SELECT * WHERE {?s ?p <http://example.com/o1>}", [&](const SparqlService::RowType& row) { + svc.TupleQuery("SELECT * WHERE {?s ?p <http://example.com/o1>}", base_uri, + [&](const SparqlService::RowType& row) { count++; s = row.at("s"); p = row.at("p"); @@ -198,7 +206,8 @@ TEST(SPARQLTest, Filter) { int count = 0; rdf::Value s, p, o; - svc.TupleQuery("SELECT * WHERE {?s ?p ?o . FILTER(?o = <http://example.com/o1>)}", [&](const SparqlService::RowType& row) { + svc.TupleQuery("SELECT * WHERE {?s ?p ?o . FILTER(?o = <http://example.com/o1>)}", base_uri, + [&](const SparqlService::RowType& row) { count++; s = row.at("s"); p = row.at("p"); @@ -223,7 +232,8 @@ TEST(SPARQLTest, Join) { int count = 0; rdf::Value s, o; - svc.TupleQuery("SELECT * WHERE {?s ?p1 ?o1 . ?o1 ?p2 ?o }", [&](const SparqlService::RowType& row) { + svc.TupleQuery("SELECT * WHERE {?s ?p1 ?o1 . ?o1 ?p2 ?o }", base_uri, + [&](const SparqlService::RowType& row) { count++; s = row.at("s"); o = row.at("o"); @@ -236,6 +246,26 @@ TEST(SPARQLTest, Join) { EXPECT_EQ("http://example.com/o2";, o.stringValue()); } +TEST(SPARQLTest, Graph) { + rdf::Statement stmt = rdf::Statement(rdf::URI("http://example.com/s1";), + rdf::URI("http://example.com/p1";), + rdf::URI("http://example.com/o1";)); + SparqlService svc(std::unique_ptr<TripleSource>(new MockTripleSource({stmt}))); + + int count = 0; + rdf::Value s, p, o; + svc.GraphQuery("CONSTRUCT { ?s ?p ?o . } WHERE {?s ?p ?o}", base_uri, + [&](const rdf::Statement& row) { + count++; + + EXPECT_EQ(stmt, row); + + return true; + }); + + EXPECT_EQ(1, count); +} + } // namespace sparql } // namespace marmotta \ No newline at end of file http://git-wip-us.apache.org/repos/asf/marmotta/blob/e764c2bd/libraries/ostrich/backend/util/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/util/CMakeLists.txt b/libraries/ostrich/backend/util/CMakeLists.txt index ac87cd8..73710ec 100644 --- a/libraries/ostrich/backend/util/CMakeLists.txt +++ b/libraries/ostrich/backend/util/CMakeLists.txt @@ -1,3 +1,6 @@ include_directories(.. ${CMAKE_CURRENT_BINARY_DIR}/..) -add_library(marmotta_util murmur3.cc murmur3.h split.cc split.h iterator.h unique.h) \ No newline at end of file +add_library(marmotta_util murmur3.cc murmur3.h split.cc split.h iterator.h unique.h time_logger.cc time_logger.h) + +add_library(marmotta_raptor_util raptor_util.h raptor_util.cc) +target_link_libraries(marmotta_raptor_util marmotta_model ${CMAKE_THREAD_LIBS_INIT} ${RAPTOR_LIBRARY}) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/marmotta/blob/e764c2bd/libraries/ostrich/backend/util/raptor_util.cc ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/util/raptor_util.cc b/libraries/ostrich/backend/util/raptor_util.cc new file mode 100644 index 0000000..bbeaecd --- /dev/null +++ b/libraries/ostrich/backend/util/raptor_util.cc @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "raptor_util.h" +#include <glog/logging.h> + +namespace marmotta { +namespace util { +namespace raptor { + +// Helper macros. Some Rasqal functions copy the input string themselves, others don't. +#define STR(s) (const unsigned char*)s.c_str() +#define CPSTR(s) (const unsigned char*)strdup(s.c_str()) + +rdf::Resource ConvertResource(raptor_term *node) { + if (node == nullptr) { + return rdf::Resource(); + } + + switch (node->type) { + case RAPTOR_TERM_TYPE_URI: + return rdf::URI(std::string((const char*)raptor_uri_as_string(node->value.uri))); + case RAPTOR_TERM_TYPE_BLANK: + return rdf::BNode(std::string((const char*)node->value.blank.string, + node->value.blank.string_len)); + default: + LOG(INFO) << "Error: unsupported resource type " << node->type; + return rdf::Resource(); + } +} + + +rdf::Value ConvertValue(raptor_term *node) { + if (node == nullptr) { + return rdf::Value(); + } + + switch (node->type) { + case RAPTOR_TERM_TYPE_URI: + return rdf::URI((const char*)raptor_uri_as_string(node->value.uri)); + case RAPTOR_TERM_TYPE_BLANK: + return rdf::BNode(std::string((const char*)node->value.blank.string, + node->value.blank.string_len)); + case RAPTOR_TERM_TYPE_LITERAL: + if(node->value.literal.language != nullptr) { + return rdf::StringLiteral( + std::string((const char*)node->value.literal.string, node->value.literal.string_len), + std::string((const char*)node->value.literal.language, node->value.literal.language_len) + ); + } else if(node->value.literal.datatype != nullptr) { + return rdf::DatatypeLiteral( + std::string((const char*)node->value.literal.string, node->value.literal.string_len), + rdf::URI((const char*)raptor_uri_as_string(node->value.literal.datatype)) + ); + } else { + return rdf::StringLiteral( + std::string((const char*)node->value.literal.string, node->value.literal.string_len) + ); + } + default: + LOG(INFO) << "Error: unsupported node type " << node->type; + return rdf::Value(); + } +} + + +rdf::URI ConvertURI(raptor_term *node) { + if (node == nullptr) { + return rdf::URI(); + } + + switch (node->type) { + case RAPTOR_TERM_TYPE_URI: + return rdf::URI((const char*)raptor_uri_as_string(node->value.uri)); + default: + return rdf::URI(); + } +} + + +rdf::Statement ConvertStatement(raptor_statement *triple) { + if (triple->graph != nullptr) { + return rdf::Statement( + ConvertResource(triple->subject), + ConvertURI(triple->predicate), + ConvertValue(triple->object), + ConvertResource(triple->graph) + ); + } else { + return rdf::Statement( + ConvertResource(triple->subject), + ConvertURI(triple->predicate), + ConvertValue(triple->object) + ); + + } +} +} // namespace raptor +} // namespace util +} // namespace marmotta + http://git-wip-us.apache.org/repos/asf/marmotta/blob/e764c2bd/libraries/ostrich/backend/util/raptor_util.h ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/util/raptor_util.h b/libraries/ostrich/backend/util/raptor_util.h new file mode 100644 index 0000000..1899235 --- /dev/null +++ b/libraries/ostrich/backend/util/raptor_util.h @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MARMOTTA_RAPTOR_MODEL_H +#define MARMOTTA_RAPTOR_MODEL_H + +#include <memory> +#include <raptor2/raptor2.h> + +#include "model/rdf_model.h" + +namespace marmotta { +namespace util { +namespace raptor { + +/* + * Convert a raptor term into a Marmotta Resource. Returns empty in case + * the node cannot be converted. + */ +rdf::Resource ConvertResource(raptor_term* node); + +/* + * Convert a raptor term into a Marmotta Value. Returns empty in case + * the node cannot be converted. + */ +rdf::Value ConvertValue(raptor_term* node); + +/* + * Convert a raptor term into a Marmotta URI. Returns empty in case + * the node cannot be converted. + */ +rdf::URI ConvertURI(raptor_term* node); + +/* + * Convert a raptor triple into a Marmotta Statement. Returns empty in case + * the node cannot be converted. + */ +rdf::Statement ConvertStatement(raptor_statement* triple); + +/* + * Convert a Marmotta Resource into a raptor term. + */ +raptor_term* AsLiteral(raptor_world* world, const rdf::Resource& r); + +/* + * Convert a Marmotta Value into a raptor term. + */ +raptor_term* AsLiteral(raptor_world* world, const rdf::Value& v); + +/* + * Convert a Marmotta URI into a raptor term. + */ +raptor_term* AsLiteral(raptor_world* world, const rdf::URI& u); + +} // namespace raptor +} // namespace util +} // namespace marmotta + + +#endif //MARMOTTA_RAPTOR_MODEL_H http://git-wip-us.apache.org/repos/asf/marmotta/blob/e764c2bd/libraries/ostrich/backend/util/time_logger.cc ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/util/time_logger.cc b/libraries/ostrich/backend/util/time_logger.cc new file mode 100644 index 0000000..e836b61 --- /dev/null +++ b/libraries/ostrich/backend/util/time_logger.cc @@ -0,0 +1,17 @@ +// +// Created by wastl on 13.02.16. +// + +#include <glog/logging.h> +#include "time_logger.h" + +marmotta::util::TimeLogger::TimeLogger(const std::string &name) + : start_(std::chrono::steady_clock::now()) + , name_(name) { + LOG(INFO) << name << " started."; +} + +marmotta::util::TimeLogger::~TimeLogger() { + LOG(INFO) << name_ << " finished (time=" << std::chrono::duration <double, std::milli> ( + std::chrono::steady_clock::now() - start_).count() << "ms)."; +} http://git-wip-us.apache.org/repos/asf/marmotta/blob/e764c2bd/libraries/ostrich/backend/util/time_logger.h ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/util/time_logger.h b/libraries/ostrich/backend/util/time_logger.h new file mode 100644 index 0000000..cf1929b --- /dev/null +++ b/libraries/ostrich/backend/util/time_logger.h @@ -0,0 +1,32 @@ +// +// Created by wastl on 13.02.16. +// + +#ifndef MARMOTTA_TIME_LOGGER_H +#define MARMOTTA_TIME_LOGGER_H + +#include <string> +#include <chrono> + +namespace marmotta { +namespace util { + +/** + * A time logger, writes a logging message when initialised and timing + * information when destructed. + */ +class TimeLogger { + public: + TimeLogger(const std::string& name); + + ~TimeLogger(); + + private: + std::string name_; + std::chrono::time_point<std::chrono::steady_clock> start_; +}; + +} // namespace util +} // namespace marmotta + +#endif //MARMOTTA_TIME_LOGGER_H
