Repository: marmotta Updated Branches: refs/heads/develop e0e5145de -> 134abbb8c
optional compressed input support for command line client Project: http://git-wip-us.apache.org/repos/asf/marmotta/repo Commit: http://git-wip-us.apache.org/repos/asf/marmotta/commit/134abbb8 Tree: http://git-wip-us.apache.org/repos/asf/marmotta/tree/134abbb8 Diff: http://git-wip-us.apache.org/repos/asf/marmotta/diff/134abbb8 Branch: refs/heads/develop Commit: 134abbb8ccc16124f06cd55fc533982bec7cd926 Parents: e0e5145 Author: Sebastian Schaffert <[email protected]> Authored: Sat Dec 12 19:17:33 2015 +0100 Committer: Sebastian Schaffert <[email protected]> Committed: Sat Dec 12 19:17:33 2015 +0100 ---------------------------------------------------------------------- libraries/ostrich/backend/CMakeLists.txt | 6 ++++ libraries/ostrich/backend/client/CMakeLists.txt | 2 +- libraries/ostrich/backend/client/client.cc | 31 ++++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/marmotta/blob/134abbb8/libraries/ostrich/backend/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/CMakeLists.txt b/libraries/ostrich/backend/CMakeLists.txt index d8232b7..608cb2a 100644 --- a/libraries/ostrich/backend/CMakeLists.txt +++ b/libraries/ostrich/backend/CMakeLists.txt @@ -14,10 +14,16 @@ find_package (Protobuf REQUIRED) find_package (GRPC REQUIRED) find_package (LevelDB REQUIRED) find_package (GLog REQUIRED) +find_package (Boost 1.54.0 COMPONENTS iostreams) find_package (Tcmalloc) add_definitions(-DNDEBUG) +if(Boost_IOSTREAMS_FOUND) + message(STATUS "Enabling gzip/bzip2 support (Boost iostreams found)") + add_definitions(-DHAVE_IOSTREAMS) +endif(Boost_IOSTREAMS_FOUND) + add_subdirectory(util) add_subdirectory(model) add_subdirectory(sparql) http://git-wip-us.apache.org/repos/asf/marmotta/blob/134abbb8/libraries/ostrich/backend/client/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/client/CMakeLists.txt b/libraries/ostrich/backend/client/CMakeLists.txt index 88ad11a..220799f 100644 --- a/libraries/ostrich/backend/client/CMakeLists.txt +++ b/libraries/ostrich/backend/client/CMakeLists.txt @@ -3,6 +3,6 @@ include_directories(.. ${CMAKE_CURRENT_BINARY_DIR}/.. ${CMAKE_CURRENT_BINARY_DIR add_executable(marmotta_client client.cc) target_link_libraries(marmotta_client marmotta_model marmotta_service marmotta_parser marmotta_serializer - ${GFLAGS_LIBRARY} + ${GFLAGS_LIBRARY} ${Boost_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${PROTOBUF_LIBRARIES} ${GRPC_LIBRARIES}) http://git-wip-us.apache.org/repos/asf/marmotta/blob/134abbb8/libraries/ostrich/backend/client/client.cc ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/client/client.cc b/libraries/ostrich/backend/client/client.cc index 16c9022..56f317c 100644 --- a/libraries/ostrich/backend/client/client.cc +++ b/libraries/ostrich/backend/client/client.cc @@ -17,6 +17,14 @@ */ #include <fstream> +#ifdef HAVE_IOSTREAMS +// support b/gzipped files +#include <boost/iostreams/filtering_streambuf.hpp> +#include <boost/iostreams/copy.hpp> +#include <boost/iostreams/filter/gzip.hpp> +#include <boost/iostreams/filter/bzip2.hpp> +#endif + #include <grpc/grpc.h> #include <grpc++/channel.h> #include <grpc++/client_context.h> @@ -51,6 +59,10 @@ using namespace marmotta; namespace svc = marmotta::service::proto; namespace spq = marmotta::sparql::proto; +#ifdef HAVE_IOSTREAMS +using namespace boost::iostreams; +#endif + // A STL iterator wrapper around a client reader. template <class T, class Proto> class ClientReaderIterator : public util::CloseableIterator<T> { @@ -212,6 +224,11 @@ DEFINE_string(host, "localhost", "Address/name of server to access."); DEFINE_string(port, "10000", "Port of server to access."); DEFINE_string(output, "", "File to write result to."); +#ifdef HAVE_IOSTREAMS +DEFINE_bool(gzip, false, "Input files are gzip compressed."); +DEFINE_bool(bzip2, false, "Input files are bzip2 compressed."); +#endif + int main(int argc, char** argv) { GOOGLE_PROTOBUF_VERIFY_VERSION; @@ -220,7 +237,21 @@ int main(int argc, char** argv) { MarmottaClient client(FLAGS_host + ":" + FLAGS_port); if ("import" == std::string(argv[1])) { +#ifdef HAVE_IOSTREAMS + std::ifstream file(argv[2]); + filtering_streambuf<input> cin; + if (FLAGS_bzip2) { + cin.push(bzip2_decompressor()); + } + if (FLAGS_gzip) { + cin.push(gzip_decompressor()); + } + cin.push(file); + + std::istream in(&cin); +#else std::ifstream in(argv[2]); +#endif std::cout << "Importing " << argv[2] << " ... " << std::endl; client.importDataset(in, parser::FormatFromString(FLAGS_format)); std::cout << "Finished!" << std::endl;
