Repository: marmotta Updated Branches: refs/heads/develop 8f60cf64d -> ea28ddb1a
some fine tuning Project: http://git-wip-us.apache.org/repos/asf/marmotta/repo Commit: http://git-wip-us.apache.org/repos/asf/marmotta/commit/ea28ddb1 Tree: http://git-wip-us.apache.org/repos/asf/marmotta/tree/ea28ddb1 Diff: http://git-wip-us.apache.org/repos/asf/marmotta/diff/ea28ddb1 Branch: refs/heads/develop Commit: ea28ddb1af6b6cdd2aa58ac792337d378925295a Parents: 8f60cf6 Author: Sebastian Schaffert <[email protected]> Authored: Sun Dec 20 13:12:04 2015 +0100 Committer: Sebastian Schaffert <[email protected]> Committed: Sun Dec 20 13:12:04 2015 +0100 ---------------------------------------------------------------------- libraries/ostrich/backend/persistence/leveldb_persistence.cc | 7 +++++-- .../apache/marmotta/loader/ostrich/OstrichLoaderBackend.java | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/marmotta/blob/ea28ddb1/libraries/ostrich/backend/persistence/leveldb_persistence.cc ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/persistence/leveldb_persistence.cc b/libraries/ostrich/backend/persistence/leveldb_persistence.cc index 365397e..759acc6 100644 --- a/libraries/ostrich/backend/persistence/leveldb_persistence.cc +++ b/libraries/ostrich/backend/persistence/leveldb_persistence.cc @@ -302,6 +302,9 @@ leveldb::Options* buildOptions(KeyComparator* cmp, leveldb::Cache* cache) { // Cache reads in memory. options->block_cache = cache; + // Write buffer size 16MB (fast bulk imports) + options->write_buffer_size = 16384 * 1024; + // Set a bloom filter of 10 bits. options->filter_policy = leveldb::NewBloomFilterPolicy(10); return options; @@ -549,7 +552,7 @@ int64_t LevelDBPersistence::RemoveStatements(const rdf::proto::Statement& patter UpdateStatistics LevelDBPersistence::Update(LevelDBPersistence::UpdateIterator &it) { auto start = std::chrono::steady_clock::now(); - DLOG(INFO) << "Starting batch update operation."; + LOG(INFO) << "Starting batch update operation."; UpdateStatistics stats; WriteBatch b_spoc, b_cspo, b_opsc, b_pcos, b_prefix, b_url; @@ -592,7 +595,7 @@ UpdateStatistics LevelDBPersistence::Update(LevelDBPersistence::UpdateIterator & t.join(); } - DLOG(INFO) << "Batch update complete. (statements added: " << stats.added_stmts + LOG(INFO) << "Batch update complete. (statements added: " << stats.added_stmts << ", statements removed: " << stats.removed_stmts << ", namespaces added: " << stats.added_ns << ", namespaces removed: " << stats.removed_ns http://git-wip-us.apache.org/repos/asf/marmotta/blob/ea28ddb1/loader/marmotta-loader-ostrich/src/main/java/org/apache/marmotta/loader/ostrich/OstrichLoaderBackend.java ---------------------------------------------------------------------- diff --git a/loader/marmotta-loader-ostrich/src/main/java/org/apache/marmotta/loader/ostrich/OstrichLoaderBackend.java b/loader/marmotta-loader-ostrich/src/main/java/org/apache/marmotta/loader/ostrich/OstrichLoaderBackend.java index bd6e49e..ddb4bce 100644 --- a/loader/marmotta-loader-ostrich/src/main/java/org/apache/marmotta/loader/ostrich/OstrichLoaderBackend.java +++ b/loader/marmotta-loader-ostrich/src/main/java/org/apache/marmotta/loader/ostrich/OstrichLoaderBackend.java @@ -45,7 +45,7 @@ public class OstrichLoaderBackend implements LoaderBackend { return new OstrichLoaderHandler( configuration.getString("backend.ostrich.host", "localhost"), configuration.getInt("backend.ostrich.port", 10000), - configuration.getLong("backend.ostrich.batchsize", 500000)); + configuration.getLong("backend.ostrich.batchsize", 1000000)); } /** @@ -90,7 +90,7 @@ public class OstrichLoaderBackend implements LoaderBackend { Option batchSize = OptionBuilder.withArgName("batchsize") .hasArgs(1) - .withDescription("maximum number of statements to commit in one batch") + .withDescription("maximum number of statements to commit in one batch (default 1M)") .withLongOpt("batchsize") .create('B'); options.add(batchSize);
