merged develop into MARMOTTA-584
Project: http://git-wip-us.apache.org/repos/asf/marmotta/repo Commit: http://git-wip-us.apache.org/repos/asf/marmotta/commit/653706d0 Tree: http://git-wip-us.apache.org/repos/asf/marmotta/tree/653706d0 Diff: http://git-wip-us.apache.org/repos/asf/marmotta/diff/653706d0 Branch: refs/heads/MARMOTTA-584 Commit: 653706d003644490b0d4a526ddb676cd3d0236a0 Parents: fc5bb28 1d20483 Author: Sergio Fernández <[email protected]> Authored: Mon May 2 09:35:07 2016 +0200 Committer: Sergio Fernández <[email protected]> Committed: Mon May 2 09:35:07 2016 +0200 ---------------------------------------------------------------------- .dockerignore | 2 + Dockerfile | 81 + README.md | 127 + README.txt | 119 - launchers/marmotta-webapp/README.md | 50 + launchers/marmotta-webapp/pom.xml | 22 + .../marmotta-webapp/src/deb/control/control | 2 +- .../marmotta-webapp/src/docker/entrypoint.sh | 23 + .../kiwi/reasoner/parser/KWRLProgramParser.jj | 26 +- .../test/parser/KWRLRuleParserTest.java | 41 +- .../persistence/KWRLProgramPersistenceTest.java | 2 - .../builder/eval/ValueExpressionEvaluator.java | 6 +- .../sparql/optimizer/NativeFilterOptimizer.java | 198 + .../sparql/sail/KiWiSparqlSailConnection.java | 5 +- .../sparql/test/ComplexKiWiSparqlQueryTest.java | 14 + .../kiwi/sparql/test/KiWiSparqlTest.java | 56 +- .../marmotta/kiwi/sail/KiWiValueFactory.java | 43 +- .../services/test/ng/BaseLDCacheTest.java | 7 +- .../test/rdf/TestLinkedDataProvider.java | 1 + libraries/ostrich/Dockerfile | 77 + libraries/ostrich/README.md | 105 + libraries/ostrich/backend/.dockerignore | 1 + libraries/ostrich/backend/CMakeLists.txt | 42 + libraries/ostrich/backend/client/CMakeLists.txt | 9 + libraries/ostrich/backend/client/client.cc | 379 + .../ostrich/backend/cmake/FindGFlags.cmake | 48 + libraries/ostrich/backend/cmake/FindGLog.cmake | 18 + libraries/ostrich/backend/cmake/FindGRPC.cmake | 67 + .../ostrich/backend/cmake/FindLevelDB.cmake | 18 + .../ostrich/backend/cmake/FindRAPTOR.cmake | 103 + .../ostrich/backend/cmake/FindRasqal.cmake | 99 + .../ostrich/backend/cmake/FindTcmalloc.cmake | 39 + libraries/ostrich/backend/model/CMakeLists.txt | 6 + libraries/ostrich/backend/model/model.proto | 77 + libraries/ostrich/backend/model/rdf_model.cc | 348 + libraries/ostrich/backend/model/rdf_model.h | 709 + .../ostrich/backend/model/rdf_operators.cc | 65 + libraries/ostrich/backend/model/rdf_operators.h | 240 + libraries/ostrich/backend/parser/CMakeLists.txt | 6 + libraries/ostrich/backend/parser/rdf_parser.cc | 157 + libraries/ostrich/backend/parser/rdf_parser.h | 94 + .../ostrich/backend/persistence/CMakeLists.txt | 21 + .../backend/persistence/leveldb_persistence.cc | 729 + .../backend/persistence/leveldb_persistence.h | 185 + .../backend/persistence/leveldb_server.cc | 73 + .../backend/persistence/leveldb_service.cc | 307 + .../backend/persistence/leveldb_service.h | 128 + .../backend/persistence/leveldb_sparql.cc | 85 + .../backend/persistence/leveldb_sparql.h | 52 + .../backend/persistence/marmotta_updatedb.cc | 215 + .../ostrich/backend/serializer/CMakeLists.txt | 10 + .../ostrich/backend/serializer/serializer.cc | 49 + .../ostrich/backend/serializer/serializer.h | 54 + .../backend/serializer/serializer_base.cc | 76 + .../backend/serializer/serializer_base.h | 104 + .../backend/serializer/serializer_proto.cc | 56 + .../backend/serializer/serializer_proto.h | 50 + .../backend/serializer/serializer_raptor.cc | 175 + .../backend/serializer/serializer_raptor.h | 55 + .../ostrich/backend/service/CMakeLists.txt | 9 + libraries/ostrich/backend/service/sail.proto | 102 + libraries/ostrich/backend/service/sparql.proto | 53 + .../ostrich/backend/sharding/CMakeLists.txt | 11 + libraries/ostrich/backend/sharding/server.cc | 66 + libraries/ostrich/backend/sharding/sharding.cc | 335 + libraries/ostrich/backend/sharding/sharding.h | 174 + libraries/ostrich/backend/sparql/CMakeLists.txt | 8 + .../ostrich/backend/sparql/rasqal_adapter.cc | 382 + .../ostrich/backend/sparql/rasqal_adapter.h | 124 + .../ostrich/backend/sparql/rasqal_model.cc | 205 + libraries/ostrich/backend/sparql/rasqal_model.h | 73 + libraries/ostrich/backend/test/CMakeLists.txt | 20 + .../ostrich/backend/test/PersistenceTest.cc | 268 + libraries/ostrich/backend/test/SparqlTest.cc | 291 + libraries/ostrich/backend/test/StatementTest.cc | 135 + .../ostrich/backend/test/gmock-gtest-all.cc | 12243 ++++++++++ libraries/ostrich/backend/test/gmock/gmock.h | 14978 ++++++++++++ libraries/ostrich/backend/test/gtest/gtest.h | 21197 +++++++++++++++++ libraries/ostrich/backend/test/main.cc | 11 + libraries/ostrich/backend/util/CMakeLists.txt | 6 + libraries/ostrich/backend/util/iterator.h | 315 + libraries/ostrich/backend/util/murmur3.cc | 313 + libraries/ostrich/backend/util/murmur3.h | 18 + libraries/ostrich/backend/util/raptor_util.cc | 179 + libraries/ostrich/backend/util/raptor_util.h | 74 + libraries/ostrich/backend/util/split.cc | 40 + libraries/ostrich/backend/util/split.h | 38 + libraries/ostrich/backend/util/time_logger.cc | 17 + libraries/ostrich/backend/util/time_logger.h | 32 + libraries/ostrich/backend/util/unique.h | 23 + libraries/ostrich/client/pom.xml | 234 + .../ostrich/sail/ClosableResponseStream.java | 163 + .../marmotta/ostrich/sail/OstrichSail.java | 87 + .../ostrich/sail/OstrichSailConnection.java | 618 + .../ostrich/sail/OstrichValueFactory.java | 260 + .../ostrich/sail/test/CMarmottaSailTest.java | 74 + .../ostrich/sail/test/TestSailConnection.java | 80 + libraries/ostrich/model/pom.xml | 219 + .../marmotta/ostrich/model/ProtoBNode.java | 79 + .../ostrich/model/ProtoDatatypeLiteral.java | 106 + .../ostrich/model/ProtoLiteralBase.java | 187 + .../marmotta/ostrich/model/ProtoNamespace.java | 86 + .../marmotta/ostrich/model/ProtoStatement.java | 221 + .../ostrich/model/ProtoStringLiteral.java | 101 + .../apache/marmotta/ostrich/model/ProtoURI.java | 113 + .../ostrich/model/test/StatementTest.java | 68 + .../marmotta/ostrich/model/test/URITest.java | 57 + .../model/src/test/resources/logback.xml | 28 + libraries/ostrich/pom.xml | 60 + libraries/pom.xml | 9 + loader/marmotta-loader-ostrich/pom.xml | 83 + .../loader/ostrich/OstrichLoaderBackend.java | 100 + .../loader/ostrich/OstrichLoaderHandler.java | 156 + ...org.apache.marmotta.loader.api.LoaderBackend | 18 + .../src/main/resources/logback.xml | 32 + loader/pom.xml | 10 + parent/pom.xml | 6 +- .../src/main/resources/web/admin/about.html | 41 +- .../main/resources/web/admin/configuration.html | 7 +- .../src/main/resources/web/admin/console.html | 2 +- .../src/main/resources/web/admin/database.html | 2 +- .../backends/marmotta-backend-ostrich/pom.xml | 176 + .../backend/ostrich/OstrichProvider.java | 97 + .../backend/ostrich/OstrichSailRepository.java | 168 + .../src/main/resources/META-INF/beans.xml | 28 + .../main/resources/config-defaults.properties | 25 + .../resources/config-descriptions.properties | 31 + .../src/main/resources/kiwi-module.properties | 38 + .../src/main/resources/web/admin/about.html | 36 + .../main/resources/web/admin/configuration.html | 52 + platform/backends/pom.xml | 6 + .../triplestore/ContextServiceImpl.java | 82 +- .../test/triplestore/ContextServiceTest.java | 78 + 133 files changed, 61616 insertions(+), 204 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/marmotta/blob/653706d0/libraries/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/builder/eval/ValueExpressionEvaluator.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/marmotta/blob/653706d0/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiValueFactory.java ---------------------------------------------------------------------- diff --cc libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiValueFactory.java index e02d82b,3a717a2..a5cb8d3 --- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiValueFactory.java +++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiValueFactory.java @@@ -566,14 -556,16 +566,16 @@@ public class KiWiValueFactory implement public Statement createStatement(Resource subject, URI predicate, Value object, Resource context, KiWiConnection connection) { try { -- IntArray cacheKey = IntArray.createSPOCKey(subject, predicate, object, context); ++ final IntArray cacheKey = IntArray.createSPOCKey(subject, predicate, object, context); - KiWiResource ksubject = convert(subject); - KiWiResource ksubject = convert(subject); -- KiWiUriResource kpredicate = convert(predicate); - KiWiNode kobject = convert(object); - KiWiResource kcontext = convert(context); - KiWiNode kobject = convert(object); - KiWiResource kcontext = convert(context); ++ final KiWiResource ksubject = convert(subject); ++ final KiWiUriResource kpredicate = convert(predicate); ++ final KiWiNode kobject = convert(object); ++ final KiWiResource kcontext = convert(context); - KiWiTriple result = new KiWiTriple(ksubject, kpredicate, kobject, kcontext); - KiWiTriple result = new KiWiTriple(ksubject,kpredicate,kobject,kcontext); ++ final KiWiTriple result = new KiWiTriple(ksubject, kpredicate, kobject, kcontext); + + boolean needsDBLookup = false; synchronized (registry) { long tripleId = registry.lookupKey(cacheKey); @@@ -585,13 -577,29 +587,29 @@@ registry.registerKey(cacheKey, connection.getTransactionId(), result.getId()); } else { // not found in registry, try loading from database - result.setId(connection.getTripleId(ksubject, kpredicate, kobject, kcontext)); + needsDBLookup = true; } + } + - if(needsDBLookup) { - result.setId(connection.getTripleId(ksubject,kpredicate,kobject,kcontext)); ++ if (needsDBLookup) { ++ result.setId(connection.getTripleId(ksubject, kpredicate, kobject, kcontext)); + } - // triple has no id from registry or database, so we create one and flag it for reasoning - if (result.getId() < 0) { - result.setId(connection.getNextSequence()); - result.setNewTriple(true); + // triple has no id from registry or database, so we create one and flag it for reasoning - if(result.getId() < 0) { ++ if (result.getId() < 0) { + synchronized (registry) { + // It's possible a concurrent thread might have created this + // triple while we were blocked. Check the registry again. + long tripleId = registry.lookupKey(cacheKey); + + if(tripleId >= 0) { + // A concurrent thread got in first. Take the one it created. + result.setId(tripleId); + } else { + // Create the new triple + result.setId(connection.getNextSequence()); + result.setNewTriple(true); + } registry.registerKey(cacheKey, connection.getTransactionId(), result.getId()); } @@@ -652,19 -659,10 +670,20 @@@ } - public void close() { + } + /** + * + * @param value + * @return + */ + public boolean isGeometry(String value) { + if (value.contains("POINT") || value.contains("MULTIPOINT") || value.contains("LINESTRING") || value.contains("MULTILINESTRING") || value.contains("POLYGON") || value.contains("MULTIPOLYGON")) { + return true; + } + return false; + } - } + } http://git-wip-us.apache.org/repos/asf/marmotta/blob/653706d0/parent/pom.xml ----------------------------------------------------------------------
