Repository: marmotta Updated Branches: refs/heads/develop c8f1d6c85 -> bc27036d0
experimental fix for MARMOTTA-506 (unit tests are running but needs more testing) Project: http://git-wip-us.apache.org/repos/asf/marmotta/repo Commit: http://git-wip-us.apache.org/repos/asf/marmotta/commit/d6c49ac1 Tree: http://git-wip-us.apache.org/repos/asf/marmotta/tree/d6c49ac1 Diff: http://git-wip-us.apache.org/repos/asf/marmotta/diff/d6c49ac1 Branch: refs/heads/develop Commit: d6c49ac1edaf1b0ff3cf3bab44f2a1e0ea4cf27d Parents: 5cfbf7a Author: Sebastian Schaffert <[email protected]> Authored: Mon Sep 22 14:51:17 2014 +0200 Committer: Sebastian Schaffert <[email protected]> Committed: Mon Sep 22 14:51:17 2014 +0200 ---------------------------------------------------------------------- .../kiwi/persistence/KiWiConnection.java | 136 +++++++++++++++++++ .../marmotta/kiwi/sail/KiWiSailConnection.java | 42 ++++-- .../kiwi/persistence/h2/statements.properties | 3 + .../persistence/mysql/statements.properties | 3 + .../persistence/pgsql/statements.properties | 3 + 5 files changed, 175 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/marmotta/blob/d6c49ac1/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java ---------------------------------------------------------------------- diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java index 727877a..11e64cd 100644 --- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java +++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java @@ -1283,6 +1283,142 @@ public class KiWiConnection implements AutoCloseable { } /** + * Mark all triples contained in the context passed as argument as deleted, setting the "deleted" flag to true and + * updating the timestamp value of "deletedAt". + * <p/> + * The triple remains in the database, because other entities might still reference it (e.g. a version). + * Use the method cleanupTriples() to fully remove all deleted triples without references. + * <p/> + * Warning: this method skips some concurrency and transaction safeguards for performance and therefore should + * only be called if run in an isolated transaction! + * + * @param ctx resource identifying the context to be deleted + */ + public void deleteContext(final KiWiResource ctx) throws SQLException { + requireJDBCConnection(); + + RetryExecution<Void> execution = new RetryExecution<>("DELETE"); + execution.setUseSavepoint(true); + execution.execute(connection, new RetryCommand<Void>() { + @Override + public Void run() throws SQLException { + // mutual exclusion: prevent parallel adding and removing of the same triple + + if (ctx.getId() < 0) { + log.warn("attempting to remove non-persistent context: {}", ctx); + } else { + if (batchCommit) { + // need to remove from triple batch and from database + commitLock.lock(); + try { + if (tripleBatch == null || tripleBatch.size() == 0) { + + PreparedStatement deleteTriple = getPreparedStatement("delete.context"); + synchronized (deleteTriple) { + deleteTriple.setLong(1, ctx.getId()); + deleteTriple.executeUpdate(); + } + // deletedStatementsLog.put(triple.getId()); + } else { + // delete all triples from triple batch with a matching context + for (Iterator<KiWiTriple> it = tripleBatch.iterator(); it.hasNext(); ) { + if (it.next().getContext().equals(ctx)) { + it.remove(); + } + } + } + } finally { + commitLock.unlock(); + } + } else { + requireJDBCConnection(); + + PreparedStatement deleteTriple = getPreparedStatement("delete.context"); + synchronized (deleteTriple) { + deleteTriple.setLong(1, ctx.getId()); + deleteTriple.executeUpdate(); + } + //deletedStatementsLog.put(triple.getId()); + + + } + } + //removeCachedTriple(triple); + + // that's radical but safe, and the improved delete performance might be worth it + tripleCache.clear(); + + return null; + } + }); + + + } + + /** + * Mark all triples contained in the triple store as deleted, setting the "deleted" flag to true and + * updating the timestamp value of "deletedAt". + * <p/> + * The triple remains in the database, because other entities might still reference it (e.g. a version). + * Use the method cleanupTriples() to fully remove all deleted triples without references. + * <p/> + * Warning: this method skips some concurrency and transaction safeguards for performance and therefore should + * only be called if run in an isolated transaction! + * + */ + public void deleteAll() throws SQLException { + requireJDBCConnection(); + + RetryExecution<Void> execution = new RetryExecution<>("DELETE"); + execution.setUseSavepoint(true); + execution.execute(connection, new RetryCommand<Void>() { + @Override + public Void run() throws SQLException { + // mutual exclusion: prevent parallel adding and removing of the same triple + + if (batchCommit) { + // need to remove from triple batch and from database + commitLock.lock(); + try { + if (tripleBatch == null || tripleBatch.size() == 0) { + + PreparedStatement deleteTriple = getPreparedStatement("delete.repository"); + synchronized (deleteTriple) { + deleteTriple.executeUpdate(); + } + // deletedStatementsLog.put(triple.getId()); + } else { + // delete all triples from triple batch with a matching context + tripleBatch.clear(); + } + } finally { + commitLock.unlock(); + } + } else { + requireJDBCConnection(); + + PreparedStatement deleteTriple = getPreparedStatement("delete.repository"); + synchronized (deleteTriple) { + deleteTriple.executeUpdate(); + } + //deletedStatementsLog.put(triple.getId()); + + + } + //removeCachedTriple(triple); + + // that's radical but safe, and the improved delete performance might be worth it + tripleCache.clear(); + + return null; + } + }); + + + } + + + /** * Mark the triple passed as argument as not deleted, setting the "deleted" flag to false and * clearing the timestamp value of "deletedAt". * <p/> http://git-wip-us.apache.org/repos/asf/marmotta/blob/d6c49ac1/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiSailConnection.java ---------------------------------------------------------------------- diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiSailConnection.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiSailConnection.java index 3e3dfaf..ba2131d 100644 --- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiSailConnection.java +++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiSailConnection.java @@ -56,7 +56,7 @@ import java.util.HashSet; import java.util.Set; /** - * Add file description here! + * A sail connection wrapping a KiWi database connection. Mostly delegates method calls to the underlying connection. * <p/> * Author: Sebastian Schaffert */ @@ -258,16 +258,7 @@ public class KiWiSailConnection extends NotifyingSailConnectionBase implements I contextSet.addAll(Lists.transform(Arrays.asList(contexts), new Function<Resource, KiWiResource>() { @Override public KiWiResource apply(Resource input) { - if(input == null) { - if(defaultContext != null) { - // null value for context means statements without context; in KiWi, this means "default context" - return (KiWiUriResource)valueFactory.createURI(defaultContext); - } else { - return null; - } - } else { - return valueFactory.convert(input); - } + return resolveContextInternal(input); } })); @@ -457,7 +448,21 @@ public class KiWiSailConnection extends NotifyingSailConnectionBase implements I @Override protected void clearInternal(Resource... contexts) throws SailException { - removeStatementsInternal(null, null, null, contexts); + // call the internal high-performance delete for this case; note that this is experimental and might have + // problems with long transactions or concurrency. + try { + if(contexts.length > 0) { + for (Resource context : contexts) { + KiWiResource kctx = resolveContextInternal(context); + databaseConnection.deleteContext(kctx); + } + } else { + databaseConnection.deleteAll(); + } + } catch(SQLException ex) { + throw new SailException("error while deleting context",ex); + } + //removeStatementsInternal(null, null, null, contexts); } /** @@ -551,6 +556,19 @@ public class KiWiSailConnection extends NotifyingSailConnectionBase implements I } + protected KiWiResource resolveContextInternal(Resource input) { + if(input == null) { + if(defaultContext != null) { + // null value for context means statements without context; in KiWi, this means "default context" + return (KiWiUriResource)valueFactory.createURI(defaultContext); + } else { + return null; + } + } else { + return valueFactory.convert(input); + } + } + public KiWiValueFactory getValueFactory() { return valueFactory; } http://git-wip-us.apache.org/repos/asf/marmotta/blob/d6c49ac1/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/statements.properties ---------------------------------------------------------------------- diff --git a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/statements.properties b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/statements.properties index 7a83ccf..9ceb131 100644 --- a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/statements.properties +++ b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/statements.properties @@ -77,6 +77,9 @@ delete.triple = UPDATE triples SET deleted = true, deletedAt = now() WHER undelete.triple = UPDATE triples SET deleted = false, deletedAt = NULL WHERE id = ? delete.namespace = DELETE FROM namespaces WHERE id = ? +delete.context = UPDATE triples SET deleted = true, deletedAt = now() WHERE context = ? +delete.repository = UPDATE triples SET deleted = true, deletedAt = now() + gc.check_consistency = SELECT svalue, ntype, count(id), max(id) FROM nodes group by svalue, ntype having count(id) > 1 gc.list_node_ids = SELECT id FROM nodes WHERE svalue = ? AND ntype = ? AND id != ? http://git-wip-us.apache.org/repos/asf/marmotta/blob/d6c49ac1/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/mysql/statements.properties ---------------------------------------------------------------------- diff --git a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/mysql/statements.properties b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/mysql/statements.properties index 4c04d4b..2c498d4 100644 --- a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/mysql/statements.properties +++ b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/mysql/statements.properties @@ -75,6 +75,9 @@ delete.triple = UPDATE triples SET deleted = true, deletedAt = now() WHER undelete.triple = UPDATE triples SET deleted = false, deletedAt = NULL WHERE id = ? delete.namespace = DELETE FROM namespaces WHERE id = ? +delete.context = UPDATE triples SET deleted = true, deletedAt = now() WHERE context = ? +delete.repository = UPDATE triples SET deleted = true, deletedAt = now() + gc.check_consistency = SELECT svalue, ntype, count(id), max(id) FROM nodes group by svalue, ntype having count(id) > 1 gc.list_node_ids = SELECT id FROM nodes WHERE svalue = ? AND ntype = ? AND id != ? http://git-wip-us.apache.org/repos/asf/marmotta/blob/d6c49ac1/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/pgsql/statements.properties ---------------------------------------------------------------------- diff --git a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/pgsql/statements.properties b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/pgsql/statements.properties index 62d5b2a..415f7ad 100644 --- a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/pgsql/statements.properties +++ b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/pgsql/statements.properties @@ -72,6 +72,9 @@ delete.triple = UPDATE triples SET deleted = true, deletedAt = now() WHER undelete.triple = UPDATE triples SET deleted = false, deletedAt = NULL WHERE id = ? delete.namespace = DELETE FROM namespaces WHERE id = ? +delete.context = UPDATE triples SET deleted = true, deletedAt = now() WHERE context = ? +delete.repository = UPDATE triples SET deleted = true, deletedAt = now() + gc.check_consistency = SELECT svalue, ntype, count(id), max(id) FROM nodes group by svalue, ntype having count(id) > 1 gc.list_node_ids = SELECT id FROM nodes WHERE svalue = ? AND ntype = CAST(? AS nodetype) AND id != ?
