Author: stefanegli
Date: Wed Aug 12 12:32:35 2015
New Revision: 1695492
URL: http://svn.apache.org/r1695492
Log:
OAK-2682 : Introducing time difference detection for DocumentNodeStore: now
startup will fail if oak and mongo server are off by more than 2 seconds
(configurable via system property 'oak.documentMK.maxServerTimeDiffMillis')
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreMBean.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/memory/MemoryDocumentStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreDB.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/LoggingDocumentStoreWrapper.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/SynchronizingDocumentStoreWrapper.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/TimingDocumentStoreWrapper.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BasicDocumentStoreTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/CountingDocumentStore.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java?rev=1695492&r1=1695491&r2=1695492&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
Wed Aug 12 12:32:35 2015
@@ -2516,6 +2516,11 @@ public final class DocumentNodeStore
}
return sdf.format(r.getTimestamp());
}
+
+ @Override
+ public long determineServerTimeDifferenceMillis() {
+ return store.determineServerTimeDifferenceMillis();
+ }
}
static abstract class NodeStoreTask implements Runnable {
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreMBean.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreMBean.java?rev=1695492&r1=1695491&r2=1695492&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreMBean.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreMBean.java
Wed Aug 12 12:32:35 2015
@@ -40,4 +40,15 @@ public interface DocumentNodeStoreMBean
String[] getLastKnownRevisions();
String formatRevision(@Name("revision") String rev, @Name("UTC")boolean
utc);
+
+ /**
+ * @return the estimated time difference in milliseconds between
+ * the local instance and the (typically common, shared) document server
system.
+ * The value can be zero if the times are estimated to be equal,
+ * positive when the local instance is ahead of the remote server
+ * and negative when the local instance is behind the remote server. An
invocation is not cached
+ * and typically requires a round-trip to the server (but that is not a
requirement).
+ * @throws UnsupportedOperationException if this DocumentStore does not
support this method
+ */
+ long determineServerTimeDifferenceMillis();
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java?rev=1695492&r1=1695491&r2=1695492&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
Wed Aug 12 12:32:35 2015
@@ -447,6 +447,19 @@ public class DocumentNodeStoreService {
observerTracker.start(context.getBundleContext());
DocumentStore ds = mk.getDocumentStore();
+
+ // OAK-2682: time difference detection applied at startup with a
default max time diff of 2000 millis (2sec)
+ final long maxDiff =
Long.parseLong(System.getProperty("oak.documentMK.maxServerTimeDiffMillis",
"2000"));
+ try{
+ final long timeDiff = ds.determineServerTimeDifferenceMillis();
+ log.info("registerNodeStore: server time difference: {}ms (max
allowed: {}ms)", timeDiff, maxDiff);
+ if (Math.abs(timeDiff)>Math.abs(maxDiff)) {
+ throw new IllegalStateException("Server clock seems off
("+timeDiff+"ms) by more than configured amount ("+maxDiff+"ms)");
+ }
+ } catch(RuntimeException e) { // no checked exception
+ // in case of a RuntimeException, just log but continue
+ log.warn("registerNodeStore: got RuntimeException while trying
to determine time difference to server: "+e, e);
+ }
Dictionary<String, Object> props = new Hashtable<String, Object>();
props.put(Constants.SERVICE_PID, DocumentNodeStore.class.getName());
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentStore.java?rev=1695492&r1=1695491&r2=1695492&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentStore.java
Wed Aug 12 12:32:35 2015
@@ -308,4 +308,15 @@ public interface DocumentStore {
* @return description of the underlying storage.
*/
Map<String, String> getMetadata();
+
+ /**
+ * @return the estimated time difference in milliseconds between
+ * the local instance and the (typically common, shared) document server
system.
+ * The value can be zero if the times are estimated to be equal,
+ * positive when the local instance is ahead of the remote server
+ * and negative when the local instance is behind the remote server. An
invocation is not cached
+ * and typically requires a round-trip to the server (but that is not a
requirement).
+ * @throws UnsupportedOperationException if this DocumentStore does not
support this method
+ */
+ long determineServerTimeDifferenceMillis();
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/memory/MemoryDocumentStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/memory/MemoryDocumentStore.java?rev=1695492&r1=1695491&r2=1695492&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/memory/MemoryDocumentStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/memory/MemoryDocumentStore.java
Wed Aug 12 12:32:35 2015
@@ -401,5 +401,11 @@ public class MemoryDocumentStore impleme
public Map<String, String> getMetadata() {
return metadata;
}
+
+ @Override
+ public long determineServerTimeDifferenceMillis() {
+ // the MemoryDocumentStore has no delays, thus return 0
+ return 0;
+ }
}
\ No newline at end of file
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java?rev=1695492&r1=1695491&r2=1695492&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java
Wed Aug 12 12:32:35 2015
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
+import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -122,7 +123,9 @@ public class MongoDocumentStore implemen
private final DBCollection settings;
private final DBCollection journal;
- private final Cache<CacheValue, NodeDocument> nodesCache;
+ private final DB db;
+
+ private final Cache<CacheValue, NodeDocument> nodesCache;
private final CacheStats cacheStats;
/**
@@ -200,6 +203,7 @@ public class MongoDocumentStore implemen
.put("version", version)
.build();
+ this.db = db;
nodes = db.getCollection(Collection.NODES.toString());
clusterNodes = db.getCollection(Collection.CLUSTER_NODES.toString());
settings = db.getCollection(Collection.SETTINGS.toString());
@@ -1483,4 +1487,34 @@ public class MongoDocumentStore implemen
parentLock.unlock();
}
}
+
+ @Override
+ public long determineServerTimeDifferenceMillis() {
+ // the assumption is that the network delay from this instance
+ // to the server, and from the server back to this instance
+ // are (more or less) equal.
+ // taking this assumption into account allows to remove
+ // the network delays from the picture: the difference
+ // between end and start time is exactly this network
+ // delay (plus some server time, but that's neglected).
+ // so if the clocks are in perfect sync and the above
+ // mentioned assumption holds, then the server time should
+ // be exactly at the midPoint between start and end.
+ // this should allow a more accurate picture of the diff.
+ final long start = System.currentTimeMillis();
+ // assumption here: server returns UTC - ie the returned
+ // date object is correctly taking care of time zones.
+ final Date serverLocalTime =
db.command("serverStatus").getDate("localTime");
+ final long end = System.currentTimeMillis();
+
+ final long midPoint = (start + end) / 2;
+ final long serverLocalTimeMillis = serverLocalTime.getTime();
+
+ // the difference should be
+ // * positive when local instance is ahead
+ // * and negative when the local instance is behind
+ final long diff = midPoint - serverLocalTimeMillis;
+
+ return diff;
+ }
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java?rev=1695492&r1=1695491&r2=1695492&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
Wed Aug 12 12:32:35 2015
@@ -294,6 +294,53 @@ public class RDBDocumentStore implements
invalidateCache(collection, id, false);
}
+ @Override
+ public long determineServerTimeDifferenceMillis() {
+ Connection connection = null;
+ PreparedStatement stmt = null;
+ ResultSet rs = null;
+ String tableName = getTable(Collection.NODES);
+ long result;
+ try {
+ connection = this.ch.getROConnection();
+ String t = "select ";
+ if (this.db.getFetchFirstSyntax() == FETCHFIRSTSYNTAX.TOP) {
+ t += "TOP 1 ";
+ }
+ t += this.db.getCurrentTimeStampInMsSyntax() + " from " +
tableName;
+ switch (this.db.getFetchFirstSyntax()) {
+ case LIMIT:
+ t += " LIMIT 1";
+ break;
+ case FETCHFIRST:
+ t += " FETCH FIRST 1 ROWS ONLY";
+ break;
+ default:
+ break;
+ }
+
+ stmt = connection.prepareStatement(t);
+
+ long start = System.currentTimeMillis();
+ rs = stmt.executeQuery();
+ if (rs.next()) {
+ long roundtrip = System.currentTimeMillis() - start;
+ long serverTime = rs.getTimestamp(1).getTime();
+ result = (start + roundtrip / 2) - serverTime;
+ } else {
+ throw new DocumentStoreException("failed to determine server
timestamp");
+ }
+ return result;
+ } catch (Exception ex) {
+ LOG.error("", ex);
+ throw new DocumentStoreException(ex);
+ } finally {
+ this.ch.closeResultSet(rs);
+ this.ch.closeStatement(stmt);
+ this.ch.closeConnection(connection);
+ }
+ }
+
private <T extends Document> void invalidateCache(Collection<T>
collection, String id, boolean remove) {
if (collection == Collection.NODES) {
invalidateNodesCache(id, remove);
@@ -1858,4 +1905,5 @@ public class RDBDocumentStore implements
}
return false;
}
+
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreDB.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreDB.java?rev=1695492&r1=1695491&r2=1695492&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreDB.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreDB.java
Wed Aug 12 12:32:35 2015
@@ -52,6 +52,12 @@ public enum RDBDocumentStoreDB {
return RDBJDBCTools.versionCheck(md, 10, 11, description);
}
+ @Override
+ public String getCurrentTimeStampInMsSyntax() {
+ return "CURRENT_TIMESTAMP";
+ }
+
+ @Override
public boolean allowsCaseInSelect() {
return false;
}
@@ -261,6 +267,11 @@ public enum RDBDocumentStoreDB {
}
@Override
+ public String getCurrentTimeStampInMsSyntax() {
+ return "CURRENT_TIMESTAMP";
+ }
+
+ @Override
public String getAdditionalDiagnostics(RDBConnectionHandler ch, String
tableName) {
Connection con = null;
PreparedStatement stmt = null;
@@ -320,6 +331,13 @@ public enum RDBDocumentStoreDB {
}
/**
+ * Query syntax for current time in ms
+ */
+ public String getCurrentTimeStampInMsSyntax() {
+ return "CURRENT_TIMESTAMP(4)";
+ }
+
+ /**
* Returns the CONCAT function or its equivalent function or sub-query.
Note
* that the function MUST NOT cause a truncated value to be written!
*
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/LoggingDocumentStoreWrapper.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/LoggingDocumentStoreWrapper.java?rev=1695492&r1=1695491&r2=1695492&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/LoggingDocumentStoreWrapper.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/LoggingDocumentStoreWrapper.java
Wed Aug 12 12:32:35 2015
@@ -333,6 +333,14 @@ public class LoggingDocumentStoreWrapper
public Map<String, String> getMetadata() {
return store.getMetadata();
}
+
+ @Override
+ public long determineServerTimeDifferenceMillis() {
+ logMethod("determineServerTimeDifferenceMillis", "start");
+ long result = store.determineServerTimeDifferenceMillis();
+ logMethod("determineServerTimeDifferenceMillis", "end", result);
+ return result;
+ }
private void logMethod(String methodName, Object... args) {
StringBuilder buff = new StringBuilder("ds");
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/SynchronizingDocumentStoreWrapper.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/SynchronizingDocumentStoreWrapper.java?rev=1695492&r1=1695491&r2=1695492&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/SynchronizingDocumentStoreWrapper.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/SynchronizingDocumentStoreWrapper.java
Wed Aug 12 12:32:35 2015
@@ -135,6 +135,11 @@ public class SynchronizingDocumentStoreW
public synchronized CacheStats getCacheStats() {
return store.getCacheStats();
}
+
+ @Override
+ public synchronized long determineServerTimeDifferenceMillis() {
+ return store.determineServerTimeDifferenceMillis();
+ }
@Override
public Map<String, String> getMetadata() {
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/TimingDocumentStoreWrapper.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/TimingDocumentStoreWrapper.java?rev=1695492&r1=1695491&r2=1695492&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/TimingDocumentStoreWrapper.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/TimingDocumentStoreWrapper.java
Wed Aug 12 12:32:35 2015
@@ -357,6 +357,18 @@ public class TimingDocumentStoreWrapper
public Map<String, String> getMetadata() {
return base.getMetadata();
}
+
+ @Override
+ public long determineServerTimeDifferenceMillis() {
+ try {
+ long start = now();
+ long result = base.determineServerTimeDifferenceMillis();
+ updateAndLogTimes("determineServerTimeDifferenceMillis", start, 0,
0);
+ return result;
+ } catch (Exception e) {
+ throw convert(e);
+ }
+ }
private void logCommonCall(long start, String key) {
int time = (int) (System.currentTimeMillis() - start);
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BasicDocumentStoreTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BasicDocumentStoreTest.java?rev=1695492&r1=1695491&r2=1695492&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BasicDocumentStoreTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BasicDocumentStoreTest.java
Wed Aug 12 12:32:35 2015
@@ -34,6 +34,7 @@ import java.util.UUID;
import org.apache.jackrabbit.oak.plugins.document.UpdateOp.Condition;
import org.apache.jackrabbit.oak.plugins.document.UpdateOp.Key;
+import org.apache.jackrabbit.oak.plugins.document.rdb.RDBDocumentStore;
import org.apache.jackrabbit.oak.plugins.document.util.Utils;
import org.junit.Test;
import org.slf4j.Logger;
@@ -587,6 +588,18 @@ public class BasicDocumentStoreTest exte
}
@Test
+ public void testServerTimeDiff() throws Exception {
+ if (super.ds instanceof RDBDocumentStore) {
+ UpdateOp up = new UpdateOp("0:/", true);
+ up.set("_id", "0:/");
+ super.ds.create(Collection.NODES, Collections.singletonList(up));
+ removeMe.add("0:/");
+ long td =
((RDBDocumentStore)super.ds).determineServerTimeDifferenceMillis();
+ LOG.info("Server time difference on " + super.dsname + ": " + td +
"ms");
+ }
+ }
+
+ @Test
public void removeWithCondition() throws Exception {
List<UpdateOp> docs = Lists.newArrayList();
docs.add(newDocument("/foo", 100));
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/CountingDocumentStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/CountingDocumentStore.java?rev=1695492&r1=1695491&r2=1695492&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/CountingDocumentStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/CountingDocumentStore.java
Wed Aug 12 12:32:35 2015
@@ -219,4 +219,8 @@ public class CountingDocumentStore imple
return delegate.getMetadata();
}
+ @Override
+ public long determineServerTimeDifferenceMillis() {
+ return delegate.determineServerTimeDifferenceMillis();
+ }
}
