Author: amitj
Date: Mon Oct 24 04:53:09 2016
New Revision: 1766346
URL: http://svn.apache.org/viewvc?rev=1766346&view=rev
Log:
OAK-4979: Caching sub-system implementation for DataStore
* Fix for getReference for FileCacheDataRecord
* Consolidated calls to the Backend for getOrCreateReference
* Introduced new AbstractDataRecord & AbstractSharedbackend with those
implementations
Added:
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/AbstractDataRecord.java
(with props)
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/AbstractSharedBackend.java
(with props)
Modified:
jackrabbit/oak/trunk/oak-blob/pom.xml
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/AbstractSharedCachingDataStore.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/AbstractDataStoreCacheTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/CachingDataStoreTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/ConsolidatedDataStoreStatsTest.java
Modified: jackrabbit/oak/trunk/oak-blob/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob/pom.xml?rev=1766346&r1=1766345&r2=1766346&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob/pom.xml (original)
+++ jackrabbit/oak/trunk/oak-blob/pom.xml Mon Oct 24 04:53:09 2016
@@ -104,6 +104,11 @@
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <version>1.5</version>
+ </dependency>
<!-- Optional dependency for jclouds s3 to enable s3 cloud store -->
<dependency>
Added:
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/AbstractDataRecord.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/AbstractDataRecord.java?rev=1766346&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/AbstractDataRecord.java
(added)
+++
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/AbstractDataRecord.java
Mon Oct 24 04:53:09 2016
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.spi.blob;
+
+import org.apache.jackrabbit.core.data.DataIdentifier;
+import org.apache.jackrabbit.core.data.DataRecord;
+
+/**
+ * Implements {@link DataRecord}
+ */
+public abstract class AbstractDataRecord implements DataRecord {
+
+ /**
+ * The data store that contains this record.
+ */
+ private final AbstractSharedBackend backend;
+
+ /**
+ * The binary identifier;
+ */
+ private final DataIdentifier identifier;
+
+ /**
+ * Creates a data record with the given identifier.
+ *
+ * @param identifier data identifier
+ */
+ public AbstractDataRecord(
+ AbstractSharedBackend backend, DataIdentifier identifier) {
+ this.backend = backend;
+ this.identifier = identifier;
+ }
+
+ /**
+ * Returns the data identifier.
+ *
+ * @return data identifier
+ */
+ public DataIdentifier getIdentifier() {
+ return identifier;
+ }
+
+ /**
+ * Delegates the call to the backend to retrieve reference.
+ *
+ * @return
+ */
+ public String getReference() {
+ return backend.getReferenceFromIdentifier(identifier);
+ }
+
+ /**
+ * Returns the string representation of the data identifier.
+ *
+ * @return string representation
+ */
+ public String toString() {
+ return identifier.toString();
+ }
+
+ /**
+ * Checks if the given object is a data record with the same identifier
+ * as this one.
+ *
+ * @param object other object
+ * @return <code>true</code> if the other object is a data record and has
+ * the same identifier as this one, <code>false</code> otherwise
+ */
+ public boolean equals(Object object) {
+ return (object instanceof DataRecord)
+ && identifier.equals(((DataRecord) object).getIdentifier());
+ }
+
+ /**
+ * Returns the hash code of the data identifier.
+ *
+ * @return hash code
+ */
+ public int hashCode() {
+ return identifier.hashCode();
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/AbstractDataRecord.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/AbstractSharedBackend.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/AbstractSharedBackend.java?rev=1766346&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/AbstractSharedBackend.java
(added)
+++
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/AbstractSharedBackend.java
Mon Oct 24 04:53:09 2016
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.spi.blob;
+
+import java.security.SecureRandom;
+
+import javax.crypto.Mac;
+import javax.crypto.spec.SecretKeySpec;
+
+import org.apache.jackrabbit.core.data.DataIdentifier;
+import org.apache.jackrabbit.core.data.DataStoreException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.commons.codec.binary.Hex.encodeHexString;
+
+/**
+ */
+public abstract class AbstractSharedBackend implements SharedBackend {
+ private static Logger LOG =
LoggerFactory.getLogger(AbstractSharedBackend.class);
+
+ private static final String ALGORITHM = "HmacSHA1";
+
+ /**
+ * Cached copy of the reference key of this data store. Initialized in
+ * {@link #getReferenceKey()} when the key is first accessed.
+ */
+ private byte[] referenceKey = null;
+
+
+ protected String getReferenceFromIdentifier(DataIdentifier identifier) {
+ try {
+ String id = identifier.toString();
+
+ Mac mac = Mac.getInstance(ALGORITHM);
+ mac.init(new SecretKeySpec(getReferenceKey(), ALGORITHM));
+ byte[] hash = mac.doFinal(id.getBytes("UTF-8"));
+
+ return id + ':' + encodeHexString(hash);
+ } catch (Exception e) {
+ LOG.error("Failed to hash identifier using MAC (Message
Authentication Code) algorithm.", e);
+ }
+ return null;
+ }
+
+ /**
+ * Returns the reference key of this backend. If one does not already
+ * exist, it is automatically created in an implementation-specific way.
+ * The default implementation simply creates a temporary random key that's
+ * valid only until the data store gets restarted. Subclasses can override
+ * and/or decorate this method to support a more persistent reference key.
+ * <p>
+ * This method is called only once during the lifetime of a backend
+ * instance and the return value is cached in memory, so it's no problem
+ * if the implementation is slow.
+ *
+ * @return reference key
+ * @throws DataStoreException if the key is not available
+ */
+ public byte[] getOrCreateReferenceKey() throws DataStoreException {
+ byte[] referenceKeyValue = new byte[256];
+ new SecureRandom().nextBytes(referenceKeyValue);
+ return referenceKeyValue;
+ }
+
+ //-----------------------------------------------------------< private >--
+
+ /**
+ * Returns the reference key of this data store. Synchronized to
+ * control concurrent access to the cached {@link #referenceKey} value.
+ *
+ * @return reference key
+ * @throws DataStoreException if the key is not available
+ */
+ private synchronized byte[] getReferenceKey() throws DataStoreException {
+ if (referenceKey == null) {
+ referenceKey = getOrCreateReferenceKey();
+ }
+ return referenceKey;
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/AbstractSharedBackend.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/AbstractSharedCachingDataStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/AbstractSharedCachingDataStore.java?rev=1766346&r1=1766345&r2=1766346&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/AbstractSharedCachingDataStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/AbstractSharedCachingDataStore.java
Mon Oct 24 04:53:09 2016
@@ -41,13 +41,13 @@ import com.google.common.io.Closeables;
import com.google.common.util.concurrent.ListeningExecutorService;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
-import org.apache.jackrabbit.core.data.AbstractDataRecord;
import org.apache.jackrabbit.core.data.AbstractDataStore;
import org.apache.jackrabbit.core.data.DataIdentifier;
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.core.data.MultiDataStoreAware;
-import org.apache.jackrabbit.oak.spi.blob.SharedBackend;
+import org.apache.jackrabbit.oak.spi.blob.AbstractDataRecord;
+import org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend;
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
import org.apache.jackrabbit.util.TransientFileFactory;
import org.slf4j.Logger;
@@ -129,7 +129,7 @@ public abstract class AbstractSharedCach
/**
* The delegate backend
*/
- protected SharedBackend backend;
+ protected AbstractSharedBackend backend;
protected ListeningExecutorService listeningExecutor;
@@ -171,7 +171,19 @@ public abstract class AbstractSharedCach
}, statisticsProvider, listeningExecutor, schedulerExecutor,
stagingPurgeInterval);
}
- protected abstract SharedBackend createBackend();
+ protected abstract AbstractSharedBackend createBackend();
+
+ @Override
+ public DataRecord getRecord(DataIdentifier identifier)
+ throws DataStoreException {
+ DataRecord record = getRecordIfStored(identifier);
+ if (record != null) {
+ return record;
+ } else {
+ throw new DataStoreException(
+ "Record " + identifier + " does not exist");
+ }
+ }
@Override
@Nullable
@@ -181,7 +193,7 @@ public abstract class AbstractSharedCach
// This avoids downloading the file for just accessing the meta data
File cached = cache.getIfPresent(dataIdentifier.toString());
if (cached != null && cached.exists()) {
- return new FileCacheDataRecord(this, dataIdentifier,
cached.length(),
+ return new FileCacheDataRecord(this, backend, dataIdentifier,
cached.length(),
cached.lastModified());
}
@@ -264,9 +276,10 @@ public abstract class AbstractSharedCach
private final long lastModified;
private final AbstractSharedCachingDataStore store;
- public FileCacheDataRecord(AbstractSharedCachingDataStore store,
DataIdentifier identifier, long length,
+ public FileCacheDataRecord(AbstractSharedCachingDataStore store,
AbstractSharedBackend backend,
+ DataIdentifier identifier, long length,
long lastModified) {
- super(store, identifier);
+ super(backend, identifier);
this.length = length;
this.lastModified = lastModified;
this.store = store;
@@ -394,6 +407,11 @@ public abstract class AbstractSharedCach
return SharedDataStore.Type.SHARED;
}
+ @Override
+ protected byte[] getOrCreateReferenceKey() throws DataStoreException {
+ return backend.getOrCreateReferenceKey();
+ }
+
/**------------------------ unimplemented methods
-------------------------------------------**/
@Override
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/AbstractDataStoreCacheTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/AbstractDataStoreCacheTest.java?rev=1766346&r1=1766345&r2=1766346&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/AbstractDataStoreCacheTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/AbstractDataStoreCacheTest.java
Mon Oct 24 04:53:09 2016
@@ -47,12 +47,12 @@ import com.google.common.util.concurrent
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import org.apache.commons.io.FileUtils;
-import org.apache.jackrabbit.core.data.AbstractDataRecord;
import org.apache.jackrabbit.core.data.DataIdentifier;
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.core.data.util.NamedThreadFactory;
-import org.apache.jackrabbit.oak.spi.blob.SharedBackend;
+import org.apache.jackrabbit.oak.spi.blob.AbstractDataRecord;
+import org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -228,7 +228,7 @@ public class AbstractDataStoreCacheTest
// A mock Backend implementation that uses a Map to keep track of what
// records have been added and removed, for test purposes only.
- static class TestMemoryBackend implements SharedBackend {
+ static class TestMemoryBackend extends AbstractSharedBackend {
final Map<DataIdentifier, File> _backend = Maps.newHashMap();
@Override public InputStream read(DataIdentifier identifier) throws
DataStoreException {
@@ -252,7 +252,7 @@ public class AbstractDataStoreCacheTest
@Override public DataRecord getRecord(DataIdentifier id) throws
DataStoreException {
if (_backend.containsKey(id)) {
final File f = _backend.get(id);
- return new AbstractDataRecord(null, id) {
+ return new AbstractDataRecord(this, id) {
@Override public long getLength() throws
DataStoreException {
return f.length();
}
@@ -320,6 +320,11 @@ public class AbstractDataStoreCacheTest
@Override public void init() throws DataStoreException {
}
+
+ @Override
+ public String getReferenceFromIdentifier(DataIdentifier identifier) {
+ return super.getReferenceFromIdentifier(identifier);
+ }
}
static InputStream randomStream(int seed, int size) {
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/CachingDataStoreTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/CachingDataStoreTest.java?rev=1766346&r1=1766345&r2=1766346&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/CachingDataStoreTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/CachingDataStoreTest.java
Mon Oct 24 04:53:09 2016
@@ -39,7 +39,7 @@ import org.apache.jackrabbit.core.data.D
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
-import org.apache.jackrabbit.oak.spi.blob.SharedBackend;
+import org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend;
import org.apache.jackrabbit.oak.stats.DefaultStatisticsProvider;
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
import org.junit.After;
@@ -79,6 +79,7 @@ public class CachingDataStoreTest extend
private CountDownLatch afterExecuteLatch;
private ScheduledExecutorService scheduledExecutor;
private AbstractSharedCachingDataStore dataStore;
+ private TestMemoryBackend backend;
@Before
public void setup() throws Exception {
@@ -100,9 +101,12 @@ public class CachingDataStoreTest extend
scheduledExecutor = Executors.newSingleThreadScheduledExecutor();
closer.register(new ExecutorCloser(scheduledExecutor, 500,
TimeUnit.MILLISECONDS));
+ final TestMemoryBackend testBackend = new TestMemoryBackend();
+ this.backend = testBackend;
+
dataStore = new AbstractSharedCachingDataStore() {
- @Override protected SharedBackend createBackend() {
- return new TestMemoryBackend();
+ @Override protected AbstractSharedBackend createBackend() {
+ return testBackend;
}
@Override public int getMinRecordLength() {
@@ -274,6 +278,63 @@ public class CachingDataStoreTest extend
assertTrue(Iterators.contains(dataStore.getAllIdentifiers(), new
DataIdentifier(id)));
}
+ @Test
+ public void reference() throws Exception {
+ File f = copyToFile(randomStream(0, 4 * 1024), folder.newFile());
+ String id = getIdForInputStream(f);
+ FileInputStream fin = new FileInputStream(f);
+ closer.register(fin);
+
+ // Record still in staging
+ DataRecord rec = dataStore.addRecord(fin);
+ assertEquals(id, rec.getIdentifier().toString());
+ assertFile(rec.getStream(), f, folder);
+ assertEquals(backend.getReferenceFromIdentifier(rec.getIdentifier()),
+ rec.getReference());
+
+ rec = dataStore.getRecordIfStored(new DataIdentifier(id));
+ assertNotNull(rec);
+ assertFile(rec.getStream(), f, folder);
+ assertEquals(backend.getReferenceFromIdentifier(rec.getIdentifier()),
+ rec.getReference());
+
+ //start & finish
+ taskLatch.countDown();
+ callbackLatch.countDown();
+ waitFinish();
+
+ // Now record in download cache
+ rec = dataStore.getRecordIfStored(new DataIdentifier(id));
+ assertNotNull(rec);
+ assertFile(rec.getStream(), f, folder);
+ assertEquals(backend.getReferenceFromIdentifier(rec.getIdentifier()),
+ rec.getReference());
+ }
+
+ @Test
+ public void referenceNoCache() throws Exception {
+ dataStore.close();
+ init(1, 0, 0);
+
+ File f = copyToFile(randomStream(0, 4 * 1024), folder.newFile());
+ String id = getIdForInputStream(f);
+ FileInputStream fin = new FileInputStream(f);
+ closer.register(fin);
+
+ // Record still in staging
+ DataRecord rec = dataStore.addRecord(fin);
+ assertEquals(id, rec.getIdentifier().toString());
+ assertFile(rec.getStream(), f, folder);
+ assertEquals(backend.getReferenceFromIdentifier(rec.getIdentifier()),
+ rec.getReference());
+
+ rec = dataStore.getRecordIfStored(new DataIdentifier(id));
+ assertNotNull(rec);
+ assertFile(rec.getStream(), f, folder);
+ assertEquals(backend.getReferenceFromIdentifier(rec.getIdentifier()),
+ rec.getReference());
+ }
+
@After
public void tear() throws Exception {
closer.close();
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/ConsolidatedDataStoreStatsTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/ConsolidatedDataStoreStatsTest.java?rev=1766346&r1=1766345&r2=1766346&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/ConsolidatedDataStoreStatsTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/ConsolidatedDataStoreStatsTest.java
Mon Oct 24 04:53:09 2016
@@ -44,7 +44,7 @@ import org.apache.jackrabbit.oak.commons
import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
import org.apache.jackrabbit.oak.plugins.memory.MultiBinaryPropertyState;
-import org.apache.jackrabbit.oak.spi.blob.SharedBackend;
+import org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
@@ -126,7 +126,7 @@ public class ConsolidatedDataStoreStatsT
scheduledExecutor = Executors.newSingleThreadScheduledExecutor();
closer.register(new ExecutorCloser(scheduledExecutor, 500,
TimeUnit.MILLISECONDS));
dataStore = new AbstractSharedCachingDataStore() {
- @Override protected SharedBackend createBackend() {
+ @Override protected AbstractSharedBackend createBackend() {
return new TestMemoryBackend();
}
