Author: amitj
Date: Mon Oct 24 04:53:21 2016
New Revision: 1766347
URL: http://svn.apache.org/viewvc?rev=1766347&view=rev
Log:
OAK-4870: Implement caching for S3DataStore
* Use fix for getReference() for S3DataStore
* Added test for reference when secret not available.
* Throwing exception when secret not available.
Modified:
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStore.java
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java
Modified:
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java?rev=1766347&r1=1766346&r2=1766347&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
Mon Oct 24 04:53:21 2016
@@ -19,6 +19,7 @@ package org.apache.jackrabbit.oak.blob.c
import java.io.File;
import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
@@ -51,6 +52,7 @@ import com.amazonaws.services.s3.transfe
import com.amazonaws.util.StringUtils;
import com.google.common.base.Function;
import com.google.common.base.Predicate;
+import com.google.common.base.Strings;
import com.google.common.collect.AbstractIterator;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -58,7 +60,8 @@ import org.apache.jackrabbit.core.data.D
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.core.data.util.NamedThreadFactory;
-import org.apache.jackrabbit.oak.spi.blob.SharedBackend;
+import org.apache.jackrabbit.oak.spi.blob.AbstractDataRecord;
+import org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -68,7 +71,7 @@ import static java.lang.Thread.currentTh
/**
* A data store backend that stores data on Amazon S3.
*/
-public class S3Backend implements SharedBackend {
+public class S3Backend extends AbstractSharedBackend {
/**
* Logger instance.
@@ -83,6 +86,8 @@ public class S3Backend implements Shared
private String bucket;
+ private String secret;
+
private TransferManager tmx;
private Properties properties;
@@ -105,9 +110,9 @@ public class S3Backend implements Shared
if (bucket == null || "".equals(bucket.trim())) {
bucket = properties.getProperty(S3Constants.S3_BUCKET);
}
-
+ secret = properties.getProperty("secret");
String region = properties.getProperty(S3Constants.S3_REGION);
- Region s3Region = null;
+ Region s3Region;
if (StringUtils.isNullOrEmpty(region)) {
com.amazonaws.regions.Region ec2Region =
Regions.getCurrentRegion();
if (ec2Region != null) {
@@ -232,9 +237,8 @@ public class S3Backend implements Shared
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
- LOG.debug(
- "write of [{}], length=[{}], in [{}]ms",
- new Object[] { identifier, file.length(),
(System.currentTimeMillis() - start) });
+ LOG.debug("write of [{}], length=[{}], in [{}]ms",
+ identifier, file.length(), (System.currentTimeMillis() - start));
}
/**
@@ -400,7 +404,7 @@ public class S3Backend implements Shared
Thread.currentThread().setContextClassLoader(
getClass().getClassLoader());
ObjectMetadata meta = s3service.getObjectMetadata(bucket,
addMetaKeyPrefix(name));
- return new S3DataRecord(s3service, bucket, name,
+ return new S3DataRecord(this, s3service, bucket, new
DataIdentifier(name),
meta.getLastModified().getTime(), meta.getContentLength(),
true);
} finally {
if (contextClassLoader != null) {
@@ -420,7 +424,8 @@ public class S3Backend implements Shared
new
ListObjectsRequest().withBucketName(bucket).withPrefix(addMetaKeyPrefix(prefix));
ObjectListing prevObjectListing =
s3service.listObjects(listObjectsRequest);
for (final S3ObjectSummary s3ObjSumm :
prevObjectListing.getObjectSummaries()) {
- metadataList.add(new S3DataRecord(s3service, bucket,
stripMetaKeyPrefix(s3ObjSumm.getKey()),
+ metadataList.add(new S3DataRecord(this, s3service, bucket,
+ new DataIdentifier(stripMetaKeyPrefix(s3ObjSumm.getKey())),
s3ObjSumm.getLastModified().getTime(),
s3ObjSumm.getSize(), true));
}
} finally {
@@ -463,7 +468,7 @@ public class S3Backend implements Shared
if (deleteList.size() > 0) {
DeleteObjectsRequest delObjsReq = new
DeleteObjectsRequest(bucket);
delObjsReq.setKeys(deleteList);
- DeleteObjectsResult dobjs =
s3service.deleteObjects(delObjsReq);
+ s3service.deleteObjects(delObjsReq);
}
} finally {
if (contextClassLoader != null) {
@@ -474,11 +479,13 @@ public class S3Backend implements Shared
@Override
public Iterator<DataRecord> getAllRecords() {
+ final AbstractSharedBackend backend = this;
return new RecordsIterator<DataRecord>(
new Function<S3ObjectSummary, DataRecord>() {
@Override
public DataRecord apply(S3ObjectSummary input) {
- return new S3DataRecord(s3service, bucket,
getIdentifierName(input.getKey()),
+ return new S3DataRecord(backend, s3service, bucket,
+ new DataIdentifier(getIdentifierName(input.getKey())),
input.getLastModified().getTime(), input.getSize());
}
});
@@ -493,10 +500,10 @@ public class S3Backend implements Shared
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
ObjectMetadata object = s3service.getObjectMetadata(bucket, key);
- S3DataRecord record = new S3DataRecord(s3service, bucket,
identifier.toString(),
+ S3DataRecord record = new S3DataRecord(this, s3service, bucket,
identifier,
object.getLastModified().getTime(), object.getContentLength());
LOG.debug("Identifier [{}]'s getRecord = [{}] took [{}]ms.",
- new Object[] {identifier, record, (System.currentTimeMillis()
- start)});
+ identifier, record, (System.currentTimeMillis() - start));
return record;
} catch (AmazonServiceException e) {
@@ -513,6 +520,19 @@ public class S3Backend implements Shared
}
}
+ @Override
+ public byte[] getOrCreateReferenceKey() throws DataStoreException {
+ try {
+ if (!Strings.isNullOrEmpty(secret)) {
+ return secret.getBytes("UTF-8");
+ }
+ LOG.warn("secret not defined");
+ throw new DataStoreException("secret not defined");
+ } catch (UnsupportedEncodingException e) {
+ throw new DataStoreException(e);
+ }
+ }
+
/**
* Returns an iterator over the S3 objects
* @param <T>
@@ -602,23 +622,24 @@ public class S3Backend implements Shared
/**
* S3DataRecord which lazily retrieves the input stream of the record.
*/
- static class S3DataRecord implements DataRecord {
+ static class S3DataRecord extends AbstractDataRecord {
private AmazonS3Client s3service;
- private DataIdentifier identifier;
private long length;
private long lastModified;
private String bucket;
private boolean isMeta;
- public S3DataRecord(AmazonS3Client s3service, String bucket, String
key, long lastModified,
+ public S3DataRecord(AbstractSharedBackend backend, AmazonS3Client
s3service, String bucket,
+ DataIdentifier key, long lastModified,
long length) {
- this(s3service, bucket, key, lastModified, length, false);
+ this(backend, s3service, bucket, key, lastModified, length, false);
}
- public S3DataRecord(AmazonS3Client s3service, String bucket, String
key, long lastModified,
+ public S3DataRecord(AbstractSharedBackend backend, AmazonS3Client
s3service, String bucket,
+ DataIdentifier key, long lastModified,
long length, boolean isMeta) {
+ super(backend, key);
this.s3service = s3service;
- this.identifier = new DataIdentifier(key);
this.lastModified = lastModified;
this.length = length;
this.bucket = bucket;
@@ -626,25 +647,15 @@ public class S3Backend implements Shared
}
@Override
- public DataIdentifier getIdentifier() {
- return identifier;
- }
-
- @Override
- public String getReference() {
- return identifier.toString();
- }
-
- @Override
public long getLength() throws DataStoreException {
return length;
}
@Override
public InputStream getStream() throws DataStoreException {
- String id = getKeyName(identifier);
+ String id = getKeyName(getIdentifier());
if (isMeta) {
- id = addMetaKeyPrefix(identifier.toString());
+ id = addMetaKeyPrefix(getIdentifier().toString());
}
return s3service.getObject(bucket, id).getObjectContent();
}
@@ -657,7 +668,7 @@ public class S3Backend implements Shared
@Override
public String toString() {
return "S3DataRecord{" +
- "identifier=" + identifier +
+ "identifier=" + getIdentifier() +
", length=" + length +
", lastModified=" + lastModified +
", bucket='" + bucket + '\'' +
@@ -723,10 +734,8 @@ public class S3Backend implements Shared
delObjsReq.setKeys(Collections.unmodifiableList(deleteList.subList(
startIndex, endIndex)));
DeleteObjectsResult dobjs =
s3service.deleteObjects(delObjsReq);
- LOG.info(
- "Records[{}] deleted in datastore from index [{}] to
[{}]",
- new Object[] { dobjs.getDeletedObjects().size(),
- startIndex, (endIndex - 1) });
+ LOG.info("Records[{}] deleted in datastore from index [{}]
to [{}]",
+ dobjs.getDeletedObjects().size(), startIndex,
(endIndex - 1));
if (endIndex == size) {
break;
} else {
@@ -799,8 +808,7 @@ public class S3Backend implements Shared
copy.waitForCopyResult();
LOG.debug("[{}] renamed to [{}] ", oldKey, newS3Key);
} catch (InterruptedException ie) {
- LOG.error(" Exception in renaming [{}] to [{}] ",
- new Object[] { ie, oldKey, newS3Key });
+ LOG.error(" Exception in renaming [{}] to [{}] ", ie,
oldKey, newS3Key);
}
} finally {
Modified:
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStore.java?rev=1766347&r1=1766346&r2=1766347&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStore.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStore.java
Mon Oct 24 04:53:21 2016
@@ -18,10 +18,8 @@ package org.apache.jackrabbit.oak.blob.c
import java.util.Properties;
-import com.google.common.base.Strings;
-import org.apache.jackrabbit.core.data.DataIdentifier;
-import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.plugins.blob.AbstractSharedCachingDataStore;
+import org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend;
import org.apache.jackrabbit.oak.spi.blob.SharedBackend;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,7 +44,7 @@ public class S3DataStore extends Abstrac
private String secret;
@Override
- protected SharedBackend createBackend() {
+ protected AbstractSharedBackend createBackend() {
S3Backend backend = new S3Backend();
if(properties != null){
backend.setProperties(properties);
@@ -54,36 +52,6 @@ public class S3DataStore extends Abstrac
return backend;
}
- @Override
- protected byte[] getOrCreateReferenceKey() throws DataStoreException {
- try {
- return secret.getBytes("UTF-8");
- } catch (Exception e) {
- LOG.info("Error in creating reference key", e);
- throw new DataStoreException(e);
- }
- }
-
- /**
- * Look in the backend for a record matching the given identifier.
Returns true
- * if such a record exists.
- *
- * @param identifier - An identifier for the record.
- * @return true if a record for the provided identifier can be found.
- */
- public boolean haveRecordForIdentifier(final String identifier) {
- try {
- if (!Strings.isNullOrEmpty(identifier)) {
- return backend.exists(new DataIdentifier(identifier));
- }
- }
- catch (DataStoreException e) {
- LOG.warn(String.format("Data Store Exception caught checking for
%s in pending uploads",
- identifier), e);
- }
- return false;
- }
-
/**------------------------------------------- Getters &
Setters-----------------------------**/
/**
@@ -105,8 +73,4 @@ public class S3DataStore extends Abstrac
public void setMinRecordLength(int minRecordLength) {
this.minRecordLength = minRecordLength;
}
-
- public void setSecret(String secret) {
- this.secret = secret;
- }
}
Modified:
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java?rev=1766347&r1=1766346&r2=1766347&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java
Mon Oct 24 04:53:21 2016
@@ -16,14 +16,17 @@
*/
package org.apache.jackrabbit.oak.blob.cloud.s3;
+import java.io.ByteArrayInputStream;
import java.io.File;
-import java.util.Date;
import java.util.List;
import java.util.Properties;
+import java.util.Random;
import javax.jcr.RepositoryException;
+import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStore;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -36,6 +39,9 @@ import org.slf4j.LoggerFactory;
import static
org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getFixtures;
import static
org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getS3DataStore;
+import static
org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.isS3Configured;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assume.assumeTrue;
/**
* Simple tests for S3DataStore.
@@ -44,15 +50,13 @@ import static org.apache.jackrabbit.oak.
public class TestS3DataStore {
protected static final Logger LOG =
LoggerFactory.getLogger(TestS3Ds.class);
- private Date startTime = null;
-
@Rule
public ExpectedException expectedEx = ExpectedException.none();
@Rule
public TemporaryFolder folder = new TemporaryFolder(new File("target"));
- protected Properties props;
+ private Properties props;
@Parameterized.Parameter
public String s3Class;
@@ -70,7 +74,6 @@ public class TestS3DataStore {
public void setUp() throws Exception {
dataStoreDir = folder.newFolder();
props = new Properties();
- startTime = new Date();
}
@Test
@@ -83,4 +86,18 @@ public class TestS3DataStore {
props.put(S3Constants.S3_REGION, "us-standard");
ds = getS3DataStore(s3Class, props, dataStoreDir.getAbsolutePath());
}
+
+ @Test
+ public void testNoSecretDefined() throws Exception {
+ assumeTrue(isS3Configured());
+ Random randomGen = new Random();
+
+ props = S3DataStoreUtils.getS3Config();
+ ds = getS3DataStore(s3Class, props, dataStoreDir.getAbsolutePath());
+ byte[] data = new byte[4096];
+ randomGen.nextBytes(data);
+ DataRecord rec = ds.addRecord(new ByteArrayInputStream(data));
+ Assert.assertEquals(data.length, rec.getLength());
+ assertNull(rec.getReference());
+ }
}
