Author: amitj
Date: Thu Sep 29 03:25:53 2016
New Revision: 1762733
URL: http://svn.apache.org/viewvc?rev=1762733&view=rev
Log:
OAK-4848: Improve oak-blob-cloud tests
Made tests parametrized on S3 implementation class
Use the same logic in all test to initialize S3DataStore
Rearranged packages for classes
Added:
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStoreUtils.java
(contents, props changed)
- copied, changed from r1762635,
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/S3DataStoreUtils.java
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSES3.java
(contents, props changed)
- copied, changed from r1762635,
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSWithSSES3.java
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSmallCache.java
(contents, props changed)
- copied, changed from r1762635,
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSWithSmallCache.java
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java
(contents, props changed)
- copied, changed from r1762635,
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DataStore.java
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java
(contents, props changed)
- copied, changed from r1762635,
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3Ds.java
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DsCacheOff.java
(contents, props changed)
- copied, changed from r1762635,
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DsCacheOff.java
Removed:
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/S3DataStoreUtils.java
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSWithSSES3.java
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSWithSmallCache.java
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DataStore.java
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3Ds.java
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DsCacheOff.java
Modified:
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSAsyncTouch.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/ds/MongoDataStoreBlobGCTest.java
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoS3DataStoreBlobGCTest.java
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/S3SharedBlobStoreGCTest.java
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedS3DataStoreUtilsTest.java
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreBlobGCIT.java
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreBlobGCIT.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java
jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java
Modified:
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSAsyncTouch.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSAsyncTouch.java?rev=1762733&r1=1762732&r2=1762733&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSAsyncTouch.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSAsyncTouch.java
Thu Sep 29 03:25:53 2016
@@ -16,12 +16,20 @@
*/
package org.apache.jackrabbit.oak.blob.cloud.aws.s3;
+import java.util.List;
+
import javax.jcr.RepositoryException;
+import com.google.common.collect.Lists;
import org.apache.jackrabbit.core.data.CachingDataStore;
-import org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils;
+import org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils;
+import org.apache.jackrabbit.oak.blob.cloud.s3.TestS3Ds;
+import org.junit.runners.Parameterized;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
+import static
org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getFixtures;
+
/**
* Test {@link org.apache.jackrabbit.core.data.CachingDataStore} with
* {@link
org.apache.jackrabbit.core.data.CachingDataStore#setTouchAsync(boolean) set to
true.
@@ -34,12 +42,17 @@ public class TestS3DSAsyncTouch extends
protected static final Logger LOG =
LoggerFactory.getLogger(TestS3DSAsyncTouch.class);
+ @Parameterized.Parameters(name = "{index}: ({0})")
+ public static List<String> fixtures() {
+ return Lists.newArrayList(getFixtures().get(0));
+ }
+
@Override
protected CachingDataStore createDataStore() throws RepositoryException {
S3DataStore s3ds = new S3DataStore();
s3ds.setProperties(props);
- s3ds.setTouchAsync(true);
s3ds.setSecret("123456");
+ s3ds.setTouchAsync(true);
s3ds.setAsyncUploadLimit(0);
s3ds.init(dataStoreDir);
s3ds.updateModifiedDateOnAccess(System.currentTimeMillis() + 50 *
1000);
Copied:
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStoreUtils.java
(from r1762635,
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/S3DataStoreUtils.java)
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStoreUtils.java?p2=jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStoreUtils.java&p1=jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/S3DataStoreUtils.java&r1=1762635&r2=1762733&rev=1762733&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/S3DataStoreUtils.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStoreUtils.java
Thu Sep 29 03:25:53 2016
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.jackrabbit.oak.blob.cloud;
+package org.apache.jackrabbit.oak.blob.cloud.s3;
import java.io.FileInputStream;
import java.io.InputStream;
@@ -33,6 +33,7 @@ import com.amazonaws.services.s3.model.S
import com.amazonaws.services.s3.transfer.TransferManager;
import com.google.common.base.Predicate;
import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
import org.apache.commons.io.IOUtils;
import org.apache.jackrabbit.core.data.Backend;
@@ -40,8 +41,6 @@ import org.apache.jackrabbit.core.data.D
import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3Backend;
import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3DataStore;
import org.apache.jackrabbit.oak.blob.cloud.aws.s3.SharedS3DataStore;
-import org.apache.jackrabbit.oak.blob.cloud.s3.S3Constants;
-import org.apache.jackrabbit.oak.blob.cloud.s3.Utils;
import org.apache.jackrabbit.oak.commons.PropertiesUtil;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils;
import org.slf4j.Logger;
@@ -55,6 +54,10 @@ public class S3DataStoreUtils extends Da
private static final String DEFAULT_CONFIG_PATH =
"./src/test/resources/aws.properties";
+ public static List<String> getFixtures() {
+ return ImmutableList.of(SharedS3DataStore.class.getName());
+ }
+
public static boolean isS3DataStore() {
String dsName = System.getProperty(DS_CLASS_NAME);
boolean s3Class = (dsName != null) &&
(dsName.equals(S3DataStore.class.getName()) || dsName
@@ -113,13 +116,22 @@ public class S3DataStoreUtils extends Da
return props;
}
- public static DataStore getS3DataStore(String className, String homeDir)
throws Exception {
+ public static DataStore getS3DataStore(String className, Properties props,
String homeDir) throws Exception {
DataStore ds =
Class.forName(className).asSubclass(DataStore.class).newInstance();
- PropertiesUtil.populate(ds, Maps.fromProperties(getS3Config()), false);
+ PropertiesUtil.populate(ds, Maps.fromProperties(props), false);
+ // Set the props object
+ if (SharedS3DataStore.class.getName().equals(className)) {
+ ((SharedS3DataStore) ds).setProperties(props);
+ }
ds.init(homeDir);
+
return ds;
}
+ public static DataStore getS3DataStore(String className, String homeDir)
throws Exception {
+ return getS3DataStore(className, getS3Config(), homeDir);
+ }
+
/**
* S3 specific cleanup
*
@@ -137,7 +149,7 @@ public class S3DataStoreUtils extends Da
}
}
- private static void deleteBucket(String bucket, Date date) throws
Exception {
+ public static void deleteBucket(String bucket, Date date) throws Exception
{
log.info("cleaning bucket [" + bucket + "]");
Properties props = getS3Config();
AmazonS3Client s3service = Utils.openService(props);
Propchange:
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStoreUtils.java
------------------------------------------------------------------------------
svn:eol-style = native
Copied:
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSES3.java
(from r1762635,
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSWithSSES3.java)
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSES3.java?p2=jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSES3.java&p1=jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSWithSSES3.java&r1=1762635&r2=1762733&rev=1762733&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSWithSSES3.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSES3.java
Thu Sep 29 03:25:53 2016
@@ -15,17 +15,13 @@
* limitations under the License.
*/
-package org.apache.jackrabbit.oak.blob.cloud.aws.s3;
+package org.apache.jackrabbit.oak.blob.cloud.s3;
import java.io.ByteArrayInputStream;
-import javax.jcr.RepositoryException;
-
-import org.apache.jackrabbit.core.data.CachingDataStore;
import org.apache.jackrabbit.core.data.DataRecord;
-import org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils;
-import org.apache.jackrabbit.oak.blob.cloud.s3.S3Constants;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,15 +41,11 @@ public class TestS3DSWithSSES3 extends T
protected static final Logger LOG =
LoggerFactory.getLogger(TestS3DSWithSSES3.class);
@Override
- protected CachingDataStore createDataStore() throws RepositoryException {
- props.setProperty(S3Constants.S3_ENCRYPTION,
- S3Constants.S3_ENCRYPTION_SSE_S3);
- S3DataStore s3ds = new S3DataStore();
- s3ds.setProperties(props);
- s3ds.setSecret("123456");
- s3ds.init(dataStoreDir);
- sleep(1000);
- return s3ds;
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ props.setProperty(S3Constants.S3_ENCRYPTION,
S3Constants.S3_ENCRYPTION_SSE_S3);
+ props.setProperty("cacheSize", "0");
}
/**
@@ -62,40 +54,33 @@ public class TestS3DSWithSSES3 extends T
@Test
public void testDataMigration() {
try {
- String bucket = props.getProperty(S3Constants.S3_BUCKET);
- S3DataStore s3ds = new S3DataStore();
- s3ds.setProperties(props);
- s3ds.setCacheSize(0);
- s3ds.init(dataStoreDir);
+ //manually close the setup ds and remove encryption
+ ds.close();
+ props.remove(S3Constants.S3_ENCRYPTION);
+ ds = createDataStore();
+
byte[] data = new byte[dataLength];
randomGen.nextBytes(data);
- DataRecord rec = s3ds.addRecord(new ByteArrayInputStream(data));
+ DataRecord rec = ds.addRecord(new ByteArrayInputStream(data));
Assert.assertEquals(data.length, rec.getLength());
assertRecord(data, rec);
- s3ds.close();
+ ds.close();
- // turn encryption now.
- props.setProperty(S3Constants.S3_BUCKET, bucket);
- props.setProperty(S3Constants.S3_ENCRYPTION,
- S3Constants.S3_ENCRYPTION_SSE_S3);
+ // turn encryption now anc recreate datastore instance
+ props.setProperty(S3Constants.S3_ENCRYPTION,
S3Constants.S3_ENCRYPTION_SSE_S3);
props.setProperty(S3Constants.S3_RENAME_KEYS, "true");
- s3ds = new S3DataStore();
- s3ds.setProperties(props);
- s3ds.setCacheSize(0);
- s3ds.init(dataStoreDir);
+ ds = createDataStore();
- rec = s3ds.getRecord(rec.getIdentifier());
+ rec = ds.getRecord(rec.getIdentifier());
Assert.assertEquals(data.length, rec.getLength());
assertRecord(data, rec);
randomGen.nextBytes(data);
- rec = s3ds.addRecord(new ByteArrayInputStream(data));
- s3ds.close();
-
+ ds.addRecord(new ByteArrayInputStream(data));
+ ds.close();
} catch (Exception e) {
LOG.error("error:", e);
fail(e.getMessage());
}
}
-
}
Propchange:
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSES3.java
------------------------------------------------------------------------------
svn:eol-style = native
Copied:
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSmallCache.java
(from r1762635,
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSWithSmallCache.java)
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSmallCache.java?p2=jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSmallCache.java&p1=jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSWithSmallCache.java&r1=1762635&r2=1762733&rev=1762733&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSWithSmallCache.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSmallCache.java
Thu Sep 29 03:25:53 2016
@@ -14,13 +14,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.jackrabbit.oak.blob.cloud.aws.s3;
-
-import javax.jcr.RepositoryException;
+package org.apache.jackrabbit.oak.blob.cloud.s3;
import org.apache.jackrabbit.core.data.CachingDataStore;
import org.apache.jackrabbit.core.data.LocalCache;
-import org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils;
+import org.junit.Before;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,15 +36,9 @@ public class TestS3DSWithSmallCache exte
protected static final Logger LOG =
LoggerFactory.getLogger(TestS3DSWithSmallCache.class);
@Override
- protected CachingDataStore createDataStore() throws RepositoryException {
- S3DataStore s3ds = new S3DataStore();
- s3ds.setProperties(props);
- s3ds.setCacheSize(dataLength * 10);
- s3ds.setCachePurgeTrigFactor(0.5d);
- s3ds.setCachePurgeResizeFactor(0.4d);
- s3ds.setSecret("123456");
- s3ds.init(dataStoreDir);
- sleep(1000);
- return s3ds;
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ props.setProperty("cacheSize", String.valueOf(dataLength * 10));
}
}
Propchange:
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSmallCache.java
------------------------------------------------------------------------------
svn:eol-style = native
Copied:
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java
(from r1762635,
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DataStore.java)
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java?p2=jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java&p1=jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DataStore.java&r1=1762635&r2=1762733&rev=1762733&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DataStore.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java
Thu Sep 29 03:25:53 2016
@@ -14,43 +14,88 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.jackrabbit.oak.blob.cloud.aws.s3;
+package org.apache.jackrabbit.oak.blob.cloud.s3;
import java.io.File;
-import java.io.IOException;
+import java.util.Date;
+import java.util.List;
import java.util.Properties;
import javax.jcr.RepositoryException;
-import org.apache.jackrabbit.oak.blob.cloud.s3.S3Constants;
+import com.google.common.base.Strings;
+import org.apache.jackrabbit.core.data.DataStore;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static
org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getFixtures;
+import static
org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getS3DataStore;
/**
* Simple tests for S3DataStore.
*/
+@RunWith(Parameterized.class)
public class TestS3DataStore {
+ protected static final Logger LOG =
LoggerFactory.getLogger(TestS3Ds.class);
+
+ private Date startTime = null;
+
@Rule
public ExpectedException expectedEx = ExpectedException.none();
@Rule
public TemporaryFolder folder = new TemporaryFolder(new File("target"));
+ protected Properties props;
+
+ @Parameterized.Parameter
+ public String s3Class;
+
+ private File dataStoreDir;
+
+ private String bucket;
+
+ private DataStore ds;
+
+ @Parameterized.Parameters(name = "{index}: ({0})")
+ public static List<String> fixtures() {
+ return getFixtures();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ dataStoreDir = folder.newFolder();
+ props = new Properties();
+ startTime = new Date();
+ }
+
+ @After
+ public void tearDown() {
+ try {
+ if (ds != null && !Strings.isNullOrEmpty(bucket)) {
+ S3DataStoreUtils.deleteBucket(bucket, startTime);
+ }
+ } catch (Exception ignore) {
+
+ }
+ }
+
@Test
- public void testAccessParamLeakOnError() throws RepositoryException,
IOException {
+ public void testAccessParamLeakOnError() throws Exception {
expectedEx.expect(RepositoryException.class);
expectedEx.expectMessage("Could not initialize S3 from
{s3Region=us-standard}");
- Properties props = new Properties();
props.put(S3Constants.ACCESS_KEY, "abcd");
props.put(S3Constants.SECRET_KEY, "123456");
props.put(S3Constants.S3_REGION, "us-standard");
-
- S3DataStore s3ds = new S3DataStore();
- s3ds.setProperties(props);
- s3ds.setSecret("123456");
- s3ds.init(folder.newFolder().getAbsolutePath());
+ ds = getS3DataStore(s3Class, props, dataStoreDir.getAbsolutePath());
}
}
Propchange:
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java
------------------------------------------------------------------------------
svn:eol-style = native
Copied:
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java
(from r1762635,
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3Ds.java)
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java?p2=jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java&p1=jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3Ds.java&r1=1762635&r2=1762733&rev=1762733&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3Ds.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java
Thu Sep 29 03:25:53 2016
@@ -14,34 +14,41 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.jackrabbit.oak.blob.cloud.aws.s3;
+package org.apache.jackrabbit.oak.blob.cloud.s3;
import java.util.Date;
+import java.util.List;
import java.util.Properties;
import javax.jcr.RepositoryException;
-import org.apache.jackrabbit.core.data.CachingDataStore;
-import org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils;
-import org.apache.jackrabbit.oak.blob.cloud.s3.S3Constants;
+import org.apache.jackrabbit.core.data.DataStore;
+import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3DataStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.AbstractDataStoreTest;
-import org.junit.After;
+import org.junit.After;;
+import org.junit.Assume;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.mockito.internal.matchers.Equals;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static
org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils.getS3Config;
-import static
org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils.isS3Configured;
+import static
org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getFixtures;
+import static
org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getS3Config;
+import static
org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getS3DataStore;
+import static
org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.isS3Configured;
import static org.junit.Assume.assumeTrue;
/**
- * Test {@link org.apache.jackrabbit.core.data.CachingDataStore} with
S3Backend and local cache on.
+ * Test {@link S3DataStore} with S3Backend and local cache on.
* It requires to pass aws config file via system property or system
properties by prefixing with 'ds.'.
* See details @ {@link S3DataStoreUtils}.
* For e.g. -Dconfig=/opt/cq/aws.properties. Sample aws properties located at
* src/test/resources/aws.properties
*/
+@RunWith(Parameterized.class)
public class TestS3Ds extends AbstractDataStoreTest {
protected static final Logger LOG =
LoggerFactory.getLogger(TestS3Ds.class);
@@ -50,6 +57,16 @@ public class TestS3Ds extends AbstractDa
protected Properties props;
+ protected String bucket;
+
+ @Parameterized.Parameter
+ public String s3Class;
+
+ @Parameterized.Parameters(name = "{index}: ({0})")
+ public static List<String> fixtures() {
+ return getFixtures();
+ }
+
@BeforeClass
public static void assumptions() {
assumeTrue(isS3Configured());
@@ -64,6 +81,7 @@ public class TestS3Ds extends AbstractDa
String.valueOf(randomGen.nextInt(9999)) + "-" +
String.valueOf(randomGen.nextInt(9999))
+ "-test";
props.setProperty(S3Constants.S3_BUCKET, bucket);
+ props.setProperty("secret", "123456");
super.setUp();
}
@@ -72,19 +90,33 @@ public class TestS3Ds extends AbstractDa
public void tearDown() {
try {
super.tearDown();
- S3DataStoreUtils.cleanup(ds, startTime);
+ S3DataStoreUtils.deleteBucket(bucket, startTime);
} catch (Exception ignore) {
}
}
- @Override
- protected CachingDataStore createDataStore() throws RepositoryException {
- S3DataStore s3ds = new S3DataStore();
- s3ds.setProperties(props);
- s3ds.setSecret("123456");
- s3ds.init(dataStoreDir);
+ protected DataStore createDataStore() throws RepositoryException {
+ DataStore s3ds = null;
+ try {
+ s3ds = getS3DataStore(s3Class, props, dataStoreDir);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
sleep(1000);
return s3ds;
}
+
+ /**----------Only run with
org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3DataStore-----------**/
+ @Override
+ public void testUpdateLastModifiedOnAccess() {
+ Assume.assumeThat(s3Class, new Equals(fixtures().get(0)));
+ super.testUpdateLastModifiedOnAccess();
+ }
+
+ @Override
+ public void testDeleteAllOlderThan() {
+ Assume.assumeThat(s3Class, new Equals(fixtures().get(0)));
+ super.testDeleteAllOlderThan();
+ }
}
Propchange:
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java
------------------------------------------------------------------------------
svn:eol-style = native
Copied:
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DsCacheOff.java
(from r1762635,
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DsCacheOff.java)
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DsCacheOff.java?p2=jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DsCacheOff.java&p1=jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DsCacheOff.java&r1=1762635&r2=1762733&rev=1762733&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DsCacheOff.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DsCacheOff.java
Thu Sep 29 03:25:53 2016
@@ -14,12 +14,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.jackrabbit.oak.blob.cloud.aws.s3;
+package org.apache.jackrabbit.oak.blob.cloud.s3;
-import javax.jcr.RepositoryException;
-
-import org.apache.jackrabbit.core.data.CachingDataStore;
-import org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils;
+import org.junit.Before;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,13 +34,9 @@ public class TestS3DsCacheOff extends Te
protected static final Logger LOG =
LoggerFactory.getLogger(TestS3DsCacheOff.class);
@Override
- protected CachingDataStore createDataStore() throws RepositoryException {
- S3DataStore s3ds = new S3DataStore();
- s3ds.setProperties(props);
- s3ds.setCacheSize(0);
- s3ds.setSecret("123456");
- s3ds.init(dataStoreDir);
- sleep(1000);
- return s3ds;
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ props.setProperty("cacheSize", "0");
}
}
Propchange:
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DsCacheOff.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java?rev=1762733&r1=1762732&r2=1762733&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java
Thu Sep 29 03:25:53 2016
@@ -21,8 +21,6 @@ package org.apache.jackrabbit.oak.plugin
import static com.google.common.collect.Sets.newHashSet;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertTrue;
-import static
org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils.getBlobStore;
-import static org.hamcrest.CoreMatchers.instanceOf;
import java.io.ByteArrayInputStream;
import java.io.File;
@@ -43,7 +41,6 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import junit.framework.Assert;
-import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.jackrabbit.core.data.DataIdentifier;
import org.apache.jackrabbit.core.data.DataRecord;
@@ -53,9 +50,6 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils;
import
org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils.SharedStoreRecordType;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils;
-import org.junit.After;
-import org.junit.Assume;
-import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -73,18 +67,14 @@ public class SharedDataStoreUtilsTest {
protected DataStoreBlobStore dataStore;
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- try {
- Assume.assumeThat(getBlobStore(),
instanceOf(SharedDataStore.class));
- } catch (Exception e) {
- Assume.assumeNoException(e);
- }
+ protected DataStoreBlobStore getBlobStore(File root) throws Exception {
+ return DataStoreUtils.getBlobStore(root);
}
@Test
public void test() throws Exception {
- dataStore = getBlobStore();
+ File rootFolder = folder.newFolder();
+ dataStore = getBlobStore(rootFolder);
String repoId1 = UUID.randomUUID().toString();
String repoId2 = UUID.randomUUID().toString();
@@ -163,7 +153,8 @@ public class SharedDataStoreUtilsTest {
@Test
public void testAddMetadata() throws Exception {
- dataStore = getBlobStore();
+ File rootFolder = folder.newFolder();
+ dataStore = getBlobStore(rootFolder);
String repoId = UUID.randomUUID().toString();
Set<String> refs = Sets.newHashSet("1_1", "1_2");
File f = folder.newFile();
@@ -189,8 +180,9 @@ public class SharedDataStoreUtilsTest {
@Test
public void testGetAllChunkIds() throws Exception {
- dataStore = getBlobStore();
- int number = 1010;
+ File rootFolder = folder.newFolder();
+ dataStore = getBlobStore(rootFolder);
+ int number = 10;
Set<String> added = newHashSet();
for (int i = 0; i < number; i++) {
String rec = dataStore.writeBlob(randomStream(i, 16516));
@@ -203,8 +195,9 @@ public class SharedDataStoreUtilsTest {
@Test
public void testGetAllRecords() throws Exception {
- dataStore = getBlobStore();
- int number = 1010;
+ File rootFolder = folder.newFolder();
+ dataStore = getBlobStore(rootFolder);
+ int number = 10;
Set<String> added = newHashSet();
for (int i = 0; i < number; i++) {
String rec = dataStore.addRecord(randomStream(i, 16516))
@@ -223,7 +216,8 @@ public class SharedDataStoreUtilsTest {
@Test
public void testStreamFromGetAllRecords() throws Exception {
- dataStore = getBlobStore();
+ File rootFolder = folder.newFolder();
+ dataStore = getBlobStore(rootFolder);
int number = 10;
Set<DataRecord> added = newHashSet();
for (int i = 0; i < number; i++) {
@@ -236,7 +230,8 @@ public class SharedDataStoreUtilsTest {
@Test
public void testGetRecordForId() throws Exception {
- dataStore = getBlobStore();
+ File rootFolder = folder.newFolder();
+ dataStore = getBlobStore(rootFolder);
int number = 10;
Set<DataRecord> added = newHashSet();
for (int i = 0; i < number; i++) {
@@ -276,10 +271,5 @@ public class SharedDataStoreUtilsTest {
r.nextBytes(data);
return new ByteArrayInputStream(data);
}
-
- @After
- public void close() throws IOException {
- FileUtils.cleanDirectory(new File(DataStoreUtils.getHomeDir()));
- }
}
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreTest.java?rev=1762733&r1=1762732&r2=1762733&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreTest.java
Thu Sep 29 03:25:53 2016
@@ -171,6 +171,7 @@ public abstract class AbstractDataStoreT
+ (System.currentTimeMillis() - start) + "]ms");
} catch (Exception e) {
LOG.error("error:", e);
+ fail(e.getMessage());
}
}
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java?rev=1762733&r1=1762732&r2=1762733&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
Thu Sep 29 03:25:53 2016
@@ -16,10 +16,6 @@
*/
package org.apache.jackrabbit.oak.plugins.document;
-import static
org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils.SharedStoreRecordType.REPOSITORY;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
@@ -35,6 +31,8 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import javax.annotation.Nullable;
+
import ch.qos.logback.classic.Level;
import com.google.common.base.Splitter;
import com.google.common.base.Stopwatch;
@@ -46,7 +44,6 @@ import com.google.common.io.Closeables;
import com.mongodb.BasicDBObject;
import com.mongodb.DBCollection;
import junit.framework.Assert;
-import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.commons.FileIOUtils;
@@ -71,7 +68,10 @@ import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.annotation.Nullable;
+import static
org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils
+ .SharedStoreRecordType.REPOSITORY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
/**
* Tests for MongoMK GC
@@ -200,17 +200,17 @@ public class MongoBlobGCTest extends Abs
@Test
public void checkMark() throws Exception {
+ String rootFolder = folder.newFolder().getAbsolutePath();
LogCustomizer customLogs = LogCustomizer
.forLogger(MarkSweepGarbageCollector.class.getName())
.enable(Level.TRACE)
.filter(Level.TRACE)
.create();
- DataStoreState state = setUp(true, 1000);
+ DataStoreState state = setUp(true, 10);
log.info("{} blobs available : {}", state.blobsPresent.size(),
state.blobsPresent);
customLogs.starting();
ThreadPoolExecutor executor = (ThreadPoolExecutor)
Executors.newFixedThreadPool(10);
- String rootFolder = folder.newFolder().getAbsolutePath();
MarkSweepGarbageCollector gcObj = init(0, executor, rootFolder);
gcObj.collectGarbage(true);
customLogs.finished();
@@ -334,9 +334,10 @@ public class MongoBlobGCTest extends Abs
new ByteArrayInputStream(new byte[0]),
REPOSITORY.getNameFromId(repoId));
}
- TestGarbageCollector gc = new TestGarbageCollector(
- new DocumentBlobReferenceRetriever(store),
- (GarbageCollectableBlobStore) store.getBlobStore(), executor,
"./target", 5, 5000, repoId);
+ TestGarbageCollector gc =
+ new TestGarbageCollector(new DocumentBlobReferenceRetriever(store),
+ (GarbageCollectableBlobStore) store.getBlobStore(), executor,
+ folder.newFolder().getAbsolutePath(), 5, 5000, repoId);
gc.collectGarbage(false);
Set<String> existingAfterGC = iterate();
log.info("{} Blobs existing after gc {}", existingAfterGC.size(),
existingAfterGC);
@@ -347,6 +348,7 @@ public class MongoBlobGCTest extends Abs
@Test
public void checkGcPathLogging() throws Exception {
+ String rootFolder = folder.newFolder().getAbsolutePath();
LogCustomizer customLogs = LogCustomizer
.forLogger(MarkSweepGarbageCollector.class.getName())
.enable(Level.TRACE)
@@ -356,7 +358,6 @@ public class MongoBlobGCTest extends Abs
setUp(false);
customLogs.starting();
ThreadPoolExecutor executor = (ThreadPoolExecutor)
Executors.newFixedThreadPool(10);
- String rootFolder = folder.newFolder().getAbsolutePath();
MarkSweepGarbageCollector gcObj = init(0, executor, rootFolder);
gcObj.collectGarbage(true);
customLogs.finished();
@@ -366,6 +367,7 @@ public class MongoBlobGCTest extends Abs
@Test
public void checkConsistencyPathLogging() throws Exception {
+ String rootFolder = folder.newFolder().getAbsolutePath();
LogCustomizer customLogs = LogCustomizer
.forLogger(MarkSweepGarbageCollector.class.getName())
.enable(Level.TRACE)
@@ -375,7 +377,6 @@ public class MongoBlobGCTest extends Abs
setUp(false);
customLogs.starting();
ThreadPoolExecutor executor = (ThreadPoolExecutor)
Executors.newFixedThreadPool(10);
- String rootFolder = folder.newFolder().getAbsolutePath();
MarkSweepGarbageCollector gcObj = init(86400, executor, rootFolder);
gcObj.checkConsistency();
customLogs.finished();
@@ -443,7 +444,7 @@ public class MongoBlobGCTest extends Abs
REPOSITORY.getNameFromId(repoId));
}
if (Strings.isNullOrEmpty(root)) {
- root = "./target";
+ root = folder.newFolder().getAbsolutePath();
}
MarkSweepGarbageCollector gc = new MarkSweepGarbageCollector(
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java?rev=1762733&r1=1762732&r2=1762733&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java
Thu Sep 29 03:25:53 2016
@@ -58,7 +58,9 @@ import org.apache.jackrabbit.oak.spi.sta
import org.apache.jackrabbit.oak.stats.Clock;
import org.junit.After;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -68,6 +70,9 @@ import org.slf4j.LoggerFactory;
public class SharedBlobStoreGCTest {
private static final Logger log =
LoggerFactory.getLogger(SharedBlobStoreGCTest.class);
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder(new File("target"));
+
protected Cluster cluster1;
protected Cluster cluster2;
private Clock clock;
@@ -80,7 +85,8 @@ public class SharedBlobStoreGCTest {
clock.waitUntil(Revision.getCurrentTimestamp());
DataStoreUtils.time = clock.getTime();
- BlobStore blobeStore1 = DataStoreUtils.getBlobStore();
+ File rootFolder = folder.newFolder();
+ BlobStore blobeStore1 = getBlobStore(rootFolder);
DocumentNodeStore ds1 = new DocumentMK.Builder()
.setAsyncDelay(0)
.setDocumentStore(new MemoryDocumentStore())
@@ -92,7 +98,7 @@ public class SharedBlobStoreGCTest {
((SharedDataStore) blobeStore1).addMetadataRecord(new
ByteArrayInputStream(new byte[0]),
SharedStoreRecordType.REPOSITORY.getNameFromId(repoId1));
- BlobStore blobeStore2 = DataStoreUtils.getBlobStore();
+ BlobStore blobeStore2 = getBlobStore(rootFolder);
DocumentNodeStore ds2 = new DocumentMK.Builder()
.setAsyncDelay(0)
.setDocumentStore(new MemoryDocumentStore())
@@ -218,12 +224,15 @@ public class SharedBlobStoreGCTest {
@After
public void tearDown() throws Exception {
- FileUtils.cleanDirectory((new
File(DataStoreUtils.getHomeDir())).getParentFile());
DataStoreUtils.time = -1;
cluster1.getDocumentNodeStore().dispose();
cluster2.getDocumentNodeStore().dispose();
}
+ protected DataStoreBlobStore getBlobStore(File root) throws Exception {
+ return DataStoreUtils.getBlobStore(root);
+ }
+
public class Cluster {
private DocumentNodeStore ds;
private int seed;
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/ds/MongoDataStoreBlobGCTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/ds/MongoDataStoreBlobGCTest.java?rev=1762733&r1=1762732&r2=1762733&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/ds/MongoDataStoreBlobGCTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/ds/MongoDataStoreBlobGCTest.java
Thu Sep 29 03:25:53 2016
@@ -16,10 +16,8 @@
*/
package org.apache.jackrabbit.oak.plugins.document.blob.ds;
-import java.io.File;
import java.util.Date;
-import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils;
import org.apache.jackrabbit.oak.plugins.document.DocumentMK;
@@ -53,7 +51,7 @@ public class MongoDataStoreBlobGCTest ex
startDate = new Date();
mongoConnection = connectionFactory.getConnection();
MongoUtils.dropCollections(mongoConnection.getDB());
- blobStore = DataStoreUtils.getBlobStore();
+ blobStore = DataStoreUtils.getBlobStore(folder.newFolder());
mk = new
DocumentMK.Builder().clock(getTestClock()).setMongoDB(mongoConnection.getDB())
.setBlobStore(blobStore).open();
}
@@ -61,7 +59,6 @@ public class MongoDataStoreBlobGCTest ex
@After
@Override
public void tearDownConnection() throws Exception {
- FileUtils.deleteDirectory(new File(DataStoreUtils.getHomeDir()));
mk.dispose();
MongoUtils.dropCollections(connectionFactory.getConnection().getDB());
}
Modified:
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoS3DataStoreBlobGCTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoS3DataStoreBlobGCTest.java?rev=1762733&r1=1762732&r2=1762733&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoS3DataStoreBlobGCTest.java
(original)
+++
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoS3DataStoreBlobGCTest.java
Thu Sep 29 03:25:53 2016
@@ -18,26 +18,64 @@
*/
package org.apache.jackrabbit.oak.plugins.document;
-import org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils;
+import java.io.File;
+import java.util.Date;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.jackrabbit.oak.blob.cloud.s3.S3Constants;
+import org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import
org.apache.jackrabbit.oak.plugins.document.blob.ds.MongoDataStoreBlobGCTest;
import org.junit.After;
+import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
import static org.junit.Assume.assumeTrue;
/**
* Tests DataStoreGC with Mongo and S3
*/
+@RunWith(Parameterized.class)
public class MongoS3DataStoreBlobGCTest extends MongoDataStoreBlobGCTest {
+
@BeforeClass
public static void assumptions() {
- assumeTrue(S3DataStoreUtils.isS3DataStore());
+ assumeTrue(S3DataStoreUtils.isS3Configured());
+ }
+
+ @Parameterized.Parameter
+ public String s3Class;
+
+ @Parameterized.Parameters(name = "{index}: ({0})")
+ public static List<String> fixtures() {
+ return S3DataStoreUtils.getFixtures();
+ }
+
+ protected String bucket;
+
+ @Before
+ @Override
+ public void setUpConnection() throws Exception {
+ Properties props = S3DataStoreUtils.getS3Config();
+ startDate = new Date();
+ mongoConnection = connectionFactory.getConnection();
+ MongoUtils.dropCollections(mongoConnection.getDB());
+ File root = folder.newFolder();
+ bucket = root.getName();
+ props.setProperty(S3Constants.S3_BUCKET, bucket);
+ blobStore = new DataStoreBlobStore(
+ S3DataStoreUtils.getS3DataStore(s3Class, props,
root.getAbsolutePath()));
+ mk = new
DocumentMK.Builder().clock(getTestClock()).setMongoDB(mongoConnection.getDB())
+ .setBlobStore(blobStore).open();
}
@After
@Override
public void tearDownConnection() throws Exception {
- S3DataStoreUtils.cleanup(blobStore.getDataStore(), startDate);
+ S3DataStoreUtils.deleteBucket(bucket, startDate);
super.tearDownConnection();
}
}
Modified:
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/S3SharedBlobStoreGCTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/S3SharedBlobStoreGCTest.java?rev=1762733&r1=1762732&r2=1762733&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/S3SharedBlobStoreGCTest.java
(original)
+++
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/S3SharedBlobStoreGCTest.java
Thu Sep 29 03:25:53 2016
@@ -18,28 +18,57 @@
*/
package org.apache.jackrabbit.oak.plugins.document;
-import org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils;
+import java.io.File;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.jackrabbit.oak.blob.cloud.s3.S3Constants;
+import org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import org.junit.After;
import org.junit.BeforeClass;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
import static org.junit.Assume.assumeTrue;
/**
* Shared BlobStoreGCTest for S3.
*/
+@RunWith(Parameterized.class)
public class S3SharedBlobStoreGCTest extends SharedBlobStoreGCTest {
+
+ @Parameterized.Parameter
+ public String s3Class;
+
+ protected String bucket;
+
+ @Parameterized.Parameters(name = "{index}: ({0})")
+ public static List<String> fixtures() {
+ return S3DataStoreUtils.getFixtures();
+ }
+
@BeforeClass
public static void assumptions() {
- assumeTrue(S3DataStoreUtils.isS3DataStore());
+ assumeTrue(S3DataStoreUtils.isS3Configured());
}
@After
public void tearDown() throws Exception {
- S3DataStoreUtils.cleanup(cluster1.getDataStore(), cluster1.getDate());
+ S3DataStoreUtils.deleteBucket(bucket, cluster1.getDate());
super.tearDown();
}
@Override
+ protected DataStoreBlobStore getBlobStore(File rootFolder) throws
Exception {
+ Properties props = S3DataStoreUtils.getS3Config();
+ bucket = rootFolder.getName();
+ props.setProperty(S3Constants.S3_BUCKET, bucket);
+ return new DataStoreBlobStore(
+ S3DataStoreUtils.getS3DataStore(s3Class, props,
rootFolder.getAbsolutePath()));
+ }
+
+ @Override
protected void sleep() throws InterruptedException {
if (S3DataStoreUtils.isS3DataStore()) {
Thread.sleep(1000);
Modified:
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedS3DataStoreUtilsTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedS3DataStoreUtilsTest.java?rev=1762733&r1=1762732&r2=1762733&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedS3DataStoreUtilsTest.java
(original)
+++
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedS3DataStoreUtilsTest.java
Thu Sep 29 03:25:53 2016
@@ -18,35 +18,61 @@
*/
package org.apache.jackrabbit.oak.plugins.document;
+import java.io.File;
import java.io.IOException;
import java.util.Date;
+import java.util.List;
+import java.util.Properties;
-import org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils;
+import org.apache.jackrabbit.oak.blob.cloud.s3.S3Constants;
+import org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils;
import org.apache.jackrabbit.oak.plugins.blob.SharedDataStoreUtilsTest;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import org.junit.After;
import org.junit.BeforeClass;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils.cleanup;
+import static
org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.deleteBucket;
import static org.junit.Assume.assumeTrue;
/**
* SharedDataStoreUtilsTest for S3.
*/
+@RunWith(Parameterized.class)
public class SharedS3DataStoreUtilsTest extends SharedDataStoreUtilsTest {
private static final Logger log =
LoggerFactory.getLogger(SharedS3DataStoreUtilsTest.class);
+ @Parameterized.Parameter
+ public String s3Class;
+
+ protected String bucket;
+
+ @Parameterized.Parameters(name = "{index}: ({0})")
+ public static List<String> fixtures() {
+ return S3DataStoreUtils.getFixtures();
+ }
+
@BeforeClass
public static void assumptions() {
- assumeTrue(S3DataStoreUtils.isS3DataStore());
+ assumeTrue(S3DataStoreUtils.isS3Configured());
+ }
+
+ @Override
+ protected DataStoreBlobStore getBlobStore(File rootFolder) throws
Exception {
+ Properties props = S3DataStoreUtils.getS3Config();
+ bucket = rootFolder.getName();
+ props.setProperty(S3Constants.S3_BUCKET, bucket);
+ return new DataStoreBlobStore(
+ S3DataStoreUtils.getS3DataStore(s3Class, props,
rootFolder.getAbsolutePath()));
}
@After
public void close() throws IOException {
- super.close();
try {
- cleanup(dataStore.getDataStore(), new Date());
+ deleteBucket(bucket, new Date());
} catch (Exception e) {
log.error("Error closing data store", e);
}
Modified:
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreBlobGCIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreBlobGCIT.java?rev=1762733&r1=1762732&r2=1762733&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreBlobGCIT.java
(original)
+++
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreBlobGCIT.java
Thu Sep 29 03:25:53 2016
@@ -19,9 +19,17 @@
package org.apache.jackrabbit.oak.plugins.segment;
-import org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils;
+import java.io.File;
+import java.util.List;
+import java.util.Properties;
+
+import com.google.common.collect.Lists;
+import org.apache.jackrabbit.oak.blob.cloud.s3.S3Constants;
+import org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import org.junit.After;
import org.junit.BeforeClass;
+import org.junit.runners.Parameterized;
import static
org.apache.jackrabbit.oak.commons.FixturesHelper.Fixture.SEGMENT_MK;
import static org.apache.jackrabbit.oak.commons.FixturesHelper.getFixtures;
@@ -31,17 +39,38 @@ import static org.junit.Assume.assumeTru
* Tests for SegmentNodeStore on S3DataStore GC
*/
public class SegmentS3DataStoreBlobGCIT extends SegmentDataStoreBlobGCIT {
+ @Parameterized.Parameter(0)
+ public boolean usePersistedMap;
- public SegmentS3DataStoreBlobGCIT(boolean usePersistedMap) {
- super(usePersistedMap);
+ @Parameterized.Parameter(1)
+ public String s3Class;
+
+ protected String bucket;
+
+ @Parameterized.Parameters(name = "{index}: ({0}, {1})")
+ public static List<Object[]> fixtures1() {
+ List<String> classes = S3DataStoreUtils.getFixtures();
+ List<Object[]> params = Lists.newArrayList();
+ for (String cl : classes) {
+ params.add(new Object[] {true, cl});
+ params.add(new Object[] {false, cl});
+ }
+ return params;
}
@BeforeClass
public static void assumptions() {
assumeTrue(getFixtures().contains(SEGMENT_MK));
- assumeTrue(S3DataStoreUtils.isS3DataStore());
+ assumeTrue(S3DataStoreUtils.isS3Configured());
}
+ protected DataStoreBlobStore getBlobStore(File rootFolder) throws
Exception {
+ Properties props = S3DataStoreUtils.getS3Config();
+ bucket = rootFolder.getName();
+ props.setProperty(S3Constants.S3_BUCKET, bucket);
+ return new DataStoreBlobStore(
+ S3DataStoreUtils.getS3DataStore(s3Class, props,
rootFolder.getAbsolutePath()));
+ }
@After
public void close() throws Exception {
Modified:
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreBlobGCIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreBlobGCIT.java?rev=1762733&r1=1762732&r2=1762733&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreBlobGCIT.java
(original)
+++
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreBlobGCIT.java
Thu Sep 29 03:25:53 2016
@@ -19,9 +19,17 @@
package org.apache.jackrabbit.oak.segment;
-import org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils;
+import java.io.File;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.jackrabbit.oak.blob.cloud.s3.S3Constants;
+import org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import org.junit.After;
import org.junit.BeforeClass;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
import static
org.apache.jackrabbit.oak.commons.FixturesHelper.Fixture.SEGMENT_TAR;
import static org.apache.jackrabbit.oak.commons.FixturesHelper.getFixtures;
@@ -30,19 +38,36 @@ import static org.junit.Assume.assumeTru
/**
* Tests for SegmentNodeStore on S3DataStore GC
*/
+@RunWith(Parameterized.class)
public class SegmentS3DataStoreBlobGCIT extends SegmentDataStoreBlobGCIT {
+ @Parameterized.Parameter
+ public String s3Class;
+
+ protected String bucket;
+
+ @Parameterized.Parameters(name = "{index}: ({0})")
+ public static List<String> fixtures() {
+ return S3DataStoreUtils.getFixtures();
+ }
@BeforeClass
public static void assumptions() {
assumeTrue(getFixtures().contains(SEGMENT_TAR));
- assumeTrue(S3DataStoreUtils.isS3DataStore());
+ assumeTrue(S3DataStoreUtils.isS3Configured());
}
+ protected DataStoreBlobStore getBlobStore(File rootFolder) throws
Exception {
+ Properties props = S3DataStoreUtils.getS3Config();
+ bucket = rootFolder.getName();
+ props.setProperty(S3Constants.S3_BUCKET, bucket);
+ return new DataStoreBlobStore(
+ S3DataStoreUtils.getS3DataStore(s3Class, props,
rootFolder.getAbsolutePath()));
+ }
@After
public void close() throws Exception {
super.close();
- S3DataStoreUtils.cleanup(blobStore.getDataStore(), startDate);
+ S3DataStoreUtils.deleteBucket(bucket, startDate);
}
}
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java?rev=1762733&r1=1762732&r2=1762733&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java
Thu Sep 29 03:25:53 2016
@@ -119,9 +119,13 @@ public class SegmentDataStoreBlobGCIT {
return setUp(10);
}
+ protected DataStoreBlobStore getBlobStore(File folder) throws Exception {
+ return DataStoreUtils.getBlobStore(folder);
+ }
+
public DataStoreState setUp(int count) throws Exception {
if (blobStore == null) {
- blobStore = DataStoreUtils.getBlobStore(folder.newFolder());
+ blobStore = getBlobStore(folder.newFolder());
}
nodeStore = getNodeStore(blobStore);
startDate = new Date();
Modified:
jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java?rev=1762733&r1=1762732&r2=1762733&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java
(original)
+++
jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java
Thu Sep 29 03:25:53 2016
@@ -90,7 +90,8 @@ import org.slf4j.LoggerFactory;
public class SegmentDataStoreBlobGCIT {
private static final Logger log =
LoggerFactory.getLogger(SegmentDataStoreBlobGCIT.class);
- private final boolean usePersistedMap;
+ @Parameterized.Parameter
+ public boolean usePersistedMap;
SegmentNodeStore nodeStore;
FileStore store;
@@ -110,8 +111,8 @@ public class SegmentDataStoreBlobGCIT {
return ImmutableList.of(new Boolean[] {true}, new Boolean[] {false});
}
- public SegmentDataStoreBlobGCIT(boolean usePersistedMap) {
- this.usePersistedMap = usePersistedMap;
+ protected DataStoreBlobStore getBlobStore(File rootFolder) throws
Exception {
+ return DataStoreUtils.getBlobStore(rootFolder);
}
protected SegmentNodeStore getNodeStore(BlobStore blobStore) throws
Exception {
@@ -145,7 +146,7 @@ public class SegmentDataStoreBlobGCIT {
public DataStoreState setUp(int count) throws Exception {
if (blobStore == null) {
- blobStore = DataStoreUtils.getBlobStore(folder.newFolder());
+ blobStore = getBlobStore(folder.newFolder());
}
nodeStore = getNodeStore(blobStore);
startDate = new Date();
@@ -275,7 +276,7 @@ public class SegmentDataStoreBlobGCIT {
.filter(Level.TRACE)
.create();
- DataStoreState state = setUp(2000);
+ DataStoreState state = setUp(5);
log.info("{} blobs available : {}", state.blobsPresent.size(),
state.blobsPresent);
customLogs.starting();
ThreadPoolExecutor executor = (ThreadPoolExecutor)
Executors.newFixedThreadPool(10);
