Author: amitj
Date: Thu Sep 22 05:38:16 2016
New Revision: 1761852
URL: http://svn.apache.org/viewvc?rev=1761852&view=rev
Log:
OAK-4838: Move S3 classes to oak-blob-cloud module
* Module dependency flipped to oak-core -> oak-blob-cloud
* S3 classes moved to oak-blob-cloud
* Moved some S3 tests to oak-it
Added:
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreService.java
(contents, props changed)
- copied, changed from r1761799,
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreService.java
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreStats.java
(contents, props changed)
- copied, changed from r1761799,
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStats.java
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3DataStore.java
(contents, props changed)
- copied, changed from r1761799,
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3DataStoreService.java
(contents, props changed)
- copied, changed from r1761799,
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStoreService.java
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/S3DataStoreUtils.java
(with props)
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreStatsTest.java
(contents, props changed)
- copied, changed from r1761799,
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStatsTest.java
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java
(contents, props changed)
- copied, changed from r1761799,
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoS3DataStoreBlobGCTest.java
(with props)
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/S3SharedBlobStoreGCTest.java
(with props)
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedS3DataStoreUtilsTest.java
(with props)
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreBlobGCIT.java
(with props)
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java
- copied, changed from r1761799,
jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreBlobGCIT.java
(with props)
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreStatsTest.java
(contents, props changed)
- copied, changed from r1761799,
jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java
Removed:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreService.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStats.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStoreService.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStatsTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java
jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java
Modified:
jackrabbit/oak/trunk/oak-blob-cloud/pom.xml
jackrabbit/oak/trunk/oak-core/pom.xml
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreUtils.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/ds/MongoDataStoreBlobGCTest.java
jackrabbit/oak/trunk/oak-it/pom.xml
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java
jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java
Modified: jackrabbit/oak/trunk/oak-blob-cloud/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/pom.xml?rev=1761852&r1=1761851&r2=1761852&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-cloud/pom.xml (original)
+++ jackrabbit/oak/trunk/oak-blob-cloud/pom.xml Thu Sep 22 05:38:16 2016
@@ -108,6 +108,11 @@
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<!-- Amazon AWS dependency -->
<dependency>
<groupId>com.amazonaws</groupId>
@@ -133,6 +138,13 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-core</artifactId>
+ <version>${project.version}</version>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
@@ -147,6 +159,17 @@
<artifactId>logback-classic</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.sling</groupId>
+ <artifactId>org.apache.sling.testing.osgi-mock</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>1.10.19</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
Copied:
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreService.java
(from r1761799,
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreService.java)
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreService.java?p2=jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreService.java&p1=jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreService.java&r1=1761799&r2=1761852&rev=1761852&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreService.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreService.java
Thu Sep 22 05:38:16 2016
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.jackrabbit.oak.plugins.blob.datastore;
+package org.apache.jackrabbit.oak.blob.cloud.aws.s3;
import java.util.Dictionary;
import java.util.Hashtable;
@@ -28,12 +28,13 @@ import org.apache.felix.scr.annotations.
import org.apache.felix.scr.annotations.ConfigurationPolicy;
import org.apache.jackrabbit.core.data.DataStore;
import org.apache.jackrabbit.core.data.DataStoreException;
+import
org.apache.jackrabbit.oak.plugins.blob.datastore.AbstractDataStoreService;
import org.osgi.framework.Constants;
import org.osgi.framework.ServiceRegistration;
import org.osgi.service.component.ComponentContext;
@Component(policy = ConfigurationPolicy.REQUIRE, name =
S3DataStoreService.NAME)
-public class S3DataStoreService extends AbstractDataStoreService{
+public class S3DataStoreService extends AbstractDataStoreService {
public static final String NAME =
"org.apache.jackrabbit.oak.plugins.blob.datastore.S3DataStore";
private static final String DESCRIPTION = "oak.datastore.description";
Propchange:
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreService.java
------------------------------------------------------------------------------
svn:eol-style = native
Copied:
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreStats.java
(from r1761799,
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStats.java)
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreStats.java?p2=jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreStats.java&p1=jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStats.java&r1=1761799&r2=1761852&rev=1761852&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStats.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreStats.java
Thu Sep 22 05:38:16 2016
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.jackrabbit.oak.plugins.blob.datastore;
+package org.apache.jackrabbit.oak.blob.cloud.aws.s3;
import com.google.common.base.Strings;
import org.apache.felix.scr.annotations.Activate;
@@ -27,6 +27,8 @@ import org.apache.jackrabbit.oak.api.Typ
import org.apache.jackrabbit.oak.blob.cloud.aws.s3.stats.S3DataStoreStatsMBean;
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.osgi.OsgiWhiteboard;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.InMemoryDataRecord;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.apache.jackrabbit.oak.spi.whiteboard.Registration;
@@ -166,7 +168,7 @@ public class S3DataStoreStats implements
final String fullBlobId = blob.getContentIdentity();
if (!Strings.isNullOrEmpty(fullBlobId)
&& !InMemoryDataRecord.isInstance(fullBlobId)) {
- String blobId = DataStoreBlobStore.BlobId.of(fullBlobId).blobId;
+ String blobId =
DataStoreBlobStore.BlobId.of(fullBlobId).getBlobId();
return s3ds.haveRecordForIdentifier(blobId);
}
return false;
Propchange:
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreStats.java
------------------------------------------------------------------------------
svn:eol-style = native
Copied:
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3DataStore.java
(from r1761799,
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java)
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3DataStore.java?p2=jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3DataStore.java&p1=jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java&r1=1761799&r2=1761852&rev=1761852&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3DataStore.java
Thu Sep 22 05:38:16 2016
@@ -16,14 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.jackrabbit.oak.plugins.blob.datastore;
+package org.apache.jackrabbit.oak.blob.cloud.aws.s3;
import org.apache.jackrabbit.core.data.Backend;
import org.apache.jackrabbit.core.data.DataIdentifier;
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
-import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3Backend;
-import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3DataStore;
import org.apache.jackrabbit.oak.plugins.blob.SharedDataStore;
import java.io.File;
Propchange:
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3DataStore.java
------------------------------------------------------------------------------
svn:eol-style = native
Copied:
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3DataStoreService.java
(from r1761799,
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStoreService.java)
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3DataStoreService.java?p2=jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3DataStoreService.java&p1=jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStoreService.java&r1=1761799&r2=1761852&rev=1761852&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStoreService.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3DataStoreService.java
Thu Sep 22 05:38:16 2016
@@ -17,12 +17,13 @@
* under the License.
*/
-package org.apache.jackrabbit.oak.plugins.blob.datastore;
+package org.apache.jackrabbit.oak.blob.cloud.aws.s3;
import org.apache.felix.scr.annotations.Component;
import org.apache.felix.scr.annotations.ConfigurationPolicy;
import org.apache.jackrabbit.core.data.DataStore;
import org.apache.jackrabbit.core.data.DataStoreException;
+import
org.apache.jackrabbit.oak.plugins.blob.datastore.AbstractDataStoreService;
import org.osgi.framework.Constants;
import org.osgi.framework.ServiceRegistration;
import org.osgi.service.component.ComponentContext;
@@ -33,7 +34,7 @@ import java.util.Map;
import java.util.Properties;
@Component(policy = ConfigurationPolicy.REQUIRE, name =
SharedS3DataStoreService.NAME)
-public class SharedS3DataStoreService extends AbstractDataStoreService{
+public class SharedS3DataStoreService extends AbstractDataStoreService {
public static final String NAME =
"org.apache.jackrabbit.oak.plugins.blob.datastore.SharedS3DataStore";
private static final String DESCRIPTION = "oak.datastore.description";
Propchange:
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3DataStoreService.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/S3DataStoreUtils.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/S3DataStoreUtils.java?rev=1761852&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/S3DataStoreUtils.java
(added)
+++
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/S3DataStoreUtils.java
Thu Sep 22 05:38:16 2016
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.blob.cloud;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Properties;
+
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.DeleteObjectsRequest;
+import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.amazonaws.services.s3.transfer.TransferManager;
+import org.apache.jackrabbit.core.data.Backend;
+import org.apache.jackrabbit.core.data.DataStore;
+import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3Backend;
+import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3DataStore;
+import org.apache.jackrabbit.oak.blob.cloud.aws.s3.SharedS3DataStore;
+import org.apache.jackrabbit.oak.blob.cloud.aws.s3.Utils;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Extension to {@link DataStoreUtils} to enable S3 extensions for cleaning.
+ */
+public class S3DataStoreUtils extends DataStoreUtils {
+ private static final Logger log =
LoggerFactory.getLogger(S3DataStoreUtils.class);
+
+ public static boolean isS3DataStore() {
+ String dsName = System.getProperty(DS_CLASS_NAME);
+ return (dsName != null) && (dsName.equals(S3DataStore.class.getName())
|| dsName
+ .equals(SharedS3DataStore.class.getName()));
+ }
+
+ /**
+ * S3 specific cleanup
+ *
+ * @param dataStore the underlying DataStore instance
+ * @param date the date of initialization
+ * @throws Exception
+ */
+ public static void cleanup(DataStore dataStore, Date date) throws
Exception {
+ if (dataStore instanceof S3DataStore) {
+ Backend backend = ((S3DataStore) dataStore).getBackend();
+ if (backend instanceof S3Backend) {
+ String bucket = ((S3Backend) backend).getBucket();
+ deleteBucket(bucket, date);
+ }
+ }
+ }
+
+ private static void deleteBucket(String bucket, Date date) throws
Exception {
+ log.info("cleaning bucket [" + bucket + "]");
+ Properties props = Utils.readConfig((String)
getConfig().get("config"));
+ AmazonS3Client s3service = Utils.openService(props);
+ TransferManager tmx = new TransferManager(s3service);
+ if (s3service.doesBucketExist(bucket)) {
+ for (int i = 0; i < 4; i++) {
+ tmx.abortMultipartUploads(bucket, date);
+ ObjectListing prevObjectListing =
s3service.listObjects(bucket);
+ while (prevObjectListing != null ) {
+ List<DeleteObjectsRequest.KeyVersion>
+ deleteList = new
ArrayList<DeleteObjectsRequest.KeyVersion>();
+ for (S3ObjectSummary s3ObjSumm :
prevObjectListing.getObjectSummaries()) {
+ deleteList.add(new DeleteObjectsRequest.KeyVersion(
+ s3ObjSumm.getKey()));
+ }
+ if (deleteList.size() > 0) {
+ DeleteObjectsRequest delObjsReq = new
DeleteObjectsRequest(
+ bucket);
+ delObjsReq.setKeys(deleteList);
+ s3service.deleteObjects(delObjsReq);
+ }
+ if (!prevObjectListing.isTruncated()) break;
+ prevObjectListing =
s3service.listNextBatchOfObjects(prevObjectListing);
+ }
+ }
+ //s3service.deleteBucket(bucket);
+ log.info("bucket [ " + bucket + "] cleaned");
+ } else {
+ log.info("bucket [" + bucket + "] doesn't exists");
+ }
+ tmx.shutdownNow();
+ s3service.shutdown();
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/S3DataStoreUtils.java
------------------------------------------------------------------------------
svn:eol-style = native
Copied:
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreStatsTest.java
(from r1761799,
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStatsTest.java)
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreStatsTest.java?p2=jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreStatsTest.java&p1=jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStatsTest.java&r1=1761799&r2=1761852&rev=1761852&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStatsTest.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreStatsTest.java
Thu Sep 22 05:38:16 2016
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.jackrabbit.oak.plugins.blob.datastore;
+package org.apache.jackrabbit.oak.blob.cloud.aws.s3;
import java.io.ByteArrayInputStream;
import java.io.File;
Propchange:
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreStatsTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-core/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/pom.xml?rev=1761852&r1=1761851&r2=1761852&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/pom.xml (original)
+++ jackrabbit/oak/trunk/oak-core/pom.xml Thu Sep 22 05:38:16 2016
@@ -184,12 +184,6 @@
<artifactId>oak-blob</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.jackrabbit</groupId>
- <artifactId>oak-blob-cloud</artifactId>
- <version>${project.version}</version>
- <optional>true</optional>
- </dependency>
<!-- General utility libraries -->
<dependency>
<groupId>com.google.guava</groupId>
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java?rev=1761852&r1=1761851&r2=1761852&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
Thu Sep 22 05:38:16 2016
@@ -623,8 +623,13 @@ public class DataStoreBlobStore implemen
return encodedBlobId;
}
- static class BlobId {
+ public static class BlobId {
static final String SEP = "#";
+
+ public String getBlobId() {
+ return blobId;
+ }
+
final String blobId;
final long length;
@@ -672,7 +677,7 @@ public class DataStoreBlobStore implemen
return encodedBlobId.contains(SEP);
}
- static BlobId of(String encodedValue) {
+ public static BlobId of(String encodedValue) {
return new BlobId(encodedValue);
}
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java?rev=1761852&r1=1761851&r2=1761852&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java
Thu Sep 22 05:38:16 2016
@@ -21,7 +21,6 @@ package org.apache.jackrabbit.oak.plugin
import static com.google.common.collect.Sets.newHashSet;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertTrue;
-import static
org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils.cleanup;
import static
org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils.getBlobStore;
import static org.hamcrest.CoreMatchers.instanceOf;
@@ -30,7 +29,6 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.util.Date;
import java.util.Map;
import java.util.Random;
import java.util.Set;
@@ -73,7 +71,7 @@ public class SharedDataStoreUtilsTest {
@Rule
public TemporaryFolder folder = new TemporaryFolder(new File("target"));
- DataStoreBlobStore dataStore;
+ protected DataStoreBlobStore dataStore;
@BeforeClass
public static void setUpBeforeClass() throws Exception {
@@ -282,11 +280,6 @@ public class SharedDataStoreUtilsTest {
@After
public void close() throws IOException {
FileUtils.cleanDirectory(new File(DataStoreUtils.getHomeDir()));
- try {
- cleanup(dataStore.getDataStore(), new Date());
- } catch (Exception e) {
- log.error("Error closing data store", e);
- }
}
}
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreUtils.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreUtils.java?rev=1761852&r1=1761851&r2=1761852&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreUtils.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreUtils.java
Thu Sep 22 05:38:16 2016
@@ -17,24 +17,12 @@
package org.apache.jackrabbit.oak.plugins.blob.datastore;
import java.io.File;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
import java.util.Map;
-import java.util.Properties;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.amazonaws.services.s3.transfer.TransferManager;
import com.google.common.collect.Maps;
-import org.apache.jackrabbit.core.data.Backend;
import org.apache.jackrabbit.core.data.DataStore;
import org.apache.jackrabbit.core.data.FileDataStore;
-import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3Backend;
-import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3DataStore;
-import org.apache.jackrabbit.oak.blob.cloud.aws.s3.Utils;
+
import org.apache.jackrabbit.oak.commons.PropertiesUtil;
import org.junit.Test;
import org.slf4j.Logger;
@@ -81,13 +69,7 @@ public class DataStoreUtils {
return getBlobStore(getHomeDir());
}
- public static boolean isS3DataStore() {
- String dsName = System.getProperty(DS_CLASS_NAME);
- return (dsName != null) && (dsName.equals(S3DataStore.class.getName())
|| dsName
- .equals(SharedS3DataStore.class.getName()));
- }
-
- private static Map<String, ?> getConfig() {
+ public static Map<String, ?> getConfig() {
Map<String, Object> result = Maps.newHashMap();
for (Map.Entry<String, ?> e :
Maps.fromProperties(System.getProperties()).entrySet()) {
String key = e.getKey();
@@ -120,56 +102,4 @@ public class DataStoreUtils {
DataStoreBlobStore dbs = getBlobStore();
assertEquals(1000, dbs.getDataStore().getMinRecordLength());
}
-
- /**
- * S3 specific cleanup
- *
- * @param dataStore the underlying DataStore instance
- * @param date the date of initialization
- * @throws Exception
- */
- public static void cleanup(DataStore dataStore, Date date) throws
Exception {
- if (dataStore instanceof S3DataStore) {
- Backend backend = ((S3DataStore) dataStore).getBackend();
- if (backend instanceof S3Backend) {
- String bucket = ((S3Backend) backend).getBucket();
- deleteBucket(bucket, date);
- }
- }
- }
-
- private static void deleteBucket(String bucket, Date date) throws
Exception {
- log.info("cleaning bucket [" + bucket + "]");
- Properties props = Utils.readConfig((String)
getConfig().get("config"));
- AmazonS3Client s3service = Utils.openService(props);
- TransferManager tmx = new TransferManager(s3service);
- if (s3service.doesBucketExist(bucket)) {
- for (int i = 0; i < 4; i++) {
- tmx.abortMultipartUploads(bucket, date);
- ObjectListing prevObjectListing =
s3service.listObjects(bucket);
- while (prevObjectListing != null ) {
- List<DeleteObjectsRequest.KeyVersion>
- deleteList = new
ArrayList<DeleteObjectsRequest.KeyVersion>();
- for (S3ObjectSummary s3ObjSumm :
prevObjectListing.getObjectSummaries()) {
- deleteList.add(new DeleteObjectsRequest.KeyVersion(
- s3ObjSumm.getKey()));
- }
- if (deleteList.size() > 0) {
- DeleteObjectsRequest delObjsReq = new
DeleteObjectsRequest(
- bucket);
- delObjsReq.setKeys(deleteList);
- s3service.deleteObjects(delObjsReq);
- }
- if (!prevObjectListing.isTruncated()) break;
- prevObjectListing =
s3service.listNextBatchOfObjects(prevObjectListing);
- }
- }
- //s3service.deleteBucket(bucket);
- log.info("bucket [ " + bucket + "] cleaned");
- } else {
- log.info("bucket [" + bucket + "] doesn't exists");
- }
- tmx.shutdownNow();
- s3service.shutdown();
- }
}
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java?rev=1761852&r1=1761851&r2=1761852&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java
Thu Sep 22 05:38:16 2016
@@ -68,8 +68,8 @@ import org.slf4j.LoggerFactory;
public class SharedBlobStoreGCTest {
private static final Logger log =
LoggerFactory.getLogger(SharedBlobStoreGCTest.class);
- private Cluster cluster1;
- private Cluster cluster2;
+ protected Cluster cluster1;
+ protected Cluster cluster2;
private Clock clock;
@Before
@@ -218,14 +218,13 @@ public class SharedBlobStoreGCTest {
@After
public void tearDown() throws Exception {
- DataStoreUtils.cleanup(cluster1.getDataStore(), cluster1.getDate());
FileUtils.cleanDirectory((new
File(DataStoreUtils.getHomeDir())).getParentFile());
DataStoreUtils.time = -1;
cluster1.getDocumentNodeStore().dispose();
cluster2.getDocumentNodeStore().dispose();
}
- class Cluster {
+ public class Cluster {
private DocumentNodeStore ds;
private int seed;
private BlobGarbageCollector gc;
@@ -294,10 +293,7 @@ public class SharedBlobStoreGCTest {
VersionGarbageCollector vGC = ds.getVersionGarbageCollector();
VersionGCStats stats = vGC.gc(0, TimeUnit.MILLISECONDS);
Assert.assertEquals(deletes.size(), stats.deletedDocGCCount);
-
- if (DataStoreUtils.isS3DataStore()) {
- Thread.sleep(1000);
- }
+ sleep();
}
private HashSet<String> addNodeSpecialChars() throws Exception {
@@ -342,5 +338,8 @@ public class SharedBlobStoreGCTest {
return ds;
}
}
+
+ protected void sleep() throws InterruptedException {
+ }
}
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/ds/MongoDataStoreBlobGCTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/ds/MongoDataStoreBlobGCTest.java?rev=1761852&r1=1761851&r2=1761852&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/ds/MongoDataStoreBlobGCTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/ds/MongoDataStoreBlobGCTest.java
Thu Sep 22 05:38:16 2016
@@ -35,8 +35,8 @@ import org.junit.BeforeClass;
*
*/
public class MongoDataStoreBlobGCTest extends MongoBlobGCTest {
- Date startDate;
- DataStoreBlobStore blobStore;
+ protected Date startDate;
+ protected DataStoreBlobStore blobStore;
@BeforeClass
public static void setUpBeforeClass() throws Exception {
@@ -61,7 +61,6 @@ public class MongoDataStoreBlobGCTest ex
@After
@Override
public void tearDownConnection() throws Exception {
- DataStoreUtils.cleanup(blobStore.getDataStore(), startDate);
FileUtils.deleteDirectory(new File(DataStoreUtils.getHomeDir()));
mk.dispose();
MongoUtils.dropCollections(connectionFactory.getConnection().getDB());
Modified: jackrabbit/oak/trunk/oak-it/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-it/pom.xml?rev=1761852&r1=1761851&r2=1761852&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-it/pom.xml (original)
+++ jackrabbit/oak/trunk/oak-it/pom.xml Thu Sep 22 05:38:16 2016
@@ -51,6 +51,12 @@
</dependency>
<dependency>
<groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-blob-cloud</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
<artifactId>oak-core</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
@@ -72,6 +78,13 @@
</dependency>
<dependency>
<groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-blob-cloud</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
<artifactId>oak-commons</artifactId>
<version>${project.version}</version>
<scope>test</scope>
@@ -125,5 +138,22 @@
<artifactId>commons-math3</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.sling</groupId>
+ <artifactId>org.apache.sling.testing.osgi-mock</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>1.10.19</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.googlecode.json-simple</groupId>
+ <artifactId>json-simple</artifactId>
+ <version>1.1.1</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
-</project>
\ No newline at end of file
+</project>
Copied:
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java
(from r1761799,
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java)
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java?p2=jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java&p1=jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java&r1=1761799&r2=1761852&rev=1761852&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java
(original)
+++
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java
Thu Sep 22 05:38:16 2016
@@ -20,9 +20,9 @@ package org.apache.jackrabbit.oak.plugin
import java.util.Map;
+import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3DataStoreStats;
+import org.apache.jackrabbit.oak.blob.cloud.aws.s3.SharedS3DataStore;
import org.apache.jackrabbit.oak.blob.cloud.aws.s3.stats.S3DataStoreStatsMBean;
-import org.apache.jackrabbit.oak.plugins.blob.datastore.S3DataStoreStats;
-import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedS3DataStore;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
Propchange:
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoS3DataStoreBlobGCTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoS3DataStoreBlobGCTest.java?rev=1761852&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoS3DataStoreBlobGCTest.java
(added)
+++
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoS3DataStoreBlobGCTest.java
Thu Sep 22 05:38:16 2016
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document;
+
+import org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils;
+import
org.apache.jackrabbit.oak.plugins.document.blob.ds.MongoDataStoreBlobGCTest;
+import org.junit.After;
+import org.junit.BeforeClass;
+
+import static org.junit.Assume.assumeTrue;
+
+/**
+ * Tests DataStoreGC with Mongo and S3
+ */
+public class MongoS3DataStoreBlobGCTest extends MongoDataStoreBlobGCTest {
+ @BeforeClass
+ public static void assumptions() {
+ assumeTrue(S3DataStoreUtils.isS3DataStore());
+ }
+
+ @After
+ @Override
+ public void tearDownConnection() throws Exception {
+ S3DataStoreUtils.cleanup(blobStore.getDataStore(), startDate);
+ super.tearDownConnection();
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoS3DataStoreBlobGCTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/S3SharedBlobStoreGCTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/S3SharedBlobStoreGCTest.java?rev=1761852&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/S3SharedBlobStoreGCTest.java
(added)
+++
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/S3SharedBlobStoreGCTest.java
Thu Sep 22 05:38:16 2016
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document;
+
+import org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils;
+import org.junit.After;
+import org.junit.BeforeClass;
+
+import static org.junit.Assume.assumeTrue;
+
+/**
+ * Shared BlobStoreGCTest for S3.
+ */
+public class S3SharedBlobStoreGCTest extends SharedBlobStoreGCTest {
+ @BeforeClass
+ public static void assumptions() {
+ assumeTrue(S3DataStoreUtils.isS3DataStore());
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ S3DataStoreUtils.cleanup(cluster1.getDataStore(), cluster1.getDate());
+ super.tearDown();
+ }
+
+ @Override
+ protected void sleep() throws InterruptedException {
+ if (S3DataStoreUtils.isS3DataStore()) {
+ Thread.sleep(1000);
+ }
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/S3SharedBlobStoreGCTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedS3DataStoreUtilsTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedS3DataStoreUtilsTest.java?rev=1761852&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedS3DataStoreUtilsTest.java
(added)
+++
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedS3DataStoreUtilsTest.java
Thu Sep 22 05:38:16 2016
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document;
+
+import java.io.IOException;
+import java.util.Date;
+
+import org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils;
+import org.apache.jackrabbit.oak.plugins.blob.SharedDataStoreUtilsTest;
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils.cleanup;
+import static org.junit.Assume.assumeTrue;
+
+/**
+ * SharedDataStoreUtilsTest for S3.
+ */
+public class SharedS3DataStoreUtilsTest extends SharedDataStoreUtilsTest {
+ private static final Logger log =
LoggerFactory.getLogger(SharedS3DataStoreUtilsTest.class);
+
+ @BeforeClass
+ public static void assumptions() {
+ assumeTrue(S3DataStoreUtils.isS3DataStore());
+ }
+
+ @After
+ public void close() throws IOException {
+ super.close();
+ try {
+ cleanup(dataStore.getDataStore(), new Date());
+ } catch (Exception e) {
+ log.error("Error closing data store", e);
+ }
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedS3DataStoreUtilsTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreBlobGCIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreBlobGCIT.java?rev=1761852&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreBlobGCIT.java
(added)
+++
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreBlobGCIT.java
Thu Sep 22 05:38:16 2016
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.segment;
+
+import org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils;
+import org.junit.After;
+import org.junit.BeforeClass;
+
+import static
org.apache.jackrabbit.oak.commons.FixturesHelper.Fixture.SEGMENT_MK;
+import static org.apache.jackrabbit.oak.commons.FixturesHelper.getFixtures;
+import static org.junit.Assume.assumeTrue;
+
+/**
+ * Tests for SegmentNodeStore on S3DataStore GC
+ */
+public class SegmentS3DataStoreBlobGCIT extends SegmentDataStoreBlobGCIT {
+
+ public SegmentS3DataStoreBlobGCIT(boolean usePersistedMap) {
+ super(usePersistedMap);
+ }
+
+ @BeforeClass
+ public static void assumptions() {
+ assumeTrue(getFixtures().contains(SEGMENT_MK));
+ assumeTrue(S3DataStoreUtils.isS3DataStore());
+ }
+
+
+ @After
+ public void close() throws Exception {
+ super.close();
+ S3DataStoreUtils.cleanup(blobStore.getDataStore(), startDate);
+ }
+}
+
Propchange:
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreBlobGCIT.java
------------------------------------------------------------------------------
svn:eol-style = native
Copied:
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java
(from r1761799,
jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java)
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java?p2=jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java&p1=jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java&r1=1761799&r2=1761852&rev=1761852&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java
(original)
+++
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java
Thu Sep 22 05:38:16 2016
@@ -21,9 +21,9 @@ package org.apache.jackrabbit.oak.plugin
import java.io.File;
import java.util.Map;
+import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3DataStoreStats;
+import org.apache.jackrabbit.oak.blob.cloud.aws.s3.SharedS3DataStore;
import org.apache.jackrabbit.oak.blob.cloud.aws.s3.stats.S3DataStoreStatsMBean;
-import org.apache.jackrabbit.oak.plugins.blob.datastore.S3DataStoreStats;
-import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedS3DataStore;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
@@ -37,6 +37,8 @@ import org.junit.rules.TemporaryFolder;
import org.osgi.framework.ServiceRegistration;
import static com.google.common.collect.Maps.newHashMap;
+import static
org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStoreService.CUSTOM_BLOB_STORE;
+import static
org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.DIRECTORY;
import static org.apache.sling.testing.mock.osgi.MockOsgi.deactivate;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
@@ -104,8 +106,8 @@ public class SegmentS3DataStoreStatsTest
private void registerSegmentNodeStoreService(boolean customBlobStore) {
Map<String, Object> properties = newHashMap();
- properties.put(SegmentNodeStoreService.CUSTOM_BLOB_STORE,
customBlobStore);
- properties.put(SegmentNodeStoreService.DIRECTORY,
folder.getRoot().getAbsolutePath());
+ properties.put(CUSTOM_BLOB_STORE, customBlobStore);
+ properties.put(DIRECTORY, folder.getRoot().getAbsolutePath());
segmentNodeStoreService = context.registerInjectActivateService(new
SegmentNodeStoreService(), properties);
}
Added:
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreBlobGCIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreBlobGCIT.java?rev=1761852&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreBlobGCIT.java
(added)
+++
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreBlobGCIT.java
Thu Sep 22 05:38:16 2016
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment;
+
+import org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils;
+import org.junit.After;
+import org.junit.BeforeClass;
+
+import static
org.apache.jackrabbit.oak.commons.FixturesHelper.Fixture.SEGMENT_TAR;
+import static org.apache.jackrabbit.oak.commons.FixturesHelper.getFixtures;
+import static org.junit.Assume.assumeTrue;
+
+/**
+ * Tests for SegmentNodeStore on S3DataStore GC
+ */
+public class SegmentS3DataStoreBlobGCIT extends SegmentDataStoreBlobGCIT {
+
+ @BeforeClass
+ public static void assumptions() {
+ assumeTrue(getFixtures().contains(SEGMENT_TAR));
+ assumeTrue(S3DataStoreUtils.isS3DataStore());
+ }
+
+
+ @After
+ public void close() throws Exception {
+ super.close();
+ S3DataStoreUtils.cleanup(blobStore.getDataStore(), startDate);
+ }
+}
+
Propchange:
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreBlobGCIT.java
------------------------------------------------------------------------------
svn:eol-style = native
Copied:
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreStatsTest.java
(from r1761799,
jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java)
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreStatsTest.java?p2=jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreStatsTest.java&p1=jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java&r1=1761799&r2=1761852&rev=1761852&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java
(original)
+++
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreStatsTest.java
Thu Sep 22 05:38:16 2016
@@ -16,14 +16,14 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.jackrabbit.oak.plugins.segment;
+package org.apache.jackrabbit.oak.segment;
import java.io.File;
import java.util.Map;
+import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3DataStoreStats;
+import org.apache.jackrabbit.oak.blob.cloud.aws.s3.SharedS3DataStore;
import org.apache.jackrabbit.oak.blob.cloud.aws.s3.stats.S3DataStoreStatsMBean;
-import org.apache.jackrabbit.oak.plugins.blob.datastore.S3DataStoreStats;
-import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedS3DataStore;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
@@ -37,6 +37,8 @@ import org.junit.rules.TemporaryFolder;
import org.osgi.framework.ServiceRegistration;
import static com.google.common.collect.Maps.newHashMap;
+import static
org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.CUSTOM_BLOB_STORE;
+import static
org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.DIRECTORY;
import static org.apache.sling.testing.mock.osgi.MockOsgi.deactivate;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
@@ -99,13 +101,14 @@ public class SegmentS3DataStoreStatsTest
unregisterBlobStore();
}
- private SegmentNodeStoreService segmentNodeStoreService;
+ private SegmentNodeStoreService
+ segmentNodeStoreService;
private void registerSegmentNodeStoreService(boolean customBlobStore) {
Map<String, Object> properties = newHashMap();
- properties.put(SegmentNodeStoreService.CUSTOM_BLOB_STORE,
customBlobStore);
- properties.put(SegmentNodeStoreService.DIRECTORY,
folder.getRoot().getAbsolutePath());
+ properties.put(CUSTOM_BLOB_STORE, customBlobStore);
+ properties.put(DIRECTORY, folder.getRoot().getAbsolutePath());
segmentNodeStoreService = context.registerInjectActivateService(new
SegmentNodeStoreService(), properties);
}
Propchange:
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreStatsTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java?rev=1761852&r1=1761851&r2=1761852&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
Thu Sep 22 05:38:16 2016
@@ -44,7 +44,7 @@ import org.apache.jackrabbit.core.data.D
import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.OakFileDataStore;
-import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedS3DataStore;
+import org.apache.jackrabbit.oak.blob.cloud.aws.s3.SharedS3DataStore;
import org.apache.jackrabbit.oak.plugins.document.DocumentMK;
import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
import org.apache.jackrabbit.oak.plugins.document.util.MongoConnection;
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java?rev=1761852&r1=1761851&r2=1761852&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java
Thu Sep 22 05:38:16 2016
@@ -443,7 +443,6 @@ public class SegmentDataStoreBlobGCIT {
if (store != null) {
store.close();
}
- DataStoreUtils.cleanup(blobStore.getDataStore(), startDate);
}
static InputStream randomStream(int seed, int size) {
Modified:
jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java?rev=1761852&r1=1761851&r2=1761852&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java
(original)
+++
jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java
Thu Sep 22 05:38:16 2016
@@ -461,7 +461,6 @@ public class SegmentDataStoreBlobGCIT {
if (store != null) {
store.close();
}
- DataStoreUtils.cleanup(blobStore.getDataStore(), startDate);
}
static InputStream randomStream(int seed, int size) {
@@ -474,7 +473,7 @@ public class SegmentDataStoreBlobGCIT {
/**
* Waits for some time and adds additional blobs after blob referenced
identified to simulate
* long running blob id collection phase.
- */
+ */
class TestGarbageCollector extends MarkSweepGarbageCollector {
long maxLastModifiedInterval;
String root;
