Author: amitj Date: Thu Sep 22 05:38:16 2016 New Revision: 1761852 URL: http://svn.apache.org/viewvc?rev=1761852&view=rev Log: OAK-4838: Move S3 classes to oak-blob-cloud module
* Module dependency flipped to oak-core -> oak-blob-cloud * S3 classes moved to oak-blob-cloud * Moved some S3 tests to oak-it Added: jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreService.java (contents, props changed) - copied, changed from r1761799, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreService.java jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreStats.java (contents, props changed) - copied, changed from r1761799, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStats.java jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3DataStore.java (contents, props changed) - copied, changed from r1761799, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3DataStoreService.java (contents, props changed) - copied, changed from r1761799, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStoreService.java jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/S3DataStoreUtils.java (with props) jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreStatsTest.java (contents, props changed) - copied, changed from r1761799, jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStatsTest.java jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/ jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java (contents, props changed) - copied, changed from r1761799, jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoS3DataStoreBlobGCTest.java (with props) jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/S3SharedBlobStoreGCTest.java (with props) jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedS3DataStoreUtilsTest.java (with props) jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/ jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreBlobGCIT.java (with props) jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java - copied, changed from r1761799, jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/ jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreBlobGCIT.java (with props) jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreStatsTest.java (contents, props changed) - copied, changed from r1761799, jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java Removed: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreService.java jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStats.java jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStoreService.java jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStatsTest.java jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java Modified: jackrabbit/oak/trunk/oak-blob-cloud/pom.xml jackrabbit/oak/trunk/oak-core/pom.xml jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreUtils.java jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/ds/MongoDataStoreBlobGCTest.java jackrabbit/oak/trunk/oak-it/pom.xml jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java Modified: jackrabbit/oak/trunk/oak-blob-cloud/pom.xml URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/pom.xml?rev=1761852&r1=1761851&r2=1761852&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-blob-cloud/pom.xml (original) +++ jackrabbit/oak/trunk/oak-blob-cloud/pom.xml Thu Sep 22 05:38:16 2016 @@ -108,6 +108,11 @@ <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.apache.jackrabbit</groupId> + <artifactId>oak-core</artifactId> + <version>${project.version}</version> + </dependency> <!-- Amazon AWS dependency --> <dependency> <groupId>com.amazonaws</groupId> @@ -133,6 +138,13 @@ <scope>test</scope> </dependency> <dependency> + <groupId>org.apache.jackrabbit</groupId> + <artifactId>oak-core</artifactId> + <version>${project.version}</version> + <classifier>tests</classifier> + <scope>test</scope> + </dependency> + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <scope>test</scope> @@ -147,6 +159,17 @@ <artifactId>logback-classic</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.sling</groupId> + <artifactId>org.apache.sling.testing.osgi-mock</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <version>1.10.19</version> + <scope>test</scope> + </dependency> </dependencies> </project> Copied: jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreService.java (from r1761799, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreService.java) URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreService.java?p2=jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreService.java&p1=jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreService.java&r1=1761799&r2=1761852&rev=1761852&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreService.java (original) +++ jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreService.java Thu Sep 22 05:38:16 2016 @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.jackrabbit.oak.plugins.blob.datastore; +package org.apache.jackrabbit.oak.blob.cloud.aws.s3; import java.util.Dictionary; import java.util.Hashtable; @@ -28,12 +28,13 @@ import org.apache.felix.scr.annotations. import org.apache.felix.scr.annotations.ConfigurationPolicy; import org.apache.jackrabbit.core.data.DataStore; import org.apache.jackrabbit.core.data.DataStoreException; +import org.apache.jackrabbit.oak.plugins.blob.datastore.AbstractDataStoreService; import org.osgi.framework.Constants; import org.osgi.framework.ServiceRegistration; import org.osgi.service.component.ComponentContext; @Component(policy = ConfigurationPolicy.REQUIRE, name = S3DataStoreService.NAME) -public class S3DataStoreService extends AbstractDataStoreService{ +public class S3DataStoreService extends AbstractDataStoreService { public static final String NAME = "org.apache.jackrabbit.oak.plugins.blob.datastore.S3DataStore"; private static final String DESCRIPTION = "oak.datastore.description"; Propchange: jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreService.java ------------------------------------------------------------------------------ svn:eol-style = native Copied: jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreStats.java (from r1761799, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStats.java) URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreStats.java?p2=jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreStats.java&p1=jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStats.java&r1=1761799&r2=1761852&rev=1761852&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStats.java (original) +++ jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreStats.java Thu Sep 22 05:38:16 2016 @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.jackrabbit.oak.plugins.blob.datastore; +package org.apache.jackrabbit.oak.blob.cloud.aws.s3; import com.google.common.base.Strings; import org.apache.felix.scr.annotations.Activate; @@ -27,6 +27,8 @@ import org.apache.jackrabbit.oak.api.Typ import org.apache.jackrabbit.oak.blob.cloud.aws.s3.stats.S3DataStoreStatsMBean; import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.osgi.OsgiWhiteboard; +import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore; +import org.apache.jackrabbit.oak.plugins.blob.datastore.InMemoryDataRecord; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.apache.jackrabbit.oak.spi.whiteboard.Registration; @@ -166,7 +168,7 @@ public class S3DataStoreStats implements final String fullBlobId = blob.getContentIdentity(); if (!Strings.isNullOrEmpty(fullBlobId) && !InMemoryDataRecord.isInstance(fullBlobId)) { - String blobId = DataStoreBlobStore.BlobId.of(fullBlobId).blobId; + String blobId = DataStoreBlobStore.BlobId.of(fullBlobId).getBlobId(); return s3ds.haveRecordForIdentifier(blobId); } return false; Propchange: jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreStats.java ------------------------------------------------------------------------------ svn:eol-style = native Copied: jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3DataStore.java (from r1761799, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java) URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3DataStore.java?p2=jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3DataStore.java&p1=jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java&r1=1761799&r2=1761852&rev=1761852&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java (original) +++ jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3DataStore.java Thu Sep 22 05:38:16 2016 @@ -16,14 +16,12 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.jackrabbit.oak.plugins.blob.datastore; +package org.apache.jackrabbit.oak.blob.cloud.aws.s3; import org.apache.jackrabbit.core.data.Backend; import org.apache.jackrabbit.core.data.DataIdentifier; import org.apache.jackrabbit.core.data.DataRecord; import org.apache.jackrabbit.core.data.DataStoreException; -import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3Backend; -import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3DataStore; import org.apache.jackrabbit.oak.plugins.blob.SharedDataStore; import java.io.File; Propchange: jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3DataStore.java ------------------------------------------------------------------------------ svn:eol-style = native Copied: jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3DataStoreService.java (from r1761799, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStoreService.java) URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3DataStoreService.java?p2=jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3DataStoreService.java&p1=jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStoreService.java&r1=1761799&r2=1761852&rev=1761852&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStoreService.java (original) +++ jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3DataStoreService.java Thu Sep 22 05:38:16 2016 @@ -17,12 +17,13 @@ * under the License. */ -package org.apache.jackrabbit.oak.plugins.blob.datastore; +package org.apache.jackrabbit.oak.blob.cloud.aws.s3; import org.apache.felix.scr.annotations.Component; import org.apache.felix.scr.annotations.ConfigurationPolicy; import org.apache.jackrabbit.core.data.DataStore; import org.apache.jackrabbit.core.data.DataStoreException; +import org.apache.jackrabbit.oak.plugins.blob.datastore.AbstractDataStoreService; import org.osgi.framework.Constants; import org.osgi.framework.ServiceRegistration; import org.osgi.service.component.ComponentContext; @@ -33,7 +34,7 @@ import java.util.Map; import java.util.Properties; @Component(policy = ConfigurationPolicy.REQUIRE, name = SharedS3DataStoreService.NAME) -public class SharedS3DataStoreService extends AbstractDataStoreService{ +public class SharedS3DataStoreService extends AbstractDataStoreService { public static final String NAME = "org.apache.jackrabbit.oak.plugins.blob.datastore.SharedS3DataStore"; private static final String DESCRIPTION = "oak.datastore.description"; Propchange: jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3DataStoreService.java ------------------------------------------------------------------------------ svn:eol-style = native Added: jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/S3DataStoreUtils.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/S3DataStoreUtils.java?rev=1761852&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/S3DataStoreUtils.java (added) +++ jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/S3DataStoreUtils.java Thu Sep 22 05:38:16 2016 @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.blob.cloud; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Properties; + +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.DeleteObjectsRequest; +import com.amazonaws.services.s3.model.ObjectListing; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.amazonaws.services.s3.transfer.TransferManager; +import org.apache.jackrabbit.core.data.Backend; +import org.apache.jackrabbit.core.data.DataStore; +import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3Backend; +import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3DataStore; +import org.apache.jackrabbit.oak.blob.cloud.aws.s3.SharedS3DataStore; +import org.apache.jackrabbit.oak.blob.cloud.aws.s3.Utils; +import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Extension to {@link DataStoreUtils} to enable S3 extensions for cleaning. + */ +public class S3DataStoreUtils extends DataStoreUtils { + private static final Logger log = LoggerFactory.getLogger(S3DataStoreUtils.class); + + public static boolean isS3DataStore() { + String dsName = System.getProperty(DS_CLASS_NAME); + return (dsName != null) && (dsName.equals(S3DataStore.class.getName()) || dsName + .equals(SharedS3DataStore.class.getName())); + } + + /** + * S3 specific cleanup + * + * @param dataStore the underlying DataStore instance + * @param date the date of initialization + * @throws Exception + */ + public static void cleanup(DataStore dataStore, Date date) throws Exception { + if (dataStore instanceof S3DataStore) { + Backend backend = ((S3DataStore) dataStore).getBackend(); + if (backend instanceof S3Backend) { + String bucket = ((S3Backend) backend).getBucket(); + deleteBucket(bucket, date); + } + } + } + + private static void deleteBucket(String bucket, Date date) throws Exception { + log.info("cleaning bucket [" + bucket + "]"); + Properties props = Utils.readConfig((String) getConfig().get("config")); + AmazonS3Client s3service = Utils.openService(props); + TransferManager tmx = new TransferManager(s3service); + if (s3service.doesBucketExist(bucket)) { + for (int i = 0; i < 4; i++) { + tmx.abortMultipartUploads(bucket, date); + ObjectListing prevObjectListing = s3service.listObjects(bucket); + while (prevObjectListing != null ) { + List<DeleteObjectsRequest.KeyVersion> + deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>(); + for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) { + deleteList.add(new DeleteObjectsRequest.KeyVersion( + s3ObjSumm.getKey())); + } + if (deleteList.size() > 0) { + DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest( + bucket); + delObjsReq.setKeys(deleteList); + s3service.deleteObjects(delObjsReq); + } + if (!prevObjectListing.isTruncated()) break; + prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing); + } + } + //s3service.deleteBucket(bucket); + log.info("bucket [ " + bucket + "] cleaned"); + } else { + log.info("bucket [" + bucket + "] doesn't exists"); + } + tmx.shutdownNow(); + s3service.shutdown(); + } +} Propchange: jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/S3DataStoreUtils.java ------------------------------------------------------------------------------ svn:eol-style = native Copied: jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreStatsTest.java (from r1761799, jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStatsTest.java) URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreStatsTest.java?p2=jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreStatsTest.java&p1=jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStatsTest.java&r1=1761799&r2=1761852&rev=1761852&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStatsTest.java (original) +++ jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreStatsTest.java Thu Sep 22 05:38:16 2016 @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.jackrabbit.oak.plugins.blob.datastore; +package org.apache.jackrabbit.oak.blob.cloud.aws.s3; import java.io.ByteArrayInputStream; import java.io.File; Propchange: jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStoreStatsTest.java ------------------------------------------------------------------------------ svn:eol-style = native Modified: jackrabbit/oak/trunk/oak-core/pom.xml URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/pom.xml?rev=1761852&r1=1761851&r2=1761852&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/pom.xml (original) +++ jackrabbit/oak/trunk/oak-core/pom.xml Thu Sep 22 05:38:16 2016 @@ -184,12 +184,6 @@ <artifactId>oak-blob</artifactId> <version>${project.version}</version> </dependency> - <dependency> - <groupId>org.apache.jackrabbit</groupId> - <artifactId>oak-blob-cloud</artifactId> - <version>${project.version}</version> - <optional>true</optional> - </dependency> <!-- General utility libraries --> <dependency> <groupId>com.google.guava</groupId> Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java?rev=1761852&r1=1761851&r2=1761852&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java (original) +++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java Thu Sep 22 05:38:16 2016 @@ -623,8 +623,13 @@ public class DataStoreBlobStore implemen return encodedBlobId; } - static class BlobId { + public static class BlobId { static final String SEP = "#"; + + public String getBlobId() { + return blobId; + } + final String blobId; final long length; @@ -672,7 +677,7 @@ public class DataStoreBlobStore implemen return encodedBlobId.contains(SEP); } - static BlobId of(String encodedValue) { + public static BlobId of(String encodedValue) { return new BlobId(encodedValue); } Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java?rev=1761852&r1=1761851&r2=1761852&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java (original) +++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java Thu Sep 22 05:38:16 2016 @@ -21,7 +21,6 @@ package org.apache.jackrabbit.oak.plugin import static com.google.common.collect.Sets.newHashSet; import static junit.framework.Assert.assertEquals; import static junit.framework.Assert.assertTrue; -import static org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils.cleanup; import static org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils.getBlobStore; import static org.hamcrest.CoreMatchers.instanceOf; @@ -30,7 +29,6 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; -import java.util.Date; import java.util.Map; import java.util.Random; import java.util.Set; @@ -73,7 +71,7 @@ public class SharedDataStoreUtilsTest { @Rule public TemporaryFolder folder = new TemporaryFolder(new File("target")); - DataStoreBlobStore dataStore; + protected DataStoreBlobStore dataStore; @BeforeClass public static void setUpBeforeClass() throws Exception { @@ -282,11 +280,6 @@ public class SharedDataStoreUtilsTest { @After public void close() throws IOException { FileUtils.cleanDirectory(new File(DataStoreUtils.getHomeDir())); - try { - cleanup(dataStore.getDataStore(), new Date()); - } catch (Exception e) { - log.error("Error closing data store", e); - } } } Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreUtils.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreUtils.java?rev=1761852&r1=1761851&r2=1761852&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreUtils.java (original) +++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreUtils.java Thu Sep 22 05:38:16 2016 @@ -17,24 +17,12 @@ package org.apache.jackrabbit.oak.plugins.blob.datastore; import java.io.File; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; import java.util.Map; -import java.util.Properties; -import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.amazonaws.services.s3.transfer.TransferManager; import com.google.common.collect.Maps; -import org.apache.jackrabbit.core.data.Backend; import org.apache.jackrabbit.core.data.DataStore; import org.apache.jackrabbit.core.data.FileDataStore; -import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3Backend; -import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3DataStore; -import org.apache.jackrabbit.oak.blob.cloud.aws.s3.Utils; + import org.apache.jackrabbit.oak.commons.PropertiesUtil; import org.junit.Test; import org.slf4j.Logger; @@ -81,13 +69,7 @@ public class DataStoreUtils { return getBlobStore(getHomeDir()); } - public static boolean isS3DataStore() { - String dsName = System.getProperty(DS_CLASS_NAME); - return (dsName != null) && (dsName.equals(S3DataStore.class.getName()) || dsName - .equals(SharedS3DataStore.class.getName())); - } - - private static Map<String, ?> getConfig() { + public static Map<String, ?> getConfig() { Map<String, Object> result = Maps.newHashMap(); for (Map.Entry<String, ?> e : Maps.fromProperties(System.getProperties()).entrySet()) { String key = e.getKey(); @@ -120,56 +102,4 @@ public class DataStoreUtils { DataStoreBlobStore dbs = getBlobStore(); assertEquals(1000, dbs.getDataStore().getMinRecordLength()); } - - /** - * S3 specific cleanup - * - * @param dataStore the underlying DataStore instance - * @param date the date of initialization - * @throws Exception - */ - public static void cleanup(DataStore dataStore, Date date) throws Exception { - if (dataStore instanceof S3DataStore) { - Backend backend = ((S3DataStore) dataStore).getBackend(); - if (backend instanceof S3Backend) { - String bucket = ((S3Backend) backend).getBucket(); - deleteBucket(bucket, date); - } - } - } - - private static void deleteBucket(String bucket, Date date) throws Exception { - log.info("cleaning bucket [" + bucket + "]"); - Properties props = Utils.readConfig((String) getConfig().get("config")); - AmazonS3Client s3service = Utils.openService(props); - TransferManager tmx = new TransferManager(s3service); - if (s3service.doesBucketExist(bucket)) { - for (int i = 0; i < 4; i++) { - tmx.abortMultipartUploads(bucket, date); - ObjectListing prevObjectListing = s3service.listObjects(bucket); - while (prevObjectListing != null ) { - List<DeleteObjectsRequest.KeyVersion> - deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>(); - for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) { - deleteList.add(new DeleteObjectsRequest.KeyVersion( - s3ObjSumm.getKey())); - } - if (deleteList.size() > 0) { - DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest( - bucket); - delObjsReq.setKeys(deleteList); - s3service.deleteObjects(delObjsReq); - } - if (!prevObjectListing.isTruncated()) break; - prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing); - } - } - //s3service.deleteBucket(bucket); - log.info("bucket [ " + bucket + "] cleaned"); - } else { - log.info("bucket [" + bucket + "] doesn't exists"); - } - tmx.shutdownNow(); - s3service.shutdown(); - } } Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java?rev=1761852&r1=1761851&r2=1761852&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java (original) +++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java Thu Sep 22 05:38:16 2016 @@ -68,8 +68,8 @@ import org.slf4j.LoggerFactory; public class SharedBlobStoreGCTest { private static final Logger log = LoggerFactory.getLogger(SharedBlobStoreGCTest.class); - private Cluster cluster1; - private Cluster cluster2; + protected Cluster cluster1; + protected Cluster cluster2; private Clock clock; @Before @@ -218,14 +218,13 @@ public class SharedBlobStoreGCTest { @After public void tearDown() throws Exception { - DataStoreUtils.cleanup(cluster1.getDataStore(), cluster1.getDate()); FileUtils.cleanDirectory((new File(DataStoreUtils.getHomeDir())).getParentFile()); DataStoreUtils.time = -1; cluster1.getDocumentNodeStore().dispose(); cluster2.getDocumentNodeStore().dispose(); } - class Cluster { + public class Cluster { private DocumentNodeStore ds; private int seed; private BlobGarbageCollector gc; @@ -294,10 +293,7 @@ public class SharedBlobStoreGCTest { VersionGarbageCollector vGC = ds.getVersionGarbageCollector(); VersionGCStats stats = vGC.gc(0, TimeUnit.MILLISECONDS); Assert.assertEquals(deletes.size(), stats.deletedDocGCCount); - - if (DataStoreUtils.isS3DataStore()) { - Thread.sleep(1000); - } + sleep(); } private HashSet<String> addNodeSpecialChars() throws Exception { @@ -342,5 +338,8 @@ public class SharedBlobStoreGCTest { return ds; } } + + protected void sleep() throws InterruptedException { + } } Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/ds/MongoDataStoreBlobGCTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/ds/MongoDataStoreBlobGCTest.java?rev=1761852&r1=1761851&r2=1761852&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/ds/MongoDataStoreBlobGCTest.java (original) +++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/ds/MongoDataStoreBlobGCTest.java Thu Sep 22 05:38:16 2016 @@ -35,8 +35,8 @@ import org.junit.BeforeClass; * */ public class MongoDataStoreBlobGCTest extends MongoBlobGCTest { - Date startDate; - DataStoreBlobStore blobStore; + protected Date startDate; + protected DataStoreBlobStore blobStore; @BeforeClass public static void setUpBeforeClass() throws Exception { @@ -61,7 +61,6 @@ public class MongoDataStoreBlobGCTest ex @After @Override public void tearDownConnection() throws Exception { - DataStoreUtils.cleanup(blobStore.getDataStore(), startDate); FileUtils.deleteDirectory(new File(DataStoreUtils.getHomeDir())); mk.dispose(); MongoUtils.dropCollections(connectionFactory.getConnection().getDB()); Modified: jackrabbit/oak/trunk/oak-it/pom.xml URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-it/pom.xml?rev=1761852&r1=1761851&r2=1761852&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-it/pom.xml (original) +++ jackrabbit/oak/trunk/oak-it/pom.xml Thu Sep 22 05:38:16 2016 @@ -51,6 +51,12 @@ </dependency> <dependency> <groupId>org.apache.jackrabbit</groupId> + <artifactId>oak-blob-cloud</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.jackrabbit</groupId> <artifactId>oak-core</artifactId> <version>${project.version}</version> <type>test-jar</type> @@ -72,6 +78,13 @@ </dependency> <dependency> <groupId>org.apache.jackrabbit</groupId> + <artifactId>oak-blob-cloud</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.jackrabbit</groupId> <artifactId>oak-commons</artifactId> <version>${project.version}</version> <scope>test</scope> @@ -125,5 +138,22 @@ <artifactId>commons-math3</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.sling</groupId> + <artifactId>org.apache.sling.testing.osgi-mock</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <version>1.10.19</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.googlecode.json-simple</groupId> + <artifactId>json-simple</artifactId> + <version>1.1.1</version> + <scope>test</scope> + </dependency> </dependencies> -</project> \ No newline at end of file +</project> Copied: jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java (from r1761799, jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java) URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java?p2=jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java&p1=jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java&r1=1761799&r2=1761852&rev=1761852&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java (original) +++ jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java Thu Sep 22 05:38:16 2016 @@ -20,9 +20,9 @@ package org.apache.jackrabbit.oak.plugin import java.util.Map; +import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3DataStoreStats; +import org.apache.jackrabbit.oak.blob.cloud.aws.s3.SharedS3DataStore; import org.apache.jackrabbit.oak.blob.cloud.aws.s3.stats.S3DataStoreStatsMBean; -import org.apache.jackrabbit.oak.plugins.blob.datastore.S3DataStoreStats; -import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedS3DataStore; import org.apache.jackrabbit.oak.spi.blob.BlobStore; import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.apache.jackrabbit.oak.stats.StatisticsProvider; Propchange: jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java ------------------------------------------------------------------------------ svn:eol-style = native Added: jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoS3DataStoreBlobGCTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoS3DataStoreBlobGCTest.java?rev=1761852&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoS3DataStoreBlobGCTest.java (added) +++ jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoS3DataStoreBlobGCTest.java Thu Sep 22 05:38:16 2016 @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.plugins.document; + +import org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils; +import org.apache.jackrabbit.oak.plugins.document.blob.ds.MongoDataStoreBlobGCTest; +import org.junit.After; +import org.junit.BeforeClass; + +import static org.junit.Assume.assumeTrue; + +/** + * Tests DataStoreGC with Mongo and S3 + */ +public class MongoS3DataStoreBlobGCTest extends MongoDataStoreBlobGCTest { + @BeforeClass + public static void assumptions() { + assumeTrue(S3DataStoreUtils.isS3DataStore()); + } + + @After + @Override + public void tearDownConnection() throws Exception { + S3DataStoreUtils.cleanup(blobStore.getDataStore(), startDate); + super.tearDownConnection(); + } +} Propchange: jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoS3DataStoreBlobGCTest.java ------------------------------------------------------------------------------ svn:eol-style = native Added: jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/S3SharedBlobStoreGCTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/S3SharedBlobStoreGCTest.java?rev=1761852&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/S3SharedBlobStoreGCTest.java (added) +++ jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/S3SharedBlobStoreGCTest.java Thu Sep 22 05:38:16 2016 @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.plugins.document; + +import org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils; +import org.junit.After; +import org.junit.BeforeClass; + +import static org.junit.Assume.assumeTrue; + +/** + * Shared BlobStoreGCTest for S3. + */ +public class S3SharedBlobStoreGCTest extends SharedBlobStoreGCTest { + @BeforeClass + public static void assumptions() { + assumeTrue(S3DataStoreUtils.isS3DataStore()); + } + + @After + public void tearDown() throws Exception { + S3DataStoreUtils.cleanup(cluster1.getDataStore(), cluster1.getDate()); + super.tearDown(); + } + + @Override + protected void sleep() throws InterruptedException { + if (S3DataStoreUtils.isS3DataStore()) { + Thread.sleep(1000); + } + } +} Propchange: jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/S3SharedBlobStoreGCTest.java ------------------------------------------------------------------------------ svn:eol-style = native Added: jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedS3DataStoreUtilsTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedS3DataStoreUtilsTest.java?rev=1761852&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedS3DataStoreUtilsTest.java (added) +++ jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedS3DataStoreUtilsTest.java Thu Sep 22 05:38:16 2016 @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.plugins.document; + +import java.io.IOException; +import java.util.Date; + +import org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils; +import org.apache.jackrabbit.oak.plugins.blob.SharedDataStoreUtilsTest; +import org.junit.After; +import org.junit.BeforeClass; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils.cleanup; +import static org.junit.Assume.assumeTrue; + +/** + * SharedDataStoreUtilsTest for S3. + */ +public class SharedS3DataStoreUtilsTest extends SharedDataStoreUtilsTest { + private static final Logger log = LoggerFactory.getLogger(SharedS3DataStoreUtilsTest.class); + + @BeforeClass + public static void assumptions() { + assumeTrue(S3DataStoreUtils.isS3DataStore()); + } + + @After + public void close() throws IOException { + super.close(); + try { + cleanup(dataStore.getDataStore(), new Date()); + } catch (Exception e) { + log.error("Error closing data store", e); + } + } +} Propchange: jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedS3DataStoreUtilsTest.java ------------------------------------------------------------------------------ svn:eol-style = native Added: jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreBlobGCIT.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreBlobGCIT.java?rev=1761852&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreBlobGCIT.java (added) +++ jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreBlobGCIT.java Thu Sep 22 05:38:16 2016 @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.segment; + +import org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils; +import org.junit.After; +import org.junit.BeforeClass; + +import static org.apache.jackrabbit.oak.commons.FixturesHelper.Fixture.SEGMENT_MK; +import static org.apache.jackrabbit.oak.commons.FixturesHelper.getFixtures; +import static org.junit.Assume.assumeTrue; + +/** + * Tests for SegmentNodeStore on S3DataStore GC + */ +public class SegmentS3DataStoreBlobGCIT extends SegmentDataStoreBlobGCIT { + + public SegmentS3DataStoreBlobGCIT(boolean usePersistedMap) { + super(usePersistedMap); + } + + @BeforeClass + public static void assumptions() { + assumeTrue(getFixtures().contains(SEGMENT_MK)); + assumeTrue(S3DataStoreUtils.isS3DataStore()); + } + + + @After + public void close() throws Exception { + super.close(); + S3DataStoreUtils.cleanup(blobStore.getDataStore(), startDate); + } +} + Propchange: jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreBlobGCIT.java ------------------------------------------------------------------------------ svn:eol-style = native Copied: jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java (from r1761799, jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java) URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java?p2=jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java&p1=jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java&r1=1761799&r2=1761852&rev=1761852&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java (original) +++ jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java Thu Sep 22 05:38:16 2016 @@ -21,9 +21,9 @@ package org.apache.jackrabbit.oak.plugin import java.io.File; import java.util.Map; +import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3DataStoreStats; +import org.apache.jackrabbit.oak.blob.cloud.aws.s3.SharedS3DataStore; import org.apache.jackrabbit.oak.blob.cloud.aws.s3.stats.S3DataStoreStatsMBean; -import org.apache.jackrabbit.oak.plugins.blob.datastore.S3DataStoreStats; -import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedS3DataStore; import org.apache.jackrabbit.oak.spi.blob.BlobStore; import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.apache.jackrabbit.oak.stats.StatisticsProvider; @@ -37,6 +37,8 @@ import org.junit.rules.TemporaryFolder; import org.osgi.framework.ServiceRegistration; import static com.google.common.collect.Maps.newHashMap; +import static org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStoreService.CUSTOM_BLOB_STORE; +import static org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.DIRECTORY; import static org.apache.sling.testing.mock.osgi.MockOsgi.deactivate; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; @@ -104,8 +106,8 @@ public class SegmentS3DataStoreStatsTest private void registerSegmentNodeStoreService(boolean customBlobStore) { Map<String, Object> properties = newHashMap(); - properties.put(SegmentNodeStoreService.CUSTOM_BLOB_STORE, customBlobStore); - properties.put(SegmentNodeStoreService.DIRECTORY, folder.getRoot().getAbsolutePath()); + properties.put(CUSTOM_BLOB_STORE, customBlobStore); + properties.put(DIRECTORY, folder.getRoot().getAbsolutePath()); segmentNodeStoreService = context.registerInjectActivateService(new SegmentNodeStoreService(), properties); } Added: jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreBlobGCIT.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreBlobGCIT.java?rev=1761852&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreBlobGCIT.java (added) +++ jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreBlobGCIT.java Thu Sep 22 05:38:16 2016 @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.segment; + +import org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils; +import org.junit.After; +import org.junit.BeforeClass; + +import static org.apache.jackrabbit.oak.commons.FixturesHelper.Fixture.SEGMENT_TAR; +import static org.apache.jackrabbit.oak.commons.FixturesHelper.getFixtures; +import static org.junit.Assume.assumeTrue; + +/** + * Tests for SegmentNodeStore on S3DataStore GC + */ +public class SegmentS3DataStoreBlobGCIT extends SegmentDataStoreBlobGCIT { + + @BeforeClass + public static void assumptions() { + assumeTrue(getFixtures().contains(SEGMENT_TAR)); + assumeTrue(S3DataStoreUtils.isS3DataStore()); + } + + + @After + public void close() throws Exception { + super.close(); + S3DataStoreUtils.cleanup(blobStore.getDataStore(), startDate); + } +} + Propchange: jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreBlobGCIT.java ------------------------------------------------------------------------------ svn:eol-style = native Copied: jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreStatsTest.java (from r1761799, jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java) URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreStatsTest.java?p2=jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreStatsTest.java&p1=jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java&r1=1761799&r2=1761852&rev=1761852&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java (original) +++ jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreStatsTest.java Thu Sep 22 05:38:16 2016 @@ -16,14 +16,14 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.jackrabbit.oak.plugins.segment; +package org.apache.jackrabbit.oak.segment; import java.io.File; import java.util.Map; +import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3DataStoreStats; +import org.apache.jackrabbit.oak.blob.cloud.aws.s3.SharedS3DataStore; import org.apache.jackrabbit.oak.blob.cloud.aws.s3.stats.S3DataStoreStatsMBean; -import org.apache.jackrabbit.oak.plugins.blob.datastore.S3DataStoreStats; -import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedS3DataStore; import org.apache.jackrabbit.oak.spi.blob.BlobStore; import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.apache.jackrabbit.oak.stats.StatisticsProvider; @@ -37,6 +37,8 @@ import org.junit.rules.TemporaryFolder; import org.osgi.framework.ServiceRegistration; import static com.google.common.collect.Maps.newHashMap; +import static org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.CUSTOM_BLOB_STORE; +import static org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.DIRECTORY; import static org.apache.sling.testing.mock.osgi.MockOsgi.deactivate; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; @@ -99,13 +101,14 @@ public class SegmentS3DataStoreStatsTest unregisterBlobStore(); } - private SegmentNodeStoreService segmentNodeStoreService; + private SegmentNodeStoreService + segmentNodeStoreService; private void registerSegmentNodeStoreService(boolean customBlobStore) { Map<String, Object> properties = newHashMap(); - properties.put(SegmentNodeStoreService.CUSTOM_BLOB_STORE, customBlobStore); - properties.put(SegmentNodeStoreService.DIRECTORY, folder.getRoot().getAbsolutePath()); + properties.put(CUSTOM_BLOB_STORE, customBlobStore); + properties.put(DIRECTORY, folder.getRoot().getAbsolutePath()); segmentNodeStoreService = context.registerInjectActivateService(new SegmentNodeStoreService(), properties); } Propchange: jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/segment/SegmentS3DataStoreStatsTest.java ------------------------------------------------------------------------------ svn:eol-style = native Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java?rev=1761852&r1=1761851&r2=1761852&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java (original) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java Thu Sep 22 05:38:16 2016 @@ -44,7 +44,7 @@ import org.apache.jackrabbit.core.data.D import org.apache.jackrabbit.core.data.DataStoreException; import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore; import org.apache.jackrabbit.oak.plugins.blob.datastore.OakFileDataStore; -import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedS3DataStore; +import org.apache.jackrabbit.oak.blob.cloud.aws.s3.SharedS3DataStore; import org.apache.jackrabbit.oak.plugins.document.DocumentMK; import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore; import org.apache.jackrabbit.oak.plugins.document.util.MongoConnection; Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java?rev=1761852&r1=1761851&r2=1761852&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java Thu Sep 22 05:38:16 2016 @@ -443,7 +443,6 @@ public class SegmentDataStoreBlobGCIT { if (store != null) { store.close(); } - DataStoreUtils.cleanup(blobStore.getDataStore(), startDate); } static InputStream randomStream(int seed, int size) { Modified: jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java?rev=1761852&r1=1761851&r2=1761852&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java (original) +++ jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java Thu Sep 22 05:38:16 2016 @@ -461,7 +461,6 @@ public class SegmentDataStoreBlobGCIT { if (store != null) { store.close(); } - DataStoreUtils.cleanup(blobStore.getDataStore(), startDate); } static InputStream randomStream(int seed, int size) { @@ -474,7 +473,7 @@ public class SegmentDataStoreBlobGCIT { /** * Waits for some time and adds additional blobs after blob referenced identified to simulate * long running blob id collection phase. - */ + */ class TestGarbageCollector extends MarkSweepGarbageCollector { long maxLastModifiedInterval; String root;