Author: tomekr
Date: Tue Mar 20 10:54:26 2018
New Revision: 1827293
URL: http://svn.apache.org/viewvc?rev=1827293&view=rev
Log:
OAK-7297: New fixture for the Azure Segment Store
Modified:
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FixturesHelper.java
jackrabbit/oak/trunk/oak-it/pom.xml
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/NodeStoreFixtures.java
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/spi/state/NodeStoreTest.java
jackrabbit/oak/trunk/oak-jcr/pom.xml
jackrabbit/oak/trunk/oak-parent/pom.xml
jackrabbit/oak/trunk/oak-run-commons/pom.xml
jackrabbit/oak/trunk/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/OakFixture.java
jackrabbit/oak/trunk/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java
jackrabbit/oak/trunk/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/SegmentTarFixture.java
Modified:
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java?rev=1827293&r1=1827292&r2=1827293&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
(original)
+++
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
Tue Mar 20 10:54:26 2018
@@ -83,6 +83,14 @@ public class BenchmarkRunner {
.withOptionalArg().defaultsTo("");
OptionSpec<String> rdbjdbctableprefix =
parser.accepts("rdbjdbctableprefix", "RDB JDBC table prefix")
.withOptionalArg().defaultsTo("");
+
+ OptionSpec<String> azureConnectionString = parser.accepts("azure",
"Azure Connection String")
+
.withOptionalArg().defaultsTo("DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;";);
+ OptionSpec<String> azureContainerName =
parser.accepts("azureContainerName", "Azure container name")
+ .withOptionalArg().defaultsTo("oak");
+ OptionSpec<String> azureRootPath = parser.accepts("azureRootPath",
"Azure root path")
+ .withOptionalArg().defaultsTo("/oak");
+
OptionSpec<Boolean> mmap = parser.accepts("mmap", "TarMK memory
mapping")
.withOptionalArg().ofType(Boolean.class)
.defaultsTo("64".equals(System.getProperty("sun.arch.data.model")));
@@ -230,6 +238,11 @@ public class BenchmarkRunner {
mmap.value(options), coldUseDataStore.value(options),
fdsCache.value(options),
coldSyncInterval.value(options),
coldShareDataStore.value(options), coldSecure.value(options),
coldOneShotRun.value(options)),
+
OakRepositoryFixture.getSegmentTarWithAzureSegmentStore(base.value(options),
+ azureConnectionString.value(options),
+ azureContainerName.value(options),
+ azureRootPath.value(options),
+ 256, cacheSize, true, fdsCache.value(options)),
OakRepositoryFixture.getRDB(rdbjdbcuri.value(options),
rdbjdbcuser.value(options),
rdbjdbcpasswd.value(options),
rdbjdbctableprefix.value(options),
dropDBAfterTest.value(options), cacheSize * MB,
vgcMaxAge.value(options)),
Modified:
jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FixturesHelper.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FixturesHelper.java?rev=1827293&r1=1827292&r2=1827293&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FixturesHelper.java
(original)
+++
jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FixturesHelper.java
Tue Mar 20 10:54:26 2018
@@ -43,7 +43,7 @@ public final class FixturesHelper {
* default fixtures when no {@code nsfixtures} is provided
*/
public enum Fixture {
- DOCUMENT_NS, @Deprecated SEGMENT_MK, DOCUMENT_RDB, MEMORY_NS,
DOCUMENT_MEM, SEGMENT_TAR, COMPOSITE_SEGMENT, COMPOSITE_MEM, COW_DOCUMENT
+ DOCUMENT_NS, @Deprecated SEGMENT_MK, DOCUMENT_RDB, MEMORY_NS,
DOCUMENT_MEM, SEGMENT_TAR, SEGMENT_AZURE, COMPOSITE_SEGMENT, COMPOSITE_MEM,
COW_DOCUMENT
}
private static final Set<Fixture> FIXTURES;
Modified: jackrabbit/oak/trunk/oak-it/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-it/pom.xml?rev=1827293&r1=1827292&r2=1827293&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-it/pom.xml (original)
+++ jackrabbit/oak/trunk/oak-it/pom.xml Tue Mar 20 10:54:26 2018
@@ -64,6 +64,12 @@
</dependency>
<dependency>
<groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-segment-azure</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
<artifactId>oak-store-document</artifactId>
<version>${project.version}</version>
<scope>test</scope>
@@ -114,6 +120,13 @@
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-segment-azure</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.jackrabbit</groupId>
Modified:
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/NodeStoreFixtures.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/NodeStoreFixtures.java?rev=1827293&r1=1827292&r2=1827293&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/NodeStoreFixtures.java
(original)
+++
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/NodeStoreFixtures.java
Tue Mar 20 10:54:26 2018
@@ -31,6 +31,7 @@ import org.apache.jackrabbit.oak.fixture
import org.apache.jackrabbit.oak.fixture.NodeStoreFixture;
import org.apache.jackrabbit.oak.composite.CompositeMemoryStoreFixture;
import org.apache.jackrabbit.oak.composite.CompositeSegmentStoreFixture;
+import org.apache.jackrabbit.oak.segment.azure.fixture.SegmentAzureFixture;
import org.apache.jackrabbit.oak.segment.fixture.SegmentTarFixture;
public class NodeStoreFixtures {
@@ -39,6 +40,8 @@ public class NodeStoreFixtures {
public static final NodeStoreFixture SEGMENT_TAR = new SegmentTarFixture();
+ public static final NodeStoreFixture SEGMENT_AZURE = new
SegmentAzureFixture();
+
public static final NodeStoreFixture DOCUMENT_NS = new
DocumentMongoFixture();
public static final NodeStoreFixture DOCUMENT_RDB = new
DocumentRdbFixture();
@@ -68,6 +71,9 @@ public class NodeStoreFixtures {
if (fixtures.contains(FixturesHelper.Fixture.SEGMENT_TAR)) {
configuredFixtures.add(SEGMENT_TAR);
}
+ if (fixtures.contains(FixturesHelper.Fixture.SEGMENT_AZURE)) {
+ configuredFixtures.add(SEGMENT_AZURE);
+ }
if (fixtures.contains(FixturesHelper.Fixture.COMPOSITE_SEGMENT)) {
configuredFixtures.add(COMPOSITE_SEGMENT);
}
Modified:
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/spi/state/NodeStoreTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/spi/state/NodeStoreTest.java?rev=1827293&r1=1827292&r2=1827293&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/spi/state/NodeStoreTest.java
(original)
+++
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/spi/state/NodeStoreTest.java
Tue Mar 20 10:54:26 2018
@@ -458,7 +458,7 @@ public class NodeStoreTest extends OakBa
NodeBuilder x = test.getChildNode("x");
if (fixture == NodeStoreFixtures.SEGMENT_TAR || fixture ==
NodeStoreFixtures.MEMORY_NS
|| fixture == NodeStoreFixtures.COMPOSITE_MEM || fixture ==
NodeStoreFixtures.COMPOSITE_SEGMENT
- || fixture == NodeStoreFixtures.COW_DOCUMENT) {
+ || fixture == NodeStoreFixtures.COW_DOCUMENT || fixture ==
NodeStoreFixtures.SEGMENT_AZURE) {
assertTrue(x.moveTo(x, "xx"));
assertFalse(x.exists());
assertFalse(test.hasChildNode("x"));
Modified: jackrabbit/oak/trunk/oak-jcr/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-jcr/pom.xml?rev=1827293&r1=1827292&r2=1827293&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-jcr/pom.xml (original)
+++ jackrabbit/oak/trunk/oak-jcr/pom.xml Tue Mar 20 10:54:26 2018
@@ -263,6 +263,12 @@
</dependency>
<dependency>
<groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-segment-azure</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
<artifactId>oak-store-document</artifactId>
<version>${project.version}</version>
<scope>test</scope>
@@ -292,6 +298,13 @@
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-segment-azure</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.jackrabbit</groupId>
Modified: jackrabbit/oak/trunk/oak-parent/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-parent/pom.xml?rev=1827293&r1=1827292&r2=1827293&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-parent/pom.xml (original)
+++ jackrabbit/oak/trunk/oak-parent/pom.xml Tue Mar 20 10:54:26 2018
@@ -66,7 +66,7 @@
<!-- specifies on which fixture to run the integration testing tests.
override in profiles or provide from command line to change behaviour.
Provide
more fixtures space separated. See
org.apache.jackrabbit.oak.jcr.FixturesHelper#AVAILABLE_FIXTURES
- for the possible values: SEGMENT_MK SEGMENT_TAR DOCUMENT_NS DOCUMENT_RDB
-->
+ for the possible values: SEGMENT_MK SEGMENT_TAR SEGMENT_AZURE
DOCUMENT_NS DOCUMENT_RDB -->
<fixtures>SEGMENT_TAR</fixtures>
<!-- whether skip the surefire unit testing during the integration testing.
Modified: jackrabbit/oak/trunk/oak-run-commons/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run-commons/pom.xml?rev=1827293&r1=1827292&r2=1827293&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run-commons/pom.xml (original)
+++ jackrabbit/oak/trunk/oak-run-commons/pom.xml Tue Mar 20 10:54:26 2018
@@ -59,6 +59,11 @@
</dependency>
<dependency>
<groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-segment-azure</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
<artifactId>oak-store-document</artifactId>
<version>${project.version}</version>
</dependency>
Modified:
jackrabbit/oak/trunk/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/OakFixture.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/OakFixture.java?rev=1827293&r1=1827292&r2=1827293&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/OakFixture.java
(original)
+++
jackrabbit/oak/trunk/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/OakFixture.java
Tue Mar 20 10:54:26 2018
@@ -66,6 +66,7 @@ public abstract class OakFixture {
public static final String OAK_RDB_DS = "Oak-RDB-DS";
public static final String OAK_SEGMENT_TAR = "Oak-Segment-Tar";
+ public static final String OAK_SEGMENT_AZURE = "Oak-Segment-Azure";
public static final String OAK_SEGMENT_TAR_DS = "Oak-Segment-Tar-DS";
public static final String OAK_SEGMENT_TAR_COLD = "Oak-Segment-Tar-Cold";
@@ -346,6 +347,17 @@ public abstract class OakFixture {
dsCacheInMB, true, syncInterval, shareBlobStore, secure,
oneShotRun);
}
+ public static OakFixture getSegmentTarWithAzureSegmentStore(final File
base, final String azureConnectionString, final String azureContainerName,
final String azureRootPath,
+ final int
maxFileSizeMB, final int cacheSizeMB, final boolean useBlobStore, final int
dsCacheInMB) {
+ return SegmentTarFixtureBuilder
+ .segmentTarFixtureBuilder(OakFixture.OAK_SEGMENT_AZURE, base)
+ .withAzure(azureConnectionString, azureContainerName,
azureRootPath)
+ .withMaxFileSize(maxFileSizeMB)
+ .withSegmentCacheSize(cacheSizeMB)
+ .withBlobStore(useBlobStore)
+ .withDSCacheSize(dsCacheInMB).build();
+ }
+
public static OakFixture getCompositeStore(final String name, final File
base,
final int maxFileSizeMB, final
int cacheSizeMB, final boolean memoryMapping,
final int mounts, final int
pathsPerMount) {
Modified:
jackrabbit/oak/trunk/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java?rev=1827293&r1=1827292&r2=1827293&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java
(original)
+++
jackrabbit/oak/trunk/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java
Tue Mar 20 10:54:26 2018
@@ -113,6 +113,14 @@ public class OakRepositoryFixture implem
memoryMapping, useBlobStore, dsCacheInMB, syncInterval,
shareBlobStore, secure, oneShotRun));
}
+ public static RepositoryFixture getSegmentTarWithAzureSegmentStore(final
File base, final String azureConnectionString,
+ final
String azureContainerName, final String azureRootPath,
+ final
int maxFileSizeMB, final int cacheSizeMB, final boolean useBlobStore,
+ final
int dsCacheInMB) {
+ return new
OakRepositoryFixture(OakFixture.getSegmentTarWithAzureSegmentStore(base,
azureConnectionString,
+ azureContainerName, azureRootPath, maxFileSizeMB, cacheSizeMB,
useBlobStore, dsCacheInMB));
+ }
+
public static RepositoryFixture getCompositeStore(File base, int
maxFileSizeMB, int cacheSizeMB,
final boolean
memoryMapping, int mounts, int pathsPerMount) {
return new
OakRepositoryFixture(OakFixture.getCompositeStore(OakFixture.OAK_COMPOSITE_STORE,
Modified:
jackrabbit/oak/trunk/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/SegmentTarFixture.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/SegmentTarFixture.java?rev=1827293&r1=1827292&r2=1827293&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/SegmentTarFixture.java
(original)
+++
jackrabbit/oak/trunk/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/SegmentTarFixture.java
Tue Mar 20 10:54:26 2018
@@ -30,6 +30,9 @@ import java.util.concurrent.TimeUnit;
import javax.annotation.Nonnull;
import com.google.common.base.StandardSystemProperty;
+import com.microsoft.azure.storage.CloudStorageAccount;
+import com.microsoft.azure.storage.blob.CloudBlobContainer;
+import com.microsoft.azure.storage.blob.CloudBlobDirectory;
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.core.data.FileDataStore;
import org.apache.jackrabbit.oak.Oak;
@@ -38,6 +41,7 @@ import org.apache.jackrabbit.oak.segment
import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
import org.apache.jackrabbit.oak.segment.SegmentNotFoundException;
import org.apache.jackrabbit.oak.segment.SegmentNotFoundExceptionListener;
+import org.apache.jackrabbit.oak.segment.azure.AzurePersistence;
import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
@@ -68,6 +72,9 @@ public class SegmentTarFixture extends O
private boolean memoryMapping;
private boolean useBlobStore;
private int dsCacheSize;
+ private String azureConnectionString;
+ private String azureContainerName;
+ private String azureRootPath;
public static SegmentTarFixtureBuilder segmentTarFixtureBuilder(String
name, File directory) {
return new SegmentTarFixtureBuilder(name, directory);
@@ -102,6 +109,13 @@ public class SegmentTarFixture extends O
this.dsCacheSize = dsCacheSize;
return this;
}
+
+ public SegmentTarFixtureBuilder withAzure(String
azureConnectionString, String azureContainerName, String azureRootPath) {
+ this.azureConnectionString = azureConnectionString;
+ this.azureContainerName = azureContainerName;
+ this.azureRootPath = azureRootPath;
+ return this;
+ }
public SegmentTarFixture build() {
return new SegmentTarFixture(this);
@@ -120,7 +134,11 @@ public class SegmentTarFixture extends O
private final boolean shareBlobStore;
private final boolean oneShotRun;
private final boolean secure;
-
+
+ private final String azureConnectionString;
+ private final String azureContainerName;
+ private final String azureRootPath;
+
private final File parentPath;
private FileStore[] stores;
@@ -149,6 +167,9 @@ public class SegmentTarFixture extends O
this.memoryMapping = builder.memoryMapping;
this.useBlobStore = builder.useBlobStore;
this.dsCacheSize = builder.dsCacheSize;
+ this.azureConnectionString = builder.azureConnectionString;
+ this.azureContainerName = builder.azureContainerName;
+ this.azureRootPath = builder.azureRootPath;
this.withColdStandby = withColdStandby;
this.syncInterval = syncInterval;
@@ -163,7 +184,14 @@ public class SegmentTarFixture extends O
.withMaxFileSize(maxFileSize)
.withSegmentCacheSize(segmentCacheSize)
.withMemoryMapping(memoryMapping);
-
+
+ if (azureConnectionString != null) {
+ CloudStorageAccount cloud =
CloudStorageAccount.parse(azureConnectionString);
+ CloudBlobContainer container =
cloud.createCloudBlobClient().getContainerReference(azureContainerName);
+ container.createIfNotExists();
+ CloudBlobDirectory directory =
container.getDirectoryReference(azureRootPath);
+ fileStoreBuilder.withCustomPersistence(new
AzurePersistence(directory));
+ }
if (useBlobStore) {
FileDataStore fds = new FileDataStore();
@@ -192,6 +220,15 @@ public class SegmentTarFixture extends O
}
FileStoreBuilder builder = fileStoreBuilder(new File(parentPath,
"primary-" + i));
+
+ if (azureConnectionString != null) {
+ CloudStorageAccount cloud =
CloudStorageAccount.parse(azureConnectionString);
+ CloudBlobContainer container =
cloud.createCloudBlobClient().getContainerReference(azureContainerName);
+ container.createIfNotExists();
+ CloudBlobDirectory directory =
container.getDirectoryReference(azureRootPath + "/primary-" + i);
+ builder.withCustomPersistence(new AzurePersistence(directory));
+ }
+
if (blobStore != null) {
builder.withBlobStore(blobStore);
}
