Author: amitj
Date: Wed May 3 05:44:52 2017
New Revision: 1793611
URL: http://svn.apache.org/viewvc?rev=1793611&view=rev
Log:
OAK-4933: Create a data store implementation that integrates with Microsoft
Azure Blob Storage
Merged r1788389 from trunk
Modified:
jackrabbit/oak/branches/1.6/ (props changed)
jackrabbit/oak/branches/1.6/oak-run/pom.xml
jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/BlobStoreFixture.java
jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java
jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
Propchange: jackrabbit/oak/branches/1.6/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 3 05:44:52 2017
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1781068,1781075,1781248,1781386,1781846,1781907,1782000,1782029,1782196,1782447,1782476,1782770,1782945,1782966,1782973,1782990,1783061,1783066,1783089,1783104-1783105,1783619,1783720,1783731,1783733,1783738,1783742,1783773,1783855,1783891,1784023,1784130,1784162,1784251,1784401,1784551,1784574,1785095,1785108,1785283,1785838,1785919,1785946,1787074,1787217,1788378,1788387-1788388,1790382,1792463,1792742,1793088
+/jackrabbit/oak/trunk:1781068,1781075,1781248,1781386,1781846,1781907,1782000,1782029,1782196,1782447,1782476,1782770,1782945,1782966,1782973,1782990,1783061,1783066,1783089,1783104-1783105,1783619,1783720,1783731,1783733,1783738,1783742,1783773,1783855,1783891,1784023,1784130,1784162,1784251,1784401,1784551,1784574,1785095,1785108,1785283,1785838,1785919,1785946,1787074,1787217,1788378,1788387-1788389,1790382,1792463,1792742,1793088
/jackrabbit/trunk:1345480
Modified: jackrabbit/oak/branches/1.6/oak-run/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-run/pom.xml?rev=1793611&r1=1793610&r2=1793611&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-run/pom.xml (original)
+++ jackrabbit/oak/branches/1.6/oak-run/pom.xml Wed May 3 05:44:52 2017
@@ -383,6 +383,13 @@
</dependency>
<dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-blob-cloud-azure</artifactId>
+ <version>${project.version}</version>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.sling</groupId>
<artifactId>org.apache.sling.testing.osgi-mock</artifactId>
<scope>compile</scope>
Modified:
jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/BlobStoreFixture.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/BlobStoreFixture.java?rev=1793611&r1=1793610&r2=1793611&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/BlobStoreFixture.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/BlobStoreFixture.java
Wed May 3 05:44:52 2017
@@ -45,7 +45,7 @@ import org.apache.jackrabbit.oak.stats.S
import static com.google.common.base.Preconditions.checkNotNull;
import static org.apache.jackrabbit.oak.fixture.DataStoreUtils.cleanup;
-import static
org.apache.jackrabbit.oak.fixture.DataStoreUtils.configureIfS3DataStore;
+import static
org.apache.jackrabbit.oak.fixture.DataStoreUtils.configureIfCloudDataStore;
public abstract class BlobStoreFixture implements Closeable{
private final String name;
@@ -85,7 +85,7 @@ public abstract class BlobStoreFixture i
String className = System.getProperty("dataStore");
if (className != null) {
- return getDataStore(basedir, fdsCacheInMB);
+ return getDataStore(basedir, fdsCacheInMB, statisticsProvider);
}
String blobStore = System.getProperty("blobStoreType");
@@ -178,7 +178,8 @@ public abstract class BlobStoreFixture i
};
}
- public static BlobStoreFixture getDataStore(final File basedir, final int
fdsCacheInMB) {
+ public static BlobStoreFixture getDataStore(final File basedir, final int
fdsCacheInMB,
+ final StatisticsProvider
statisticsProvider) {
return new BlobStoreFixture("DS") {
private DataStore dataStore;
private BlobStore blobStore;
@@ -193,7 +194,8 @@ public abstract class BlobStoreFixture i
dataStore =
Class.forName(className).asSubclass(DataStore.class).newInstance();
config = getConfig();
configure(dataStore, config);
- dataStore = configureIfS3DataStore(className, dataStore,
config, unique.toLowerCase());
+
+ dataStore = configureIfCloudDataStore(className,
dataStore, config, unique.toLowerCase(), statisticsProvider);
storeDir = new File(basedir, unique);
dataStore.init(storeDir.getAbsolutePath());
blobStore = new DataStoreBlobStore(dataStore, true,
fdsCacheInMB);
@@ -209,7 +211,7 @@ public abstract class BlobStoreFixture i
if (blobStore instanceof DataStoreBlobStore) {
try {
((DataStoreBlobStore) blobStore).close();
- cleanup(storeDir, config);
+ cleanup(storeDir, config, unique.toLowerCase());
} catch (Exception e) {
throw new RuntimeException(e);
}
Modified:
jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java?rev=1793611&r1=1793610&r2=1793611&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java
Wed May 3 05:44:52 2017
@@ -31,61 +31,84 @@ import com.amazonaws.services.s3.model.O
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.google.common.base.Strings;
+import com.microsoft.azure.storage.blob.CloudBlobContainer;
+
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.core.data.DataStore;
import org.apache.jackrabbit.oak.blob.cloud.aws.s3.SharedS3DataStore;
+import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants;
+import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureDataStore;
import org.apache.jackrabbit.oak.blob.cloud.s3.S3Constants;
import org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStore;
import org.apache.jackrabbit.oak.blob.cloud.s3.Utils;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Extension to {@link DataStoreUtils} to enable S3 extensions for cleaning
and initialization.
+ * Extension to {@link DataStoreUtils} to enable S3 / AzureBlob extensions for
cleaning and initialization.
*/
public class DataStoreUtils {
private static final Logger log =
LoggerFactory.getLogger(DataStoreUtils.class);
private static Class JR2_S3 = SharedS3DataStore.class;
private static Class S3 = S3DataStore.class;
+ private static Class AZURE = AzureDataStore.class;
public static boolean isS3DataStore(String dsName) {
return (dsName != null) && (dsName.equals(S3.getName()) ||
dsName.equals(JR2_S3.getName()));
}
- public static DataStore configureIfS3DataStore(String className, DataStore
ds,
- Map<String, ?> config, String bucket) throws Exception {
+ public static boolean isAzureDataStore(String dsName) {
+ return (dsName != null) &&
+ (dsName.equals(AZURE.getName()));
+ }
+
+ public static DataStore configureIfCloudDataStore(String className,
DataStore ds,
+ Map<String, ?> config,
String bucket,
+ StatisticsProvider
statisticsProvider) throws Exception {
// Add bucket info
Properties props = new Properties();
props.putAll(config);
- props.setProperty(S3Constants.S3_BUCKET, bucket);
- // Set the props object
- if (S3.getName().equals(className)) {
- ((S3DataStore) ds).setProperties(props);
- } else
- if (JR2_S3.getName().equals(className)) {
- ((org.apache.jackrabbit.oak.blob.cloud.aws.s3.SharedS3DataStore)
ds).setProperties(props);
+ log.info("Using bucket [ {} ]", bucket);
+
+ if (isS3DataStore(className)) {
+ props.setProperty(S3Constants.S3_BUCKET, bucket);
+
+ // Set the props object
+ if (S3.getName().equals(className)) {
+ ((S3DataStore) ds).setProperties(props);
+ ((S3DataStore) ds).setStatisticsProvider(statisticsProvider);
+ } else if (JR2_S3.getName().equals(className)) {
+
((org.apache.jackrabbit.oak.blob.cloud.aws.s3.SharedS3DataStore)
ds).setProperties(props);
+ }
+ } else if (isAzureDataStore(className)) {
+ props.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME,
bucket);
+ ((AzureDataStore) ds).setProperties(props);
+ ((AzureDataStore) ds).setStatisticsProvider(statisticsProvider);
}
+
return ds;
}
/**
- * Clean directory and if S3 bucket configured delete that.
+ * Clean directory and if S3 bucket/Azure container is configured delete
that.
*
* @param storeDir the local directory
* @param config the datastore config
+ * @param bucket the S3 bucket name / Azure container name
* @throws Exception
*/
- public static void cleanup(File storeDir, Map<String, ?> config) throws
Exception {
+ public static void cleanup(File storeDir, Map<String, ?> config, String
bucket) throws Exception {
FileUtils.deleteQuietly(storeDir);
- String bucket = null;
if (config.containsKey(S3Constants.S3_BUCKET)) {
- bucket = (String) config.get(S3Constants.S3_BUCKET);
if (!Strings.isNullOrEmpty(bucket)) {
deleteBucket(bucket, config, new Date());
}
+ } else if
(config.containsKey(AzureConstants.AZURE_BLOB_CONTAINER_NAME)) {
+ deleteAzureContainer(config, bucket);
}
}
@@ -124,4 +147,22 @@ public class DataStoreUtils {
tmx.shutdownNow();
s3service.shutdown();
}
+
+ public static void deleteAzureContainer(Map<String, ?> config, String
containerName) throws Exception {
+ String accountName =
(String)config.get(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME);
+ String accountKey =
(String)config.get(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY);
+ if (Strings.isNullOrEmpty(containerName) ||
+ Strings.isNullOrEmpty(accountName) ||
+ Strings.isNullOrEmpty(accountKey)) {
+ return;
+ }
+ log.info("deleting container [" + containerName + "]");
+ CloudBlobContainer container =
org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.Utils
+
.getBlobContainer(org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.Utils.getConnectionString(accountName,
accountKey), containerName);
+ if (container.deleteIfExists()) {
+ log.info("container [ " + containerName + "] deleted");
+ } else {
+ log.info("container [" + containerName + "] doesn't exists");
+ }
+ }
}
Modified:
jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java?rev=1793611&r1=1793610&r2=1793611&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
Wed May 3 05:44:52 2017
@@ -79,7 +79,7 @@ public class DataStoreCheckCommand imple
String helpStr =
"datastorecheck [--id] [--ref] [--consistency] [--store
<path>|<mongo_uri>] "
- + "[--s3ds <s3ds_config>|--fds <fds_config>] [--dump <path>]";
+ + "[--s3ds <s3ds_config>|--fds <fds_config>|--azureblobds
<azureblobds_config>] [--dump <path>]";
Closer closer = Closer.create();
try {
Modified:
jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java?rev=1793611&r1=1793610&r2=1793611&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
Wed May 3 05:44:52 2017
@@ -21,6 +21,7 @@ import static java.util.Arrays.asList;
import static org.apache.jackrabbit.oak.commons.PropertiesUtil.populate;
import java.io.Closeable;
+import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Dictionary;
@@ -31,8 +32,10 @@ import java.util.Properties;
import javax.annotation.Nullable;
import javax.jcr.RepositoryException;
+import org.apache.commons.io.FileUtils;
import com.google.common.collect.Maps;
import com.google.common.io.Closer;
+import com.google.common.io.Files;
import com.mongodb.MongoClientURI;
import com.mongodb.MongoURI;
import joptsimple.ArgumentAcceptingOptionSpec;
@@ -42,9 +45,10 @@ import joptsimple.OptionSpec;
import org.apache.felix.cm.file.ConfigurationHandler;
import org.apache.jackrabbit.core.data.DataStore;
import org.apache.jackrabbit.core.data.DataStoreException;
+import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureDataStore;
+import org.apache.jackrabbit.oak.blob.cloud.aws.s3.SharedS3DataStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.OakFileDataStore;
-import org.apache.jackrabbit.oak.blob.cloud.aws.s3.SharedS3DataStore;
import org.apache.jackrabbit.oak.plugins.document.DocumentMK;
import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
import org.apache.jackrabbit.oak.plugins.document.util.MongoConnection;
@@ -131,10 +135,13 @@ class Utils {
parser.accepts("s3ds", "S3DataStore
config").withRequiredArg().ofType(String.class);
ArgumentAcceptingOptionSpec<String> fdsConfig =
parser.accepts("fds", "FileDataStore
config").withRequiredArg().ofType(String.class);
+ ArgumentAcceptingOptionSpec<String> azureBlobDSConfig =
+ parser.accepts("azureblobds", "AzureBlobStorageDataStore
config").withRequiredArg().ofType(String.class);
+
OptionSet options = parser.parse(args);
- if (!options.has(s3dsConfig) && !options.has(fdsConfig)) {
+ if (!options.has(s3dsConfig) && !options.has(fdsConfig) &&
!options.has(azureBlobDSConfig)) {
return null;
}
@@ -146,6 +153,15 @@ class Utils {
s3ds.setProperties(props);
s3ds.init(null);
delegate = s3ds;
+ } else if (options.has(azureBlobDSConfig)) {
+ AzureDataStore azureds = new AzureDataStore();
+ String cfgPath = azureBlobDSConfig.value(options);
+ Properties props = loadAndTransformProps(cfgPath);
+ azureds.setProperties(props);
+ File homeDir = Files.createTempDir();
+ azureds.init(homeDir.getAbsolutePath());
+ closer.register(asCloseable(homeDir));
+ delegate = azureds;
} else {
delegate = new OakFileDataStore();
String cfgPath = fdsConfig.value(options);
@@ -202,6 +218,16 @@ class Utils {
}
};
}
+
+ static Closeable asCloseable(final File dir) {
+ return new Closeable() {
+
+ @Override
+ public void close() throws IOException {
+ FileUtils.deleteDirectory(dir);
+ }
+ };
+ }
private static Properties loadAndTransformProps(String cfgPath) throws
IOException {
