Author: mattryan
Date: Tue Jul 21 18:41:31 2020
New Revision: 1880114
URL: http://svn.apache.org/viewvc?rev=1880114&view=rev
Log:
OAK-9142: Use concurrent request limit option for all Azure interactions
Modified:
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/Utils.java
Modified:
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java?rev=1880114&r1=1880113&r2=1880114&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
Tue Jul 21 18:41:31 2020
@@ -54,7 +54,6 @@ import com.google.common.collect.Abstrac
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.microsoft.azure.storage.AccessCondition;
-import com.microsoft.azure.storage.RequestOptions;
import com.microsoft.azure.storage.ResultContinuation;
import com.microsoft.azure.storage.ResultSegment;
import com.microsoft.azure.storage.RetryPolicy;
@@ -77,11 +76,11 @@ import org.apache.jackrabbit.core.data.D
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.commons.PropertiesUtil;
+import
org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordDownloadOptions;
+import
org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUpload;
import
org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadException;
import
org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadOptions;
import
org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadToken;
-import
org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordDownloadOptions;
-import
org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUpload;
import org.apache.jackrabbit.oak.spi.blob.AbstractDataRecord;
import org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend;
import org.apache.jackrabbit.util.Base64;
@@ -108,11 +107,13 @@ public class AzureBlobStoreBackend exten
static final long MAX_BINARY_UPLOAD_SIZE = (long) Math.floor(1024L * 1024L
* 1024L * 1024L * 4.75); // 4.75TB, Azure limit
private static final int MAX_ALLOWABLE_UPLOAD_URIS = 50000; // Azure limit
private static final int MAX_UNIQUE_RECORD_TRIES = 10;
+ private static final int DEFAULT_CONCURRENT_REQUEST_COUNT = 2;
+ private static final int MAX_CONCURRENT_REQUEST_COUNT = 50;
private Properties properties;
private String containerName;
private String connectionString;
- private int concurrentRequestCount = 1;
+ private int concurrentRequestCount = DEFAULT_CONCURRENT_REQUEST_COUNT;
private RetryPolicy retryPolicy;
private Integer requestTimeout;
private int httpDownloadURIExpirySeconds = 0; // disabled by default
@@ -131,15 +132,16 @@ public class AzureBlobStoreBackend exten
}
protected CloudBlobContainer getAzureContainer() throws DataStoreException
{
- CloudBlobContainer container =
Utils.getBlobContainer(connectionString, containerName);
- RequestOptions requestOptions =
container.getServiceClient().getDefaultRequestOptions();
- if (retryPolicy != null) {
+ BlobRequestOptions requestOptions = new BlobRequestOptions();
+ if (null != retryPolicy) {
requestOptions.setRetryPolicyFactory(retryPolicy);
}
- if (requestTimeout != null) {
+ if (null != requestTimeout) {
requestOptions.setTimeoutIntervalInMs(requestTimeout);
}
- return container;
+ requestOptions.setConcurrentRequestCount(concurrentRequestCount);
+
+ return Utils.getBlobContainer(connectionString, containerName,
requestOptions);
}
@Override
@@ -164,8 +166,23 @@ public class AzureBlobStoreBackend exten
containerName = (String)
properties.get(AzureConstants.AZURE_BLOB_CONTAINER_NAME);
createBlobContainer =
PropertiesUtil.toBoolean(properties.getProperty(AzureConstants.AZURE_CREATE_CONTAINER),
true);
connectionString =
Utils.getConnectionStringFromProperties(properties);
- concurrentRequestCount =
PropertiesUtil.toInteger(properties.get(AzureConstants.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION),
1);
+
+ concurrentRequestCount = PropertiesUtil.toInteger(
+
properties.get(AzureConstants.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION),
+ DEFAULT_CONCURRENT_REQUEST_COUNT);
+ if (concurrentRequestCount < DEFAULT_CONCURRENT_REQUEST_COUNT)
{
+ LOG.warn("Invalid setting [{}] for
concurrentRequestsPerOperation (too low); resetting to {}",
+ concurrentRequestCount,
+ DEFAULT_CONCURRENT_REQUEST_COUNT);
+ concurrentRequestCount = DEFAULT_CONCURRENT_REQUEST_COUNT;
+ } else if (concurrentRequestCount >
MAX_CONCURRENT_REQUEST_COUNT) {
+ LOG.warn("Invalid setting [{}] for
concurrentRequestsPerOperation (too high); resetting to {}",
+ concurrentRequestCount,
+ MAX_CONCURRENT_REQUEST_COUNT);
+ concurrentRequestCount = MAX_CONCURRENT_REQUEST_COUNT;
+ }
LOG.info("Using concurrentRequestsPerOperation={}",
concurrentRequestCount);
+
retryPolicy =
Utils.getRetryPolicy((String)properties.get(AzureConstants.AZURE_BLOB_MAX_REQUEST_RETRY));
if
(properties.getProperty(AzureConstants.AZURE_BLOB_REQUEST_TIMEOUT) != null) {
requestTimeout =
PropertiesUtil.toInteger(properties.getProperty(AzureConstants.AZURE_BLOB_REQUEST_TIMEOUT),
RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT);
Modified:
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/Utils.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/Utils.java?rev=1880114&r1=1880113&r2=1880114&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/Utils.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/Utils.java
Tue Jul 21 18:41:31 2020
@@ -19,6 +19,17 @@
package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetSocketAddress;
+import java.net.Proxy;
+import java.net.SocketAddress;
+import java.net.URISyntaxException;
+import java.security.InvalidKeyException;
+import java.util.Properties;
+
import com.google.common.base.Strings;
import com.microsoft.azure.storage.CloudStorageAccount;
import com.microsoft.azure.storage.OperationContext;
@@ -26,21 +37,13 @@ import com.microsoft.azure.storage.Retry
import com.microsoft.azure.storage.RetryNoRetry;
import com.microsoft.azure.storage.RetryPolicy;
import com.microsoft.azure.storage.StorageException;
+import com.microsoft.azure.storage.blob.BlobRequestOptions;
import com.microsoft.azure.storage.blob.CloudBlobClient;
import com.microsoft.azure.storage.blob.CloudBlobContainer;
import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.commons.PropertiesUtil;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.InetSocketAddress;
-import java.net.Proxy;
-import java.net.SocketAddress;
-import java.net.URISyntaxException;
-import java.security.InvalidKeyException;
-import java.util.Properties;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
public final class Utils {
@@ -60,15 +63,34 @@ public final class Utils {
* @param connectionString connectionString to configure @link {@link
CloudBlobClient}
* @return {@link CloudBlobClient}
*/
- public static CloudBlobClient getBlobClient(final String connectionString)
throws URISyntaxException, InvalidKeyException {
+ public static CloudBlobClient getBlobClient(@NotNull final String
connectionString) throws URISyntaxException, InvalidKeyException {
+ return getBlobClient(connectionString, null);
+ }
+
+ public static CloudBlobClient getBlobClient(@NotNull final String
connectionString,
+ @Nullable final
BlobRequestOptions requestOptions) throws URISyntaxException,
InvalidKeyException {
CloudStorageAccount account =
CloudStorageAccount.parse(connectionString);
CloudBlobClient client = account.createCloudBlobClient();
+ if (null != requestOptions) {
+ client.setDefaultRequestOptions(requestOptions);
+ }
return client;
}
- public static CloudBlobContainer getBlobContainer(final String
connectionString, final String containerName) throws DataStoreException {
+ public static CloudBlobContainer getBlobContainer(@NotNull final String
connectionString,
+ @NotNull final String
containerName) throws DataStoreException {
+ return getBlobContainer(connectionString, containerName, null);
+ }
+
+ public static CloudBlobContainer getBlobContainer(@NotNull final String
connectionString,
+ @NotNull final String
containerName,
+ @Nullable final
BlobRequestOptions requestOptions) throws DataStoreException {
try {
- CloudBlobClient client = Utils.getBlobClient(connectionString);
+ CloudBlobClient client = (
+ (null == requestOptions)
+ ? Utils.getBlobClient(connectionString)
+ : Utils.getBlobClient(connectionString,
requestOptions)
+ );
return client.getContainerReference(containerName);
} catch (InvalidKeyException | URISyntaxException | StorageException
e) {
throw new DataStoreException(e);
