This is an automated email from the ASF dual-hosted git repository. daim pushed a commit to branch DetailedGC/OAK-10199 in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit ec69f660427b025a84dd756b30a004db0760f36e Author: Miroslav Smiljanic <[email protected]> AuthorDate: Thu Dec 21 15:55:48 2023 +0100 Issue/oak 10573 azure write timeout (#1244) * OAK-10573 request options optimised for write operations * OAK-10573 introduce AzureRequestOptions util class * OAK-10573 adding license header * OAK-10573 add write check in AzureArchiveManager * OAK-10573 remove unused imports --------- Co-authored-by: Miroslav Smiljanic <[email protected]> --- .../oak/segment/azure/AzureArchiveManager.java | 2 + .../oak/segment/azure/AzureJournalFile.java | 15 +++- .../oak/segment/azure/AzurePersistence.java | 37 +-------- .../segment/azure/AzureSegmentArchiveWriter.java | 13 ++- .../segment/azure/util/AzureRequestOptions.java | 96 ++++++++++++++++++++++ .../azure/util/AzureRequestOptionsTest.java | 78 ++++++++++++++++++ 6 files changed, 198 insertions(+), 43 deletions(-) diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManager.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManager.java index 10c5ce485f..13eeab83d9 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManager.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManager.java @@ -139,6 +139,7 @@ public class AzureArchiveManager implements SegmentArchiveManager { getBlobs(archiveName) .forEach(cloudBlob -> { try { + writeAccessController.checkWritingAllowed(); cloudBlob.delete(); } catch (StorageException e) { log.error("Can't delete segment {}", cloudBlob.getUri().getPath(), e); @@ -158,6 +159,7 @@ public class AzureArchiveManager implements SegmentArchiveManager { getBlobs(from) .forEach(cloudBlob -> { try { + writeAccessController.checkWritingAllowed(); renameBlob(cloudBlob, targetDirectory); } catch (IOException e) { log.error("Can't rename segment {}", cloudBlob.getUri().getPath(), e); diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureJournalFile.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureJournalFile.java index b468e42b44..76b52de113 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureJournalFile.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureJournalFile.java @@ -22,6 +22,9 @@ import com.microsoft.azure.storage.blob.CloudAppendBlob; import com.microsoft.azure.storage.blob.CloudBlob; import com.microsoft.azure.storage.blob.CloudBlobDirectory; import com.microsoft.azure.storage.blob.ListBlobItem; +import com.microsoft.azure.storage.blob.DeleteSnapshotsOption; +import com.microsoft.azure.storage.blob.BlobRequestOptions; +import org.apache.jackrabbit.oak.segment.azure.util.AzureRequestOptions; import org.apache.jackrabbit.oak.segment.azure.util.CaseInsensitiveKeysMapAccess; import org.apache.jackrabbit.oak.segment.remote.WriteAccessController; import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile; @@ -162,7 +165,11 @@ public class AzureJournalFile implements JournalFile { private int lineCount; + private final BlobRequestOptions writeOptimisedBlobRequestOptions; + public AzureJournalWriter() throws IOException { + writeOptimisedBlobRequestOptions = AzureRequestOptions.optimiseForWriteOperations(directory.getServiceClient().getDefaultRequestOptions()); + List<CloudAppendBlob> blobs = getJournalBlobs(); if (blobs.isEmpty()) { try { @@ -190,7 +197,7 @@ public class AzureJournalFile implements JournalFile { writeAccessController.checkWritingAllowed(); for (CloudAppendBlob cloudAppendBlob : getJournalBlobs()) { - cloudAppendBlob.delete(); + cloudAppendBlob.delete(DeleteSnapshotsOption.NONE, null, writeOptimisedBlobRequestOptions, null); } createNextFile(0); @@ -229,11 +236,11 @@ public class AzureJournalFile implements JournalFile { text.append(line).append("\n"); } try { - currentBlob.appendText(text.toString()); + currentBlob.appendText(text.toString(), null, null, writeOptimisedBlobRequestOptions, null); currentBlob.getMetadata().put("lastEntry", entries.get(entries.size() - 1)); lineCount += entries.size(); currentBlob.getMetadata().put("lineCount", Integer.toString(lineCount)); - currentBlob.uploadMetadata(); + currentBlob.uploadMetadata(null, writeOptimisedBlobRequestOptions, null); } catch (StorageException e) { throw new IOException(e); } @@ -243,7 +250,7 @@ public class AzureJournalFile implements JournalFile { private void createNextFile(int suffix) throws IOException { try { currentBlob = directory.getAppendBlobReference(getJournalFileName(suffix + 1)); - currentBlob.createOrReplace(); + currentBlob.createOrReplace(null, writeOptimisedBlobRequestOptions, null); lineCount = 0; } catch (URISyntaxException | StorageException e) { throw new IOException(e); diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistence.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistence.java index eed07004e5..dc41fbff1e 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistence.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistence.java @@ -25,15 +25,14 @@ import java.util.concurrent.TimeUnit; import com.microsoft.azure.storage.OperationContext; import com.microsoft.azure.storage.RequestCompletedEvent; -import com.microsoft.azure.storage.RetryLinearRetry; import com.microsoft.azure.storage.StorageEvent; import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.blob.BlobListingDetails; -import com.microsoft.azure.storage.blob.BlobRequestOptions; import com.microsoft.azure.storage.blob.CloudAppendBlob; import com.microsoft.azure.storage.blob.CloudBlobDirectory; import com.microsoft.azure.storage.blob.CloudBlockBlob; import com.microsoft.azure.storage.blob.ListBlobItem; +import org.apache.jackrabbit.oak.segment.azure.util.AzureRequestOptions; import org.apache.jackrabbit.oak.segment.remote.WriteAccessController; import org.apache.jackrabbit.oak.segment.spi.monitor.FileStoreMonitor; import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitor; @@ -48,19 +47,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class AzurePersistence implements SegmentNodeStorePersistence { - - private static final String RETRY_ATTEMPTS_PROP = "segment.azure.retry.attempts"; - private static final int DEFAULT_RETRY_ATTEMPTS = 5; - - private static final String RETRY_BACKOFF_PROP = "segment.azure.retry.backoff"; - private static final int DEFAULT_RETRY_BACKOFF_SECONDS = 5; - - private static final String TIMEOUT_EXECUTION_PROP = "segment.timeout.execution"; - private static final int DEFAULT_TIMEOUT_EXECUTION = 30; - - private static final String TIMEOUT_INTERVAL_PROP = "segment.timeout.interval"; - private static final int DEFAULT_TIMEOUT_INTERVAL = 1; - private static final Logger log = LoggerFactory.getLogger(AzurePersistence.class); protected final CloudBlobDirectory segmentstoreDirectory; @@ -70,26 +56,7 @@ public class AzurePersistence implements SegmentNodeStorePersistence { public AzurePersistence(CloudBlobDirectory segmentStoreDirectory) { this.segmentstoreDirectory = segmentStoreDirectory; - BlobRequestOptions defaultRequestOptions = segmentStoreDirectory.getServiceClient().getDefaultRequestOptions(); - if (defaultRequestOptions.getRetryPolicyFactory() == null) { - int retryAttempts = Integer.getInteger(RETRY_ATTEMPTS_PROP, DEFAULT_RETRY_ATTEMPTS); - if (retryAttempts > 0) { - Integer retryBackoffSeconds = Integer.getInteger(RETRY_BACKOFF_PROP, DEFAULT_RETRY_BACKOFF_SECONDS); - defaultRequestOptions.setRetryPolicyFactory(new RetryLinearRetry((int) TimeUnit.SECONDS.toMillis(retryBackoffSeconds), retryAttempts)); - } - } - if (defaultRequestOptions.getMaximumExecutionTimeInMs() == null) { - int timeoutExecution = Integer.getInteger(TIMEOUT_EXECUTION_PROP, DEFAULT_TIMEOUT_EXECUTION); - if (timeoutExecution > 0) { - defaultRequestOptions.setMaximumExecutionTimeInMs((int) TimeUnit.SECONDS.toMillis(timeoutExecution)); - } - } - if (defaultRequestOptions.getTimeoutIntervalInMs() == null) { - int timeoutInterval = Integer.getInteger(TIMEOUT_INTERVAL_PROP, DEFAULT_TIMEOUT_INTERVAL); - if (timeoutInterval > 0) { - defaultRequestOptions.setTimeoutIntervalInMs((int) TimeUnit.SECONDS.toMillis(timeoutInterval)); - } - } + AzureRequestOptions.applyDefaultRequestOptions(segmentStoreDirectory.getServiceClient().getDefaultRequestOptions()); } @Override diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentArchiveWriter.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentArchiveWriter.java index 3df0eaafa6..e9bbbb322a 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentArchiveWriter.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentArchiveWriter.java @@ -25,12 +25,14 @@ import java.io.IOException; import java.net.URISyntaxException; import java.util.concurrent.TimeUnit; +import com.microsoft.azure.storage.blob.BlobRequestOptions; import org.apache.jackrabbit.guava.common.base.Stopwatch; import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.blob.CloudBlobDirectory; import com.microsoft.azure.storage.blob.CloudBlockBlob; import org.apache.jackrabbit.oak.commons.Buffer; +import org.apache.jackrabbit.oak.segment.azure.util.AzureRequestOptions; import org.apache.jackrabbit.oak.segment.remote.WriteAccessController; import org.apache.jackrabbit.oak.segment.azure.util.Retrier; import org.apache.jackrabbit.oak.segment.remote.AbstractRemoteSegmentArchiveWriter; @@ -47,10 +49,13 @@ public class AzureSegmentArchiveWriter extends AbstractRemoteSegmentArchiveWrite Integer.getInteger("azure.segment.archive.writer.retries.intervalMs", 5000) ); + private final BlobRequestOptions writeOptimisedBlobRequestOptions; + public AzureSegmentArchiveWriter(CloudBlobDirectory archiveDirectory, IOMonitor ioMonitor, FileStoreMonitor monitor, WriteAccessController writeAccessController) { super(ioMonitor, monitor); this.archiveDirectory = archiveDirectory; this.writeAccessController = writeAccessController; + this.writeOptimisedBlobRequestOptions = AzureRequestOptions.optimiseForWriteOperations(archiveDirectory.getServiceClient().getDefaultRequestOptions()); } @Override @@ -71,8 +76,8 @@ public class AzureSegmentArchiveWriter extends AbstractRemoteSegmentArchiveWrite Stopwatch stopwatch = Stopwatch.createStarted(); try { blob.setMetadata(AzureBlobMetadata.toSegmentMetadata(indexEntry)); - blob.uploadFromByteArray(data, offset, size); - blob.uploadMetadata(); + blob.uploadFromByteArray(data, offset, size, null, writeOptimisedBlobRequestOptions, null); + blob.uploadMetadata(null, writeOptimisedBlobRequestOptions, null); } catch (StorageException e) { throw new IOException(e); } @@ -97,7 +102,7 @@ public class AzureSegmentArchiveWriter extends AbstractRemoteSegmentArchiveWrite try { writeAccessController.checkWritingAllowed(); - getBlob(getName() + extension).uploadFromByteArray(data, 0, data.length); + getBlob(getName() + extension).uploadFromByteArray(data, 0, data.length, null, writeOptimisedBlobRequestOptions, null); } catch (StorageException e) { throw new IOException(e); } @@ -110,7 +115,7 @@ public class AzureSegmentArchiveWriter extends AbstractRemoteSegmentArchiveWrite try { writeAccessController.checkWritingAllowed(); - getBlob("closed").uploadFromByteArray(new byte[0], 0, 0); + getBlob("closed").uploadFromByteArray(new byte[0], 0, 0, null, writeOptimisedBlobRequestOptions, null); } catch (StorageException e) { throw new IOException(e); } diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/util/AzureRequestOptions.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/util/AzureRequestOptions.java new file mode 100644 index 0000000000..24452d15ea --- /dev/null +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/util/AzureRequestOptions.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.segment.azure.util; + +import com.microsoft.azure.storage.RetryLinearRetry; +import com.microsoft.azure.storage.blob.BlobRequestOptions; + +import java.util.concurrent.TimeUnit; + +public class AzureRequestOptions { + + static final String RETRY_ATTEMPTS_PROP = "segment.azure.retry.attempts"; + static final int DEFAULT_RETRY_ATTEMPTS = 5; + + static final String RETRY_BACKOFF_PROP = "segment.azure.retry.backoff"; + static final int DEFAULT_RETRY_BACKOFF_SECONDS = 5; + + static final String TIMEOUT_EXECUTION_PROP = "segment.timeout.execution"; + static final int DEFAULT_TIMEOUT_EXECUTION = 30; + + static final String TIMEOUT_INTERVAL_PROP = "segment.timeout.interval"; + static final int DEFAULT_TIMEOUT_INTERVAL = 1; + + static final String WRITE_TIMEOUT_EXECUTION_PROP = "segment.write.timeout.execution"; + + static final String WRITE_TIMEOUT_INTERVAL_PROP = "segment.write.timeout.interval"; + + private AzureRequestOptions() { + } + + /** + * Apply default request options to the blobRequestOptions if they are not already set. + * @param blobRequestOptions + */ + public static void applyDefaultRequestOptions(BlobRequestOptions blobRequestOptions) { + if (blobRequestOptions.getRetryPolicyFactory() == null) { + int retryAttempts = Integer.getInteger(RETRY_ATTEMPTS_PROP, DEFAULT_RETRY_ATTEMPTS); + if (retryAttempts > 0) { + Integer retryBackoffSeconds = Integer.getInteger(RETRY_BACKOFF_PROP, DEFAULT_RETRY_BACKOFF_SECONDS); + blobRequestOptions.setRetryPolicyFactory(new RetryLinearRetry((int) TimeUnit.SECONDS.toMillis(retryBackoffSeconds), retryAttempts)); + } + } + if (blobRequestOptions.getMaximumExecutionTimeInMs() == null) { + int timeoutExecution = Integer.getInteger(TIMEOUT_EXECUTION_PROP, DEFAULT_TIMEOUT_EXECUTION); + if (timeoutExecution > 0) { + blobRequestOptions.setMaximumExecutionTimeInMs((int) TimeUnit.SECONDS.toMillis(timeoutExecution)); + } + } + if (blobRequestOptions.getTimeoutIntervalInMs() == null) { + int timeoutInterval = Integer.getInteger(TIMEOUT_INTERVAL_PROP, DEFAULT_TIMEOUT_INTERVAL); + if (timeoutInterval > 0) { + blobRequestOptions.setTimeoutIntervalInMs((int) TimeUnit.SECONDS.toMillis(timeoutInterval)); + } + } + } + + /** + * Optimise the blob request options for write operations. This method does not change the original blobRequestOptions. + * This method also applies the default request options if they are not already set, by calling {@link #applyDefaultRequestOptions(BlobRequestOptions)} + * @param blobRequestOptions + * @return write optimised blobRequestOptions + */ + public static BlobRequestOptions optimiseForWriteOperations(BlobRequestOptions blobRequestOptions) { + BlobRequestOptions writeOptimisedBlobRequestOptions = new BlobRequestOptions(blobRequestOptions); + applyDefaultRequestOptions(writeOptimisedBlobRequestOptions); + + Integer writeTimeoutExecution = Integer.getInteger(WRITE_TIMEOUT_EXECUTION_PROP); + if (writeTimeoutExecution != null) { + writeOptimisedBlobRequestOptions.setMaximumExecutionTimeInMs((int) TimeUnit.SECONDS.toMillis(writeTimeoutExecution)); + } + + Integer writeTimeoutInterval = Integer.getInteger(WRITE_TIMEOUT_INTERVAL_PROP); + if (writeTimeoutInterval != null) { + writeOptimisedBlobRequestOptions.setTimeoutIntervalInMs((int) TimeUnit.SECONDS.toMillis(writeTimeoutInterval)); + } + + return writeOptimisedBlobRequestOptions; + } +} diff --git a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/util/AzureRequestOptionsTest.java b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/util/AzureRequestOptionsTest.java new file mode 100644 index 0000000000..7bbfd391a1 --- /dev/null +++ b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/util/AzureRequestOptionsTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.segment.azure.util; + +import com.microsoft.azure.storage.blob.BlobRequestOptions; +import org.junit.Before; +import org.junit.Test; + +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; + +public class AzureRequestOptionsTest { + + private BlobRequestOptions blobRequestOptions; + + @Before + public void setUp() { + blobRequestOptions = new BlobRequestOptions(); + } + + @Test + public void testApplyDefaultRequestOptions() { + AzureRequestOptions.applyDefaultRequestOptions(blobRequestOptions); + assertEquals(Long.valueOf(TimeUnit.SECONDS.toMillis(AzureRequestOptions.DEFAULT_TIMEOUT_EXECUTION)), Long.valueOf(blobRequestOptions.getMaximumExecutionTimeInMs())); + assertEquals(Long.valueOf(TimeUnit.SECONDS.toMillis(AzureRequestOptions.DEFAULT_TIMEOUT_INTERVAL)), Long.valueOf(blobRequestOptions.getTimeoutIntervalInMs())); + } + + @Test + public void testApplyDefaultRequestOptionsWithCustomTimeouts() { + System.setProperty(AzureRequestOptions.TIMEOUT_EXECUTION_PROP, "10"); + System.setProperty(AzureRequestOptions.TIMEOUT_INTERVAL_PROP, "5"); + + AzureRequestOptions.applyDefaultRequestOptions(blobRequestOptions); + assertEquals(Long.valueOf(TimeUnit.SECONDS.toMillis(10)), Long.valueOf(blobRequestOptions.getMaximumExecutionTimeInMs())); + assertEquals(Long.valueOf(TimeUnit.SECONDS.toMillis(5)), Long.valueOf(blobRequestOptions.getTimeoutIntervalInMs())); + + System.clearProperty(AzureRequestOptions.TIMEOUT_EXECUTION_PROP); + System.clearProperty(AzureRequestOptions.TIMEOUT_INTERVAL_PROP); + } + + @Test + public void testOptimiseForWriteOperations() { + BlobRequestOptions writeBlobRequestoptions = AzureRequestOptions.optimiseForWriteOperations(blobRequestOptions); + assertEquals(Long.valueOf(TimeUnit.SECONDS.toMillis(AzureRequestOptions.DEFAULT_TIMEOUT_EXECUTION)), Long.valueOf(writeBlobRequestoptions.getMaximumExecutionTimeInMs())); + assertEquals(Long.valueOf(TimeUnit.SECONDS.toMillis(AzureRequestOptions.DEFAULT_TIMEOUT_INTERVAL)), Long.valueOf(writeBlobRequestoptions.getTimeoutIntervalInMs())); + } + + @Test + public void testOptimiseForWriteOperationsWithCustomTimeouts() { + System.setProperty(AzureRequestOptions.WRITE_TIMEOUT_EXECUTION_PROP, "10"); + System.setProperty(AzureRequestOptions.WRITE_TIMEOUT_INTERVAL_PROP, "5"); + + BlobRequestOptions writeBlobRequestoptions = AzureRequestOptions.optimiseForWriteOperations(blobRequestOptions); + assertEquals(Long.valueOf(TimeUnit.SECONDS.toMillis(10)), Long.valueOf(writeBlobRequestoptions.getMaximumExecutionTimeInMs())); + assertEquals(Long.valueOf(TimeUnit.SECONDS.toMillis(5)), Long.valueOf(writeBlobRequestoptions.getTimeoutIntervalInMs())); + + System.clearProperty(AzureRequestOptions.WRITE_TIMEOUT_EXECUTION_PROP); + System.clearProperty(AzureRequestOptions.WRITE_TIMEOUT_INTERVAL_PROP); + } +}
