This is an automated email from the ASF dual-hosted git repository.
miroslav pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 117fc0559d OAK-11506 Update oak-segment-azure tools to use azure sdk
v12 implementation (#2160)
117fc0559d is described below
commit 117fc0559db3965de8ffe98e97207dffc84bd063
Author: Ieran Bogdan <[email protected]>
AuthorDate: Wed Mar 19 10:57:50 2025 +0200
OAK-11506 Update oak-segment-azure tools to use azure sdk v12
implementation (#2160)
* OAK-11506 Update oak-segment-azure tools to use azure sdk v12
implementation
* OAK-11506 Update oak-segment-azure tools to use azure sdk v12
implementation
* OAK-11506 Update oak-segment-azure tools to use azure sdk v12
implementation - refactor AzurePersistenceManager
* OAK-11506 Update oak-segment-azure tools to use azure sdk v12
implementation - fix unit tests
* OAK-11506 Update oak-segment-azure tools to use azure sdk v12
implementation - fix unit tests
* OAK-11506 Update oak-segment-azure tools to use azure sdk v12
implementation
* OAK-11506 Update oak-segment-azure tools to use azure sdk v12
implementation - remove AZURE_STORAGE_ACCOUNT_NAME
* OAK-11506 Update oak-segment-azure tools to use azure sdk v12
implementation - remove AZURE_STORAGE_ACCOUNT_NAME
* OAK-11506 Update oak-segment-azure tools to use azure sdk v12
implementation - PR review change
* OAK-11506 Update oak-segment-azure tools to use azure sdk v12
implementation - PR review change
---
.../oak/run/cli/SegmentTarFixtureProvider.java | 5 +-
.../explorer/AzureSegmentStoreExplorerBackend.java | 6 +-
.../jackrabbit/oak/run/FileStoreDiffCommand.java | 31 ++---
.../jackrabbit/oak/run/DataStoreCommandTest.java | 9 +-
.../oak/segment/azure/AzurePersistence.java | 4 +
.../oak/segment/azure/AzurePersistenceManager.java | 77 +++++++++--
.../oak/segment/azure/AzureUtilities.java | 4 +-
.../oak/segment/azure/tool/AzureCheck.java | 45 ++++---
.../oak/segment/azure/tool/AzureCompact.java | 120 ++++++++++-------
.../oak/segment/azure/tool/SegmentCopy.java | 13 +-
.../segment/azure/tool/SegmentStoreMigrator.java | 19 ++-
.../oak/segment/azure/tool/ToolUtils.java | 73 +++++------
.../azure/util/AzureConfigurationParserUtils.java | 5 +-
.../segment/azure/AzurePersistenceManagerTest.java | 142 +++++++++++++++++++++
.../SegmentCopyAzureServicePrincipalToTarTest.java | 20 +--
.../segment/azure/tool/SegmentCopyTestBase.java | 2 +-
.../oak/segment/azure/tool/ToolUtilsTest.java | 96 ++++----------
...entAzureServicePrincipalNodeStoreContainer.java | 34 ++---
.../upgrade/cli/node/SegmentAzureFactoryTest.java | 16 +--
19 files changed, 426 insertions(+), 295 deletions(-)
diff --git
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/SegmentTarFixtureProvider.java
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/SegmentTarFixtureProvider.java
index 6918658f99..0d70319717 100644
---
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/SegmentTarFixtureProvider.java
+++
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/SegmentTarFixtureProvider.java
@@ -30,7 +30,6 @@ import java.nio.file.Files;
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.commons.pio.Closer;
import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
-import
org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8;
import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
@@ -53,12 +52,10 @@ class SegmentTarFixtureProvider {
FileStoreBuilder builder;
if (segmentStoreType == ToolUtils.SegmentStoreType.AZURE) {
- final AzureStorageCredentialManagerV8
azureStorageCredentialManagerV8 = new AzureStorageCredentialManagerV8();
SegmentNodeStorePersistence segmentNodeStorePersistence =
- ToolUtils.newSegmentNodeStorePersistence(segmentStoreType,
pathOrUri, azureStorageCredentialManagerV8);
+ ToolUtils.newSegmentNodeStorePersistence(segmentStoreType,
pathOrUri);
File tempDir =
Files.createTempDirectory("azure-segment-store").toFile();
closer.register(() -> FileUtils.deleteQuietly(tempDir));
- closer.register(azureStorageCredentialManagerV8);
builder =
fileStoreBuilder(tempDir).withCustomPersistence(segmentNodeStorePersistence);
} else {
builder = fileStoreBuilder(new
File(pathOrUri)).withMaxFileSize(256);
diff --git
a/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AzureSegmentStoreExplorerBackend.java
b/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AzureSegmentStoreExplorerBackend.java
index 742576a066..a3a116ecb5 100644
---
a/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AzureSegmentStoreExplorerBackend.java
+++
b/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AzureSegmentStoreExplorerBackend.java
@@ -18,7 +18,6 @@
*/
package org.apache.jackrabbit.oak.explorer;
-import
org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8;
import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils;
import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile;
@@ -39,16 +38,14 @@ import static
org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreB
public class AzureSegmentStoreExplorerBackend extends
AbstractSegmentTarExplorerBackend {
private final String path;
private SegmentNodeStorePersistence persistence;
- private final AzureStorageCredentialManagerV8
azureStorageCredentialManagerV8;
public AzureSegmentStoreExplorerBackend(String path) {
this.path = path;
- this.azureStorageCredentialManagerV8 = new
AzureStorageCredentialManagerV8();
}
@Override
public void open() throws IOException {
- this.persistence =
newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, path,
azureStorageCredentialManagerV8);
+ this.persistence =
newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, path);
try {
this.store =
fileStoreBuilder(Files.createTempDirectory(getClass().getSimpleName() +
"-").toFile())
@@ -63,7 +60,6 @@ public class AzureSegmentStoreExplorerBackend extends
AbstractSegmentTarExplorer
@Override
public void close() {
super.close();
- azureStorageCredentialManagerV8.close();
}
@Override
diff --git
a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java
b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java
index cf01abbf43..dd49502329 100644
---
a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java
+++
b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java
@@ -27,7 +27,6 @@ import joptsimple.OptionParser;
import joptsimple.OptionSet;
import joptsimple.OptionSpec;
import org.apache.jackrabbit.oak.run.commons.Command;
-import
org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8;
import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils;
import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore;
import
org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence;
@@ -86,22 +85,20 @@ class FileStoreDiffCommand implements Command {
}
} else {
if (pathOrURI.startsWith("az:")) {
- try (AzureStorageCredentialManagerV8
azureStorageCredentialManagerV8 = new AzureStorageCredentialManagerV8()) {
- SegmentNodeStorePersistence azurePersistence =
ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE,
pathOrURI, azureStorageCredentialManagerV8);
- ReadOnlyFileStore store =
fileStoreBuilder(Files.createTempDirectory(getClass().getSimpleName() +
"-").toFile())
-
.withCustomPersistence(azurePersistence).withBlobStore(newBasicReadOnlyBlobStore()).buildReadOnly();
- statusCode = Diff.builder()
- .withPath(pathOrURI)
- .withReadOnlyFileStore(store)
- .withOutput(out)
- .withInterval(interval)
- .withIncremental(incremental)
- .withFilter(path)
- .withIgnoreMissingSegments(ignoreSNFEs)
- .withRevisionsProcessor(ToolUtils::readRevisions)
- .build()
- .run();
- }
+ SegmentNodeStorePersistence azurePersistence =
ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE,
pathOrURI);
+ ReadOnlyFileStore store =
fileStoreBuilder(Files.createTempDirectory(getClass().getSimpleName() +
"-").toFile())
+
.withCustomPersistence(azurePersistence).withBlobStore(newBasicReadOnlyBlobStore()).buildReadOnly();
+ statusCode = Diff.builder()
+ .withPath(pathOrURI)
+ .withReadOnlyFileStore(store)
+ .withOutput(out)
+ .withInterval(interval)
+ .withIncremental(incremental)
+ .withFilter(path)
+ .withIgnoreMissingSegments(ignoreSNFEs)
+ .withRevisionsProcessor(ToolUtils::readRevisions)
+ .build()
+ .run();
} else {
ReadOnlyFileStore store = fileStoreBuilder(new
File(pathOrURI)).withBlobStore(newBasicReadOnlyBlobStore()).buildReadOnly();
statusCode = Diff.builder()
diff --git
a/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java
b/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java
index 7bf9c2a274..cc17485b2c 100644
---
a/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java
+++
b/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java
@@ -71,8 +71,7 @@ import
org.apache.jackrabbit.oak.plugins.index.lucene.directory.OakDirectory;
import org.apache.jackrabbit.oak.run.cli.BlobStoreOptions.Type;
import org.apache.jackrabbit.oak.segment.SegmentNodeStore;
import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
-import
org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8;
-import org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8;
+import org.apache.jackrabbit.oak.segment.azure.AzurePersistenceManager;
import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils;
import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
import org.apache.jackrabbit.oak.segment.file.FileStore;
@@ -1128,7 +1127,6 @@ public class DataStoreCommandTest {
class AzureSegmentStoreFixture extends SegmentStoreFixture {
private static final String AZURE_DIR = "repository";
private String container;
- private final AzureStorageCredentialManagerV8
azureStorageCredentialManagerV8 = new AzureStorageCredentialManagerV8();
@Override public NodeStore init(DataStoreBlobStore blobStore, File
storeFile) throws Exception {
Properties props = AzureDataStoreUtils.getAzureConfig();
@@ -1138,14 +1136,14 @@ public class DataStoreCommandTest {
container = container + System.currentTimeMillis();
// Create the azure segment container
String connectionString = getAzureConnectionString(accessKey,
secretKey, container, AZURE_DIR);
- AzureUtilitiesV8.cloudBlobDirectoryFrom(connectionString,
container, AZURE_DIR);
+
AzurePersistenceManager.createAzurePersistence(connectionString, null,
accessKey, container, AZURE_DIR, false, true);
// get the azure uri expected by the command
storePath = getAzureUri(accessKey, container, AZURE_DIR);
// initialize azure segment for test setup
SegmentNodeStorePersistence segmentNodeStorePersistence =
-
ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE,
storePath, azureStorageCredentialManagerV8);
+
ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE,
storePath);
fileStore =
fileStoreBuilder(storeFile).withBlobStore(blobStore)
.withCustomPersistence(segmentNodeStorePersistence).build();
@@ -1177,7 +1175,6 @@ public class DataStoreCommandTest {
public void after() {
try {
AzureDataStoreUtils.deleteContainer(container);
- azureStorageCredentialManagerV8.close();
} catch(Exception e) {
log.error("Error in cleaning the container {}", container,
e);
}
diff --git
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistence.java
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistence.java
index a0f0ddded5..9a6f75595c 100644
---
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistence.java
+++
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistence.java
@@ -53,6 +53,10 @@ public class AzurePersistence implements
SegmentNodeStorePersistence {
protected WriteAccessController writeAccessController = new
WriteAccessController();
+ public AzurePersistence(BlobContainerClient blobContainerClient, String
rootPrefix) {
+ this(blobContainerClient, blobContainerClient, blobContainerClient,
rootPrefix);
+ }
+
public AzurePersistence(BlobContainerClient readBlobContainerClient,
BlobContainerClient writeBlobContainerClient, BlobContainerClient
noRetryBlobContainerClient, String rootPrefix) {
this(readBlobContainerClient, writeBlobContainerClient,
noRetryBlobContainerClient, rootPrefix, null);
}
diff --git
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistenceManager.java
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistenceManager.java
index 12fee4c31c..e16179741e 100644
---
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistenceManager.java
+++
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistenceManager.java
@@ -41,14 +41,18 @@ public class AzurePersistenceManager {
private AzurePersistenceManager() {
}
- public static AzurePersistence createAzurePersistenceFrom(@NotNull String
accountName, @NotNull String containerName, @NotNull String rootPrefix,
@NotNull Environment environment) throws IOException {
+ public static AzurePersistence createAzurePersistenceFrom(String
accountName, String containerName, String rootPrefix, String sasToken) throws
IOException {
+ return createAzurePersistence(null, sasToken, accountName,
containerName, rootPrefix, false, false);
+ }
+
+ public static AzurePersistence createAzurePersistenceFrom(String
accountName, String containerName, String rootPrefix, Environment environment)
throws IOException {
final String clientId = environment.getVariable(AZURE_CLIENT_ID);
final String clientSecret =
environment.getVariable(AZURE_CLIENT_SECRET);
final String tenantId = environment.getVariable(AZURE_TENANT_ID);
if (StringUtils.isNoneBlank(clientId, clientSecret, tenantId)) {
try {
- return
createPersistenceFromServicePrincipalCredentials(accountName, containerName,
rootPrefix, clientId, clientSecret, tenantId, false, false);
+ return
createPersistenceFromServicePrincipalCredentials(accountName, containerName,
rootPrefix, clientId, clientSecret, tenantId, false, true);
} catch (IllegalArgumentException |
StringIndexOutOfBoundsException e) {
log.error("Error occurred while connecting to Azure Storage
using service principals: ", e);
throw new IllegalArgumentException(
@@ -86,6 +90,7 @@ public class AzurePersistenceManager {
}
private static AzurePersistence createPersistenceFromAccessKey(String
accountName, String containerName, String accessKey, String blobEndpoint,
String rootPrefix, boolean enableSecondaryLocation, boolean createContainer)
throws IOException {
+ checkIfEmpty(accessKey, "accessKey");
StringBuilder connectionString = new StringBuilder();
connectionString.append("DefaultEndpointsProtocol=https;");
connectionString.append("AccountName=").append(accountName).append(';');
@@ -93,7 +98,7 @@ public class AzurePersistenceManager {
if (!StringUtils.isBlank(blobEndpoint)) {
connectionString.append("BlobEndpoint=").append(blobEndpoint).append(';');
}
- return createAzurePersistence(connectionString.toString(),
accountName, containerName, rootPrefix, enableSecondaryLocation,
createContainer);
+ return createAzurePersistence(connectionString.toString(), null,
accountName, containerName, rootPrefix, enableSecondaryLocation,
createContainer);
}
@NotNull
@@ -114,11 +119,12 @@ public class AzurePersistenceManager {
@NotNull
- private static AzurePersistence
createPersistenceFromServicePrincipalCredentials(Configuration configuration)
throws IOException {
+ private static AzurePersistence
createPersistenceFromServicePrincipalCredentials(Configuration configuration) {
return
createPersistenceFromServicePrincipalCredentials(configuration.accountName(),
configuration.containerName(), configuration.rootPath(),
configuration.clientId(), configuration.clientSecret(),
configuration.tenantId(), configuration.enableSecondaryLocation(), true);
}
- private static AzurePersistence
createPersistenceFromServicePrincipalCredentials(String accountName, String
containerName, String rootPrefix, String clientId, String clientSecret, String
tenantId, boolean enableSecondaryLocation, boolean createContainer) throws
IOException {
+ public static AzurePersistence
createPersistenceFromServicePrincipalCredentials(String accountName, String
containerName, String rootPrefix, String clientId, String clientSecret, String
tenantId, boolean enableSecondaryLocation, boolean createContainer) {
+ checkArguments(accountName, containerName, rootPrefix);
AzureHttpRequestLoggingPolicy azureHttpRequestLoggingPolicy = new
AzureHttpRequestLoggingPolicy();
ClientSecretCredential clientSecretCredential = new
ClientSecretCredentialBuilder()
@@ -146,21 +152,41 @@ public class AzurePersistenceManager {
@NotNull
private static AzurePersistence createAzurePersistence(String
connectionString, Configuration configuration, boolean createContainer) throws
IOException {
- return createAzurePersistence(connectionString,
configuration.accountName(), configuration.containerName(),
configuration.rootPath(), configuration.enableSecondaryLocation(),
createContainer);
+ return createAzurePersistence(connectionString, null,
configuration.accountName(), configuration.containerName(),
configuration.rootPath(), configuration.enableSecondaryLocation(),
createContainer);
}
@NotNull
- private static AzurePersistence createAzurePersistence(String
connectionString, String accountName, String containerName, String rootPrefix,
boolean enableSecondaryLocation, boolean createContainer) throws IOException {
+ public static AzurePersistence createAzurePersistence(String
connectionString, String sasToken, String accountName, String containerName,
String rootPrefix, boolean enableSecondaryLocation, boolean createContainer)
throws IOException {
+ if (StringUtils.isBlank(connectionString) &&
StringUtils.isBlank(sasToken)) {
+ throw new IllegalArgumentException("Both connectionString and
sasToken are not configured. Please configure one of them.");
+ }
+ checkArguments(accountName, containerName, rootPrefix);
+
try {
AzureHttpRequestLoggingPolicy azureHttpRequestLoggingPolicy = new
AzureHttpRequestLoggingPolicy();
RequestRetryOptions retryOptions =
readRequestRetryOptions(enableSecondaryLocation, accountName);
- BlobContainerClient blobContainerClient =
getBlobContainerClient(accountName, containerName, retryOptions,
azureHttpRequestLoggingPolicy, connectionString);
+ BlobContainerClient blobContainerClient;
+ if (sasToken != null) {
+ blobContainerClient =
getBlobContainerClientWithSas(accountName, containerName, retryOptions,
azureHttpRequestLoggingPolicy, sasToken);
+ } else {
+ blobContainerClient = getBlobContainerClient(accountName,
containerName, retryOptions, azureHttpRequestLoggingPolicy, connectionString);
+ }
RequestRetryOptions writeRetryOptions =
AzureRequestOptions.getRetryOperationsOptimiseForWriteOperations();
- BlobContainerClient writeBlobContainerClient =
getBlobContainerClient(accountName, containerName, writeRetryOptions,
azureHttpRequestLoggingPolicy, connectionString);
+ BlobContainerClient writeBlobContainerClient;
+ if (sasToken != null) {
+ writeBlobContainerClient =
getBlobContainerClientWithSas(accountName, containerName, writeRetryOptions,
azureHttpRequestLoggingPolicy, sasToken);
+ } else {
+ writeBlobContainerClient = getBlobContainerClient(accountName,
containerName, writeRetryOptions, azureHttpRequestLoggingPolicy,
connectionString);
+ }
- BlobContainerClient noRetryBlobContainerClient =
getBlobContainerClient(accountName, containerName, null,
azureHttpRequestLoggingPolicy, connectionString);
+ BlobContainerClient noRetryBlobContainerClient;
+ if (sasToken != null) {
+ noRetryBlobContainerClient =
getBlobContainerClientWithSas(accountName, containerName, null,
azureHttpRequestLoggingPolicy, sasToken);
+ } else {
+ noRetryBlobContainerClient =
getBlobContainerClient(accountName, containerName, null,
azureHttpRequestLoggingPolicy, connectionString);
+ }
if (createContainer) {
blobContainerClient.createIfNotExists();
@@ -174,8 +200,15 @@ public class AzurePersistenceManager {
}
}
+ private static BlobContainerClient getBlobContainerClientWithSas(String
accountName, String containerName, RequestRetryOptions requestRetryOptions,
AzureHttpRequestLoggingPolicy azureHttpRequestLoggingPolicy, String sasToken) {
+ BlobServiceClient blobServiceClient =
blobServiceClientBuilder(accountName, requestRetryOptions,
azureHttpRequestLoggingPolicy, sasToken)
+ .buildClient();
+
+ return blobServiceClient.getBlobContainerClient(containerName);
+ }
+
private static BlobContainerClient getBlobContainerClient(String
accountName, String containerName, RequestRetryOptions requestRetryOptions,
AzureHttpRequestLoggingPolicy azureHttpRequestLoggingPolicy, String
connectionString) {
- BlobServiceClient blobServiceClient =
blobServiceClientBuilder(accountName, requestRetryOptions,
azureHttpRequestLoggingPolicy)
+ BlobServiceClient blobServiceClient =
blobServiceClientBuilder(accountName, requestRetryOptions,
azureHttpRequestLoggingPolicy, null)
.connectionString(connectionString)
.buildClient();
@@ -183,15 +216,20 @@ public class AzurePersistenceManager {
}
private static BlobContainerClient getBlobContainerClient(String
accountName, String containerName, RequestRetryOptions requestRetryOptions,
AzureHttpRequestLoggingPolicy azureHttpRequestLoggingPolicy,
ClientSecretCredential clientSecretCredential) {
- BlobServiceClient blobServiceClient =
blobServiceClientBuilder(accountName, requestRetryOptions,
azureHttpRequestLoggingPolicy)
+ BlobServiceClient blobServiceClient =
blobServiceClientBuilder(accountName, requestRetryOptions,
azureHttpRequestLoggingPolicy, null)
.credential(clientSecretCredential)
.buildClient();
return blobServiceClient.getBlobContainerClient(containerName);
}
- private static BlobServiceClientBuilder blobServiceClientBuilder(String
accountName, RequestRetryOptions requestRetryOptions,
AzureHttpRequestLoggingPolicy azureHttpRequestLoggingPolicy) {
- String endpoint = String.format("https://%s.blob.core.windows.net";,
accountName);
+ private static BlobServiceClientBuilder blobServiceClientBuilder(String
accountName, RequestRetryOptions requestRetryOptions,
AzureHttpRequestLoggingPolicy azureHttpRequestLoggingPolicy, String sasToken) {
+ if (sasToken == null) {
+ sasToken = "";
+ } else {
+ sasToken = "?" + sasToken;
+ }
+ String endpoint = String.format("https://%s.blob.core.windows.net%s";,
accountName, sasToken);
BlobServiceClientBuilder builder = new BlobServiceClientBuilder()
.endpoint(endpoint)
@@ -221,4 +259,15 @@ public class AzurePersistenceManager {
return rootPath;
}
+ private static void checkArguments(String accountName, String
containerName, String rootPrefix){
+ checkIfEmpty(accountName, "Account name");
+ checkIfEmpty(containerName, "Container name");
+ checkIfEmpty(rootPrefix, "Root prefix");
+ }
+
+ private static void checkIfEmpty(String argument, String argumentName) {
+ if (StringUtils.isEmpty(argument)) {
+ throw new IllegalArgumentException(String.format("%s must not be
empty argument", argumentName));
+ }
+ }
}
diff --git
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java
index f6743edafc..c3a7349a0e 100644
---
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java
+++
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java
@@ -60,7 +60,9 @@ public final class AzureUtilities {
public static List<BlobItem> getBlobs(BlobContainerClient
blobContainerClient, ListBlobsOptions listOptions) {
- listOptions.setDetails(new
BlobListDetails().setRetrieveMetadata(true));
+ if (listOptions != null) {
+ listOptions.setDetails(new
BlobListDetails().setRetrieveMetadata(true));
+ }
return blobContainerClient.listBlobs(listOptions,
null).stream().collect(Collectors.toList());
}
diff --git
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java
index 486fb5b132..b6479a6607 100644
---
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java
+++
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java
@@ -16,9 +16,8 @@
*/
package org.apache.jackrabbit.oak.segment.azure.tool;
-import com.microsoft.azure.storage.blob.CloudBlobDirectory;
-import org.apache.jackrabbit.oak.segment.azure.v8.AzurePersistenceV8;
-import
org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8;
+import com.azure.storage.blob.BlobContainerClient;
+import org.apache.jackrabbit.oak.segment.azure.AzurePersistence;
import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
import org.apache.jackrabbit.oak.segment.file.JournalReader;
import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore;
@@ -57,6 +56,8 @@ public class AzureCheck {
private String path;
+ private String rootPrefix;
+
private String journal;
private long debugInterval = Long.MAX_VALUE;
@@ -85,7 +86,7 @@ public class AzureCheck {
private Integer persistentCacheSizeGb;
- private CloudBlobDirectory cloudBlobDirectory;
+ private BlobContainerClient blobContainerClient;
private Builder() {
// Prevent external instantiation.
@@ -102,6 +103,17 @@ public class AzureCheck {
return this;
}
+ /**
+ * The root prefix to an existing segment store. This parameter is
required.
+ *
+ * @param rootPrefix
+ * @return this builder.
+ */
+ public Builder withRootPrefix(String rootPrefix) {
+ this.rootPrefix = rootPrefix;
+ return this;
+ }
+
/**
* The path to the journal of the segment store. This parameter is
* optional. If not provided, the journal in the default location is
@@ -278,12 +290,12 @@ public class AzureCheck {
/**
* The Azure blob directory to connect to.
- * @param cloudBlobDirectory
+ * @param blobContainerClient
* the Azure blob directory.
* @return this builder
*/
- public Builder withCloudBlobDirectory(CloudBlobDirectory
cloudBlobDirectory) {
- this.cloudBlobDirectory = requireNonNull(cloudBlobDirectory);
+ public Builder withBlobContainerClient(BlobContainerClient
blobContainerClient) {
+ this.blobContainerClient = requireNonNull(blobContainerClient);
return this;
}
@@ -293,7 +305,7 @@ public class AzureCheck {
* @return an instance of {@link Runnable}.
*/
public AzureCheck build() {
- if (cloudBlobDirectory == null) {
+ if (blobContainerClient == null) {
requireNonNull(path);
}
return new AzureCheck(this);
@@ -320,6 +332,8 @@ public class AzureCheck {
private final String path;
+ private final String rootPrefix;
+
private final String journal;
private final long debugInterval;
@@ -348,11 +362,11 @@ public class AzureCheck {
private final Integer persistentCacheSizeGb;
- private final CloudBlobDirectory cloudBlobDirectory;
- private final AzureStorageCredentialManagerV8
azureStorageCredentialManagerV8;
+ private final BlobContainerClient blobContainerClient;
private AzureCheck(Builder builder) {
this.path = builder.path;
+ this.rootPrefix = builder.rootPrefix;
this.debugInterval = builder.debugInterval;
this.checkHead = builder.checkHead;
this.checkBinaries = builder.checkBinaries;
@@ -367,8 +381,7 @@ public class AzureCheck {
this.failFast = builder.failFast;
this.persistentCachePath = builder.persistentCachePath;
this.persistentCacheSizeGb = builder.persistentCacheSizeGb;
- this.cloudBlobDirectory = builder.cloudBlobDirectory;
- this.azureStorageCredentialManagerV8 = new
AzureStorageCredentialManagerV8();
+ this.blobContainerClient = builder.blobContainerClient;
}
private static Integer revisionsToCheckCount(Integer revisionsCount) {
@@ -379,10 +392,10 @@ public class AzureCheck {
StatisticsIOMonitor ioMonitor = new StatisticsIOMonitor();
SegmentNodeStorePersistence persistence;
- if (cloudBlobDirectory != null) {
- persistence = new AzurePersistenceV8(cloudBlobDirectory);
+ if (blobContainerClient != null) {
+ persistence = new AzurePersistence(blobContainerClient,
rootPrefix);
} else {
- persistence =
ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE,
path, azureStorageCredentialManagerV8);
+ persistence =
ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE,
path);
}
if (persistentCachePath != null) {
@@ -434,8 +447,6 @@ public class AzureCheck {
} catch (Exception e) {
e.printStackTrace(err);
return 1;
- } finally {
- azureStorageCredentialManagerV8.close();
}
}
diff --git
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
index d150be0936..1a61d9fcb6 100644
---
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
+++
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
@@ -19,24 +19,19 @@ package org.apache.jackrabbit.oak.segment.azure.tool;
import static
org.apache.jackrabbit.oak.commons.conditions.Validate.checkArgument;
import static java.util.Objects.requireNonNull;
import static
org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.createArchiveManager;
-import static
org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.createCloudBlobDirectory;
+import static
org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.createAzurePersistence;
import static
org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.decorateWithCache;
import static
org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.newFileStore;
-import static
org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.newSegmentNodeStorePersistence;
import static
org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.printableStopwatch;
+import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.blob.models.BlobItem;
+import com.azure.storage.blob.models.BlobListDetails;
+import com.azure.storage.blob.models.ListBlobsOptions;
import org.apache.jackrabbit.guava.common.base.Stopwatch;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.blob.BlobListingDetails;
-import com.microsoft.azure.storage.blob.CloudBlob;
-import com.microsoft.azure.storage.blob.CloudBlobContainer;
-import com.microsoft.azure.storage.blob.CloudBlobDirectory;
-import com.microsoft.azure.storage.blob.ListBlobItem;
import org.apache.jackrabbit.oak.segment.SegmentCache;
-import org.apache.jackrabbit.oak.segment.azure.v8.AzurePersistenceV8;
-import
org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8;
-import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.SegmentStoreType;
+import org.apache.jackrabbit.oak.segment.azure.AzurePersistence;
import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.GCType;
import
org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.CompactorType;
import org.apache.jackrabbit.oak.segment.file.FileStore;
@@ -53,7 +48,6 @@ import java.io.PrintStream;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.util.Collections;
-import java.util.EnumSet;
import java.util.List;
/**
@@ -79,6 +73,10 @@ public class AzureCompact {
private String targetPath;
+ private String rootPrefix;
+
+ private String targetRootPrefix;
+
private boolean force;
private long gcLogInterval = 150000;
@@ -99,9 +97,9 @@ public class AzureCompact {
private int garbageThresholdPercentage;
- private CloudBlobDirectory sourceCloudBlobDirectory;
+ private BlobContainerClient sourceBlobContainerClient;
- private CloudBlobDirectory destinationCloudBlobDirectory;
+ private BlobContainerClient destinationBlobContainerClient;
private Builder() {
// Prevent external instantiation.
@@ -131,6 +129,30 @@ public class AzureCompact {
return this;
}
+ /**
+ * The root directory to an existing segment store.
+ *
+ * @param rootPrefix
+ * the directory to an existing segment store.
+ * @return the builder
+ */
+ public Builder withRootPrefix(String rootPrefix) {
+ this.rootPrefix = rootPrefix;
+ return this;
+ }
+
+ /**
+ * The root directory to the target segment store.
+ *
+ * @param targetRootPrefix
+ * the root directory to the target segmen store.
+ * @return this builder
+ */
+ public Builder withTargetRootPrefix(String targetRootPrefix) {
+ this.targetRootPrefix = targetRootPrefix;
+ return this;
+ }
+
/**
* Whether to fail if run on an older version of the store of force
upgrading
* its format.
@@ -253,13 +275,13 @@ public class AzureCompact {
return this;
}
- public Builder withSourceCloudBlobDirectory(CloudBlobDirectory
sourceCloudBlobDirectory) {
- this.sourceCloudBlobDirectory =
requireNonNull(sourceCloudBlobDirectory);
+ public Builder withSourceBlobContainerClient(BlobContainerClient
sourceBlobContainerClient) {
+ this.sourceBlobContainerClient =
requireNonNull(sourceBlobContainerClient);
return this;
}
- public Builder withDestinationCloudBlobDirectory(CloudBlobDirectory
destinationCloudBlobDirectory) {
- this.destinationCloudBlobDirectory =
requireNonNull(destinationCloudBlobDirectory);
+ public Builder withDestinationBlobContainerClient(BlobContainerClient
destinationBlobContainerClient) {
+ this.destinationBlobContainerClient =
requireNonNull(destinationBlobContainerClient);
return this;
}
@@ -269,7 +291,7 @@ public class AzureCompact {
* @return an instance of {@link Runnable}.
*/
public AzureCompact build() {
- if (sourceCloudBlobDirectory == null ||
destinationCloudBlobDirectory == null) {
+ if (sourceBlobContainerClient == null ||
destinationBlobContainerClient == null) {
requireNonNull(path);
requireNonNull(targetPath);
}
@@ -283,6 +305,10 @@ public class AzureCompact {
private final String targetPath;
+ private final String rootPrefix;
+
+ private final String targetRootPrefix;
+
private final int segmentCacheSize;
private final boolean strictVersionCheck;
@@ -303,14 +329,15 @@ public class AzureCompact {
private final int garbageThresholdPercentage;
- private final CloudBlobDirectory sourceCloudBlobDirectory;
+ private final BlobContainerClient sourceBlobContainerClient;
- private final CloudBlobDirectory destinationCloudBlobDirectory;
- private final AzureStorageCredentialManagerV8
azureStorageCredentialManagerV8;
+ private final BlobContainerClient destinationBlobContainerClient;
private AzureCompact(Builder builder) {
this.path = builder.path;
this.targetPath = builder.targetPath;
+ this.rootPrefix = builder.rootPrefix;
+ this.targetRootPrefix = builder.targetRootPrefix;
this.segmentCacheSize = builder.segmentCacheSize;
this.strictVersionCheck = !builder.force;
this.gcLogInterval = builder.gcLogInterval;
@@ -321,22 +348,26 @@ public class AzureCompact {
this.persistentCacheSizeGb = builder.persistentCacheSizeGb;
this.garbageThresholdGb = builder.garbageThresholdGb;
this.garbageThresholdPercentage = builder.garbageThresholdPercentage;
- this.sourceCloudBlobDirectory = builder.sourceCloudBlobDirectory;
- this.destinationCloudBlobDirectory =
builder.destinationCloudBlobDirectory;
- this.azureStorageCredentialManagerV8 = new
AzureStorageCredentialManagerV8();
+ this.sourceBlobContainerClient = builder.sourceBlobContainerClient;
+ this.destinationBlobContainerClient =
builder.destinationBlobContainerClient;
}
- public int run() throws IOException, StorageException, URISyntaxException {
+ public int run() throws IOException, URISyntaxException {
Stopwatch watch = Stopwatch.createStarted();
SegmentNodeStorePersistence roPersistence;
SegmentNodeStorePersistence rwPersistence;
- if (sourceCloudBlobDirectory != null && destinationCloudBlobDirectory
!= null) {
- roPersistence = new AzurePersistenceV8(sourceCloudBlobDirectory);
- rwPersistence = new
AzurePersistenceV8(destinationCloudBlobDirectory);
+ BlobContainerClient targetContainer;
+ if (sourceBlobContainerClient != null &&
destinationBlobContainerClient != null) {
+ roPersistence = new AzurePersistence(sourceBlobContainerClient,
rootPrefix);
+ rwPersistence = new
AzurePersistence(destinationBlobContainerClient, targetRootPrefix);
+ targetContainer = destinationBlobContainerClient;
} else {
- roPersistence =
newSegmentNodeStorePersistence(SegmentStoreType.AZURE, path,
azureStorageCredentialManagerV8);
- rwPersistence =
newSegmentNodeStorePersistence(SegmentStoreType.AZURE, targetPath,
azureStorageCredentialManagerV8);
+ roPersistence = createAzurePersistence(path);
+
+ AzurePersistence rwAzurePersistence =
createAzurePersistence(targetPath);
+ targetContainer = rwAzurePersistence.getReadBlobContainerClient();
+ rwPersistence = rwAzurePersistence;
}
if (persistentCachePath != null) {
@@ -348,8 +379,8 @@ public class AzureCompact {
SegmentArchiveManager roArchiveManager =
createArchiveManager(roPersistence);
SegmentArchiveManager rwArchiveManager =
createArchiveManager(rwPersistence);
- System.out.printf("Compacting %s\n", path != null ? path :
sourceCloudBlobDirectory.getUri().toString());
- System.out.printf(" to %s\n", targetPath != null ? targetPath :
destinationCloudBlobDirectory.getUri().toString());
+ System.out.printf("Compacting %s\n", path != null ? path :
sourceBlobContainerClient.getBlobContainerUrl());
+ System.out.printf(" to %s\n", targetPath != null ? targetPath :
destinationBlobContainerClient.getBlobContainerUrl());
System.out.printf(" before\n");
List<String> beforeArchives = Collections.emptyList();
try {
@@ -360,14 +391,6 @@ public class AzureCompact {
printArchives(System.out, beforeArchives);
- CloudBlobContainer targetContainer = null;
- if (targetPath != null) {
- CloudBlobDirectory targetDirectory =
createCloudBlobDirectory(targetPath.substring(3),
azureStorageCredentialManagerV8);
- targetContainer = targetDirectory.getContainer();
- } else {
- targetContainer = destinationCloudBlobDirectory.getContainer();
- }
-
GCGeneration gcGeneration = null;
String root = null;
@@ -424,8 +447,6 @@ public class AzureCompact {
long newSize = printTargetRepoSizeInfo(targetContainer);
persistGCJournal(rwPersistence, newSize, gcGeneration, root);
- // close azure storage credential manager
- azureStorageCredentialManagerV8.close();
return 0;
}
@@ -456,13 +477,14 @@ public class AzureCompact {
return true;
}
- private long printTargetRepoSizeInfo(CloudBlobContainer container) {
- System.out.printf("Calculating the size of container %s\n",
container.getName());
+ private long printTargetRepoSizeInfo(BlobContainerClient
blobContainerClient) {
+ System.out.printf("Calculating the size of container %s\n",
blobContainerClient.getBlobContainerName());
long size = 0;
- for (ListBlobItem i : container.listBlobs(null, true,
EnumSet.of(BlobListingDetails.METADATA), null, null)) {
- if (i instanceof CloudBlob) {
- size += ((CloudBlob) i).getProperties().getLength();
- }
+ ListBlobsOptions listBlobsOptions = new ListBlobsOptions();
+ listBlobsOptions.setDetails(new
BlobListDetails().setRetrieveMetadata(true));
+
+ for (BlobItem blobItem :
blobContainerClient.listBlobs(listBlobsOptions, null)) {
+ size += blobItem.getProperties().getContentLength();
}
System.out.printf("The size is: %d MB \n", size / 1024 / 1024);
return size;
diff --git
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopy.java
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopy.java
index 706f64885a..cfed41d3cf 100644
---
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopy.java
+++
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopy.java
@@ -43,7 +43,6 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.jackrabbit.oak.commons.Buffer;
-import
org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8;
import
org.apache.jackrabbit.oak.segment.azure.tool.SegmentStoreMigrator.Segment;
import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.SegmentStoreType;
import org.apache.jackrabbit.oak.segment.azure.util.Retrier;
@@ -260,7 +259,6 @@ public class SegmentCopy {
private SegmentNodeStorePersistence destPersistence;
private ExecutorService executor =
Executors.newFixedThreadPool(READ_THREADS + 1);
- private final AzureStorageCredentialManagerV8
azureStorageCredentialManagerV8;
public SegmentCopy(Builder builder) {
this.source = builder.source;
@@ -273,7 +271,6 @@ public class SegmentCopy {
this.maxSizeGb = builder.maxSizeGb;
this.outWriter = builder.outWriter;
this.errWriter = builder.errWriter;
- this.azureStorageCredentialManagerV8 = new
AzureStorageCredentialManagerV8();
}
public int run() {
@@ -288,7 +285,7 @@ public class SegmentCopy {
if (flat && destType == SegmentStoreType.TAR) {
try {
if (srcPersistence == null) {
- srcPersistence = newSegmentNodeStorePersistence(srcType,
source, azureStorageCredentialManagerV8);
+ srcPersistence = newSegmentNodeStorePersistence(srcType,
source);
}
SegmentArchiveManager sourceManager =
srcPersistence.createArchiveManager(false, false,
@@ -366,14 +363,12 @@ public class SegmentCopy {
destination);
e.printStackTrace(errWriter);
return 1;
- } finally {
- azureStorageCredentialManagerV8.close();
}
} else {
try {
if (srcPersistence == null || destPersistence == null) {
- srcPersistence = newSegmentNodeStorePersistence(srcType,
source, azureStorageCredentialManagerV8);
- destPersistence = newSegmentNodeStorePersistence(destType,
destination, azureStorageCredentialManagerV8);
+ srcPersistence = newSegmentNodeStorePersistence(srcType,
source);
+ destPersistence = newSegmentNodeStorePersistence(destType,
destination);
}
printMessage(outWriter, "Started segment-copy transfer!");
@@ -397,8 +392,6 @@ public class SegmentCopy {
destination);
e.printStackTrace(errWriter);
return 1;
- } finally {
- azureStorageCredentialManagerV8.close();
}
}
diff --git
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentStoreMigrator.java
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentStoreMigrator.java
index 6bba400c16..404be35419 100644
---
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentStoreMigrator.java
+++
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentStoreMigrator.java
@@ -19,11 +19,9 @@ package org.apache.jackrabbit.oak.segment.azure.tool;
import static
org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.fetchByteArray;
import static
org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.storeDescription;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.blob.CloudBlobDirectory;
-
+import com.azure.storage.blob.BlobContainerClient;
import org.apache.jackrabbit.oak.commons.Buffer;
-import org.apache.jackrabbit.oak.segment.azure.v8.AzurePersistenceV8;
+import org.apache.jackrabbit.oak.segment.azure.AzurePersistence;
import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.SegmentStoreType;
import org.apache.jackrabbit.oak.segment.azure.util.Retrier;
import org.apache.jackrabbit.oak.segment.file.tar.TarPersistence;
@@ -45,7 +43,6 @@ import org.slf4j.LoggerFactory;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
-import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -289,9 +286,9 @@ public class SegmentStoreMigrator implements Closeable {
return this;
}
- public Builder withSource(CloudBlobDirectory dir) throws
URISyntaxException, StorageException {
- this.source = new AzurePersistenceV8(dir);
- this.sourceName = storeDescription(SegmentStoreType.AZURE,
dir.getContainer().getName() + "/" + dir.getPrefix());
+ public Builder withSource(BlobContainerClient blobContainerClient,
String rootPrefix) {
+ this.source = new AzurePersistence(blobContainerClient,
rootPrefix);
+ this.sourceName = storeDescription(SegmentStoreType.AZURE,
blobContainerClient.getBlobContainerName() + "/" + rootPrefix);
return this;
}
@@ -313,9 +310,9 @@ public class SegmentStoreMigrator implements Closeable {
return this;
}
- public Builder withTarget(CloudBlobDirectory dir) throws
URISyntaxException, StorageException {
- this.target = new AzurePersistenceV8(dir);
- this.targetName = storeDescription(SegmentStoreType.AZURE,
dir.getContainer().getName() + "/" + dir.getPrefix());
+ public Builder withTarget(BlobContainerClient blobContainerClient,
String rootPrefix) {
+ this.target = new AzurePersistence(blobContainerClient,
rootPrefix);
+ this.targetName = storeDescription(SegmentStoreType.AZURE,
blobContainerClient.getBlobContainerUrl() + "/" + rootPrefix);
return this;
}
diff --git
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java
index 3eab847215..e41e6f5350 100644
---
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java
+++
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java
@@ -21,27 +21,24 @@ package org.apache.jackrabbit.oak.segment.azure.tool;
import static
org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_ACCOUNT_NAME;
import static
org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_DIR;
import static
org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_SHARED_ACCESS_SIGNATURE;
-import static
org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_STORAGE_URI;
+import static
org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_CONTAINER_NAME;
import static
org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.parseAzureConfigurationFromUri;
import static
org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.defaultGCOptions;
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
-import java.net.URISyntaxException;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
import java.util.concurrent.TimeUnit;
import org.apache.jackrabbit.oak.commons.Buffer;
import org.apache.jackrabbit.oak.commons.collections.ListUtils;
-import org.apache.jackrabbit.oak.segment.azure.v8.AzurePersistenceV8;
-import
org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8;
-import org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8;
+import org.apache.jackrabbit.oak.segment.azure.AzurePersistence;
+import org.apache.jackrabbit.oak.segment.azure.AzurePersistenceManager;
import org.apache.jackrabbit.oak.segment.azure.util.Environment;
import
org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.CompactorType;
import org.apache.jackrabbit.oak.segment.file.*;
@@ -60,12 +57,8 @@ import
org.apache.jackrabbit.oak.segment.spi.persistence.persistentcache.Persist
import org.apache.jackrabbit.guava.common.base.Stopwatch;
import org.apache.jackrabbit.guava.common.collect.Iterators;
-import com.microsoft.azure.storage.StorageCredentials;
-import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.blob.CloudBlobDirectory;
+
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
-import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -130,15 +123,12 @@ public class ToolUtils {
}
public static SegmentNodeStorePersistence
newSegmentNodeStorePersistence(SegmentStoreType storeType,
-
String pathOrUri,
-
@Nullable AzureStorageCredentialManagerV8 azureStorageCredentialManagerV8) {
+
String pathOrUri) {
SegmentNodeStorePersistence persistence = null;
switch (storeType) {
case AZURE:
- Objects.requireNonNull(azureStorageCredentialManagerV8, "azure
storage credentials manager instance cannot be null");
- CloudBlobDirectory cloudBlobDirectory =
createCloudBlobDirectory(pathOrUri.substring(3),
azureStorageCredentialManagerV8);
- persistence = new AzurePersistenceV8(cloudBlobDirectory);
+ persistence = createAzurePersistence(pathOrUri.substring(3));
break;
default:
persistence = new TarPersistence(new File(pathOrUri));
@@ -160,48 +150,43 @@ public class ToolUtils {
return archiveManager;
}
- public static CloudBlobDirectory createCloudBlobDirectory(String path,
AzureStorageCredentialManagerV8 azureStorageCredentialManagerV8) {
- return createCloudBlobDirectory(path, ENVIRONMENT,
azureStorageCredentialManagerV8);
+ public static AzurePersistence createAzurePersistence(String path) {
+ return createAzurePersistence(path, ENVIRONMENT);
}
- public static CloudBlobDirectory createCloudBlobDirectory(String path,
- Environment
environment,
-
AzureStorageCredentialManagerV8 azureStorageCredentialManagerV8) {
+ public static AzurePersistence createAzurePersistence(String path,
+ Environment
environment) {
Map<String, String> config = parseAzureConfigurationFromUri(path);
String accountName = config.get(KEY_ACCOUNT_NAME);
-
- StorageCredentials credentials;
- if (config.containsKey(KEY_SHARED_ACCESS_SIGNATURE)) {
- credentials = new
StorageCredentialsSharedAccessSignature(config.get(KEY_SHARED_ACCESS_SIGNATURE));
- } else {
- credentials =
azureStorageCredentialManagerV8.getStorageCredentialsFromEnvironment(accountName,
environment);
- }
-
- String uri = config.get(KEY_STORAGE_URI);
+ String containerName = config.get(KEY_CONTAINER_NAME);
String dir = config.get(KEY_DIR);
+ AzurePersistence azurePersistence;
try {
- return AzureUtilitiesV8.cloudBlobDirectoryFrom(credentials, uri,
dir);
- } catch (URISyntaxException | StorageException e) {
+ if (config.containsKey(KEY_SHARED_ACCESS_SIGNATURE)) {
+ azurePersistence =
AzurePersistenceManager.createAzurePersistenceFrom(accountName, containerName,
dir, config.get(KEY_SHARED_ACCESS_SIGNATURE));
+ } else {
+ azurePersistence =
AzurePersistenceManager.createAzurePersistenceFrom(accountName, containerName,
dir, environment);
+ }
+ } catch (IOException e) {
throw new IllegalArgumentException(
"Could not connect to the Azure Storage. Please verify the
path provided!");
}
+ return azurePersistence;
}
public static List<String> readRevisions(String uri) {
- try (AzureStorageCredentialManagerV8 azureStorageCredentialManagerV8 =
new AzureStorageCredentialManagerV8()) {
- SegmentNodeStorePersistence persistence =
newSegmentNodeStorePersistence(SegmentStoreType.AZURE, uri,
azureStorageCredentialManagerV8);
- JournalFile journal = persistence.getJournalFile();
- if (journal.exists()) {
- try (JournalReader journalReader = new JournalReader(journal))
{
- Iterator<String> revisionIterator =
Iterators.transform(journalReader,
- entry -> entry.getRevision());
- return ListUtils.toList(revisionIterator);
- } catch (Exception e) {
- log.error("Error while reading from journal file");
- e.printStackTrace();
- }
+ SegmentNodeStorePersistence persistence =
newSegmentNodeStorePersistence(SegmentStoreType.AZURE, uri);
+ JournalFile journal = persistence.getJournalFile();
+ if (journal.exists()) {
+ try (JournalReader journalReader = new JournalReader(journal)) {
+ Iterator<String> revisionIterator =
Iterators.transform(journalReader,
+ entry -> entry.getRevision());
+ return ListUtils.toList(revisionIterator);
+ } catch (Exception e) {
+ log.error("Error while reading from journal file");
+ e.printStackTrace();
}
}
diff --git
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/util/AzureConfigurationParserUtils.java
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/util/AzureConfigurationParserUtils.java
index e2279f9d81..9ee8f1098f 100644
---
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/util/AzureConfigurationParserUtils.java
+++
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/util/AzureConfigurationParserUtils.java
@@ -147,14 +147,15 @@ public class AzureConfigurationParserUtils {
int dotPosHost = host.indexOf(".");
String accountName = host.substring(0, dotPosHost);
- String container = path.substring(0, lastSlashPosPath);
- String storageUri = scheme + "://" + host + container;
+ String container = path.substring(1, lastSlashPosPath);
+ String storageUri = scheme + "://" + host + "/" + container;
String dir = path.substring(lastSlashPosPath + 1);
Map<String, String> config = new HashMap<>();
config.put(KEY_ACCOUNT_NAME, accountName);
config.put(KEY_STORAGE_URI, storageUri);
config.put(KEY_DIR, dir);
+ config.put(KEY_CONTAINER_NAME, container);
if (sasToken != null) {
config.put(KEY_SHARED_ACCESS_SIGNATURE, sasToken);
}
diff --git
a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistenceManagerTest.java
b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistenceManagerTest.java
new file mode 100644
index 0000000000..6909edc07e
--- /dev/null
+++
b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistenceManagerTest.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.segment.azure;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertThrows;
+import org.apache.jackrabbit.oak.segment.azure.util.Environment;
+import org.osgi.util.converter.Converters;
+
+import java.util.HashMap;
+
+public class AzurePersistenceManagerTest {
+
+ @Test
+ public void bothConnectionStringAndSasTokenNotConfiguredTest() {
+ assertThrows(IllegalArgumentException.class, () ->
AzurePersistenceManager.createAzurePersistence(null, null, "accountName",
"containerName", "aem", false, false));
+ }
+
+ @Test
+ public void bothConnectionStringAndSasTokenBlankTest() {
+ assertThrows(IllegalArgumentException.class, () ->
AzurePersistenceManager.createAzurePersistence("", "", "accountName",
"containerName", "aem", false, false));
+ }
+
+ @Test
+ public void connectionStringBlankAndSasTokenNullTest() {
+ assertThrows(IllegalArgumentException.class, () ->
AzurePersistenceManager.createAzurePersistence("", null, "accountName",
"containerName", "aem", false, false));
+ }
+
+ @Test
+ public void connectionStringNullAndSasTokenBlankTest() {
+ assertThrows(IllegalArgumentException.class, () ->
AzurePersistenceManager.createAzurePersistence(null, "", "accountName",
"containerName", "aem", false, false));
+ }
+
+ @Test
+ public void accountNameIsNullTest() {
+ assertThrows(IllegalArgumentException.class, () ->
AzurePersistenceManager.createAzurePersistence("connectionString", "sasToken",
null, "containerName", "aem", false, false));
+ }
+
+ @Test
+ public void accountNameIsEmptyTest() {
+ assertThrows(IllegalArgumentException.class, () ->
AzurePersistenceManager.createAzurePersistence("connectionString", "sasToken",
"", "containerName", "aem", false, false));
+ }
+
+ @Test
+ public void containerNameIsNullTest() {
+ assertThrows(IllegalArgumentException.class, () ->
AzurePersistenceManager.createAzurePersistence("connectionString", "sasToken",
"accountName", null, "aem", false, false));
+ }
+
+ @Test
+ public void containerNameIsEmptyTest() {
+ assertThrows(IllegalArgumentException.class, () ->
AzurePersistenceManager.createAzurePersistence("connectionString", "sasToken",
"accountName", "", "aem", false, false));
+ }
+
+ @Test
+ public void rootPrefixIsNullTest() {
+ assertThrows(IllegalArgumentException.class, () ->
AzurePersistenceManager.createAzurePersistence("connectionString", "sasToken",
"accountName", "containerName", null, false, false));
+ }
+
+ @Test
+ public void rootPrefixIsEmptyTest() {
+ assertThrows(IllegalArgumentException.class, () ->
AzurePersistenceManager.createAzurePersistence("connectionString", "sasToken",
"accountName", "containerName", "", false, false));
+ }
+
+ @Test
+ public void servicePrincipalsAccountNameIsNullTest() {
+ assertThrows(IllegalArgumentException.class, () ->
AzurePersistenceManager.createPersistenceFromServicePrincipalCredentials(null,
"containerName", "rootPrefix", "clientId", "clientSecret", "tenantId", false,
false));
+ }
+
+ @Test
+ public void servicePrincipalAccountNameIsEmptyTest() {
+ assertThrows(IllegalArgumentException.class, () ->
AzurePersistenceManager.createPersistenceFromServicePrincipalCredentials("",
"containerName", "rootPrefix", "clientId", "clientSecret", "tenantId", false,
false));
+ }
+
+ @Test
+ public void servicePrincipalsContainerNameIsNullTest() {
+ assertThrows(IllegalArgumentException.class, () ->
AzurePersistenceManager.createPersistenceFromServicePrincipalCredentials("accountName",
null, "rootPrefix", "clientId", "clientSecret", "tenantId", false, false));
+ }
+
+ @Test
+ public void servicePrincipalsContainerNameIsEmptyTest() {
+ assertThrows(IllegalArgumentException.class, () ->
AzurePersistenceManager.createPersistenceFromServicePrincipalCredentials("accountName",
"", "rootPrefix", "clientId", "clientSecret", "tenantId", false, false));
+ }
+
+ @Test
+ public void servicePrincipalsRootPrefixIsNullTest() {
+ assertThrows(IllegalArgumentException.class, () ->
AzurePersistenceManager.createPersistenceFromServicePrincipalCredentials("accountName",
"containerName", null, "clientId", "clientSecret", "tenantId", false, false));
+ }
+
+ @Test
+ public void servicePrincipalsRootPrefixIsEmptyTest() {
+ assertThrows(IllegalArgumentException.class, () ->
AzurePersistenceManager.createPersistenceFromServicePrincipalCredentials("accountName",
"containerName", "", "clientId", "clientSecret", "tenantId", false, false));
+ }
+
+ @Test
+ public void createAzurePersistenceAllNullTest() {
+ assertThrows(IllegalArgumentException.class, () ->
AzurePersistenceManager.createAzurePersistenceFrom(null, null, null,
"sasToken"));
+ }
+
+ @Test
+ public void createAzurePersistenceFromEnvAllNullTest() {
+ Environment env = new Environment();
+ assertThrows(IllegalArgumentException.class, () ->
AzurePersistenceManager.createAzurePersistenceFrom(null, null, null, env));
+ }
+
+ @Test
+ public void createAzurePersistenceFromConfigurationAllNullTest() {
+ assertThrows(IllegalArgumentException.class, () ->
AzurePersistenceManager.createAzurePersistenceFrom(getConfiguration()));
+ }
+
+ private static Configuration getConfiguration() {
+ return Converters.standardConverter()
+ .convert(new HashMap<Object, Object>() {{
+ put("accountName", null);
+ put("accessKey", null);
+ put("connectionURL",
"https://accounts.blob.core.windows.net";);
+ put("sharedAccessSignature", "sharedAccessSignature");
+ put("clientId", "clientId");
+ put("clientSecret", "clientSecret");
+ put("tenantId", "tenantId");
+ put("blobEndpoint", "blobEndpoint");
+ }})
+ .to(Configuration.class);
+ }
+
+}
\ No newline at end of file
diff --git
a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyAzureServicePrincipalToTarTest.java
b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyAzureServicePrincipalToTarTest.java
index 7d7ada9e3b..c6e36ab070 100644
---
a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyAzureServicePrincipalToTarTest.java
+++
b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyAzureServicePrincipalToTarTest.java
@@ -18,18 +18,16 @@
*/
package org.apache.jackrabbit.oak.segment.azure.tool;
-import com.microsoft.azure.storage.blob.CloudBlobDirectory;
-import org.apache.jackrabbit.oak.segment.azure.v8.AzurePersistenceV8;
-import
org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8;
-import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils;
import org.apache.jackrabbit.oak.segment.azure.util.Environment;
import
org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence;
import org.junit.Test;
-import static
org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8.AZURE_ACCOUNT_NAME;
-import static
org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8.AZURE_CLIENT_ID;
-import static
org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8.AZURE_CLIENT_SECRET;
-import static
org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8.AZURE_TENANT_ID;
+import java.io.IOException;
+
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_ACCOUNT_NAME;
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_ID;
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_SECRET;
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_TENANT_ID;
import static org.junit.Assume.assumeNotNull;
public class SegmentCopyAzureServicePrincipalToTarTest extends
SegmentCopyTestBase {
@@ -53,11 +51,7 @@ public class SegmentCopyAzureServicePrincipalToTarTest
extends SegmentCopyTestBa
protected SegmentNodeStorePersistence getSrcPersistence() {
String accountName = ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME);
String path = String.format(SEGMENT_STORE_PATH_FORMAT, accountName,
CONTAINER_NAME, DIR);
- CloudBlobDirectory cloudBlobDirectory;
- try (AzureStorageCredentialManagerV8 azureStorageCredentialManagerV8 =
new AzureStorageCredentialManagerV8()) {
- cloudBlobDirectory = ToolUtils.createCloudBlobDirectory(path,
ENVIRONMENT, azureStorageCredentialManagerV8);
- }
- return new AzurePersistenceV8(cloudBlobDirectory);
+ return ToolUtils.createAzurePersistence(path, ENVIRONMENT);
}
@Override
diff --git
a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java
b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java
index 05e7d32b9e..f259939e53 100644
---
a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java
+++
b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java
@@ -231,7 +231,7 @@ public abstract class SegmentCopyTestBase {
}
protected SegmentNodeStorePersistence getTarPersistence() {
- return newSegmentNodeStorePersistence(SegmentStoreType.TAR,
folder.getRoot().getAbsolutePath(), null);
+ return newSegmentNodeStorePersistence(SegmentStoreType.TAR,
folder.getRoot().getAbsolutePath());
}
protected SegmentNodeStorePersistence getAzurePersistence() throws
Exception {
diff --git
a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtilsTest.java
b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtilsTest.java
index 4fbb51555e..049cfdb5ee 100644
---
a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtilsTest.java
+++
b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtilsTest.java
@@ -23,44 +23,30 @@ import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.Appender;
import ch.qos.logback.core.read.ListAppender;
-import com.microsoft.azure.storage.StorageCredentials;
-import com.microsoft.azure.storage.StorageCredentialsAccountAndKey;
-import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature;
-import java.net.URISyntaxException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.blob.CloudBlobDirectory;
import
org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzuriteDockerRule;
-import
org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8;
-import org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8;
+import org.apache.jackrabbit.oak.segment.azure.AzurePersistence;
import org.apache.jackrabbit.oak.segment.azure.util.Environment;
import org.jetbrains.annotations.NotNull;
-import org.junit.After;
-import org.junit.Before;
import org.junit.Test;
-import org.mockito.ArgumentCaptor;
-import org.mockito.MockedStatic;
import org.slf4j.LoggerFactory;
-import static
org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8.AZURE_ACCOUNT_NAME;
-import static
org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8.AZURE_CLIENT_ID;
-import static
org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8.AZURE_CLIENT_SECRET;
-import static
org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8.AZURE_SECRET_KEY;
-import static
org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8.AZURE_TENANT_ID;
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_ACCOUNT_NAME;
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_ID;
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_SECRET;
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_SECRET_KEY;
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_TENANT_ID;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
import static org.junit.Assume.assumeNotNull;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.mockStatic;
public class ToolUtilsTest {
private static final Environment ENVIRONMENT = new Environment();
@@ -71,75 +57,56 @@ public class ToolUtilsTest {
private static final String DEFAULT_ACCOUNT_NAME = "myaccount";
private static final String DEFAULT_CONTAINER_NAME = "oak";
private static final String DEFAULT_REPO_DIR = "repository";
- private static final String DEFAULT_CONTAINER_URL =
String.format(CONTAINER_URL_FORMAT, DEFAULT_ACCOUNT_NAME,
DEFAULT_CONTAINER_NAME);
private static final String DEFAULT_SEGMENT_STORE_PATH =
String.format(SEGMENT_STORE_PATH_FORMAT, DEFAULT_ACCOUNT_NAME,
DEFAULT_CONTAINER_NAME, DEFAULT_REPO_DIR);
public static final String AZURE_SECRET_KEY_WARNING = "AZURE_CLIENT_ID,
AZURE_CLIENT_SECRET and AZURE_TENANT_ID environment variables empty or missing.
Switching to authentication with AZURE_SECRET_KEY.";
private final TestEnvironment environment = new TestEnvironment();
- private AzureStorageCredentialManagerV8 azureStorageCredentialManagerV8;
-
- @Before
- public void init() {
- this.azureStorageCredentialManagerV8 = new
AzureStorageCredentialManagerV8();
- }
-
- @After
- public void clear() {
- this.azureStorageCredentialManagerV8.close();
- }
@Test
- public void createCloudBlobDirectoryWithAccessKey() {
+ public void createAzurePersistenceWithAccessKey() {
environment.setVariable(AZURE_SECRET_KEY,
AzuriteDockerRule.ACCOUNT_KEY);
final ListAppender<ILoggingEvent> logAppender = subscribeAppender();
- StorageCredentialsAccountAndKey credentials = expectCredentials(
- StorageCredentialsAccountAndKey.class,
- () ->
ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH, environment,
azureStorageCredentialManagerV8),
- DEFAULT_CONTAINER_URL
- );
+ AzurePersistence azurePersistence =
ToolUtils.createAzurePersistence(DEFAULT_SEGMENT_STORE_PATH, environment);
assertTrue(checkLogContainsMessage(AZURE_SECRET_KEY_WARNING,
logAppender.list.stream().map(ILoggingEvent::getFormattedMessage).collect(Collectors.toList())));
assertEquals(Level.WARN, logAppender.list.get(0).getLevel());
- assertEquals(DEFAULT_ACCOUNT_NAME, credentials.getAccountName());
- assertEquals(AzuriteDockerRule.ACCOUNT_KEY,
credentials.exportBase64EncodedKey());
+ assertEquals(DEFAULT_ACCOUNT_NAME,
azurePersistence.getReadBlobContainerClient().getAccountName());
+ assertEquals(DEFAULT_CONTAINER_NAME,
azurePersistence.getReadBlobContainerClient().getBlobContainerName());
unsubscribe(logAppender);
}
@Test
- public void createCloudBlobDirectoryFailsWhenAccessKeyNotPresent() {
+ public void createAzurePersistenceFailsWhenAccessKeyNotPresent() {
environment.setVariable(AZURE_SECRET_KEY, null);
assertThrows(IllegalArgumentException.class, () ->
- ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH,
environment, azureStorageCredentialManagerV8)
+ ToolUtils.createAzurePersistence(DEFAULT_SEGMENT_STORE_PATH,
environment)
);
}
@Test
- public void createCloudBlobDirectoryFailsWhenAccessKeyIsInvalid() {
+ public void createAzurePersistenceFailsWhenAccessKeyIsInvalid() {
environment.setVariable(AZURE_SECRET_KEY, "invalid");
- assertThrows(IllegalArgumentException.class, () ->
- ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH,
environment, azureStorageCredentialManagerV8)
+ AzurePersistence azurePersistence =
ToolUtils.createAzurePersistence(DEFAULT_SEGMENT_STORE_PATH, environment);
+
+ assertThrows(Exception.class, () ->
+ azurePersistence.getReadBlobContainerClient().exists()
);
}
@Test
- public void createCloudBlobDirectoryWithSasUri() {
+ public void createAzurePersistenceWithSasUri() {
String sasToken =
"sig=qL%2Fi%2BP7J6S0sA8Ihc%2BKq75U5uJcnukpfktT2fm1ckXk%3D&se=2022-02-09T11%3A52%3A42Z&sv=2019-02-02&sp=rl&sr=c";
- StorageCredentialsSharedAccessSignature credentials =
expectCredentials(
- StorageCredentialsSharedAccessSignature.class,
- () ->
ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH + '?' + sasToken,
azureStorageCredentialManagerV8),
- DEFAULT_CONTAINER_URL
- );
+ AzurePersistence azurePersistence =
ToolUtils.createAzurePersistence(DEFAULT_SEGMENT_STORE_PATH + '?' + sasToken);
- assertEquals(sasToken, credentials.getToken());
- assertNull("AccountName should be null when SAS credentials are used",
credentials.getAccountName());
+ assertEquals(DEFAULT_CONTAINER_NAME,
azurePersistence.getReadBlobContainerClient().getBlobContainerName());
}
@Test
- public void createCloudBlobDirectoryWithServicePrincipal() throws
URISyntaxException, StorageException {
+ public void createAzurePersistenceWithServicePrincipal() {
assumeNotNull(ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME));
assumeNotNull(ENVIRONMENT.getVariable(AZURE_TENANT_ID));
assumeNotNull(ENVIRONMENT.getVariable(AZURE_CLIENT_ID));
@@ -149,24 +116,9 @@ public class ToolUtilsTest {
String containerName = "oak";
String segmentStorePath = String.format(SEGMENT_STORE_PATH_FORMAT,
accountName, containerName, DEFAULT_REPO_DIR);
- CloudBlobDirectory cloudBlobDirectory =
ToolUtils.createCloudBlobDirectory(segmentStorePath, ENVIRONMENT,
azureStorageCredentialManagerV8);
- assertNotNull(cloudBlobDirectory);
- assertEquals(containerName,
cloudBlobDirectory.getContainer().getName());
- }
-
- private static <T extends StorageCredentials> T expectCredentials(Class<T>
clazz, Runnable body, String containerUrl) {
- ArgumentCaptor<T> credentialsCaptor = ArgumentCaptor.forClass(clazz);
- try (MockedStatic<AzureUtilitiesV8> mockedAzureUtilities =
mockStatic(AzureUtilitiesV8.class)) {
- body.run();
-
- mockedAzureUtilities.verify(() ->
AzureUtilitiesV8.cloudBlobDirectoryFrom(
- credentialsCaptor.capture(),
- eq(containerUrl),
- eq(DEFAULT_REPO_DIR)
- )
- );
- return credentialsCaptor.getValue();
- }
+ AzurePersistence azurePersistence =
ToolUtils.createAzurePersistence(segmentStorePath, ENVIRONMENT);
+ assertNotNull(azurePersistence);
+ assertEquals(containerName,
azurePersistence.getReadBlobContainerClient().getBlobContainerName());
}
private boolean checkLogContainsMessage(String toCheck, List<String>
messages) {
diff --git
a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentAzureServicePrincipalNodeStoreContainer.java
b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentAzureServicePrincipalNodeStoreContainer.java
index 34f6bcc987..b3535646e3 100644
---
a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentAzureServicePrincipalNodeStoreContainer.java
+++
b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentAzureServicePrincipalNodeStoreContainer.java
@@ -16,11 +16,9 @@
*/
package org.apache.jackrabbit.oak.upgrade.cli.container;
-import com.microsoft.azure.storage.blob.CloudBlobDirectory;
import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
-import org.apache.jackrabbit.oak.segment.azure.v8.AzurePersistenceV8;
-import
org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8;
-import org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8;
+import org.apache.jackrabbit.oak.segment.azure.AzurePersistence;
+import org.apache.jackrabbit.oak.segment.azure.AzureUtilities;
import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils;
import org.apache.jackrabbit.oak.segment.azure.util.Environment;
import org.apache.jackrabbit.oak.segment.file.FileStore;
@@ -43,8 +41,7 @@ public class SegmentAzureServicePrincipalNodeStoreContainer
implements NodeStore
private final BlobStore blobStore;
private FileStore fs;
private File tmpDir;
- private AzurePersistenceV8 azurePersistenceV8;
- private final AzureStorageCredentialManagerV8
azureStorageCredentialManagerV8;
+ private AzurePersistence azurePersistence;
public SegmentAzureServicePrincipalNodeStoreContainer() {
this(null);
@@ -52,21 +49,20 @@ public class SegmentAzureServicePrincipalNodeStoreContainer
implements NodeStore
public SegmentAzureServicePrincipalNodeStoreContainer(BlobStore blobStore)
{
this.blobStore = blobStore;
- this.azureStorageCredentialManagerV8 = new
AzureStorageCredentialManagerV8();
}
@Override
public NodeStore open() throws IOException {
try {
- azurePersistenceV8 = createAzurePersistence();
+ azurePersistence = createAzurePersistence();
} catch (Exception e) {
throw new IllegalStateException(e);
}
tmpDir = Files.createTempDirectory(getClass().getSimpleName() +
"-").toFile();
FileStoreBuilder builder = FileStoreBuilder.fileStoreBuilder(tmpDir)
-
.withCustomPersistence(azurePersistenceV8).withMemoryMapping(false);
+
.withCustomPersistence(azurePersistence).withMemoryMapping(false);
if (blobStore != null) {
builder.withBlobStore(blobStore);
}
@@ -80,14 +76,13 @@ public class SegmentAzureServicePrincipalNodeStoreContainer
implements NodeStore
return new
FileStoreUtils.NodeStoreWithFileStore(SegmentNodeStoreBuilders.builder(fs).build(),
fs);
}
- private AzurePersistenceV8 createAzurePersistence() {
- if (azurePersistenceV8 != null) {
- return azurePersistenceV8;
+ private AzurePersistence createAzurePersistence() {
+ if (azurePersistence != null) {
+ return azurePersistence;
}
- String path = String.format(AZURE_SEGMENT_STORE_PATH,
ENVIRONMENT.getVariable(AzureUtilitiesV8.AZURE_ACCOUNT_NAME),
+ String path = String.format(AZURE_SEGMENT_STORE_PATH,
ENVIRONMENT.getVariable(AzureUtilities.AZURE_ACCOUNT_NAME),
CONTAINER_NAME, DIR);
- CloudBlobDirectory cloudBlobDirectory =
ToolUtils.createCloudBlobDirectory(path, ENVIRONMENT,
azureStorageCredentialManagerV8);
- return new AzurePersistenceV8(cloudBlobDirectory);
+ return ToolUtils.createAzurePersistence(path, ENVIRONMENT);
}
@Override
@@ -99,16 +94,13 @@ public class SegmentAzureServicePrincipalNodeStoreContainer
implements NodeStore
if (tmpDir != null) {
tmpDir.delete();
}
- if (azureStorageCredentialManagerV8 != null) {
- azureStorageCredentialManagerV8.close();
- }
}
@Override
public void clean() throws IOException {
- AzurePersistenceV8 azurePersistenceV8 = createAzurePersistence();
+ AzurePersistence azurePersistence = createAzurePersistence();
try {
-
AzureUtilitiesV8.deleteAllBlobs(azurePersistenceV8.getSegmentstoreDirectory());
+
AzureUtilities.deleteAllEntries(azurePersistence.getReadBlobContainerClient(),
null);
} catch (Exception e) {
throw new IOException(e);
}
@@ -116,7 +108,7 @@ public class SegmentAzureServicePrincipalNodeStoreContainer
implements NodeStore
@Override
public String getDescription() {
- return "az:" + String.format(AZURE_SEGMENT_STORE_PATH,
ENVIRONMENT.getVariable(AzureUtilitiesV8.AZURE_ACCOUNT_NAME),
+ return "az:" + String.format(AZURE_SEGMENT_STORE_PATH,
ENVIRONMENT.getVariable(AzureUtilities.AZURE_ACCOUNT_NAME),
CONTAINER_NAME, DIR);
}
}
\ No newline at end of file
diff --git
a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactoryTest.java
b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactoryTest.java
index 393cd34b28..1114121348 100644
---
a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactoryTest.java
+++
b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactoryTest.java
@@ -21,12 +21,12 @@ import
com.microsoft.azure.storage.SharedAccessAccountPermissions;
import com.microsoft.azure.storage.SharedAccessAccountPolicy;
import com.microsoft.azure.storage.SharedAccessAccountResourceType;
import com.microsoft.azure.storage.SharedAccessAccountService;
-import com.microsoft.azure.storage.blob.CloudBlobDirectory;
import org.apache.commons.lang3.StringUtils;
-import
org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzuriteDockerRule;
import org.apache.jackrabbit.oak.commons.pio.Closer;
+import
org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzuriteDockerRule;
+import org.apache.jackrabbit.oak.segment.azure.AzurePersistence;
+import org.apache.jackrabbit.oak.segment.azure.AzureUtilities;
import
org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8;
-import org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8;
import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils;
import org.apache.jackrabbit.oak.segment.azure.util.Environment;
import org.apache.jackrabbit.oak.upgrade.cli.CliUtils;
@@ -122,7 +122,7 @@ public class SegmentAzureFactoryTest {
assertEquals(1, nodeStore.getFileStore().getSegmentCount());
} finally {
closer.close();
- cleanup(uri, azureStorageCredentialManagerV8);
+ cleanup(uri);
}
}
}
@@ -149,16 +149,16 @@ public class SegmentAzureFactoryTest {
assertEquals(1, nodeStore.getFileStore().getSegmentCount());
} finally {
closer.close();
- cleanup(uri, azureStorageCredentialManagerV8);
+ cleanup(uri);
}
}
}
- private void cleanup(String uri, AzureStorageCredentialManagerV8
azureStorageCredentialManagerV8) {
+ private void cleanup(String uri) {
uri = uri + "/" + DIR;
try {
- CloudBlobDirectory cloudBlobDirectory =
ToolUtils.createCloudBlobDirectory(uri, ENVIRONMENT,
azureStorageCredentialManagerV8);
- AzureUtilitiesV8.deleteAllBlobs(cloudBlobDirectory);
+ AzurePersistence azurePersistence =
ToolUtils.createAzurePersistence(uri, ENVIRONMENT);
+
AzureUtilities.deleteAllEntries(azurePersistence.getReadBlobContainerClient(),
null);
} catch (Exception e) {
throw new IllegalStateException(e);
}
