This is an automated email from the ASF dual-hosted git repository.
adulceanu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 0a6f99367f OAK-10615 - Azure Service Principal Support in oak-run
segment-copy, compact, console (#1280)
0a6f99367f is described below
commit 0a6f99367fd20bf215b4154a76e4a6e422acd2a5
Author: Andrei Dulceanu <[email protected]>
AuthorDate: Fri Jan 26 16:00:04 2024 +0200
OAK-10615 - Azure Service Principal Support in oak-run segment-copy,
compact, console (#1280)
* OAK-10615 - Azure Service Principal Support in oak-run
First un-tested implementation
* Fixed ToolUtilsTest
* Added test for warning when connecting with AZURE_SECRET_KEY instead of
service principal
* Added SegmentCopyAzureServicePrincipalToTarTest
* Fixed warning test when connecting with AZURE_SECRET_KEY
Bumped azurite version to 3.29.0
Increased oak.segment.azure package version after refactorings
* Bumped azurite to 3.29.0
Increased again oak-run size to include apache commons lang3 needed for
running azure commands
* Updated Jackrabbit Oak Site Documentation
---
.../cloud/azure/blobstorage/AzuriteDockerRule.java | 2 +-
.../site/markdown/nodestore/segment/overview.md | 2 +-
oak-run/pom.xml | 7 +-
.../segment/azure/AzureSegmentStoreService.java | 13 +--
.../oak/segment/azure/AzureUtilities.java | 22 +++++
.../jackrabbit/oak/segment/azure/package-info.java | 2 +-
.../oak/segment/azure/tool/ToolUtils.java | 37 ++++++-
.../SegmentCopyAzureServicePrincipalToTarTest.java | 80 +++++++++++++++
.../segment/azure/tool/SegmentCopyTestBase.java | 13 ++-
.../azure/AzureSegmentStoreServiceTest.java | 10 +-
.../oak/segment/azure/tool/ToolUtilsTest.java | 108 ++++++++++++++++++---
11 files changed, 254 insertions(+), 42 deletions(-)
diff --git
a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzuriteDockerRule.java
b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzuriteDockerRule.java
index 3eaf27980a..cb709aca29 100644
---
a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzuriteDockerRule.java
+++
b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzuriteDockerRule.java
@@ -38,7 +38,7 @@ import java.util.concurrent.atomic.AtomicReference;
public class AzuriteDockerRule extends ExternalResource {
- private static final DockerImageName DOCKER_IMAGE_NAME =
DockerImageName.parse("mcr.microsoft.com/azure-storage/azurite:3.19.0");
+ private static final DockerImageName DOCKER_IMAGE_NAME =
DockerImageName.parse("mcr.microsoft.com/azure-storage/azurite:3.29.0");
public static final String ACCOUNT_KEY =
"Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==";
public static final String ACCOUNT_NAME = "devstoreaccount1";
private static final AtomicReference<Exception> STARTUP_EXCEPTION = new
AtomicReference<>();
diff --git a/oak-doc/src/site/markdown/nodestore/segment/overview.md
b/oak-doc/src/site/markdown/nodestore/segment/overview.md
index cd2133c058..7e07b02ca2 100644
--- a/oak-doc/src/site/markdown/nodestore/segment/overview.md
+++ b/oak-doc/src/site/markdown/nodestore/segment/overview.md
@@ -688,7 +688,7 @@ Besides the local storage in TAR files (previously known as
TarMK), support for
**Connection Instructions**:
-* **Microsoft Azure** The `cloud-prefix` for MS Azure is `az`, therefore a
valid connection argument would be
`az:https://myaccount.blob.core.windows.net/container/repository`, where the
part after `:` is the Azure URL identifier for the _repository_ directory
inside the specified _container_ of the _myaccount_ Azure storage account. The
last missing piece is the secret key which will be supplied as an environment
variable, i.e. `AZURE_SECRET_KEY`.
+* **Microsoft Azure** The `cloud-prefix` for MS Azure is `az`, therefore a
valid connection argument would be
`az:https://myaccount.blob.core.windows.net/container/repository`, where the
part after `:` is the Azure URL identifier for the _repository_ directory
inside the specified _container_ of the _myaccount_ Azure storage account.
Default authentication to Microsoft Entra ID with service principal credentials
supplied via `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET` and `AZURE_TENANT_ID`
[...]
* **Amazon AWS** The `cloud-prefix` for Amazon AWS is `aws`, therefore a valid
connection argument would be
`aws:bucket;root_directory;journal_table;lock_table` where the part after `:`
defines the _root_directory_ inside the specified _bucket_ in S3 and the
_journal_table_ and _lock_table_ tables within DynamoDB services. The other
portion to connect to AWS is the credentials which will be supplied by placing
a credentials file with ~/.aws folder.
diff --git a/oak-run/pom.xml b/oak-run/pom.xml
index 0c2057422f..4f2e9d5356 100644
--- a/oak-run/pom.xml
+++ b/oak-run/pom.xml
@@ -34,6 +34,7 @@
<jetty.version>9.4.53.v20231009</jetty.version>
<!--
Size History:
+ + 78.7 MB Apache Commons Lang ยป 3.0 (OAK-10615)
+ 78 MB Azure Identity client library for Java (OAK-10604)
+ 60 MB Groovy 2.5 (OAK-10066)
+ 56 MB MongoDB Java driver 3.12.7 (OAK-9357)
@@ -48,7 +49,7 @@
+ 41 MB build failing on the release profile (OAK-6250)
+ 38 MB. Initial value. Current 35MB plus a 10%
-->
- <max.jar.size>78300000</max.jar.size>
+ <max.jar.size>78700000</max.jar.size>
</properties>
<build>
@@ -314,6 +315,10 @@
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
diff --git
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java
index 2e02cae99d..1b5640c76d 100644
---
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java
+++
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java
@@ -18,9 +18,6 @@
*/
package org.apache.jackrabbit.oak.segment.azure;
-import com.azure.core.credential.TokenRequestContext;
-import com.azure.identity.ClientSecretCredential;
-import com.azure.identity.ClientSecretCredentialBuilder;
import com.microsoft.azure.storage.CloudStorageAccount;
import com.microsoft.azure.storage.LocationMode;
import com.microsoft.azure.storage.StorageCredentialsToken;
@@ -46,6 +43,7 @@ import java.security.InvalidKeyException;
import java.util.Hashtable;
import java.util.Objects;
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.storageCredentialAccessTokenFrom;
import static org.osgi.framework.Constants.SERVICE_PID;
@Component(
@@ -126,14 +124,7 @@ public class AzureSegmentStoreService {
@NotNull
private static AzurePersistence
createPersistenceFromServicePrincipalCredentials(Configuration configuration)
throws IOException {
- ClientSecretCredential clientSecretCredential = new
ClientSecretCredentialBuilder()
- .clientId(configuration.clientId())
- .clientSecret(configuration.clientSecret())
- .tenantId(configuration.tenantId())
- .build();
-
- String accessToken = clientSecretCredential.getTokenSync(new
TokenRequestContext().addScopes("https://storage.azure.com/.default";)).getToken();
- StorageCredentialsToken storageCredentialsToken = new
StorageCredentialsToken(configuration.accountName(), accessToken);
+ StorageCredentialsToken storageCredentialsToken =
storageCredentialAccessTokenFrom(configuration.accountName(),
configuration.clientId(), configuration.clientSecret(),
configuration.tenantId());
try {
CloudStorageAccount cloud = new
CloudStorageAccount(storageCredentialsToken, true, DEFAULT_ENDPOINT_SUFFIX,
configuration.accountName());
diff --git
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java
index d37ccf8b97..46e410e398 100644
---
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java
+++
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java
@@ -27,10 +27,14 @@ import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
+import com.azure.core.credential.TokenRequestContext;
+import com.azure.identity.ClientSecretCredential;
+import com.azure.identity.ClientSecretCredentialBuilder;
import com.microsoft.azure.storage.CloudStorageAccount;
import com.microsoft.azure.storage.ResultContinuation;
import com.microsoft.azure.storage.ResultSegment;
import com.microsoft.azure.storage.StorageCredentials;
+import com.microsoft.azure.storage.StorageCredentialsToken;
import com.microsoft.azure.storage.StorageException;
import com.microsoft.azure.storage.StorageUri;
import com.microsoft.azure.storage.blob.BlobListingDetails;
@@ -45,6 +49,13 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public final class AzureUtilities {
+ public static final String AZURE_ACCOUNT_NAME = "AZURE_ACCOUNT_NAME";
+ public static final String AZURE_SECRET_KEY = "AZURE_SECRET_KEY";
+ public static final String AZURE_TENANT_ID = "AZURE_TENANT_ID";
+ public static final String AZURE_CLIENT_ID = "AZURE_CLIENT_ID";
+ public static final String AZURE_CLIENT_SECRET = "AZURE_CLIENT_SECRET";
+
+ private static final String AZURE_DEFAULT_SCOPE =
"https://storage.azure.com/.default";;
private static final Logger log =
LoggerFactory.getLogger(AzureUtilities.class);
@@ -117,6 +128,17 @@ public final class AzureUtilities {
return container.getDirectoryReference(dir);
}
+ public static StorageCredentialsToken
storageCredentialAccessTokenFrom(String accountName, String clientId, String
clientSecret, String tenantId) {
+ ClientSecretCredential clientSecretCredential = new
ClientSecretCredentialBuilder()
+ .clientId(clientId)
+ .clientSecret(clientSecret)
+ .tenantId(tenantId)
+ .build();
+
+ String accessToken = clientSecretCredential.getTokenSync(new
TokenRequestContext().addScopes(AZURE_DEFAULT_SCOPE)).getToken();
+ return new StorageCredentialsToken(accountName, accessToken);
+ }
+
private static ResultSegment<ListBlobItem>
listBlobsInSegments(CloudBlobDirectory directory,
ResultContinuation token) throws IOException {
ResultSegment<ListBlobItem> result = null;
diff --git
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java
index bbe447ca4f..46bf646176 100644
---
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java
+++
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
@Internal(since = "1.0.0")
-@Version("2.1.0")
+@Version("2.2.0")
package org.apache.jackrabbit.oak.segment.azure;
import org.apache.jackrabbit.oak.commons.annotations.Internal;
diff --git
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java
index 325d648fef..51984f440c 100644
---
a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java
+++
b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java
@@ -18,7 +18,16 @@
*/
package org.apache.jackrabbit.oak.segment.azure.tool;
-import static
org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.*;
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_ID;
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_SECRET;
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_SECRET_KEY;
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_TENANT_ID;
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.storageCredentialAccessTokenFrom;
+import static
org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_ACCOUNT_NAME;
+import static
org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_DIR;
+import static
org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_SHARED_ACCESS_SIGNATURE;
+import static
org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_STORAGE_URI;
+import static
org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.parseAzureConfigurationFromUri;
import static
org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.defaultGCOptions;
import java.io.File;
@@ -29,6 +38,7 @@ import java.text.MessageFormat;
import java.util.Map;
import java.util.concurrent.TimeUnit;
+import org.apache.commons.lang3.StringUtils;
import org.apache.jackrabbit.oak.commons.Buffer;
import org.apache.jackrabbit.oak.segment.azure.AzurePersistence;
import org.apache.jackrabbit.oak.segment.azure.AzureUtilities;
@@ -54,12 +64,14 @@ import
com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature;
import com.microsoft.azure.storage.StorageException;
import com.microsoft.azure.storage.blob.CloudBlobDirectory;
import org.jetbrains.annotations.NotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Utility class for common stuff pertaining to tooling.
*/
public class ToolUtils {
-
+ private static final Logger log = LoggerFactory.getLogger(ToolUtils.class);
private static final Environment ENVIRONMENT = new Environment();
private ToolUtils() {
@@ -170,7 +182,7 @@ public class ToolUtils {
if (config.containsKey(KEY_SHARED_ACCESS_SIGNATURE)) {
credentials = new
StorageCredentialsSharedAccessSignature(config.get(KEY_SHARED_ACCESS_SIGNATURE));
} else {
- credentials = getStorageCredentialsAccountAndKey(accountName,
environment);
+ credentials = getStorageCredentialsFromAccountAndEnv(accountName,
environment);
}
String uri = config.get(KEY_STORAGE_URI);
@@ -185,8 +197,23 @@ public class ToolUtils {
}
@NotNull
- private static StorageCredentials
getStorageCredentialsAccountAndKey(String accountName, Environment environment)
{
- String key = environment.getVariable("AZURE_SECRET_KEY");
+ private static StorageCredentials
getStorageCredentialsFromAccountAndEnv(String accountName, Environment
environment) {
+ String clientId = environment.getVariable(AZURE_CLIENT_ID);
+ String clientSecret = environment.getVariable(AZURE_CLIENT_SECRET);
+ String tenantId = environment.getVariable(AZURE_TENANT_ID);
+
+ if (!StringUtils.isAnyBlank(clientId, clientSecret, tenantId)) {
+ try {
+ return storageCredentialAccessTokenFrom(accountName, clientId,
clientSecret, tenantId);
+ } catch (IllegalArgumentException |
StringIndexOutOfBoundsException e) {
+ throw new IllegalArgumentException(
+ "Could not connect to the Azure Storage. Please verify
if AZURE_CLIENT_ID, AZURE_CLIENT_SECRET and AZURE_TENANT_ID environment
variables are correctly set!");
+ }
+ } else {
+ log.warn("AZURE_CLIENT_ID, AZURE_CLIENT_SECRET and AZURE_TENANT_ID
environment variables empty or missing. Switching to authentication with
AZURE_SECRET_KEY.");
+ }
+
+ String key = environment.getVariable(AZURE_SECRET_KEY);
try {
return new StorageCredentialsAccountAndKey(accountName, key);
} catch (IllegalArgumentException | StringIndexOutOfBoundsException e)
{
diff --git
a/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyAzureServicePrincipalToTarTest.java
b/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyAzureServicePrincipalToTarTest.java
new file mode 100644
index 0000000000..892d380151
--- /dev/null
+++
b/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyAzureServicePrincipalToTarTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package oak.apache.jackrabbit.oak.segment.azure.tool;
+
+import com.microsoft.azure.storage.blob.CloudBlobDirectory;
+import org.apache.jackrabbit.oak.segment.azure.AzurePersistence;
+import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils;
+import org.apache.jackrabbit.oak.segment.azure.util.Environment;
+import
org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence;
+import org.junit.Test;
+
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_ACCOUNT_NAME;
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_ID;
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_SECRET;
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_TENANT_ID;
+import static org.junit.Assume.assumeNotNull;
+
+public class SegmentCopyAzureServicePrincipalToTarTest extends
SegmentCopyTestBase {
+ private static final Environment ENVIRONMENT = new Environment();
+ private static final String CONTAINER_NAME = "oak";
+ private static final String DIR = "repository";
+ private static final String SEGMENT_STORE_PATH_FORMAT =
"https://%s.blob.core.windows.net/%s/%s";;
+
+ @Test
+ @Override
+ public void testSegmentCopy() throws Exception {
+ assumeNotNull(ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME));
+ assumeNotNull(ENVIRONMENT.getVariable(AZURE_TENANT_ID));
+ assumeNotNull(ENVIRONMENT.getVariable(AZURE_CLIENT_ID));
+ assumeNotNull(ENVIRONMENT.getVariable(AZURE_CLIENT_SECRET));
+
+ super.testSegmentCopy();
+ }
+
+ @Override
+ protected SegmentNodeStorePersistence getSrcPersistence() {
+ String accountName = ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME);
+ String path = String.format(SEGMENT_STORE_PATH_FORMAT, accountName,
CONTAINER_NAME, DIR);
+ CloudBlobDirectory cloudBlobDirectory =
ToolUtils.createCloudBlobDirectory(path, ENVIRONMENT);
+
+ return new AzurePersistence(cloudBlobDirectory);
+ }
+
+ @Override
+ protected SegmentNodeStorePersistence getDestPersistence() {
+ return getTarPersistence();
+ }
+
+ @Override
+ protected String getSrcPathOrUri() {
+ String accountName = ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME);
+
+ StringBuilder sb = new StringBuilder();
+ sb.append("az:");
+ sb.append(String.format(SEGMENT_STORE_PATH_FORMAT, accountName,
CONTAINER_NAME, DIR));
+
+ return sb.toString();
+ }
+
+ @Override
+ protected String getDestPathOrUri() {
+ return getTarPersistencePathOrUri();
+ }
+}
diff --git
a/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java
b/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java
index 0113734b87..a732dbeb36 100644
---
a/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java
+++
b/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java
@@ -18,7 +18,11 @@
*/
package oak.apache.jackrabbit.oak.segment.azure.tool;
-import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.*;
+import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.ADD;
+import static
com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.CREATE;
+import static
com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.LIST;
+import static
com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.READ;
+import static
com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.WRITE;
import static
org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.newFileStore;
import static
org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.newSegmentNodeStorePersistence;
import static org.junit.Assert.assertEquals;
@@ -50,7 +54,12 @@ import
org.apache.jackrabbit.oak.segment.azure.tool.SegmentCopy;
import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.SegmentStoreType;
import
org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.CompactorType;
import org.apache.jackrabbit.oak.segment.file.FileStore;
-import org.apache.jackrabbit.oak.segment.spi.monitor.*;
+import org.apache.jackrabbit.oak.segment.spi.monitor.FileStoreMonitor;
+import org.apache.jackrabbit.oak.segment.spi.monitor.FileStoreMonitorAdapter;
+import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitor;
+import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitorAdapter;
+import org.apache.jackrabbit.oak.segment.spi.monitor.RemoteStoreMonitor;
+import org.apache.jackrabbit.oak.segment.spi.monitor.RemoteStoreMonitorAdapter;
import org.apache.jackrabbit.oak.segment.spi.persistence.GCJournalFile;
import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFileReader;
import org.apache.jackrabbit.oak.segment.spi.persistence.ManifestFile;
diff --git
a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreServiceTest.java
b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreServiceTest.java
index 117609d0b6..4da1f99779 100644
---
a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreServiceTest.java
+++
b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreServiceTest.java
@@ -40,6 +40,11 @@ import org.junit.Rule;
import org.junit.Test;
import org.osgi.util.converter.Converters;
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_ACCOUNT_NAME;
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_ID;
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_SECRET;
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_TENANT_ID;
+
import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.ADD;
import static
com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.CREATE;
import static
com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.LIST;
@@ -65,11 +70,6 @@ public class AzureSegmentStoreServiceTest {
private static final EnumSet<SharedAccessBlobPermissions> READ_WRITE =
EnumSet.of(READ, LIST, CREATE, WRITE, ADD);
private static final ImmutableSet<String> BLOBS = ImmutableSet.of("blob1",
"blob2");
- private static final String AZURE_ACCOUNT_NAME = "AZURE_ACCOUNT_NAME";
- private static final String AZURE_TENANT_ID = "AZURE_TENANT_ID";
- private static final String AZURE_CLIENT_ID = "AZURE_CLIENT_ID";
- private static final String AZURE_CLIENT_SECRET = "AZURE_CLIENT_SECRET";
-
private CloudBlobContainer container;
@Before
diff --git
a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtilsTest.java
b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtilsTest.java
index 11999453d1..6ae1a0abba 100644
---
a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtilsTest.java
+++
b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtilsTest.java
@@ -18,59 +18,95 @@
*/
package org.apache.jackrabbit.oak.segment.azure.tool;
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.Appender;
+import ch.qos.logback.core.read.ListAppender;
import com.microsoft.azure.storage.StorageCredentials;
import com.microsoft.azure.storage.StorageCredentialsAccountAndKey;
import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature;
+
+import java.net.URISyntaxException;
import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
+import com.microsoft.azure.storage.StorageException;
+import com.microsoft.azure.storage.blob.CloudBlobDirectory;
import
org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzuriteDockerRule;
import org.apache.jackrabbit.oak.segment.azure.AzureUtilities;
import org.apache.jackrabbit.oak.segment.azure.util.Environment;
+import org.jetbrains.annotations.NotNull;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.MockedStatic;
+import org.slf4j.LoggerFactory;
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_ACCOUNT_NAME;
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_ID;
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_SECRET;
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_SECRET_KEY;
+import static
org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_TENANT_ID;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeNotNull;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mockStatic;
public class ToolUtilsTest {
- private static final String CONTAINER_URL =
"https://myaccount.blob.core.windows.net/oak-test";;
- private static final String REPO_DIR = "repository";
- private static final String SEGMENT_STORE_PATH = CONTAINER_URL + '/' +
REPO_DIR;
+ private static final Environment ENVIRONMENT = new Environment();
+
+ private static final String CONTAINER_URL_FORMAT =
"https://%s.blob.core.windows.net/%s";;
+ private static final String SEGMENT_STORE_PATH_FORMAT =
CONTAINER_URL_FORMAT + "/%s";
+
+ private static final String DEFAULT_ACCOUNT_NAME = "myaccount";
+ private static final String DEFAULT_CONTAINER_NAME = "oak";
+ private static final String DEFAULT_REPO_DIR = "repository";
+ private static final String DEFAULT_CONTAINER_URL =
String.format(CONTAINER_URL_FORMAT, DEFAULT_ACCOUNT_NAME,
DEFAULT_CONTAINER_NAME);
+ private static final String DEFAULT_SEGMENT_STORE_PATH =
String.format(SEGMENT_STORE_PATH_FORMAT, DEFAULT_ACCOUNT_NAME,
DEFAULT_CONTAINER_NAME, DEFAULT_REPO_DIR);
+ public static final String AZURE_SECRET_KEY_WARNING = "AZURE_CLIENT_ID,
AZURE_CLIENT_SECRET and AZURE_TENANT_ID environment variables empty or missing.
Switching to authentication with AZURE_SECRET_KEY.";
private final TestEnvironment environment = new TestEnvironment();
@Test
public void createCloudBlobDirectoryWithAccessKey() {
- environment.setVariable("AZURE_SECRET_KEY",
AzuriteDockerRule.ACCOUNT_KEY);
+ environment.setVariable(AZURE_SECRET_KEY,
AzuriteDockerRule.ACCOUNT_KEY);
+
+ final ListAppender<ILoggingEvent> logAppender = subscribeAppender();
StorageCredentialsAccountAndKey credentials = expectCredentials(
StorageCredentialsAccountAndKey.class,
- () -> ToolUtils.createCloudBlobDirectory(SEGMENT_STORE_PATH,
environment)
+ () ->
ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH, environment),
+ DEFAULT_CONTAINER_URL
);
-
- assertEquals("myaccount", credentials.getAccountName());
+
+ assertTrue(checkLogContainsMessage(AZURE_SECRET_KEY_WARNING,
logAppender.list.stream().map(ILoggingEvent::getFormattedMessage).collect(Collectors.toList())));
+ assertEquals(Level.WARN, logAppender.list.get(0).getLevel());
+
+ assertEquals(DEFAULT_ACCOUNT_NAME, credentials.getAccountName());
assertEquals(AzuriteDockerRule.ACCOUNT_KEY,
credentials.exportBase64EncodedKey());
+ unsubscribe(logAppender);
}
@Test
public void createCloudBlobDirectoryFailsWhenAccessKeyNotPresent() {
- environment.setVariable("AZURE_SECRET_KEY", null);
+ environment.setVariable(AZURE_SECRET_KEY, null);
assertThrows(IllegalArgumentException.class, () ->
- ToolUtils.createCloudBlobDirectory(SEGMENT_STORE_PATH)
+ ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH,
environment)
);
}
@Test
public void createCloudBlobDirectoryFailsWhenAccessKeyIsInvalid() {
- environment.setVariable("AZURE_SECRET_KEY", "invalid");
+ environment.setVariable(AZURE_SECRET_KEY, "invalid");
assertThrows(IllegalArgumentException.class, () ->
- ToolUtils.createCloudBlobDirectory(SEGMENT_STORE_PATH)
+ ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH,
environment)
);
}
@@ -80,28 +116,70 @@ public class ToolUtilsTest {
StorageCredentialsSharedAccessSignature credentials =
expectCredentials(
StorageCredentialsSharedAccessSignature.class,
- () -> ToolUtils.createCloudBlobDirectory(SEGMENT_STORE_PATH + '?'
+ sasToken)
+ () ->
ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH + '?' + sasToken),
+ DEFAULT_CONTAINER_URL
);
assertEquals(sasToken, credentials.getToken());
assertNull("AccountName should be null when SAS credentials are used",
credentials.getAccountName());
}
- private static <T extends StorageCredentials> T expectCredentials(Class<T>
clazz, Runnable body) {
+ @Test
+ public void createCloudBlobDirectoryWithServicePrincipal() throws
URISyntaxException, StorageException {
+ assumeNotNull(ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME));
+ assumeNotNull(ENVIRONMENT.getVariable(AZURE_TENANT_ID));
+ assumeNotNull(ENVIRONMENT.getVariable(AZURE_CLIENT_ID));
+ assumeNotNull(ENVIRONMENT.getVariable(AZURE_CLIENT_SECRET));
+
+ String accountName = ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME);
+ String containerName = "oak";
+ String segmentStorePath = String.format(SEGMENT_STORE_PATH_FORMAT,
accountName, containerName, DEFAULT_REPO_DIR);
+
+ CloudBlobDirectory cloudBlobDirectory =
ToolUtils.createCloudBlobDirectory(segmentStorePath, ENVIRONMENT);
+ assertNotNull(cloudBlobDirectory);
+ assertEquals(containerName,
cloudBlobDirectory.getContainer().getName());
+ }
+
+ private static <T extends StorageCredentials> T expectCredentials(Class<T>
clazz, Runnable body, String containerUrl) {
ArgumentCaptor<T> credentialsCaptor = ArgumentCaptor.forClass(clazz);
try (MockedStatic<AzureUtilities> mockedAzureUtilities =
mockStatic(AzureUtilities.class)) {
body.run();
mockedAzureUtilities.verify(() ->
AzureUtilities.cloudBlobDirectoryFrom(
credentialsCaptor.capture(),
- eq(CONTAINER_URL),
- eq(REPO_DIR)
+ eq(containerUrl),
+ eq(DEFAULT_REPO_DIR)
)
);
return credentialsCaptor.getValue();
}
}
+ private boolean checkLogContainsMessage(String toCheck, List<String>
messages) {
+ for (String message : messages) {
+ if (message.equals(toCheck)) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ private ListAppender<ILoggingEvent> subscribeAppender() {
+ ListAppender<ILoggingEvent> appender = new
ListAppender<ILoggingEvent>();
+ appender.setContext((LoggerContext) LoggerFactory.getILoggerFactory());
+ appender.setName("asynclogcollector");
+ appender.start();
+ ((LoggerContext) LoggerFactory.getILoggerFactory()).getLogger(
+
ch.qos.logback.classic.Logger.ROOT_LOGGER_NAME).addAppender(appender);
+ return appender;
+ }
+
+ private void unsubscribe(@NotNull final Appender<ILoggingEvent> appender) {
+ ((LoggerContext) LoggerFactory.getILoggerFactory()).getLogger(
+
ch.qos.logback.classic.Logger.ROOT_LOGGER_NAME).detachAppender(appender);
+ }
+
static class TestEnvironment extends Environment {
private final Map<String, String> envs = new HashMap<>();
