This is an automated email from the ASF dual-hosted git repository. daim pushed a commit to branch DetailedGC/OAK-10199 in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 6c2acf4431634be456e41e81d7d8d6add8efe031 Author: Andrei Dulceanu <[email protected]> AuthorDate: Mon Jan 22 17:24:06 2024 +0200 OAK-10604 - Azure Service Principal Support in oak-segment-azure (#1268) * OAK-10604 - Azure Service Principal Support in oak-segment-azure Added service principal configuration properties Added Azure Identity client library for Java as a dependency and updated all oak-run* poms to adjust size * OAK-10604 - Azure Service Principal Support in oak-segment-azure First untested implementation * OAK-10604 - Azure Service Principal Support in oak-segment-azure Added scope to token request Added test case using environment variables * OAK-10604 - Azure Service Principal Support in oak-segment-azure Nitpick - avoid wildcard imports * OAK-10604 - Azure Service Principal Support in oak-segment-azure Corrected token request scope and fixed the test * OAK-10604 - Azure Service Principal Support in oak-segment-azure OSGi wiring fix - wip * OSGi wiring - wip * OAK-10604 - Azure Service Principal Support in oak-segment-azure Fixed OSGi wiring * OAK-10604 - Azure Service Principal Support in oak-segment-azure Increased size for oak-run and oak-run-elastic jars after embedding more libraries in oak-segment-azure * OAK-10604 - Azure Service Principal Support in oak-segment-azure Increased again oak-run jar size due to sonar complaining about it during CI --- oak-parent/pom.xml | 25 ++++++++ oak-run-elastic/pom.xml | 3 +- oak-run/pom.xml | 3 +- oak-segment-azure/pom.xml | 55 ++++++++++++++++- .../segment/azure/AzureSegmentStoreService.java | 70 ++++++++++++++------- .../oak/segment/azure/Configuration.java | 15 +++++ .../jackrabbit/oak/segment/azure/package-info.java | 2 +- .../azure/AzureSegmentStoreServiceTest.java | 72 +++++++++++++++++++--- 8 files changed, 209 insertions(+), 36 deletions(-) diff --git a/oak-parent/pom.xml b/oak-parent/pom.xml index 8ff41e9155..bc70193202 100644 --- a/oak-parent/pom.xml +++ b/oak-parent/pom.xml @@ -728,6 +728,31 @@ <artifactId>azure-keyvault-core</artifactId> <version>1.2.6</version> </dependency> + <dependency> + <groupId>com.azure</groupId> + <artifactId>azure-identity</artifactId> + <version>1.11.1</version> + </dependency> + <dependency> + <groupId>com.azure</groupId> + <artifactId>azure-identity-broker</artifactId> + <version>1.0.1</version> + </dependency> + <dependency> + <groupId>com.azure</groupId> + <artifactId>azure-core</artifactId> + <version>1.45.1</version> + </dependency> + <dependency> + <groupId>com.azure</groupId> + <artifactId>azure-json</artifactId> + <version>1.0.1</version> + </dependency> + <dependency> + <groupId>com.azure</groupId> + <artifactId>azure-xml</artifactId> + <version>1.0.0-beta.2</version> + </dependency> <!-- Pax Exam Integration Test Dependencies --> <dependency> diff --git a/oak-run-elastic/pom.xml b/oak-run-elastic/pom.xml index a6312ba5c8..391a02c024 100644 --- a/oak-run-elastic/pom.xml +++ b/oak-run-elastic/pom.xml @@ -38,8 +38,9 @@ 121 MB : add Elasticsearch Java client along with RHLC: the latter can be removed when the code can be fully migrated to use the new client 125 MB : shaded Guava 85 MB : remove Elasticsearch RHLC + 103.5 MB: Azure Identity client library for Java (OAK-10604) --> - <max.jar.size>85000000</max.jar.size> + <max.jar.size>103500000</max.jar.size> </properties> diff --git a/oak-run/pom.xml b/oak-run/pom.xml index b58f2d379e..791488196b 100644 --- a/oak-run/pom.xml +++ b/oak-run/pom.xml @@ -34,6 +34,7 @@ <jetty.version>9.4.53.v20231009</jetty.version> <!-- Size History: + + 78 MB Azure Identity client library for Java (OAK-10604) + 60 MB Groovy 2.5 (OAK-10066) + 56 MB MongoDB Java driver 3.12.7 (OAK-9357) + 55 MB Add support for segment persistent cache (OAK-7744) @@ -47,7 +48,7 @@ + 41 MB build failing on the release profile (OAK-6250) + 38 MB. Initial value. Current 35MB plus a 10% --> - <max.jar.size>62914560</max.jar.size> + <max.jar.size>78100000</max.jar.size> </properties> <build> diff --git a/oak-segment-azure/pom.xml b/oak-segment-azure/pom.xml index f8c7f6512f..b285fde0e6 100644 --- a/oak-segment-azure/pom.xml +++ b/oak-segment-azure/pom.xml @@ -42,6 +42,24 @@ <Import-Package> org.apache.jackrabbit.oak.segment.spi*, org.apache.jackrabbit.oak.segment.remote*, + com.fasterxml.jackson.annotation;resolution:=optional, + com.fasterxml.jackson.databind*;resolution:=optional, + com.fasterxml.jackson.dataformat.xml;resolution:=optional, + com.fasterxml.jackson.datatype*;resolution:=optional, + com.microsoft.aad.msal4j*;resolution:=optional, + com.nimbusds.common.contenttype;resolution:=optional, + com.nimbusds.jose*;resolution:=optional, + com.nimbusds.jwt;resolution:=optional, + com.nimbusds.jwt.util;resolution:=optional, + com.nimbusds.oauth2.sdk*;resolution:=optional, + com.nimbusds.jwt.proc;resolution:=optional, + com.nimbusds.langtag;resolution:=optional, + com.nimbusds.openid.connect.sdk*;resolution:=optional, + com.nimbusds.secevent.sdk*;resolution:=optional, + com.sun.jna*;resolution:=optional, + org.reactivestreams;resolution:=optional, + reactor.core*;resolution:=optional, + reactor.util*;resolution:=optional, !org.apache.jackrabbit.oak.segment*, !com.google.*, !android.os, @@ -54,11 +72,18 @@ org.apache.jackrabbit.oak.segment.azure.util, com.microsoft.azure.storage, com.microsoft.azure.storage.core, - com.microsoft.azure.storage.blob + com.microsoft.azure.storage.blob, + com.azure.core.credential, + com.azure.identity </Export-Package> <Embed-Dependency> azure-storage, azure-keyvault-core, + azure-core, + azure-identity, + azure-identity-broker, + azure-json, + azure-xml, guava, jsr305 </Embed-Dependency> @@ -151,6 +176,34 @@ <groupId>com.microsoft.azure</groupId> <artifactId>azure-keyvault-core</artifactId> </dependency> + + <!-- Azure Identity Client for Microsoft Entra ID dependency --> + <dependency> + <groupId>com.azure</groupId> + <artifactId>azure-identity</artifactId> + </dependency> + <dependency> + <groupId>com.azure</groupId> + <artifactId>azure-identity-broker</artifactId> + </dependency> + <dependency> + <groupId>com.azure</groupId> + <artifactId>azure-core</artifactId> + </dependency> + <dependency> + <groupId>com.azure</groupId> + <artifactId>azure-json</artifactId> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + </dependency> + <dependency> + <groupId>com.azure</groupId> + <artifactId>azure-xml</artifactId> + </dependency> + + <!-- Azure Guava dependency --> <dependency> <groupId>com.google.guava</groupId> diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java index e76372f99c..2e02cae99d 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java @@ -18,8 +18,12 @@ */ package org.apache.jackrabbit.oak.segment.azure; +import com.azure.core.credential.TokenRequestContext; +import com.azure.identity.ClientSecretCredential; +import com.azure.identity.ClientSecretCredentialBuilder; import com.microsoft.azure.storage.CloudStorageAccount; import com.microsoft.azure.storage.LocationMode; +import com.microsoft.azure.storage.StorageCredentialsToken; import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.blob.BlobRequestOptions; import com.microsoft.azure.storage.blob.CloudBlobClient; @@ -41,7 +45,6 @@ import java.net.URISyntaxException; import java.security.InvalidKeyException; import java.util.Hashtable; import java.util.Objects; -import java.util.Properties; import static org.osgi.framework.Constants.SERVICE_PID; @@ -57,6 +60,7 @@ public class AzureSegmentStoreService { public static final String DEFAULT_ROOT_PATH = "/oak"; public static final boolean DEFAULT_ENABLE_SECONDARY_LOCATION = false; + public static final String DEFAULT_ENDPOINT_SUFFIX = "core.windows.net"; private ServiceRegistration registration; @@ -84,6 +88,9 @@ public class AzureSegmentStoreService { if (!StringUtils.isBlank(configuration.connectionURL())) { return createPersistenceFromConnectionURL(configuration); } + if (!StringUtils.isAnyBlank(configuration.clientId(), configuration.clientSecret(), configuration.tenantId())) { + return createPersistenceFromServicePrincipalCredentials(configuration); + } if (!StringUtils.isBlank(configuration.sharedAccessSignature())) { return createPersistenceFromSasUri(configuration); } @@ -118,33 +125,53 @@ public class AzureSegmentStoreService { } @NotNull - private static AzurePersistence createAzurePersistence( - String connectionString, - Configuration configuration, - boolean createContainer - ) throws IOException { + private static AzurePersistence createPersistenceFromServicePrincipalCredentials(Configuration configuration) throws IOException { + ClientSecretCredential clientSecretCredential = new ClientSecretCredentialBuilder() + .clientId(configuration.clientId()) + .clientSecret(configuration.clientSecret()) + .tenantId(configuration.tenantId()) + .build(); + + String accessToken = clientSecretCredential.getTokenSync(new TokenRequestContext().addScopes("https://storage.azure.com/.default";)).getToken(); + StorageCredentialsToken storageCredentialsToken = new StorageCredentialsToken(configuration.accountName(), accessToken); + + try { + CloudStorageAccount cloud = new CloudStorageAccount(storageCredentialsToken, true, DEFAULT_ENDPOINT_SUFFIX, configuration.accountName()); + return createAzurePersistence(cloud, configuration, true); + } catch (StorageException | URISyntaxException e) { + throw new IOException(e); + } + } + + @NotNull + private static AzurePersistence createAzurePersistence(String connectionString, Configuration configuration, boolean createContainer) throws IOException { try { CloudStorageAccount cloud = CloudStorageAccount.parse(connectionString); log.info("Connection string: '{}'", cloud); - CloudBlobClient cloudBlobClient = cloud.createCloudBlobClient(); - BlobRequestOptions blobRequestOptions = new BlobRequestOptions(); - - if (configuration.enableSecondaryLocation()) { - blobRequestOptions.setLocationMode(LocationMode.PRIMARY_THEN_SECONDARY); - } - cloudBlobClient.setDefaultRequestOptions(blobRequestOptions); - - CloudBlobContainer container = cloudBlobClient.getContainerReference(configuration.containerName()); - if (createContainer && !container.exists()) { - container.create(); - } - String path = normalizePath(configuration.rootPath()); - return new AzurePersistence(container.getDirectoryReference(path)); + return createAzurePersistence(cloud, configuration, createContainer); } catch (StorageException | URISyntaxException | InvalidKeyException e) { throw new IOException(e); } } + @NotNull + private static AzurePersistence createAzurePersistence(CloudStorageAccount cloud, Configuration configuration, boolean createContainer) throws URISyntaxException, StorageException { + CloudBlobClient cloudBlobClient = cloud.createCloudBlobClient(); + BlobRequestOptions blobRequestOptions = new BlobRequestOptions(); + + if (configuration.enableSecondaryLocation()) { + blobRequestOptions.setLocationMode(LocationMode.PRIMARY_THEN_SECONDARY); + } + cloudBlobClient.setDefaultRequestOptions(blobRequestOptions); + + CloudBlobContainer container = cloudBlobClient.getContainerReference(configuration.containerName()); + if (createContainer && !container.exists()) { + container.create(); + } + String path = normalizePath(configuration.rootPath()); + return new AzurePersistence(container.getDirectoryReference(path)); + } + @NotNull private static String normalizePath(@NotNull String rootPath) { if (rootPath.length() > 0 && rootPath.charAt(0) == '/') { @@ -153,5 +180,4 @@ public class AzureSegmentStoreService { return rootPath; } -} - +} \ No newline at end of file diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/Configuration.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/Configuration.java index 2eaf628bc1..f0aa1e4b1c 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/Configuration.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/Configuration.java @@ -68,6 +68,21 @@ import static org.apache.jackrabbit.oak.segment.azure.Configuration.PID; description = "Blob Endpoint URL used to connect to the Azure Storage") String blobEndpoint() default ""; + @AttributeDefinition( + name = "Azure Service Principal ID (optional)", + description = "Azure Service Principal ID for Azure Storage authentication") + String clientId() default ""; + + @AttributeDefinition( + name = "Azure Service Principal Password (optional)", + description = "Azure Service Principal Password for Azure Storage authentication") + String clientSecret() default ""; + + @AttributeDefinition( + name = "Azure Active Directory ID (optional)", + description = "Azure Active Directory ID for Azure Storage authentication") + String tenantId() default ""; + @AttributeDefinition( name = "Role", description = "The role of this persistence. It should be unique and may be used to filter " + diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java index 30d378df6d..bbe447ca4f 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java @@ -15,7 +15,7 @@ * limitations under the License. */ @Internal(since = "1.0.0") -@Version("2.0.0") +@Version("2.1.0") package org.apache.jackrabbit.oak.segment.azure; import org.apache.jackrabbit.oak.commons.annotations.Internal; diff --git a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreServiceTest.java b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreServiceTest.java index 4688fcec15..117609d0b6 100644 --- a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreServiceTest.java +++ b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreServiceTest.java @@ -23,22 +23,38 @@ import java.io.IOException; import java.net.URISyntaxException; import java.time.Duration; import java.time.Instant; -import java.util.*; +import java.util.Date; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Set; import java.util.stream.StreamSupport; import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzuriteDockerRule; +import org.apache.jackrabbit.oak.segment.azure.util.Environment; import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence; import org.apache.sling.testing.mock.osgi.junit.OsgiContext; import org.jetbrains.annotations.NotNull; -import org.junit.*; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; import org.osgi.util.converter.Converters; -import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.*; +import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.ADD; +import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.CREATE; +import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.LIST; +import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.READ; +import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.WRITE; import static java.util.stream.Collectors.toSet; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; +import static org.junit.Assume.assumeNotNull; public class AzureSegmentStoreServiceTest { - + private static final Environment ENVIRONMENT = new Environment(); + @ClassRule public static AzuriteDockerRule azurite = new AzuriteDockerRule(); @@ -48,6 +64,11 @@ public class AzureSegmentStoreServiceTest { private static final EnumSet<SharedAccessBlobPermissions> READ_ONLY = EnumSet.of(READ, LIST); private static final EnumSet<SharedAccessBlobPermissions> READ_WRITE = EnumSet.of(READ, LIST, CREATE, WRITE, ADD); private static final ImmutableSet<String> BLOBS = ImmutableSet.of("blob1", "blob2"); + + private static final String AZURE_ACCOUNT_NAME = "AZURE_ACCOUNT_NAME"; + private static final String AZURE_TENANT_ID = "AZURE_TENANT_ID"; + private static final String AZURE_CLIENT_ID = "AZURE_CLIENT_ID"; + private static final String AZURE_CLIENT_SECRET = "AZURE_CLIENT_SECRET"; private CloudBlobContainer container; @@ -121,6 +142,29 @@ public class AzureSegmentStoreServiceTest { assertReadAccessGranted(persistence, concat(BLOBS, "test")); } + @Test + public void connectWithServicePrincipal() throws Exception { + // Note: make sure blob1.txt and blob2.txt are uploaded to + // AZURE_ACCOUNT_NAME/oak before running this test + + assumeNotNull(ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME)); + assumeNotNull(ENVIRONMENT.getVariable(AZURE_TENANT_ID)); + assumeNotNull(ENVIRONMENT.getVariable(AZURE_CLIENT_ID)); + assumeNotNull(ENVIRONMENT.getVariable(AZURE_CLIENT_SECRET)); + + AzureSegmentStoreService azureSegmentStoreService = new AzureSegmentStoreService(); + String accountName = ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME); + String tenantId = ENVIRONMENT.getVariable(AZURE_TENANT_ID); + String clientId = ENVIRONMENT.getVariable(AZURE_CLIENT_ID); + String clientSecret = ENVIRONMENT.getVariable(AZURE_CLIENT_SECRET); + azureSegmentStoreService.activate(context.componentContext(), getConfigurationWithServicePrincipal(accountName, clientId, clientSecret, tenantId)); + + SegmentNodeStorePersistence persistence = context.getService(SegmentNodeStorePersistence.class); + assertNotNull(persistence); + assertWriteAccessGranted(persistence); + assertReadAccessGranted(persistence, concat(BLOBS, "test")); + } + @Test public void deactivate() throws Exception { AzureSegmentStoreService azureSegmentStoreService = new AzureSegmentStoreService(); @@ -149,6 +193,7 @@ public class AzureSegmentStoreServiceTest { Set<String> actualBlobNames = StreamSupport.stream(container.listBlobs().spliterator(), false) .map(blob -> blob.getUri().getPath()) .map(path -> path.substring(path.lastIndexOf('/') + 1)) + .filter(name -> name.equals("test.txt") || name.startsWith("blob")) .collect(toSet()); Set<String> expectedBlobNames = expectedBlobs.stream().map(name -> name + ".txt").collect(toSet()); @@ -202,11 +247,11 @@ public class AzureSegmentStoreServiceTest { } private static Configuration getConfigurationWithSharedAccessSignature(String sasToken) { - return getConfiguration(sasToken, null, null); + return getConfiguration(sasToken, AzuriteDockerRule.ACCOUNT_NAME, null, null, null, null, null); } private static Configuration getConfigurationWithAccessKey(String accessKey) { - return getConfiguration(null, accessKey, null); + return getConfiguration(null, AzuriteDockerRule.ACCOUNT_NAME, accessKey, null, null, null, null); } private static Configuration getConfigurationWithConfigurationURL(String accessKey) { @@ -214,17 +259,24 @@ public class AzureSegmentStoreServiceTest { + "BlobEndpoint=" + azurite.getBlobEndpoint() + ';' + "AccountName=" + AzuriteDockerRule.ACCOUNT_NAME + ';' + "AccountKey=" + accessKey + ';'; - return getConfiguration(null, null, connectionString); + return getConfiguration(null, AzuriteDockerRule.ACCOUNT_NAME, null, connectionString, null, null, null); + } + + private static Configuration getConfigurationWithServicePrincipal(String accountName, String clientId, String clientSecret, String tenantId) { + return getConfiguration(null, accountName, null, null, clientId, clientSecret, tenantId); } @NotNull - private static Configuration getConfiguration(String sasToken, String accessKey, String connectionURL) { + private static Configuration getConfiguration(String sasToken, String accountName, String accessKey, String connectionURL, String clientId, String clientSecret, String tenantId) { return Converters.standardConverter() .convert(new HashMap<Object, Object>() {{ - put("accountName", AzuriteDockerRule.ACCOUNT_NAME); + put("accountName", accountName); put("accessKey", accessKey); put("connectionURL", connectionURL); put("sharedAccessSignature", sasToken); + put("clientId", clientId); + put("clientSecret", clientSecret); + put("tenantId", tenantId); put("blobEndpoint", azurite.getBlobEndpoint()); }}) .to(Configuration.class);
