saxenapranav commented on code in PR #6552: URL: https://github.com/apache/hadoop/pull/6552#discussion_r1495337702
########## hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java: ########## @@ -0,0 +1,156 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.azurebfs; + +import java.io.IOException; + +import org.junit.Assume; +import org.junit.Test; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.SASTokenProviderException; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException; +import org.apache.hadoop.fs.azurebfs.extensions.MockDelegationSASTokenProvider; +import org.apache.hadoop.fs.azurebfs.services.AuthType; +import org.apache.hadoop.fs.azurebfs.utils.AccountSASGenerator; +import org.apache.hadoop.fs.azurebfs.utils.Base64; +import org.apache.hadoop.fs.azurebfs.utils.TracingContext; + +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_FIXED_TOKEN; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; + +/** + * Tests to validate the choice between using a SASTokenProvider to generate + * a SAS or using a Fixed SAS Token configured by user. + */ +public class ITestAzureBlobFileSystemChooseSAS extends AbstractAbfsIntegrationTest{ + + private String accountSAS = null; + + + /** + * To differentiate which config was used we will use different type of SAS Tokens. + * For Fixed SAS Token we will use an Account SAS with permissions to do File system level operations. + * For SASTokenProvider we will use a User Delegation SAS Token Provider + * such that File System level operations are not permitted. + */ + public ITestAzureBlobFileSystemChooseSAS() throws Exception { + // SAS Token configured might not have permissions for creating file system. + // Shared Key must be configured to create one. Once created, a new instance + // of same file system will be used with SAS Authentication. + Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); + } + + @Override + public void setup() throws Exception { + createFilesystemForSASTests(); + super.setup(); + generateAccountSAS(); } + + /** + * Generates a Account SAS Token using the Account Shared Key to be used as a fixed SAS Token. + * This will be used by individual tests to set in the configurations. + * @throws AzureBlobFileSystemException + */ + private void generateAccountSAS() throws AzureBlobFileSystemException { + final String accountKey = getConfiguration().getStorageAccountKey(); + AccountSASGenerator configAccountSASGenerator = new AccountSASGenerator(Base64.decode(accountKey)); + accountSAS = configAccountSASGenerator.getAccountSAS(getAccountName()); + } + + /** + * Tests the scenario where both the SASTokenProvider and a fixed SAS token are configured. + * SASTokenProvider class should be chosen and User Delegation SAS should be used. + * @throws Exception + */ + @Test + public void testBothProviderFixedTokenConfigured() throws Exception { + AbfsConfiguration testAbfsConfig = getConfiguration(); Review Comment: Since, these tests may run in parallel with test of other classes. Lets make clone of the configuration object and use it in the test. ########## hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java: ########## @@ -332,6 +335,8 @@ public AbfsRestOperation setFilesystemProperties(final String properties, final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); abfsUriQueryBuilder.addQuery(QUERY_PARAM_RESOURCE, FILESYSTEM); + appendSASTokenToQuery(ROOT_PATH, "", abfsUriQueryBuilder); Review Comment: why operationName is not provided here. If the thing is that for container APIs, existing SAS mechanism can not work, we should still prevent them. We might have to add intelligence which SAS implementations are allowed for container APIs. ########## hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java: ########## @@ -941,33 +941,66 @@ public AccessTokenProvider getTokenProvider() throws TokenAccessProviderExceptio } } + /** + * The user can choose between a configured fixed sas token, and a user + * implementation of the SASTokenProvider interface. Preference will be given + * to SASTokenProvider class provided as the value of "fs.azure.sas.token.provider.type". + * If above config is not set, it is expected that user wants to use a + * fixed SAS Token provided as value of "fs.azure.sas.fixed.token". + * <ol> + * <li>If both the configs are not provided, + * initialization fails and {@link TokenAccessProviderException} is thrown.</li> + * <li>If both are present, SASTokenProvider class will be used to generate SAS Token.</li> + * <li>If only fixed SAS Token is configured, this will return null + * and Fixed SAS token will be used to sign requests.</li> + * </ol> + * Avoid using a tokenProvider implementation just to read the configured fixed token, + * as this could create confusion. Also,implementing the SASTokenProvider + * requires relying on the raw configurations. It is more stable to depend on the + * AbfsConfiguration with which a filesystem is initialized, + * and eliminate chances of dynamic modifications and spurious situations. + * @return sasTokenProvider object. + * @throws AzureBlobFileSystemException + */ public SASTokenProvider getSASTokenProvider() throws AzureBlobFileSystemException { AuthType authType = getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey); if (authType != AuthType.SAS) { throw new SASTokenProviderException(String.format( - "Invalid auth type: %s is being used, expecting SAS", authType)); + "Invalid auth type: %s is being used, expecting SAS.", authType)); } try { - String configKey = FS_AZURE_SAS_TOKEN_PROVIDER_TYPE; - Class<? extends SASTokenProvider> sasTokenProviderClass = - getTokenProviderClass(authType, configKey, null, + Class<? extends SASTokenProvider> sasTokenProviderImplementation = + getTokenProviderClass(authType, FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, + null, SASTokenProvider.class); - - Preconditions.checkArgument(sasTokenProviderClass != null, - String.format("The configuration value for \"%s\" is invalid.", configKey)); - - SASTokenProvider sasTokenProvider = ReflectionUtils - .newInstance(sasTokenProviderClass, rawConfig); - Preconditions.checkArgument(sasTokenProvider != null, - String.format("Failed to initialize %s", sasTokenProviderClass)); - - LOG.trace("Initializing {}", sasTokenProviderClass.getName()); - sasTokenProvider.initialize(rawConfig, accountName); - LOG.trace("{} init complete", sasTokenProviderClass.getName()); - return sasTokenProvider; + String configuredFixedToken = this.rawConfig.get(FS_AZURE_SAS_FIXED_TOKEN, + null); + + Preconditions.checkArgument( + sasTokenProviderImplementation != null || configuredFixedToken != null, + "At least one of the \"%s\" and \"%s\" must be set.", + FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, FS_AZURE_SAS_FIXED_TOKEN); + + // Prefer SASTokenProvider Implementation if configured. + if (sasTokenProviderImplementation != null) { + LOG.trace("Using SASTokenProvider class because it is given precedence when it is set."); + SASTokenProvider sasTokenProvider = ReflectionUtils.newInstance( + sasTokenProviderImplementation, rawConfig); + Preconditions.checkArgument(sasTokenProvider != null, + "Failed to initialize %s", sasTokenProviderImplementation); + + LOG.trace("Initializing {}", sasTokenProviderImplementation.getName()); + sasTokenProvider.initialize(rawConfig, accountName); + LOG.trace("{} init complete", sasTokenProviderImplementation.getName()); + return sasTokenProvider; + } else { + // Configured Fixed SAS Token will be used to sign the requests. + return null; Review Comment: Instead of returning null, can we create an implementation of SasTokenProvider interface, which implements getSasToken() method and return the value in the configuration `FS_AZURE_SAS_FIXED_TOKEN`. This would simplify code logic, remove null checks, and reduce git diffs. ########## hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java: ########## @@ -941,33 +941,66 @@ public AccessTokenProvider getTokenProvider() throws TokenAccessProviderExceptio } } + /** + * The user can choose between a configured fixed sas token, and a user + * implementation of the SASTokenProvider interface. Preference will be given + * to SASTokenProvider class provided as the value of "fs.azure.sas.token.provider.type". + * If above config is not set, it is expected that user wants to use a + * fixed SAS Token provided as value of "fs.azure.sas.fixed.token". + * <ol> + * <li>If both the configs are not provided, + * initialization fails and {@link TokenAccessProviderException} is thrown.</li> + * <li>If both are present, SASTokenProvider class will be used to generate SAS Token.</li> + * <li>If only fixed SAS Token is configured, this will return null + * and Fixed SAS token will be used to sign requests.</li> + * </ol> + * Avoid using a tokenProvider implementation just to read the configured fixed token, + * as this could create confusion. Also,implementing the SASTokenProvider + * requires relying on the raw configurations. It is more stable to depend on the + * AbfsConfiguration with which a filesystem is initialized, + * and eliminate chances of dynamic modifications and spurious situations. + * @return sasTokenProvider object. + * @throws AzureBlobFileSystemException + */ public SASTokenProvider getSASTokenProvider() throws AzureBlobFileSystemException { AuthType authType = getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey); if (authType != AuthType.SAS) { throw new SASTokenProviderException(String.format( - "Invalid auth type: %s is being used, expecting SAS", authType)); + "Invalid auth type: %s is being used, expecting SAS.", authType)); } try { - String configKey = FS_AZURE_SAS_TOKEN_PROVIDER_TYPE; - Class<? extends SASTokenProvider> sasTokenProviderClass = - getTokenProviderClass(authType, configKey, null, + Class<? extends SASTokenProvider> sasTokenProviderImplementation = + getTokenProviderClass(authType, FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, + null, SASTokenProvider.class); - - Preconditions.checkArgument(sasTokenProviderClass != null, - String.format("The configuration value for \"%s\" is invalid.", configKey)); - - SASTokenProvider sasTokenProvider = ReflectionUtils - .newInstance(sasTokenProviderClass, rawConfig); - Preconditions.checkArgument(sasTokenProvider != null, - String.format("Failed to initialize %s", sasTokenProviderClass)); - - LOG.trace("Initializing {}", sasTokenProviderClass.getName()); - sasTokenProvider.initialize(rawConfig, accountName); - LOG.trace("{} init complete", sasTokenProviderClass.getName()); - return sasTokenProvider; + String configuredFixedToken = this.rawConfig.get(FS_AZURE_SAS_FIXED_TOKEN, + null); + + Preconditions.checkArgument( Review Comment: Should it be prevented that both configs are not given by the developer. Or if allowed, should we document that the customImplementation will be taken more priority above the fixedSasToken. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
