This is an automated email from the ASF dual-hosted git repository. dimuthuupe pushed a commit to branch ISSUE-108-Path-style in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit 23ecf9dd2cc4e15f25df45c90285014bedbf07af Author: DImuthuUpe <[email protected]> AuthorDate: Sat Sep 2 07:55:51 2023 +0530 Enabling path style Access to custom S3 endpoint --- python-cli/mft_cli/airavata_mft_cli/storage/s3.py | 5 ++- python-cli/mft_cli/pyproject.toml | 4 +-- python-sdk/setup.cfg | 2 +- .../src/airavata_mft_sdk/s3/S3Storage_pb2.py | 38 +++++++++++----------- .../server/backend/sql/entity/S3StorageEntity.java | 11 +++++++ .../stub/src/main/proto/s3/S3Storage.proto | 3 ++ .../apache/airavata/mft/transport/s3/S3Util.java | 12 +++++-- 7 files changed, 49 insertions(+), 26 deletions(-) diff --git a/python-cli/mft_cli/airavata_mft_cli/storage/s3.py b/python-cli/mft_cli/airavata_mft_cli/storage/s3.py index 770d2fd..c6651b8 100644 --- a/python-cli/mft_cli/airavata_mft_cli/storage/s3.py +++ b/python-cli/mft_cli/airavata_mft_cli/storage/s3.py @@ -43,6 +43,8 @@ def handle_add_storage(): options = ["Through AWS Cli config file", "Enter manually" ] option, index = pick(options, "How do you want to load credentials", indicator="=>") + enable_path_style_access = False + if index == 1: # Manual configuration client_id = typer.prompt("Access Key ID") client_secret = typer.prompt("Secret Access Key") @@ -59,6 +61,7 @@ def handle_add_storage(): else: # If endpoint is a S3 compatible endpoint endpoint = typer.prompt("What is the S3 endpoint URL?") region = typer.prompt("What is the region of the bucket?") + enable_path_style_access = typer.confirm("Enable Path Style Access?", False) else: # Loading credentials from the aws cli config file config = configparser.RawConfigParser() @@ -93,7 +96,7 @@ def handle_add_storage(): s3_secret = S3Credential_pb2.S3Secret(accessKey=client_id, secretKey=client_secret, sessionToken = session_token) secret_wrapper = MFTAgentStubs_pb2.SecretWrapper(s3=s3_secret) - s3_storage = S3Storage_pb2.S3Storage(endpoint=endpoint, region=region) + s3_storage = S3Storage_pb2.S3Storage(endpoint=endpoint, region=region, enablePathStyleAccess=enable_path_style_access) storage_wrapper = MFTAgentStubs_pb2.StorageWrapper(s3=s3_storage) direct_req = MFTAgentStubs_pb2.GetResourceMetadataRequest(resourcePath="", secret=secret_wrapper, storage=storage_wrapper) diff --git a/python-cli/mft_cli/pyproject.toml b/python-cli/mft_cli/pyproject.toml index 9ec3419..924f904 100644 --- a/python-cli/mft_cli/pyproject.toml +++ b/python-cli/mft_cli/pyproject.toml @@ -18,7 +18,7 @@ [tool.poetry] name = "airavata-mft-cli" -version = "0.1.17" +version = "0.1.18" description = "Command Line Client for Apache Airavata MFT data transfer software" authors = [ "Dimuthu Wannipurage <[email protected]>", @@ -40,7 +40,7 @@ typer = {extras = ["all"], version = "^0.7.0"} pick = {version= "2.2.0"} grpcio= [{version="1.46.3", markers = "platform_machine != 'arm64'"},{version="1.47.0rc1", markers = "platform_machine == 'arm64'"}] grpcio-tools = [{version="1.46.3", markers = "platform_machine != 'arm64'"},{version="1.47.0rc1", markers = "platform_machine == 'arm64'"}] -airavata-mft-sdk = "0.0.1a33" +airavata-mft-sdk = "0.0.1a34" pandas = "^2.0.3" [build-system] diff --git a/python-sdk/setup.cfg b/python-sdk/setup.cfg index 9d72928..2f1caa5 100644 --- a/python-sdk/setup.cfg +++ b/python-sdk/setup.cfg @@ -16,7 +16,7 @@ # under the License. [metadata] name = airavata_mft_sdk -version = 0.0.1-alpha33 +version = 0.0.1-alpha34 author = Airavata MFT Developers author_email = [email protected] description = Python SDK for Apache Airavata Managed File Transfers (MFT) diff --git a/python-sdk/src/airavata_mft_sdk/s3/S3Storage_pb2.py b/python-sdk/src/airavata_mft_sdk/s3/S3Storage_pb2.py index f510529..3d3fcef 100644 --- a/python-sdk/src/airavata_mft_sdk/s3/S3Storage_pb2.py +++ b/python-sdk/src/airavata_mft_sdk/s3/S3Storage_pb2.py @@ -14,7 +14,7 @@ _sym_db = _symbol_database.Default() -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x12s3/S3Storage.proto\x12\x31org.apache.airavata.mft.resource.stubs.s3.storage\"r\n\tS3Storage\x12\x11\n\tstorageId\x18\x01 \x01(\t\x12\x12\n\nbucketName\x18\x02 \x01(\t\x12\x0e\n\x06region\x18\x03 \x01(\t\x12\x10\n\x08\x65ndpoint\x18\x04 \x01(\t\x12\x0e\n\x06useTLS\x18\x05 \x01(\x08\x12\x0c\n\x04name\x18\x06 \x01(\t\"5\n\x14S3StorageListRequest\x12\x0e\n\x06offset\x18\x01 \x01(\x05\x12\r\n\x05limit\x18\x02 \x01(\x05\"g\n\x15 [...] +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x12s3/S3Storage.proto\x12\x31org.apache.airavata.mft.resource.stubs.s3.storage\"\x91\x01\n\tS3Storage\x12\x11\n\tstorageId\x18\x01 \x01(\t\x12\x12\n\nbucketName\x18\x02 \x01(\t\x12\x0e\n\x06region\x18\x03 \x01(\t\x12\x10\n\x08\x65ndpoint\x18\x04 \x01(\t\x12\x0e\n\x06useTLS\x18\x05 \x01(\x08\x12\x0c\n\x04name\x18\x06 \x01(\t\x12\x1d\n\x15\x65nablePathStyleAccess\x18\x07 \x01(\x08\"5\n\x14S3StorageListRequest\x12\x0e\n\x06offse [...] @@ -94,22 +94,22 @@ if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b'P\001' - _S3STORAGE._serialized_start=73 - _S3STORAGE._serialized_end=187 - _S3STORAGELISTREQUEST._serialized_start=189 - _S3STORAGELISTREQUEST._serialized_end=242 - _S3STORAGELISTRESPONSE._serialized_start=244 - _S3STORAGELISTRESPONSE._serialized_end=347 - _S3STORAGEGETREQUEST._serialized_start=349 - _S3STORAGEGETREQUEST._serialized_end=389 - _S3STORAGECREATEREQUEST._serialized_start=391 - _S3STORAGECREATEREQUEST._serialized_end=518 - _S3STORAGEUPDATEREQUEST._serialized_start=520 - _S3STORAGEUPDATEREQUEST._serialized_end=647 - _S3STORAGEUPDATERESPONSE._serialized_start=649 - _S3STORAGEUPDATERESPONSE._serialized_end=693 - _S3STORAGEDELETEREQUEST._serialized_start=695 - _S3STORAGEDELETEREQUEST._serialized_end=738 - _S3STORAGEDELETERESPONSE._serialized_start=740 - _S3STORAGEDELETERESPONSE._serialized_end=781 + _S3STORAGE._serialized_start=74 + _S3STORAGE._serialized_end=219 + _S3STORAGELISTREQUEST._serialized_start=221 + _S3STORAGELISTREQUEST._serialized_end=274 + _S3STORAGELISTRESPONSE._serialized_start=276 + _S3STORAGELISTRESPONSE._serialized_end=379 + _S3STORAGEGETREQUEST._serialized_start=381 + _S3STORAGEGETREQUEST._serialized_end=421 + _S3STORAGECREATEREQUEST._serialized_start=424 + _S3STORAGECREATEREQUEST._serialized_end=582 + _S3STORAGEUPDATEREQUEST._serialized_start=585 + _S3STORAGEUPDATEREQUEST._serialized_end=743 + _S3STORAGEUPDATERESPONSE._serialized_start=745 + _S3STORAGEUPDATERESPONSE._serialized_end=789 + _S3STORAGEDELETEREQUEST._serialized_start=791 + _S3STORAGEDELETEREQUEST._serialized_end=834 + _S3STORAGEDELETERESPONSE._serialized_start=836 + _S3STORAGEDELETERESPONSE._serialized_end=877 # @@protoc_insertion_point(module_scope) diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/S3StorageEntity.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/S3StorageEntity.java index 402ff06..0698aea 100644 --- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/S3StorageEntity.java +++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/S3StorageEntity.java @@ -48,6 +48,9 @@ public class S3StorageEntity { @Column(name = "USE_TLS") private boolean useTLS; + @Column(name = "PATH_STYLE_ACCESS") + private boolean enablePathStyleAccess = false; + public String getStorageId() { return storageId; } @@ -95,4 +98,12 @@ public class S3StorageEntity { public void setName(String name) { this.name = name; } + + public boolean isEnablePathStyleAccess() { + return enablePathStyleAccess; + } + + public void setEnablePathStyleAccess(boolean enablePathStyleAccess) { + this.enablePathStyleAccess = enablePathStyleAccess; + } } diff --git a/services/resource-service/stub/src/main/proto/s3/S3Storage.proto b/services/resource-service/stub/src/main/proto/s3/S3Storage.proto index 76396ac..8a026d3 100644 --- a/services/resource-service/stub/src/main/proto/s3/S3Storage.proto +++ b/services/resource-service/stub/src/main/proto/s3/S3Storage.proto @@ -27,6 +27,7 @@ message S3Storage { string endpoint = 4; bool useTLS = 5; string name = 6; + bool enablePathStyleAccess = 7; } message S3StorageListRequest { @@ -49,6 +50,7 @@ message S3StorageCreateRequest { string endpoint = 4; bool useTLS = 5; string name = 6; + bool enablePathStyleAccess = 7; } message S3StorageUpdateRequest { @@ -58,6 +60,7 @@ message S3StorageUpdateRequest { string endpoint = 4; bool useTLS = 5; string name = 6; + bool enablePathStyleAccess = 7; } message S3StorageUpdateResponse { diff --git a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3Util.java b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3Util.java index 9495ec8..8f026d7 100644 --- a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3Util.java +++ b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3Util.java @@ -97,13 +97,19 @@ public class S3Util { ClientConfiguration clientConfiguration = new ClientConfiguration(); clientConfiguration.setUseTcpKeepAlive(true); - AmazonS3 s3Client = AmazonS3ClientBuilder.standard() + AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard() .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration( s3Storage.getEndpoint(), s3Storage.getRegion())) .withCredentials(new AWSStaticCredentialsProvider(awsCreds)) .withClientConfiguration(clientConfiguration) - .disableChunkedEncoding() - .build(); + .enablePathStyleAccess() + .disableChunkedEncoding(); + + if (s3Storage.getEnablePathStyleAccess()) { + amazonS3ClientBuilder = amazonS3ClientBuilder.enablePathStyleAccess(); + } + + AmazonS3 s3Client = amazonS3ClientBuilder.build(); s3ClientCache.get().put(secretKey, s3Client); return s3Client;
