This is an automated email from the ASF dual-hosted git repository. dimuthuupe pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit 8aa069fc99cf66999e2a79d7cedfb7cc5ffe7bf0 Author: Dimuthu Wannipurage <[email protected]> AuthorDate: Thu Feb 2 13:25:46 2023 -0500 Refactoring Swift transport to support auth V3 --- python-cli/README.md | 2 +- .../mft_cli/airavata_mft_cli/storage/__init__.py | 3 + .../mft_cli/airavata_mft_cli/storage/swift.py | 82 +++++++++++++ python-cli/mft_cli/pyproject.toml | 3 +- python-sdk/setup.cfg | 8 +- python-sdk/src/airavata_mft_sdk/mft_client.py | 4 + .../airavata_mft_sdk/swift/SwiftCredential_pb2.py | 58 ++++----- .../src/airavata_mft_sdk/swift/SwiftStorage_pb2.py | 36 +++--- .../backend/sql/entity/SwiftStorageEntity.java | 21 ---- .../stub/src/main/proto/swift/SwiftStorage.proto | 6 - .../server/backend/sql/SQLSecretBackend.java | 79 ++++++------ .../sql/entity/swift/SwiftSecretEntity.java | 14 ++- ...retEntity.java => SwiftV2AuthSecretEntity.java} | 46 +++---- ...retEntity.java => SwiftV3AuthSecretEntity.java} | 60 ++++++++-- ...itory.java => SwiftV2AuthSecretRepository.java} | 6 +- ...itory.java => SwiftV3AuthSecretRepository.java} | 6 +- .../src/main/proto/swift/SwiftCredential.proto | 38 +++--- .../transport/swift/SwiftIncomingConnector.java | 32 +---- .../transport/swift/SwiftMetadataCollector.java | 133 ++++++++++----------- .../transport/swift/SwiftOutgoingConnector.java | 29 +---- .../airavata/mft/transport/swift/SwiftUtil.java | 68 +++++++++++ 21 files changed, 423 insertions(+), 311 deletions(-) diff --git a/python-cli/README.md b/python-cli/README.md index b941fe5..b076417 100644 --- a/python-cli/README.md +++ b/python-cli/README.md @@ -37,7 +37,7 @@ Install dependencies ``` pip install grpcio==1.46.3 pip install grpcio-tools==1.46.3 -pip install airavata_mft_sdk==0.0.1-alpha21 +pip install airavata_mft_sdk==0.0.1-alpha24 ``` Build the binary diff --git a/python-cli/mft_cli/airavata_mft_cli/storage/__init__.py b/python-cli/mft_cli/airavata_mft_cli/storage/__init__.py index e7ba957..c125b96 100644 --- a/python-cli/mft_cli/airavata_mft_cli/storage/__init__.py +++ b/python-cli/mft_cli/airavata_mft_cli/storage/__init__.py @@ -21,6 +21,7 @@ from pick import pick import airavata_mft_cli.storage.s3 as s3 import airavata_mft_cli.storage.azure as azure import airavata_mft_cli.storage.gcs as gcs +import airavata_mft_cli.storage.swift as swift from airavata_mft_sdk import mft_client from airavata_mft_sdk.common import StorageCommon_pb2 from rich.console import Console @@ -39,6 +40,8 @@ def add_storage(): azure.handle_add_storage() elif option == "Google Cloud Storage (GCS)": gcs.handle_add_storage() + elif option == "Openstack SWIFT": + swift.handle_add_storage() @app.command("list") def list_storage(): diff --git a/python-cli/mft_cli/airavata_mft_cli/storage/swift.py b/python-cli/mft_cli/airavata_mft_cli/storage/swift.py new file mode 100644 index 0000000..0bdf5fc --- /dev/null +++ b/python-cli/mft_cli/airavata_mft_cli/storage/swift.py @@ -0,0 +1,82 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +from rich import print +from pick import pick +import typer +from airavata_mft_sdk import mft_client +from airavata_mft_sdk.swift import SwiftCredential_pb2 +from airavata_mft_sdk.swift import SwiftStorage_pb2 +from airavata_mft_sdk.common import StorageCommon_pb2 + +def handle_add_storage(): + + options = ["v3", "v2" ] + option, index = pick(options, "Select Keystone Auth Version", indicator="=>") + + secret_create_req= SwiftCredential_pb2.SwiftSecretCreateRequest() + if (option == "v3"): + + user_name = typer.prompt("User Name") + password = typer.prompt("Password") + tenant_name = typer.prompt("Tenant Name") + project_domain = typer.prompt("Project Domain Name", "Default") + user_domain = typer.prompt("User Domain Name", "Default") + v3_sec = SwiftCredential_pb2.SwiftV3AuthSecret(userDomainName=user_domain, userName=user_name, + password=password, tenantName=tenant_name, projectDomainName=project_domain) + secret_create_req.v3AuthSecret.CopyFrom(v3_sec) + else : + tenant_name = typer.prompt("Tenant Name") + user_name = typer.prompt("User Name") + password = typer.prompt("Password") + v2_sec = SwiftCredential_pb2.SwiftV2AuthSecret(tenant=tenant_name, userName=user_name, password=password) + secret_create_req.v2AuthSecret.CopyFrom(v2_sec) + + + auth_url = typer.prompt("Auth URL") + secret_create_req.endpoint = auth_url + + region_name = typer.prompt("Region Name") + container = typer.prompt("Container") + + storage_name = typer.prompt("Name of the storage ", container) + + client = mft_client.MFTClient(transfer_api_port = 7003, + transfer_api_secured = False, + resource_service_host = "localhost", + resource_service_port = 7003, + resource_service_secured = False, + secret_service_host = "localhost", + secret_service_port = 7003) + + + swift_storage_create_req = SwiftStorage_pb2.SwiftStorageCreateRequest(region=region_name, + container=container, + name=storage_name) + + created_storage = client.swift_storage_api.createSwiftStorage(swift_storage_create_req) + + created_secret = client.swift_secret_api.createSwiftSecret(secret_create_req) + + secret_for_storage_req = StorageCommon_pb2.SecretForStorage(storageId = created_storage.storageId, + secretId = created_secret.secretId, + storageType = StorageCommon_pb2.StorageType.SWIFT) + + client.common_api.registerSecretForStorage(secret_for_storage_req) + + print("Successfully added the Swift Container...") \ No newline at end of file diff --git a/python-cli/mft_cli/pyproject.toml b/python-cli/mft_cli/pyproject.toml index 8591b95..83f46f0 100644 --- a/python-cli/mft_cli/pyproject.toml +++ b/python-cli/mft_cli/pyproject.toml @@ -32,8 +32,7 @@ typer = {extras = ["all"], version = "^0.7.0"} pick = {version= "2.2.0"} grpcio= [{version="1.46.3", markers = "platform_machine != 'arm64'"},{version="1.47.0rc1", markers = "platform_machine == 'arm64'"}] grpcio-tools = [{version="1.46.3", markers = "platform_machine != 'arm64'"},{version="1.47.0rc1", markers = "platform_machine == 'arm64'"}] -airavata_mft_sdk= {version="0.0.1-alpha21"} - +airavata_mft_sdk= {version="0.0.1-alpha24"} [build-system] requires = ["poetry-core"] diff --git a/python-sdk/setup.cfg b/python-sdk/setup.cfg index b05fadd..519de26 100644 --- a/python-sdk/setup.cfg +++ b/python-sdk/setup.cfg @@ -16,7 +16,7 @@ # under the License. [metadata] name = airavata_mft_sdk -version = 0.0.1-alpha21 +version = 0.0.1-alpha24 author = Airavata MFT Developers author_email = [email protected] description = Python SDK for Apache Airavata Managed File Transfers (MFT) @@ -35,9 +35,9 @@ package_dir = packages = find: python_requires = >=3.6 install_requires= - grpcio>='1.44.0' - grpcio-tools>='1.44.0' - google-api-python-client>='2.0.0' + grpcio>=1.44.0 + grpcio-tools>=1.44.0 + google-api-python-client>=2.0.0 [options.packages.find] where = src \ No newline at end of file diff --git a/python-sdk/src/airavata_mft_sdk/mft_client.py b/python-sdk/src/airavata_mft_sdk/mft_client.py index 4463263..ca5c326 100644 --- a/python-sdk/src/airavata_mft_sdk/mft_client.py +++ b/python-sdk/src/airavata_mft_sdk/mft_client.py @@ -7,6 +7,7 @@ from airavata_mft_sdk.ftp import FTPStorageService_pb2_grpc from airavata_mft_sdk.gcs import GCSStorageService_pb2_grpc from airavata_mft_sdk.local import LocalStorageService_pb2_grpc from airavata_mft_sdk.s3 import S3StorageService_pb2_grpc +from airavata_mft_sdk.swift import SwiftStorageService_pb2_grpc from airavata_mft_sdk.scp import SCPStorageService_pb2_grpc from airavata_mft_sdk.common import StorageCommon_pb2_grpc @@ -17,6 +18,7 @@ from airavata_mft_sdk.dropbox import DropboxSecretService_pb2_grpc from airavata_mft_sdk.ftp import FTPSecretService_pb2_grpc from airavata_mft_sdk.gcs import GCSSecretService_pb2_grpc from airavata_mft_sdk.s3 import S3SecretService_pb2_grpc +from airavata_mft_sdk.swift import SwiftSecretService_pb2_grpc from airavata_mft_sdk.scp import SCPSecretService_pb2_grpc class MFTClient: @@ -47,6 +49,7 @@ class MFTClient: self.local_storage_api = LocalStorageService_pb2_grpc.LocalStorageServiceStub(self.resource_channel) self.s3_storage_api = S3StorageService_pb2_grpc.S3StorageServiceStub(self.resource_channel) self.scp_storage_api = SCPStorageService_pb2_grpc.SCPStorageServiceStub(self.resource_channel) + self.swift_storage_api = SwiftStorageService_pb2_grpc.SwiftStorageServiceStub(self.resource_channel) self.common_api = StorageCommon_pb2_grpc.StorageCommonServiceStub(self.resource_channel) if (not secret_service_secured): @@ -58,6 +61,7 @@ class MFTClient: self.ftp_secret_api = FTPSecretService_pb2_grpc.FTPSecretServiceStub(self.secret_channel) self.gcs_secret_api = GCSSecretService_pb2_grpc.GCSSecretServiceStub(self.secret_channel) self.s3_secret_api = S3SecretService_pb2_grpc.S3SecretServiceStub(self.secret_channel) + self.swift_secret_api = SwiftSecretService_pb2_grpc.SwiftSecretServiceStub(self.secret_channel) self.scp_secret_api = SCPSecretService_pb2_grpc.SCPSecretServiceStub(self.secret_channel) diff --git a/python-sdk/src/airavata_mft_sdk/swift/SwiftCredential_pb2.py b/python-sdk/src/airavata_mft_sdk/swift/SwiftCredential_pb2.py index e7580a0..9717b6f 100644 --- a/python-sdk/src/airavata_mft_sdk/swift/SwiftCredential_pb2.py +++ b/python-sdk/src/airavata_mft_sdk/swift/SwiftCredential_pb2.py @@ -15,12 +15,12 @@ _sym_db = _symbol_database.Default() import airavata_mft_sdk.CredCommon_pb2 as CredCommon__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1bswift/SwiftCredential.proto\x12.org.apache.airavata.mft.credential.stubs.swift\x1a\x10\x43redCommon.proto\"^\n\x13SwiftPasswordSecret\x12\x10\n\x08userName\x18\x01 \x01(\t\x12\x10\n\x08password\x18\x02 \x01(\t\x12\x11\n\tprojectId\x18\x03 \x01(\t\x12\x10\n\x08\x64omainId\x18\x04 \x01(\t\"K\n\x19SwiftAuthCredentialSecret\x12\x14\n\x0c\x63redentialId\x18\x01 \x01(\t\x12\x18\n\x10\x63redentialSecret\x18\x02 \x01(\t\"\xf3\x01\ [...] +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1bswift/SwiftCredential.proto\x12.org.apache.airavata.mft.credential.stubs.swift\x1a\x10\x43redCommon.proto\"G\n\x11SwiftV2AuthSecret\x12\x0e\n\x06tenant\x18\x01 \x01(\t\x12\x10\n\x08userName\x18\x02 \x01(\t\x12\x10\n\x08password\x18\x03 \x01(\t\"~\n\x11SwiftV3AuthSecret\x12\x16\n\x0euserDomainName\x18\x01 \x01(\t\x12\x10\n\x08userName\x18\x02 \x01(\t\x12\x10\n\x08password\x18\x03 \x01(\t\x12\x12\n\ntenantName\x18\x04 \x01(\ [...] -_SWIFTPASSWORDSECRET = DESCRIPTOR.message_types_by_name['SwiftPasswordSecret'] -_SWIFTAUTHCREDENTIALSECRET = DESCRIPTOR.message_types_by_name['SwiftAuthCredentialSecret'] +_SWIFTV2AUTHSECRET = DESCRIPTOR.message_types_by_name['SwiftV2AuthSecret'] +_SWIFTV3AUTHSECRET = DESCRIPTOR.message_types_by_name['SwiftV3AuthSecret'] _SWIFTSECRET = DESCRIPTOR.message_types_by_name['SwiftSecret'] _SWIFTSECRETGETREQUEST = DESCRIPTOR.message_types_by_name['SwiftSecretGetRequest'] _SWIFTSECRETCREATEREQUEST = DESCRIPTOR.message_types_by_name['SwiftSecretCreateRequest'] @@ -28,19 +28,19 @@ _SWIFTSECRETUPDATEREQUEST = DESCRIPTOR.message_types_by_name['SwiftSecretUpdateR _SWIFTSECRETUPDATERESPONSE = DESCRIPTOR.message_types_by_name['SwiftSecretUpdateResponse'] _SWIFTSECRETDELETEREQUEST = DESCRIPTOR.message_types_by_name['SwiftSecretDeleteRequest'] _SWIFTSECRETDELETERESPONSE = DESCRIPTOR.message_types_by_name['SwiftSecretDeleteResponse'] -SwiftPasswordSecret = _reflection.GeneratedProtocolMessageType('SwiftPasswordSecret', (_message.Message,), { - 'DESCRIPTOR' : _SWIFTPASSWORDSECRET, +SwiftV2AuthSecret = _reflection.GeneratedProtocolMessageType('SwiftV2AuthSecret', (_message.Message,), { + 'DESCRIPTOR' : _SWIFTV2AUTHSECRET, '__module__' : 'swift.SwiftCredential_pb2' - # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.credential.stubs.swift.SwiftPasswordSecret) + # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.credential.stubs.swift.SwiftV2AuthSecret) }) -_sym_db.RegisterMessage(SwiftPasswordSecret) +_sym_db.RegisterMessage(SwiftV2AuthSecret) -SwiftAuthCredentialSecret = _reflection.GeneratedProtocolMessageType('SwiftAuthCredentialSecret', (_message.Message,), { - 'DESCRIPTOR' : _SWIFTAUTHCREDENTIALSECRET, +SwiftV3AuthSecret = _reflection.GeneratedProtocolMessageType('SwiftV3AuthSecret', (_message.Message,), { + 'DESCRIPTOR' : _SWIFTV3AUTHSECRET, '__module__' : 'swift.SwiftCredential_pb2' - # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.credential.stubs.swift.SwiftAuthCredentialSecret) + # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.credential.stubs.swift.SwiftV3AuthSecret) }) -_sym_db.RegisterMessage(SwiftAuthCredentialSecret) +_sym_db.RegisterMessage(SwiftV3AuthSecret) SwiftSecret = _reflection.GeneratedProtocolMessageType('SwiftSecret', (_message.Message,), { 'DESCRIPTOR' : _SWIFTSECRET, @@ -95,22 +95,22 @@ if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b'P\001' - _SWIFTPASSWORDSECRET._serialized_start=97 - _SWIFTPASSWORDSECRET._serialized_end=191 - _SWIFTAUTHCREDENTIALSECRET._serialized_start=193 - _SWIFTAUTHCREDENTIALSECRET._serialized_end=268 - _SWIFTSECRET._serialized_start=271 - _SWIFTSECRET._serialized_end=514 - _SWIFTSECRETGETREQUEST._serialized_start=516 - _SWIFTSECRETGETREQUEST._serialized_end=620 - _SWIFTSECRETCREATEREQUEST._serialized_start=623 - _SWIFTSECRETCREATEREQUEST._serialized_end=924 - _SWIFTSECRETUPDATEREQUEST._serialized_start=927 - _SWIFTSECRETUPDATEREQUEST._serialized_end=1246 - _SWIFTSECRETUPDATERESPONSE._serialized_start=1248 - _SWIFTSECRETUPDATERESPONSE._serialized_end=1293 - _SWIFTSECRETDELETEREQUEST._serialized_start=1295 - _SWIFTSECRETDELETEREQUEST._serialized_end=1402 - _SWIFTSECRETDELETERESPONSE._serialized_start=1404 - _SWIFTSECRETDELETERESPONSE._serialized_end=1447 + _SWIFTV2AUTHSECRET._serialized_start=97 + _SWIFTV2AUTHSECRET._serialized_end=168 + _SWIFTV3AUTHSECRET._serialized_start=170 + _SWIFTV3AUTHSECRET._serialized_end=296 + _SWIFTSECRET._serialized_start=299 + _SWIFTSECRET._serialized_end=540 + _SWIFTSECRETGETREQUEST._serialized_start=542 + _SWIFTSECRETGETREQUEST._serialized_end=646 + _SWIFTSECRETCREATEREQUEST._serialized_start=649 + _SWIFTSECRETCREATEREQUEST._serialized_end=948 + _SWIFTSECRETUPDATEREQUEST._serialized_start=951 + _SWIFTSECRETUPDATEREQUEST._serialized_end=1268 + _SWIFTSECRETUPDATERESPONSE._serialized_start=1270 + _SWIFTSECRETUPDATERESPONSE._serialized_end=1315 + _SWIFTSECRETDELETEREQUEST._serialized_start=1317 + _SWIFTSECRETDELETEREQUEST._serialized_end=1424 + _SWIFTSECRETDELETERESPONSE._serialized_start=1426 + _SWIFTSECRETDELETERESPONSE._serialized_end=1469 # @@protoc_insertion_point(module_scope) diff --git a/python-sdk/src/airavata_mft_sdk/swift/SwiftStorage_pb2.py b/python-sdk/src/airavata_mft_sdk/swift/SwiftStorage_pb2.py index e6af4c6..6863413 100644 --- a/python-sdk/src/airavata_mft_sdk/swift/SwiftStorage_pb2.py +++ b/python-sdk/src/airavata_mft_sdk/swift/SwiftStorage_pb2.py @@ -14,7 +14,7 @@ _sym_db = _symbol_database.Default() -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18swift/SwiftStorage.proto\x12\x34org.apache.airavata.mft.resource.stubs.swift.storage\"}\n\x0cSwiftStorage\x12\x11\n\tstorageId\x18\x01 \x01(\t\x12\x11\n\tcontainer\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x10\n\x08\x65ndpoint\x18\x04 \x01(\t\x12\x0e\n\x06region\x18\x05 \x01(\t\x12\x17\n\x0fkeystoneVersion\x18\x06 \x01(\x05\"8\n\x17SwiftStorageListRequest\x12\x0e\n\x06offset\x18\x01 \x01(\x05\x12\r\n\x05limit\x [...] +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18swift/SwiftStorage.proto\x12\x34org.apache.airavata.mft.resource.stubs.swift.storage\"R\n\x0cSwiftStorage\x12\x11\n\tstorageId\x18\x01 \x01(\t\x12\x11\n\tcontainer\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x0e\n\x06region\x18\x05 \x01(\t\"8\n\x17SwiftStorageListRequest\x12\x0e\n\x06offset\x18\x01 \x01(\x05\x12\r\n\x05limit\x18\x02 \x01(\x05\"p\n\x18SwiftStorageListResponse\x12T\n\x08storages\x18\x01 \x03(\x0b\x [...] @@ -95,21 +95,21 @@ if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b'P\001' _SWIFTSTORAGE._serialized_start=82 - _SWIFTSTORAGE._serialized_end=207 - _SWIFTSTORAGELISTREQUEST._serialized_start=209 - _SWIFTSTORAGELISTREQUEST._serialized_end=265 - _SWIFTSTORAGELISTRESPONSE._serialized_start=267 - _SWIFTSTORAGELISTRESPONSE._serialized_end=379 - _SWIFTSTORAGEGETREQUEST._serialized_start=381 - _SWIFTSTORAGEGETREQUEST._serialized_end=424 - _SWIFTSTORAGECREATEREQUEST._serialized_start=427 - _SWIFTSTORAGECREATEREQUEST._serialized_end=565 - _SWIFTSTORAGEUPDATEREQUEST._serialized_start=568 - _SWIFTSTORAGEUPDATEREQUEST._serialized_end=706 - _SWIFTSTORAGEUPDATERESPONSE._serialized_start=708 - _SWIFTSTORAGEUPDATERESPONSE._serialized_end=755 - _SWIFTSTORAGEDELETEREQUEST._serialized_start=757 - _SWIFTSTORAGEDELETEREQUEST._serialized_end=803 - _SWIFTSTORAGEDELETERESPONSE._serialized_start=805 - _SWIFTSTORAGEDELETERESPONSE._serialized_end=849 + _SWIFTSTORAGE._serialized_end=164 + _SWIFTSTORAGELISTREQUEST._serialized_start=166 + _SWIFTSTORAGELISTREQUEST._serialized_end=222 + _SWIFTSTORAGELISTRESPONSE._serialized_start=224 + _SWIFTSTORAGELISTRESPONSE._serialized_end=336 + _SWIFTSTORAGEGETREQUEST._serialized_start=338 + _SWIFTSTORAGEGETREQUEST._serialized_end=381 + _SWIFTSTORAGECREATEREQUEST._serialized_start=383 + _SWIFTSTORAGECREATEREQUEST._serialized_end=478 + _SWIFTSTORAGEUPDATEREQUEST._serialized_start=480 + _SWIFTSTORAGEUPDATEREQUEST._serialized_end=575 + _SWIFTSTORAGEUPDATERESPONSE._serialized_start=577 + _SWIFTSTORAGEUPDATERESPONSE._serialized_end=624 + _SWIFTSTORAGEDELETEREQUEST._serialized_start=626 + _SWIFTSTORAGEDELETEREQUEST._serialized_end=672 + _SWIFTSTORAGEDELETERESPONSE._serialized_start=674 + _SWIFTSTORAGEDELETERESPONSE._serialized_end=718 # @@protoc_insertion_point(module_scope) diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/SwiftStorageEntity.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/SwiftStorageEntity.java index 887c874..4792795 100644 --- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/SwiftStorageEntity.java +++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/SwiftStorageEntity.java @@ -38,15 +38,9 @@ public class SwiftStorageEntity { @Column(name = "CONTAINER") String container; - @Column(name = "ENDPOINT") - String endpoint; - @Column(name = "REGION") String region; - @Column(name = "KEYSTONE_VERSION") - int keystoneVersion; - public String getStorageId() { return storageId; } @@ -71,14 +65,6 @@ public class SwiftStorageEntity { this.container = container; } - public String getEndpoint() { - return endpoint; - } - - public void setEndpoint(String endpoint) { - this.endpoint = endpoint; - } - public String getRegion() { return region; } @@ -87,11 +73,4 @@ public class SwiftStorageEntity { this.region = region; } - public int getKeystoneVersion() { - return keystoneVersion; - } - - public void setKeystoneVersion(int keystoneVersion) { - this.keystoneVersion = keystoneVersion; - } } diff --git a/services/resource-service/stub/src/main/proto/swift/SwiftStorage.proto b/services/resource-service/stub/src/main/proto/swift/SwiftStorage.proto index 824491e..d621209 100644 --- a/services/resource-service/stub/src/main/proto/swift/SwiftStorage.proto +++ b/services/resource-service/stub/src/main/proto/swift/SwiftStorage.proto @@ -24,9 +24,7 @@ message SwiftStorage { string storageId = 1; string container = 2; string name = 3; - string endpoint = 4; string region = 5; - int32 keystoneVersion = 6; } message SwiftStorageListRequest { @@ -46,18 +44,14 @@ message SwiftStorageCreateRequest { string storageId = 1; string container = 2; string name = 3; - string endpoint = 4; string region = 5; - int32 keystoneVersion = 6; } message SwiftStorageUpdateRequest { string storageId = 1; string container = 2; string name = 3; - string endpoint = 4; string region = 5; - int32 keystoneVersion = 6; } message SwiftStorageUpdateResponse { diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java index d19a1fd..2c08b98 100644 --- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java @@ -28,12 +28,13 @@ import org.apache.airavata.mft.credential.stubs.scp.*; import org.apache.airavata.mft.credential.stubs.swift.*; import org.apache.airavata.mft.secret.server.backend.SecretBackend; import org.apache.airavata.mft.secret.server.backend.sql.entity.*; -import org.apache.airavata.mft.secret.server.backend.sql.entity.swift.SwiftAuthCredentialSecretEntity; -import org.apache.airavata.mft.secret.server.backend.sql.entity.swift.SwiftPasswordSecretEntity; +import org.apache.airavata.mft.secret.server.backend.sql.entity.swift.SwiftSecretEntity.InternalSecretType; +import org.apache.airavata.mft.secret.server.backend.sql.entity.swift.SwiftV2AuthSecretEntity; +import org.apache.airavata.mft.secret.server.backend.sql.entity.swift.SwiftV3AuthSecretEntity; import org.apache.airavata.mft.secret.server.backend.sql.entity.swift.SwiftSecretEntity; import org.apache.airavata.mft.secret.server.backend.sql.repository.*; -import org.apache.airavata.mft.secret.server.backend.sql.repository.swift.SwiftAuthCredentialSecretRepository; -import org.apache.airavata.mft.secret.server.backend.sql.repository.swift.SwiftPasswordSecretRepository; +import org.apache.airavata.mft.secret.server.backend.sql.repository.swift.SwiftV2AuthSecretRepository; +import org.apache.airavata.mft.secret.server.backend.sql.repository.swift.SwiftV3AuthSecretRepository; import org.apache.airavata.mft.secret.server.backend.sql.repository.swift.SwiftSecretRepository; import org.dozer.DozerBeanMapper; import org.slf4j.Logger; @@ -64,10 +65,10 @@ public class SQLSecretBackend implements SecretBackend { private SwiftSecretRepository swiftSecretRepository; @Autowired - private SwiftPasswordSecretRepository swiftPasswordSecretRepository; + private SwiftV3AuthSecretRepository swiftV3AuthSecretRepository; @Autowired - private SwiftAuthCredentialSecretRepository swiftAuthCredentialSecretRepository; + private SwiftV2AuthSecretRepository swiftV2AuthSecretRepository; @Autowired private ODataSecretRepository odataSecretRepository; @@ -229,26 +230,27 @@ public class SQLSecretBackend implements SecretBackend { SwiftSecret.Builder secBuilder = SwiftSecret.newBuilder(); SwiftSecretEntity secEty = secEtyOp.get(); secBuilder.setSecretId(secEty.getSecretId()); + secBuilder.setEndpoint(secEty.getEndpoint()); switch (secEty.getInternalSecretType()) { - case PASSWORD: - Optional<SwiftPasswordSecretEntity> passSec = swiftPasswordSecretRepository + case V2: + Optional<SwiftV2AuthSecretEntity> v2Sec = swiftV2AuthSecretRepository .findBySecretId(secEty.getInternalSecretId()); - if (passSec.isPresent()) { - SwiftPasswordSecret.Builder passBuilder = SwiftPasswordSecret.newBuilder(); - mapper.map(passSec.get(), passBuilder); - secBuilder.setPasswordSecret(passBuilder.build()); + if (v2Sec.isPresent()) { + SwiftV2AuthSecret.Builder v2Builder = SwiftV2AuthSecret.newBuilder(); + mapper.map(v2Sec.get(), v2Builder); + secBuilder.setV2AuthSecret(v2Builder.build()); } else { throw new Exception("Can not find a swift password secret with id " + secEty.getInternalSecretId()); } break; - case AUTH_CREDENTIAL: - Optional<SwiftAuthCredentialSecretEntity> authCredSec = swiftAuthCredentialSecretRepository + case V3: + Optional<SwiftV3AuthSecretEntity> v3Sec = swiftV3AuthSecretRepository .findBySecretId(secEty.getInternalSecretId()); - if (authCredSec.isPresent()) { - SwiftAuthCredentialSecret.Builder authBuilder = SwiftAuthCredentialSecret.newBuilder(); - mapper.map(authCredSec.get(), authBuilder); - secBuilder.setAuthCredentialSecret(authBuilder.build()); + if (v3Sec.isPresent()) { + SwiftV3AuthSecret.Builder v3Builder = SwiftV3AuthSecret.newBuilder(); + mapper.map(v3Sec.get(), v3Builder); + secBuilder.setV3AuthSecret(v3Builder.build()); } else { throw new Exception("Can not find a swift auth cred secret with id " + secEty.getInternalSecretId()); } @@ -266,21 +268,22 @@ public class SQLSecretBackend implements SecretBackend { public SwiftSecret createSwiftSecret(SwiftSecretCreateRequest request) throws Exception { SwiftSecretEntity secEty = new SwiftSecretEntity(); - SwiftAuthCredentialSecretEntity authCredSaved = null; - SwiftPasswordSecretEntity passSecSaved = null; + secEty.setEndpoint(request.getEndpoint()); + SwiftV2AuthSecretEntity v2SecSaved = null; + SwiftV3AuthSecretEntity v3SecSaved = null; switch (request.getSecretCase()) { - case PASSWORDSECRET: - passSecSaved = swiftPasswordSecretRepository - .save(mapper.map(request.getPasswordSecret(), SwiftPasswordSecretEntity.class)); - secEty.setInternalSecretId(passSecSaved.getSecretId()); - secEty.setInternalSecretType(SwiftSecretEntity.InternalSecretType.PASSWORD); + case V2AUTHSECRET: + v2SecSaved = swiftV2AuthSecretRepository + .save(mapper.map(request.getV2AuthSecret(), SwiftV2AuthSecretEntity.class)); + secEty.setInternalSecretId(v2SecSaved.getSecretId()); + secEty.setInternalSecretType(InternalSecretType.V2); break; - case AUTHCREDENTIALSECRET: - authCredSaved = swiftAuthCredentialSecretRepository - .save(mapper.map(request.getAuthCredentialSecret(), SwiftAuthCredentialSecretEntity.class)); - secEty.setInternalSecretId(authCredSaved.getSecretId()); - secEty.setInternalSecretType(SwiftSecretEntity.InternalSecretType.AUTH_CREDENTIAL); + case V3AUTHSECRET: + v3SecSaved = swiftV3AuthSecretRepository + .save(mapper.map(request.getV3AuthSecret(), SwiftV3AuthSecretEntity.class)); + secEty.setInternalSecretId(v3SecSaved.getSecretId()); + secEty.setInternalSecretType(InternalSecretType.V3); break; case SECRET_NOT_SET: throw new Exception("No internal secret is set"); @@ -290,11 +293,11 @@ public class SQLSecretBackend implements SecretBackend { SwiftSecret.Builder secBuilder = SwiftSecret.newBuilder(); secBuilder.setSecretId(savedEty.getSecretId()); switch (savedEty.getInternalSecretType()) { - case PASSWORD: - secBuilder.setPasswordSecret(mapper.map(passSecSaved, SwiftPasswordSecret.newBuilder().getClass())); + case V2: + secBuilder.setV2AuthSecret(mapper.map(v2SecSaved, SwiftV2AuthSecret.newBuilder().getClass())); break; - case AUTH_CREDENTIAL: - secBuilder.setAuthCredentialSecret(mapper.map(authCredSaved, SwiftAuthCredentialSecret.newBuilder().getClass())); + case V3: + secBuilder.setV3AuthSecret(mapper.map(v3SecSaved, SwiftV3AuthSecret.newBuilder().getClass())); break; } @@ -312,11 +315,11 @@ public class SQLSecretBackend implements SecretBackend { if (secOp.isPresent()) { swiftSecretRepository.deleteById(request.getSecretId()); switch (secOp.get().getInternalSecretType()) { - case AUTH_CREDENTIAL: - swiftAuthCredentialSecretRepository.deleteById(secOp.get().getInternalSecretId()); + case V2: + swiftV2AuthSecretRepository.deleteById(secOp.get().getInternalSecretId()); break; - case PASSWORD: - swiftPasswordSecretRepository.deleteById(secOp.get().getInternalSecretId()); + case V3: + swiftV3AuthSecretRepository.deleteById(secOp.get().getInternalSecretId()); break; } return true; diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/swift/SwiftSecretEntity.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/swift/SwiftSecretEntity.java index 203edec..bec3bd3 100644 --- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/swift/SwiftSecretEntity.java +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/swift/SwiftSecretEntity.java @@ -28,7 +28,7 @@ import javax.persistence.Id; public class SwiftSecretEntity { public enum InternalSecretType { - PASSWORD, AUTH_CREDENTIAL; + V2, V3; } @Id @@ -43,6 +43,9 @@ public class SwiftSecretEntity { @Column(name = "INTERNAL_SECRET_TYPE") private InternalSecretType internalSecretType; + @Column(name = "ENDPOINT") + private String endpoint; + public String getSecretId() { return secretId; } @@ -66,4 +69,13 @@ public class SwiftSecretEntity { public void setInternalSecretType(InternalSecretType internalSecretType) { this.internalSecretType = internalSecretType; } + + public String getEndpoint() { + return endpoint; + } + + public SwiftSecretEntity setEndpoint(String endpoint) { + this.endpoint = endpoint; + return this; + } } diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/swift/SwiftPasswordSecretEntity.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/swift/SwiftV2AuthSecretEntity.java similarity index 74% rename from services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/swift/SwiftPasswordSecretEntity.java rename to services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/swift/SwiftV2AuthSecretEntity.java index 9157af5..1cccc6a 100644 --- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/swift/SwiftPasswordSecretEntity.java +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/swift/SwiftV2AuthSecretEntity.java @@ -25,7 +25,7 @@ import javax.persistence.GeneratedValue; import javax.persistence.Id; @Entity -public class SwiftPasswordSecretEntity { +public class SwiftV2AuthSecretEntity { @Id @Column(name = "SECRET_ID") @@ -33,17 +33,14 @@ public class SwiftPasswordSecretEntity { @GenericGenerator(name = "uuid", strategy = "uuid2") private String secretId; + @Column(name = "TENANT") + private String tenant; + @Column(name = "USER_NAME") - String userName; + private String userName; @Column(name = "PASSWORD") - String password; - - @Column(name = "PROJECT_ID") - String projectId; - - @Column(name = "DOMAIN_ID") - String domainId; + private String password; public String getSecretId() { return secretId; @@ -53,35 +50,30 @@ public class SwiftPasswordSecretEntity { this.secretId = secretId; } + public String getTenant() { + return tenant; + } + + public SwiftV2AuthSecretEntity setTenant(String tenant) { + this.tenant = tenant; + return this; + } + public String getUserName() { return userName; } - public void setUserName(String userName) { + public SwiftV2AuthSecretEntity setUserName(String userName) { this.userName = userName; + return this; } public String getPassword() { return password; } - public void setPassword(String password) { + public SwiftV2AuthSecretEntity setPassword(String password) { this.password = password; - } - - public String getProjectId() { - return projectId; - } - - public void setProjectId(String projectId) { - this.projectId = projectId; - } - - public String getDomainId() { - return domainId; - } - - public void setDomainId(String domainId) { - this.domainId = domainId; + return this; } } diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/swift/SwiftAuthCredentialSecretEntity.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/swift/SwiftV3AuthSecretEntity.java similarity index 51% rename from services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/swift/SwiftAuthCredentialSecretEntity.java rename to services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/swift/SwiftV3AuthSecretEntity.java index 7b07f94..af34e42 100644 --- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/swift/SwiftAuthCredentialSecretEntity.java +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/swift/SwiftV3AuthSecretEntity.java @@ -25,7 +25,7 @@ import javax.persistence.GeneratedValue; import javax.persistence.Id; @Entity -public class SwiftAuthCredentialSecretEntity { +public class SwiftV3AuthSecretEntity { @Id @Column(name = "SECRET_ID") @@ -33,11 +33,18 @@ public class SwiftAuthCredentialSecretEntity { @GenericGenerator(name = "uuid", strategy = "uuid2") private String secretId; - @Column(name = "CREDENTIAL_ID") - private String credentialId; + @Column(name = "USER_DOMAIN_NAME") + private String userDomainName; + @Column(name = "USER_NAME") + private String userName; + @Column(name = "PASSWORD") + private String password; - @Column(name = "CREDENTIAL_SECRET") - private String credentialSecret; + @Column(name = "TENANT_NAME") + private String tenantName; + + @Column(name = "PROJECT_DOMAIN_NAME") + private String projectDomainName; public String getSecretId() { return secretId; @@ -47,19 +54,46 @@ public class SwiftAuthCredentialSecretEntity { this.secretId = secretId; } - public String getCredentialId() { - return credentialId; + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public String getUserDomainName() { + return userDomainName; + } + + public SwiftV3AuthSecretEntity setUserDomainName(String userDomainName) { + this.userDomainName = userDomainName; + return this; + } + + public String getTenantName() { + return tenantName; } - public void setCredentialId(String credentialId) { - this.credentialId = credentialId; + public SwiftV3AuthSecretEntity setTenantName(String tenantName) { + this.tenantName = tenantName; + return this; } - public String getCredentialSecret() { - return credentialSecret; + public String getProjectDomainName() { + return projectDomainName; } - public void setCredentialSecret(String credentialSecret) { - this.credentialSecret = credentialSecret; + public SwiftV3AuthSecretEntity setProjectDomainName(String projectDomainName) { + this.projectDomainName = projectDomainName; + return this; } } diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/swift/SwiftAuthCredentialSecretRepository.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/swift/SwiftV2AuthSecretRepository.java similarity index 81% rename from services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/swift/SwiftAuthCredentialSecretRepository.java rename to services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/swift/SwiftV2AuthSecretRepository.java index 9cda467..7ca6375 100644 --- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/swift/SwiftAuthCredentialSecretRepository.java +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/swift/SwiftV2AuthSecretRepository.java @@ -17,11 +17,11 @@ package org.apache.airavata.mft.secret.server.backend.sql.repository.swift; -import org.apache.airavata.mft.secret.server.backend.sql.entity.swift.SwiftAuthCredentialSecretEntity; +import org.apache.airavata.mft.secret.server.backend.sql.entity.swift.SwiftV2AuthSecretEntity; import org.springframework.data.repository.CrudRepository; import java.util.Optional; -public interface SwiftAuthCredentialSecretRepository extends CrudRepository<SwiftAuthCredentialSecretEntity, String> { - Optional<SwiftAuthCredentialSecretEntity> findBySecretId(String secretId); +public interface SwiftV2AuthSecretRepository extends CrudRepository<SwiftV2AuthSecretEntity, String> { + Optional<SwiftV2AuthSecretEntity> findBySecretId(String secretId); } diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/swift/SwiftPasswordSecretRepository.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/swift/SwiftV3AuthSecretRepository.java similarity index 82% rename from services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/swift/SwiftPasswordSecretRepository.java rename to services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/swift/SwiftV3AuthSecretRepository.java index 51ef5b4..1574f70 100644 --- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/swift/SwiftPasswordSecretRepository.java +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/swift/SwiftV3AuthSecretRepository.java @@ -17,12 +17,12 @@ package org.apache.airavata.mft.secret.server.backend.sql.repository.swift; -import org.apache.airavata.mft.secret.server.backend.sql.entity.swift.SwiftPasswordSecretEntity; +import org.apache.airavata.mft.secret.server.backend.sql.entity.swift.SwiftV3AuthSecretEntity; import org.springframework.data.repository.CrudRepository; import java.util.Optional; -public interface SwiftPasswordSecretRepository extends CrudRepository<SwiftPasswordSecretEntity, String> { - Optional<SwiftPasswordSecretEntity> findBySecretId(String secretId); +public interface SwiftV3AuthSecretRepository extends CrudRepository<SwiftV3AuthSecretEntity, String> { + Optional<SwiftV3AuthSecretEntity> findBySecretId(String secretId); } diff --git a/services/secret-service/stub/src/main/proto/swift/SwiftCredential.proto b/services/secret-service/stub/src/main/proto/swift/SwiftCredential.proto index 02f34d2..0ad6ca7 100644 --- a/services/secret-service/stub/src/main/proto/swift/SwiftCredential.proto +++ b/services/secret-service/stub/src/main/proto/swift/SwiftCredential.proto @@ -22,24 +22,27 @@ package org.apache.airavata.mft.credential.stubs.swift; import "CredCommon.proto"; -message SwiftPasswordSecret { - string userName = 1; - string password = 2; - string projectId = 3; - string domainId = 4; +message SwiftV2AuthSecret { + string tenant = 1; + string userName = 2; + string password = 3; } -message SwiftAuthCredentialSecret { - string credentialId = 1; - string credentialSecret = 2; +message SwiftV3AuthSecret { + string userDomainName = 1; + string userName = 2; + string password = 3; + string tenantName = 4; + string projectDomainName = 5; } message SwiftSecret { string secretId = 1; oneof secret { - SwiftPasswordSecret passwordSecret = 2; - SwiftAuthCredentialSecret authCredentialSecret = 3; + SwiftV2AuthSecret v2AuthSecret = 2; + SwiftV3AuthSecret v3AuthSecret = 3; } + string endpoint = 4; } message SwiftSecretGetRequest { @@ -48,20 +51,23 @@ message SwiftSecretGetRequest { } message SwiftSecretCreateRequest { + oneof secret { - SwiftPasswordSecret passwordSecret = 1; - SwiftAuthCredentialSecret authCredentialSecret = 2; + SwiftV2AuthSecret v2AuthSecret = 1; + SwiftV3AuthSecret v3AuthSecret = 2; } - org.apache.airavata.mft.common.AuthToken authzToken = 3; + string endpoint = 3; + org.apache.airavata.mft.common.AuthToken authzToken = 4; } message SwiftSecretUpdateRequest { string secretId = 1; oneof secret { - SwiftPasswordSecret passwordSecret = 2; - SwiftAuthCredentialSecret authCredentialSecret = 3; + SwiftV2AuthSecret v2AuthSecret = 2; + SwiftV3AuthSecret v3AuthSecret = 3; } - org.apache.airavata.mft.common.AuthToken authzToken = 4; + string endpoint = 4; + org.apache.airavata.mft.common.AuthToken authzToken = 5; } message SwiftSecretUpdateResponse { diff --git a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java index e578b5f..5f9a415 100644 --- a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java +++ b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java @@ -50,40 +50,10 @@ public class SwiftIncomingConnector implements IncomingChunkedConnector { public void init(ConnectorConfig cc) throws Exception { SwiftStorage swiftStorage = cc.getStorage().getSwift(); - this.resourcePath = cc.getResourcePath(); - SwiftSecret swiftSecret = cc.getSecret().getSwift(); - - String provider = "openstack-swift"; - - Properties overrides = new Properties(); - overrides.put(KeystoneProperties.KEYSTONE_VERSION, swiftStorage.getKeystoneVersion() + ""); - - String identity = null; - String credential = null; - switch (swiftSecret.getSecretCase()) { - case PASSWORDSECRET: - identity = swiftSecret.getPasswordSecret().getDomainId() + ":" + swiftSecret.getPasswordSecret().getUserName(); - credential = swiftSecret.getPasswordSecret().getPassword(); - overrides.put(KeystoneProperties.SCOPE, "projectId:" + swiftSecret.getPasswordSecret().getProjectId()); - overrides.put(KeystoneProperties.CREDENTIAL_TYPE, CredentialTypes.PASSWORD_CREDENTIALS); - break; - case AUTHCREDENTIALSECRET: - identity = swiftSecret.getAuthCredentialSecret().getCredentialId(); - credential = swiftSecret.getAuthCredentialSecret().getCredentialSecret(); - overrides.put(KeystoneProperties.CREDENTIAL_TYPE, CredentialTypes.API_ACCESS_KEY_CREDENTIALS); - break; - } - - swiftApi = ContextBuilder.newBuilder(provider) - .endpoint(swiftStorage.getEndpoint()) - .credentials(identity, credential) - .overrides(overrides) - .buildApi(SwiftApi.class); - + swiftApi = SwiftUtil.createSwiftApi(swiftSecret, swiftStorage); objectApi = swiftApi.getObjectApi(swiftStorage.getRegion(), swiftStorage.getContainer()); - } @Override diff --git a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftMetadataCollector.java b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftMetadataCollector.java index b9f33e3..ad4fa85 100644 --- a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftMetadataCollector.java +++ b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftMetadataCollector.java @@ -17,14 +17,13 @@ package org.apache.airavata.mft.transport.swift; +import com.google.common.collect.FluentIterable; import org.apache.airavata.mft.agent.stub.*; import org.apache.airavata.mft.core.api.MetadataCollector; import org.apache.airavata.mft.credential.stubs.swift.SwiftSecret; import org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorage; -import org.jclouds.ContextBuilder; -import org.jclouds.openstack.keystone.auth.config.CredentialTypes; -import org.jclouds.openstack.keystone.config.KeystoneProperties; import org.jclouds.openstack.swift.v1.SwiftApi; +import org.jclouds.openstack.swift.v1.domain.Container; import org.jclouds.openstack.swift.v1.domain.ObjectList; import org.jclouds.openstack.swift.v1.domain.SwiftObject; import org.jclouds.openstack.swift.v1.features.ObjectApi; @@ -49,87 +48,81 @@ public class SwiftMetadataCollector implements MetadataCollector { } } - private SwiftApi getSwiftApi(SwiftStorage swiftStorage, SwiftSecret swiftSecret) { - String provider = "openstack-swift"; - - Properties overrides = new Properties(); - overrides.put(KeystoneProperties.KEYSTONE_VERSION, swiftStorage.getKeystoneVersion() + ""); - - String identity = null; - String credential = null; - switch (swiftSecret.getSecretCase()) { - case PASSWORDSECRET: - identity = swiftSecret.getPasswordSecret().getDomainId() + ":" + swiftSecret.getPasswordSecret().getUserName(); - credential = swiftSecret.getPasswordSecret().getPassword(); - overrides.put(KeystoneProperties.SCOPE, "projectId:" + swiftSecret.getPasswordSecret().getProjectId()); - overrides.put(KeystoneProperties.CREDENTIAL_TYPE, CredentialTypes.PASSWORD_CREDENTIALS); - break; - case AUTHCREDENTIALSECRET: - identity = swiftSecret.getAuthCredentialSecret().getCredentialId(); - credential = swiftSecret.getAuthCredentialSecret().getCredentialSecret(); - overrides.put(KeystoneProperties.CREDENTIAL_TYPE, CredentialTypes.API_ACCESS_KEY_CREDENTIALS); - break; - } - - return ContextBuilder.newBuilder(provider) - .endpoint(swiftStorage.getEndpoint()) - .credentials(identity, credential) - .overrides(overrides) - .buildApi(SwiftApi.class); - } - @Override public ResourceMetadata getResourceMetadata(String resourcePath, boolean recursiveSearch) throws Exception { checkInitialized(); - SwiftApi swiftApi = getSwiftApi(swiftStorage, swiftSecret); - - ObjectApi objectApi = swiftApi.getObjectApi(swiftStorage.getRegion(), swiftStorage.getContainer()); - - ResourceMetadata.Builder resourceBuilder = ResourceMetadata.newBuilder(); - if ("".equals(resourcePath)) { - DirectoryMetadata.Builder rootDirBuilder = DirectoryMetadata.newBuilder(); - - ObjectList objectList = objectApi.list(); - objectList.forEach(swiftObject -> { - FileMetadata.Builder fileBuilder = FileMetadata.newBuilder(); - fileBuilder.setFriendlyName(swiftObject.getName()); - fileBuilder.setResourcePath(swiftObject.getName()); - fileBuilder.setCreatedTime(swiftObject.getLastModified().getTime()); - fileBuilder.setUpdateTime(swiftObject.getLastModified().getTime()); - fileBuilder.setResourceSize(swiftObject.getPayload().getContentMetadata().getContentLength()); - rootDirBuilder.addFiles(fileBuilder); - }); - resourceBuilder.setDirectory(rootDirBuilder); - } else { - SwiftObject swiftObject = objectApi.get(resourcePath); - - if (swiftObject == null) { - resourceBuilder.setError(MetadataFetchError.NOT_FOUND); - return resourceBuilder.build(); + SwiftApi swiftApi = SwiftUtil.createSwiftApi(swiftSecret, swiftStorage); + + try { + ResourceMetadata.Builder resourceBuilder = ResourceMetadata.newBuilder(); + if ("".equals(resourcePath) && "".equals(swiftStorage.getContainer())) { + FluentIterable<Container> containers = swiftApi.getContainerApi(swiftStorage.getRegion()).list(); + DirectoryMetadata.Builder parentDir = DirectoryMetadata.newBuilder(); + parentDir.setResourcePath(""); + parentDir.setFriendlyName(""); + containers.forEach(container -> { + DirectoryMetadata.Builder bucketDir = DirectoryMetadata.newBuilder(); + bucketDir.setFriendlyName(container.getName()); + bucketDir.setResourcePath(container.getName()); + parentDir.addDirectories(bucketDir); + }); + resourceBuilder.setDirectory(parentDir); + + } else { + ObjectApi objectApi = swiftApi.getObjectApi(swiftStorage.getRegion(), swiftStorage.getContainer()); + if ("".equals(resourcePath)) { + + DirectoryMetadata.Builder rootDirBuilder = DirectoryMetadata.newBuilder(); + + ObjectList objectList = objectApi.list(); + objectList.forEach(swiftObject -> { + FileMetadata.Builder fileBuilder = FileMetadata.newBuilder(); + fileBuilder.setFriendlyName(swiftObject.getName()); + fileBuilder.setResourcePath(swiftObject.getName()); + fileBuilder.setCreatedTime(swiftObject.getLastModified().getTime()); + fileBuilder.setUpdateTime(swiftObject.getLastModified().getTime()); + fileBuilder.setResourceSize(swiftObject.getPayload().getContentMetadata().getContentLength()); + rootDirBuilder.addFiles(fileBuilder); + }); + resourceBuilder.setDirectory(rootDirBuilder); + } else { + SwiftObject swiftObject = objectApi.get(resourcePath); + + if (swiftObject == null) { + resourceBuilder.setError(MetadataFetchError.NOT_FOUND); + return resourceBuilder.build(); + } + + FileMetadata.Builder fileBuilder = FileMetadata.newBuilder(); + fileBuilder.setFriendlyName(swiftObject.getName()); + fileBuilder.setResourcePath(swiftObject.getName()); + fileBuilder.setCreatedTime(swiftObject.getLastModified().getTime()); + fileBuilder.setUpdateTime(swiftObject.getLastModified().getTime()); + fileBuilder.setResourceSize(swiftObject.getPayload().getContentMetadata().getContentLength()); + resourceBuilder.setFile(fileBuilder); + } } - - FileMetadata.Builder fileBuilder = FileMetadata.newBuilder(); - fileBuilder.setFriendlyName(swiftObject.getName()); - fileBuilder.setResourcePath(swiftObject.getName()); - fileBuilder.setCreatedTime(swiftObject.getLastModified().getTime()); - fileBuilder.setUpdateTime(swiftObject.getLastModified().getTime()); - fileBuilder.setResourceSize(swiftObject.getPayload().getContentMetadata().getContentLength()); - resourceBuilder.setFile(fileBuilder); + return resourceBuilder.build(); + } finally{ + swiftApi.close(); } - return resourceBuilder.build(); } @Override public Boolean isAvailable(String resourcePath) throws Exception { checkInitialized(); - SwiftApi swiftApi = getSwiftApi(swiftStorage, swiftSecret); + SwiftApi swiftApi = SwiftUtil.createSwiftApi(swiftSecret, swiftStorage); - ObjectApi objectApi = swiftApi.getObjectApi(swiftStorage.getRegion(), swiftStorage.getContainer()); + try { + ObjectApi objectApi = swiftApi.getObjectApi(swiftStorage.getRegion(), swiftStorage.getContainer()); - SwiftObject swiftObject = objectApi.get(resourcePath); + SwiftObject swiftObject = objectApi.get(resourcePath); - return swiftObject != null; + return swiftObject != null; + } finally { + swiftApi.close(); + } } } diff --git a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftOutgoingConnector.java b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftOutgoingConnector.java index 310143b..31132d5 100644 --- a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftOutgoingConnector.java +++ b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftOutgoingConnector.java @@ -60,34 +60,7 @@ public class SwiftOutgoingConnector implements OutgoingChunkedConnector { this.resourcePath = cc.getResourcePath(); SwiftSecret swiftSecret = cc.getSecret().getSwift(); - - String provider = "openstack-swift"; - - Properties overrides = new Properties(); - overrides.put(KeystoneProperties.KEYSTONE_VERSION, swiftStorage.getKeystoneVersion() + ""); - - String identity = null; - String credential = null; - switch (swiftSecret.getSecretCase()) { - case PASSWORDSECRET: - identity = swiftSecret.getPasswordSecret().getDomainId() + ":" + swiftSecret.getPasswordSecret().getUserName(); - credential = swiftSecret.getPasswordSecret().getPassword(); - overrides.put(KeystoneProperties.SCOPE, "projectId:" + swiftSecret.getPasswordSecret().getProjectId()); - overrides.put(KeystoneProperties.CREDENTIAL_TYPE, CredentialTypes.PASSWORD_CREDENTIALS); - break; - case AUTHCREDENTIALSECRET: - identity = swiftSecret.getAuthCredentialSecret().getCredentialId(); - credential = swiftSecret.getAuthCredentialSecret().getCredentialSecret(); - overrides.put(KeystoneProperties.CREDENTIAL_TYPE, CredentialTypes.API_ACCESS_KEY_CREDENTIALS); - break; - } - - swiftApi = ContextBuilder.newBuilder(provider) - .endpoint(swiftStorage.getEndpoint()) - .credentials(identity, credential) - .overrides(overrides) - .buildApi(SwiftApi.class); - + swiftApi = SwiftUtil.createSwiftApi(swiftSecret, swiftStorage); objectApi = swiftApi.getObjectApi(swiftStorage.getRegion(), swiftStorage.getContainer()); staticLargeObjectApi = swiftApi.getStaticLargeObjectApi(swiftStorage.getRegion(), swiftStorage.getContainer()); } diff --git a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftUtil.java b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftUtil.java new file mode 100644 index 0000000..4145854 --- /dev/null +++ b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftUtil.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.transport.swift; + +import java.util.Properties; +import org.apache.airavata.mft.credential.stubs.swift.SwiftSecret; +import org.apache.airavata.mft.credential.stubs.swift.SwiftV2AuthSecret; +import org.apache.airavata.mft.credential.stubs.swift.SwiftV3AuthSecret; +import org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorage; +import org.jclouds.ContextBuilder; +import org.jclouds.openstack.keystone.config.KeystoneProperties; +import org.jclouds.openstack.swift.v1.SwiftApi; + +// https://jclouds.apache.org/guides/openstack/ +public class SwiftUtil { + public static SwiftApi createSwiftApi(SwiftSecret swiftSecret, SwiftStorage swiftStorage) throws Exception { + String provider = "openstack-swift"; + Properties overrides = new Properties(); + switch (swiftSecret.getSecretCase()) { + case V2AUTHSECRET: + SwiftV2AuthSecret v2AuthSecret = swiftSecret.getV2AuthSecret(); + overrides.put(KeystoneProperties.KEYSTONE_VERSION, "2"); + return ContextBuilder.newBuilder(provider) + .endpoint(swiftSecret.getEndpoint()) + .credentials(v2AuthSecret.getTenant() + ":" + v2AuthSecret.getUserName(), + v2AuthSecret.getPassword()) + .overrides(overrides) + .buildApi(SwiftApi.class); + + case V3AUTHSECRET: + + SwiftV3AuthSecret v3AuthSecret = swiftSecret.getV3AuthSecret(); + overrides.put(KeystoneProperties.KEYSTONE_VERSION, "3"); + if (!"".equals(v3AuthSecret.getTenantName())) { + overrides.put(KeystoneProperties.SCOPE, "project:" + v3AuthSecret.getTenantName()); + } + + if (!"".equals(v3AuthSecret.getProjectDomainName())) { + overrides.put(KeystoneProperties.PROJECT_DOMAIN_NAME, v3AuthSecret.getProjectDomainName()); + } + + return ContextBuilder.newBuilder(provider) + .endpoint(swiftSecret.getEndpoint()) + .credentials(v3AuthSecret.getUserDomainName() + ":" + v3AuthSecret.getUserName(), + v3AuthSecret.getPassword()) + .overrides(overrides) + .buildApi(SwiftApi.class); + + default: + throw new Exception("No v2 or v3 auth secret set"); + } + } +}
