DImuthuUpe commented on code in PR #68: URL: https://github.com/apache/airavata-mft/pull/68#discussion_r1064174242
########## python-cli/mft_cli/mft_cli/storage/gcs.py: ########## @@ -0,0 +1,243 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from rich import print +from pick import pick +import typer +from airavata_mft_sdk import mft_client +from airavata_mft_sdk.gcs import GCSCredential_pb2 +from airavata_mft_sdk.gcs import GCSStorage_pb2 +from airavata_mft_sdk import MFTTransferApi_pb2 +from airavata_mft_sdk import MFTAgentStubs_pb2 +from airavata_mft_sdk.common import StorageCommon_pb2 +import json +import configparser +import os +import base64 +import google.auth +import googleapiclient.discovery + +gcs_key_path = '.mft/keys/gcs/service_account_key.json' + + +def handle_add_storage(): + session_token = "" + gcs_regions = ["us-central", "us-east1", "us-east4", "us-east5", "us-south1", Review Comment: This is not required ########## transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSMetadataCollector.java: ########## @@ -19,131 +19,224 @@ import com.google.api.gax.paging.Page; import com.google.cloud.storage.Blob; +import com.google.cloud.storage.Bucket; import com.google.cloud.storage.Storage; import com.google.auth.oauth2.ServiceAccountCredentials; import com.google.cloud.storage.StorageOptions; + import org.apache.airavata.mft.agent.stub.*; import org.apache.airavata.mft.core.api.MetadataCollector; import org.apache.airavata.mft.credential.stubs.gcs.GCSSecret; import org.apache.airavata.mft.resource.stubs.gcs.storage.GCSStorage; +import java.io.File; import java.security.PrivateKey; import java.time.temporal.ChronoField; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + -public class GCSMetadataCollector implements MetadataCollector { +public class GCSMetadataCollector implements MetadataCollector +{ boolean initialized = false; private GCSStorage gcsStorage; private GCSSecret gcsSecret; @Override - public void init(StorageWrapper storage, SecretWrapper secret) { + public void init( StorageWrapper storage, SecretWrapper secret ) + { this.gcsStorage = storage.getGcs(); this.gcsSecret = secret.getGcs(); this.initialized = true; } - private void checkInitialized() { - if (!initialized) { - throw new IllegalStateException("GCS Metadata Collector is not initialized"); + private void checkInitialized() Review Comment: private void checkInitialized() { ########## python-cli/mft_cli/mft_cli/storage/gcs.py: ########## @@ -0,0 +1,243 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from rich import print +from pick import pick +import typer +from airavata_mft_sdk import mft_client +from airavata_mft_sdk.gcs import GCSCredential_pb2 +from airavata_mft_sdk.gcs import GCSStorage_pb2 +from airavata_mft_sdk import MFTTransferApi_pb2 +from airavata_mft_sdk import MFTAgentStubs_pb2 +from airavata_mft_sdk.common import StorageCommon_pb2 +import json +import configparser +import os +import base64 +import google.auth +import googleapiclient.discovery + +gcs_key_path = '.mft/keys/gcs/service_account_key.json' + + +def handle_add_storage(): + session_token = "" + gcs_regions = ["us-central", "us-east1", "us-east4", "us-east5", "us-south1", + "us-west1", "us-west2", "us-west3", "us-west4", "northamerica-northeast1", "northamerica-northeast2", + "southamerica-east1", "southamerica-west1", + "europe-central2", "europe-north1", "europe-southwest1", "europe-west1", "europe-west2", + "europe-west3", "europe-west4", "europe-west6", "europe-west8", "europe-west9", + "asia-east1", "asia-east2", "asia-northeast1", "asia-northeast2", "asia-northeast3", + "asia-southeast1", "asia-south1", "asia-south2", "asia-southeast2", + "me-west1", "australia-southeast1", "australia-southeast2" + ] + + options = ["Through Google Cloud SDK config file", "Enter manually"] + option, index = pick(options, "How do you want to load credentials", indicator="=>") + + if index == 1: # Manual configuration + print("MFT uses service accounts to gain access to Google Cloud. You can creat a service account by going to " Review Comment: Try if you can make this print prettier. We use rich library to print so there might be very nice formatting improvements. https://www.freecodecamp.org/news/use-the-rich-library-in-python/ ########## python-cli/mft_cli/mft_cli/storage/gcs.py: ########## @@ -0,0 +1,243 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from rich import print +from pick import pick +import typer +from airavata_mft_sdk import mft_client +from airavata_mft_sdk.gcs import GCSCredential_pb2 +from airavata_mft_sdk.gcs import GCSStorage_pb2 +from airavata_mft_sdk import MFTTransferApi_pb2 +from airavata_mft_sdk import MFTAgentStubs_pb2 +from airavata_mft_sdk.common import StorageCommon_pb2 +import json +import configparser +import os +import base64 +import google.auth +import googleapiclient.discovery + +gcs_key_path = '.mft/keys/gcs/service_account_key.json' + + +def handle_add_storage(): + session_token = "" + gcs_regions = ["us-central", "us-east1", "us-east4", "us-east5", "us-south1", + "us-west1", "us-west2", "us-west3", "us-west4", "northamerica-northeast1", "northamerica-northeast2", + "southamerica-east1", "southamerica-west1", + "europe-central2", "europe-north1", "europe-southwest1", "europe-west1", "europe-west2", + "europe-west3", "europe-west4", "europe-west6", "europe-west8", "europe-west9", + "asia-east1", "asia-east2", "asia-northeast1", "asia-northeast2", "asia-northeast3", + "asia-southeast1", "asia-south1", "asia-south2", "asia-southeast2", + "me-west1", "australia-southeast1", "australia-southeast2" + ] + + options = ["Through Google Cloud SDK config file", "Enter manually"] + option, index = pick(options, "How do you want to load credentials", indicator="=>") + + if index == 1: # Manual configuration + print("MFT uses service accounts to gain access to Google Cloud. You can creat a service account by going to " + "https://console.cloud.google.com/iam-admin/serviceaccounts. Download the JSON format of the service account key." + " More information about service accounts can be found here https://cloud.google.com/iam/docs/service-accounts";) + + credential_json_path = typer.prompt("Service Account Credential JSON path") + with open(credential_json_path) as json_file: + sa_entries = json.load(json_file) + client_id = sa_entries['client_id'] + client_secret = sa_entries['private_key'] + project_id = sa_entries['project_id'] + + + else: # Loading credentials from the gcloud cli config file + service_account = None + client_email = None + default_service_account_name = 'mft-gcs-serviceaccount' + + # Find the active config for Google cloud ADC + active_config_path = os.path.join(os.path.expanduser('~'), '.config/gcloud/active_config') + with open(active_config_path) as f: + active_config = f.readline() + print('Active config : ' + active_config) Review Comment: Is this a debug print? ########## python-cli/mft_cli/mft_cli/storage/gcs.py: ########## @@ -0,0 +1,243 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from rich import print +from pick import pick +import typer +from airavata_mft_sdk import mft_client +from airavata_mft_sdk.gcs import GCSCredential_pb2 +from airavata_mft_sdk.gcs import GCSStorage_pb2 +from airavata_mft_sdk import MFTTransferApi_pb2 +from airavata_mft_sdk import MFTAgentStubs_pb2 +from airavata_mft_sdk.common import StorageCommon_pb2 +import json +import configparser +import os +import base64 +import google.auth +import googleapiclient.discovery + +gcs_key_path = '.mft/keys/gcs/service_account_key.json' + + +def handle_add_storage(): + session_token = "" + gcs_regions = ["us-central", "us-east1", "us-east4", "us-east5", "us-south1", Review Comment: @Jayancv ########## transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSMetadataCollector.java: ########## @@ -19,131 +19,224 @@ import com.google.api.gax.paging.Page; import com.google.cloud.storage.Blob; +import com.google.cloud.storage.Bucket; import com.google.cloud.storage.Storage; import com.google.auth.oauth2.ServiceAccountCredentials; import com.google.cloud.storage.StorageOptions; + import org.apache.airavata.mft.agent.stub.*; import org.apache.airavata.mft.core.api.MetadataCollector; import org.apache.airavata.mft.credential.stubs.gcs.GCSSecret; import org.apache.airavata.mft.resource.stubs.gcs.storage.GCSStorage; +import java.io.File; import java.security.PrivateKey; import java.time.temporal.ChronoField; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + -public class GCSMetadataCollector implements MetadataCollector { +public class GCSMetadataCollector implements MetadataCollector +{ boolean initialized = false; private GCSStorage gcsStorage; private GCSSecret gcsSecret; @Override - public void init(StorageWrapper storage, SecretWrapper secret) { Review Comment: public void init(StorageWrapper storage, SecretWrapper secret) { Take the { to the line where the function is defined. Keep a space between the end of the line and { -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
