This is an automated email from the ASF dual-hosted git repository. lahirujayathilake pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata.git
commit 86425b63dde80595eff7dfe4b37248b67b6f585f Author: yasith <[email protected]> AuthorDate: Sat Feb 22 20:13:45 2025 -0600 cleanup imports, fix bugs found by type checker --- .../airavata_experiments/airavata.py | 5 +- .../airavata_sdk/clients/api_server_client.py | 28 ++++---- .../clients/credential_store_client.py | 26 +++---- .../airavata_sdk/clients/file_handling_client.py | 9 +-- .../airavata_sdk/clients/group_manager_client.py | 23 +++--- .../airavata_sdk/clients/iam_admin_client.py | 25 ++++--- .../airavata_sdk/clients/keycloak_token_fetcher.py | 83 +++++++++++----------- .../clients/sftp_file_handling_client.py | 10 ++- .../clients/sharing_registry_client.py | 23 +++--- .../airavata_sdk/clients/tenant_profile_client.py | 23 +++--- .../airavata_sdk/clients/user_profile_client.py | 22 +++--- .../clients/utils/api_server_client_util.py | 34 +++++---- .../clients/utils/data_model_creation_util.py | 75 +++++++++++++------ .../clients/utils/experiment_handler_util.py | 70 +++++++++++------- .../samples/api_server_client_samples.py | 17 ++--- .../samples/create_launch_echo_experiment.py | 43 +++++------ .../samples/create_launch_gaussian_experiment.py | 50 ++++++------- .../airavata_sdk/samples/file_utils.py | 1 + .../samples/group_manager_client_samples.py | 8 +-- .../samples/iam_admin_client_samples.py | 5 +- .../airavata_sdk/samples/metadata_fetcher.py | 16 +---- .../samples/sharing_registry_client_samples.py | 6 +- .../samples/tenant_profile_client_samples.py | 5 +- .../samples/user_profile_client_samples.py | 8 +-- .../airavata_sdk/transport/settings.py | 17 ++--- 25 files changed, 320 insertions(+), 312 deletions(-) diff --git a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_experiments/airavata.py b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_experiments/airavata.py index a45dd4fb65..6f9d7ec3de 100644 --- a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_experiments/airavata.py +++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_experiments/airavata.py @@ -17,6 +17,8 @@ import logging from pathlib import Path from typing import Literal, NamedTuple + +from airavata_sdk.transport.settings import APIServerSettings from .sftp import SFTPConnector import time import warnings @@ -161,7 +163,8 @@ class AiravataOperator: self.access_token = access_token self.settings = Settings(config_file) # load api server settings and create client - self.api_server_client = APIServerClient(api_server_settings=self.settings) + api_server_settings = APIServerSettings(config_file) + self.api_server_client = APIServerClient(api_server_settings=api_server_settings) # load gateway settings gateway_id = self.default_gateway_id() self.airavata_token = self.__airavata_token__(self.access_token, gateway_id) diff --git a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/api_server_client.py b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/api_server_client.py index 7ec09f7186..a54caff4a4 100644 --- a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/api_server_client.py +++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/api_server_client.py @@ -16,8 +16,8 @@ import configparser import logging +from typing import Optional -from airavata.api.Airavata import Client from airavata_sdk.transport import utils from airavata_sdk.transport.settings import APIServerSettings @@ -27,16 +27,16 @@ logger.setLevel(logging.DEBUG) class APIServerClient(object): - def __init__(self, configuration_file_location=None, api_server_settings=None): - if configuration_file_location is not None: - self.api_server_settings = APIServerSettings(configuration_file_location) + def __init__(self, configuration_file_location: Optional[str] = None, api_server_settings: Optional[APIServerSettings] = None): + if api_server_settings is not None: + self.settings = api_server_settings + elif configuration_file_location is not None: + self.settings = APIServerSettings(configuration_file_location) self._load_settings(configuration_file_location) - elif api_server_settings is not None: - self.api_server_settings = api_server_settings - self.client: Client = utils.initialize_api_client_pool( - self.api_server_settings.API_SERVER_HOST, - self.api_server_settings.API_SERVER_PORT, - self.api_server_settings.API_SERVER_SECURE, + self.client = utils.initialize_api_client_pool( + self.settings.API_SERVER_HOST, + self.settings.API_SERVER_PORT, + self.settings.API_SERVER_SECURE, ) # expose the needed functions self.is_user_exists = self.client.isUserExists @@ -231,10 +231,10 @@ class APIServerClient(object): self.remove_parsing_template = self.client.removeParsingTemplate self.list_all_parsing_templates = self.client.listAllParsingTemplates - def _load_settings(self, configuration_file_location): + def _load_settings(self, configuration_file_location: Optional[str]): if configuration_file_location is not None: config = configparser.ConfigParser() config.read(configuration_file_location) - self.api_server_settings.API_SERVER_HOST = config.get('APIServer', 'API_HOST') - self.api_server_settings.API_SERVER_PORT = config.getint('APIServer', 'API_PORT') - self.api_server_settings.API_SERVER_SECURE = config.getboolean('APIServer', 'API_SECURE') + self.settings.API_SERVER_HOST = config.get('APIServer', 'API_HOST') + self.settings.API_SERVER_PORT = config.getint('APIServer', 'API_PORT') + self.settings.API_SERVER_SECURE = config.getboolean('APIServer', 'API_SECURE') diff --git a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/credential_store_client.py b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/credential_store_client.py index 909dccde91..3b9ddaeb7c 100644 --- a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/credential_store_client.py +++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/credential_store_client.py @@ -16,8 +16,8 @@ import configparser import logging +from typing import Optional -from airavata.api.credential.store.CredentialStoreService import Client from airavata_sdk.transport import utils from airavata_sdk.transport.settings import CredentialStoreServerSettings @@ -27,25 +27,21 @@ logger.setLevel(logging.DEBUG) class CredentialStoreClient(object): - def __init__(self, configuration_file_location=None): - self.credential_store_server_settings = CredentialStoreServerSettings(configuration_file_location) + def __init__(self, configuration_file_location: Optional[str] = None): + self.settings = CredentialStoreServerSettings(configuration_file_location) self._load_settings(configuration_file_location) - self.client: Client = utils.initialize_credential_store_client( - self.credential_store_server_settings.CREDENTIAL_STORE_API_HOST, - self.credential_store_server_settings.CREDENTIAL_STORE_API_PORT, - self.credential_store_server_settings.CREDENTIAL_STORE_API_SECURE + self.client = utils.initialize_credential_store_client( + self.settings.CREDENTIAL_STORE_API_HOST, + self.settings.CREDENTIAL_STORE_API_PORT, + self.settings.CREDENTIAL_STORE_API_SECURE, ) # expose the needed functions self.get_SSH_credential = self.client.getSSHCredential - def _load_settings(self, configuration_file_location): + def _load_settings(self, configuration_file_location: Optional[str]): if configuration_file_location is not None: config = configparser.ConfigParser() config.read(configuration_file_location) - self.credential_store_server_settings.CREDENTIAL_STORE_API_HOST = config.get('CredentialStoreServer', - 'CREDENTIAL_STORE_API_HOST') - self.credential_store_server_settings.CREDENTIAL_STORE_API_PORT = config.getint('CredentialStoreServer', - 'CREDENTIAL_STORE_API_PORT') - self.credential_store_server_settings.CREDENTIAL_STORE_API_SECURE = config.getboolean( - 'CredentialStoreServer', - 'CREDENTIAL_STORE_API_SECURE') + self.settings.CREDENTIAL_STORE_API_HOST = config.get('CredentialStoreServer', 'CREDENTIAL_STORE_API_HOST') + self.settings.CREDENTIAL_STORE_API_PORT = config.getint('CredentialStoreServer', 'CREDENTIAL_STORE_API_PORT') + self.settings.CREDENTIAL_STORE_API_SECURE = config.getboolean('CredentialStoreServer', 'CREDENTIAL_STORE_API_SECURE') diff --git a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/file_handling_client.py b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/file_handling_client.py index 94d870c7c7..df8a0a924a 100644 --- a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/file_handling_client.py +++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/file_handling_client.py @@ -15,6 +15,7 @@ # import logging +from typing import Optional import paramiko from paramiko import SSHClient @@ -38,16 +39,16 @@ logger.addHandler(handler) class FileHandler(object): - def __init__(self, host, port, username, passphrase, privateKeyFilePath): + def __init__(self, host: str, port: int, username: str, passphrase: Optional[str], pkey: Optional[paramiko.RSAKey]): self.host = host self.port = port self.username = username self.password = passphrase - self.filePath = privateKeyFilePath + self.pkey = None def upload_file(self, files, remote_path, recursive, preserve_item): try: - ssh.connect(self.host, self.port, self.username, passphrase=self.password, pkey=self.filePath) + ssh.connect(self.host, self.port, self.username, passphrase=self.password, pkey=self.pkey) transport = ssh.get_transport() assert transport is not None with SCPClient(transport) as scp: @@ -57,7 +58,7 @@ class FileHandler(object): def download_file(self, remote_path, local_path, recursive, preserve_item): try: - ssh.connect(self.host, self.port, self.username, passphrase=self.password, pkey=self.filePath) + ssh.connect(self.host, self.port, self.username, passphrase=self.password, pkey=self.pkey) transport = ssh.get_transport() assert transport is not None with SCPClient(transport) as scp: diff --git a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/group_manager_client.py b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/group_manager_client.py index 7bbee1d111..dba238bf4a 100644 --- a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/group_manager_client.py +++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/group_manager_client.py @@ -16,8 +16,8 @@ import configparser import logging +from typing import Optional -from airavata.service.profile.groupmanager.cpi.GroupManagerService import Client from airavata_sdk.transport import utils from airavata_sdk.transport.settings import ProfileServerSettings @@ -35,13 +35,13 @@ logger.addHandler(handler) class GroupManagerClient(object): - def __init__(self, configuration_file_location=None): - self.group_manager_settings = ProfileServerSettings(configuration_file_location) + def __init__(self, configuration_file_location: Optional[str] = None): + self.settings = ProfileServerSettings(configuration_file_location) self._load_settings(configuration_file_location) - self.client: Client = utils.initialize_group_manager_client( - self.group_manager_settings.PROFILE_SERVICE_HOST, - self.group_manager_settings.PROFILE_SERVICE_PORT, - self.group_manager_settings.PROFILE_SERVICE_SECURE + self.client = utils.initialize_group_manager_client( + self.settings.PROFILE_SERVICE_HOST, + self.settings.PROFILE_SERVICE_PORT, + self.settings.PROFILE_SERVICE_SECURE, ) # expose the needed functions self.get_api_version = self.client.getAPIVersion @@ -59,11 +59,10 @@ class GroupManagerClient(object): self.has_admin_access = self.client.hasAdminAccess self.has_owner_access = self.client.hasOwnerAccess - def _load_settings(self, configuration_file_location): + def _load_settings(self, configuration_file_location: Optional[str]): if configuration_file_location is not None: config = configparser.ConfigParser() config.read(configuration_file_location) - self.group_manager_settings.PROFILE_SERVICE_HOST = config.get('ProfileServer', 'PROFILE_SERVICE_HOST') - self.group_manager_settings.PROFILE_SERVICE_PORT = config.getint('ProfileServer', 'PROFILE_SERVICE_PORT') - self.group_manager_settings.PROFILE_SERVICE_SECURE = config.getboolean('ProfileServer', - 'PROFILE_SERVICE_SECURE') + self.settings.PROFILE_SERVICE_HOST = config.get('ProfileServer', 'PROFILE_SERVICE_HOST') + self.settings.PROFILE_SERVICE_PORT = config.getint('ProfileServer', 'PROFILE_SERVICE_PORT') + self.settings.PROFILE_SERVICE_SECURE = config.getboolean('ProfileServer', 'PROFILE_SERVICE_SECURE') diff --git a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/iam_admin_client.py b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/iam_admin_client.py index d1d4594d17..aeab7c01a1 100644 --- a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/iam_admin_client.py +++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/iam_admin_client.py @@ -16,8 +16,8 @@ import configparser import logging +from typing import Optional -from airavata.service.profile.iam.admin.services.cpi.IamAdminServices import Client from airavata_sdk.transport import utils from airavata_sdk.transport.settings import ProfileServerSettings @@ -35,13 +35,14 @@ logger.addHandler(handler) class IAMAdminClient(object): - def __init__(self, configuration_file_location=None): - self.iam_admin_settings = ProfileServerSettings(configuration_file_location) + def __init__(self, configuration_file_location: Optional[str] = None): + self.settings = ProfileServerSettings(configuration_file_location) self._load_settings(configuration_file_location) - self.client: Client = utils.initialize_iam_admin_client( - self.iam_admin_settings.PROFILE_SERVICE_HOST, - self.iam_admin_settings.PROFILE_SERVICE_PORT, - self.iam_admin_settings.PROFILE_SERVICE_SECURE) + self.client = utils.initialize_iam_admin_client( + self.settings.PROFILE_SERVICE_HOST, + self.settings.PROFILE_SERVICE_PORT, + self.settings.PROFILE_SERVICE_SECURE, + ) # expose the needed functions self.set_up_gateway = self.client.setUpGateway self.is_username_available = self.client.isUsernameAvailable @@ -59,12 +60,10 @@ class IAMAdminClient(object): self.remove_role_from_user = self.client.removeRoleFromUser self.get_users_with_role = self.client.getUsersWithRole - def _load_settings(self, configuration_file_location): + def _load_settings(self, configuration_file_location: Optional[str]): if configuration_file_location is not None: config = configparser.ConfigParser() config.read(configuration_file_location) - self.iam_admin_settings.PROFILE_SERVICE_HOST = config.get('ProfileServer', 'PROFILE_SERVICE_HOST') - self.iam_admin_settings.PROFILE_SERVICE_PORT = config.getint('ProfileServer', 'PROFILE_SERVICE_PORT') - self.iam_admin_settings.PROFILE_SERVICE_SECURE = config.getboolean('ProfileServer', - 'PROFILE_SERVICE_SECURE') - + self.settings.PROFILE_SERVICE_HOST = config.get('ProfileServer', 'PROFILE_SERVICE_HOST') + self.settings.PROFILE_SERVICE_PORT = config.getint('ProfileServer', 'PROFILE_SERVICE_PORT') + self.settings.PROFILE_SERVICE_SECURE = config.getboolean('ProfileServer', 'PROFILE_SERVICE_SECURE') diff --git a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/keycloak_token_fetcher.py b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/keycloak_token_fetcher.py index e1d5e11f4e..c403f75a09 100644 --- a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/keycloak_token_fetcher.py +++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/keycloak_token_fetcher.py @@ -16,6 +16,7 @@ import configparser import os +from typing import Optional from oauthlib.oauth2 import LegacyApplicationClient from requests_oauthlib import OAuth2Session @@ -27,53 +28,53 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) class Authenticator(object): - def __init__(self, configuration_file_location=None): - self.keycloak_settings = KeycloakServerSettings(configuration_file_location) + def __init__(self, configuration_file_location: Optional[str] = None): + self.settings = KeycloakServerSettings(configuration_file_location) self._load_settings(configuration_file_location) - def get_token_and_user_info_password_flow(self, username, password, gateway_id): - client_id = self.keycloak_settings.CLIENT_ID - client_secret = self.keycloak_settings.CLIENT_SECRET - token_url = self.keycloak_settings.TOKEN_URL - userinfo_url = self.keycloak_settings.USER_INFO_URL - verify_ssl = self.keycloak_settings.VERIFY_SSL - oauth2_session = OAuth2Session(client=LegacyApplicationClient( - client_id=client_id)) - token = oauth2_session.fetch_token(token_url=token_url, - username=username, - password=password, - client_id=client_id, - client_secret=client_secret, - verify=verify_ssl) - + def get_token_and_user_info_password_flow(self, username: str, password: str, gateway_id: str): + client_id = self.settings.CLIENT_ID + client_secret = self.settings.CLIENT_SECRET + token_url = self.settings.TOKEN_URL + # userinfo_url = self.keycloak_settings.USER_INFO_URL + verify_ssl = self.settings.VERIFY_SSL + oauth2_session = OAuth2Session(client=LegacyApplicationClient(client_id=client_id)) + token = oauth2_session.fetch_token( + token_url=token_url, + username=username, + password=password, + client_id=client_id, + client_secret=client_secret, + verify=verify_ssl, + ) claimsMap = { "userName": username, "gatewayID": gateway_id } return AuthzToken(accessToken=token['access_token'], claimsMap=claimsMap) - def get_airavata_authz_token(self, username, token, gateway_id): + def get_airavata_authz_token(self, username: str, token: str, gateway_id: str): claimsMap = { "userName": username, "gatewayID": gateway_id } return AuthzToken(accessToken=token, claimsMap=claimsMap) - def get_authorize_url(self, username, password, gateway_id): - client_id = self.keycloak_settings.CLIENT_ID - client_secret = self.keycloak_settings.CLIENT_SECRET - token_url = self.keycloak_settings.TOKEN_URL - userinfo_url = self.keycloak_settings.USER_INFO_URL - verify_ssl = self.keycloak_settings.VERIFY_SSL - oauth2_session = OAuth2Session(client=LegacyApplicationClient( - client_id=client_id)) - token = oauth2_session.fetch_token(token_url=token_url, - username=username, - password=password, - client_id=client_id, - client_secret=client_secret, - verify=verify_ssl) - + def get_authorize_url(self, username: str, password: str, gateway_id: str): + client_id = self.settings.CLIENT_ID + client_secret = self.settings.CLIENT_SECRET + token_url = self.settings.TOKEN_URL + # userinfo_url = self.keycloak_settings.USER_INFO_URL + verify_ssl = self.settings.VERIFY_SSL + oauth2_session = OAuth2Session(client=LegacyApplicationClient(client_id=client_id)) + token = oauth2_session.fetch_token( + token_url=token_url, + username=username, + password=password, + client_id=client_id, + client_secret=client_secret, + verify=verify_ssl, + ) claimsMap = { "userName": username, "gatewayID": gateway_id @@ -81,16 +82,16 @@ class Authenticator(object): return AuthzToken(accessToken=token['access_token'], claimsMap=claimsMap) def authenticate_with_auth_code(self): - print("Click on Login URI ", self.keycloak_settings.LOGIN_DESKTOP_URI) - return self.keycloak_settings.LOGIN_DESKTOP_URI + print("Click on Login URI ", self.settings.LOGIN_DESKTOP_URI) + return self.settings.LOGIN_DESKTOP_URI - def _load_settings(self, configuration_file_location): + def _load_settings(self, configuration_file_location: Optional[str]): if configuration_file_location is not None: config = configparser.ConfigParser() config.read(configuration_file_location) # self.keycloak_settings.KEYCLOAK_CA_CERTIFICATE = config.get("KeycloakServer", 'CERTIFICATE_FILE_PATH') - self.keycloak_settings.CLIENT_ID = config.get('KeycloakServer', 'CLIENT_ID') - self.keycloak_settings.CLIENT_SECRET = config.get('KeycloakServer', 'CLIENT_SECRET') - self.keycloak_settings.TOKEN_URL = config.get('KeycloakServer', 'TOKEN_URL') - self.keycloak_settings.USER_INFO_URL = config.get('KeycloakServer', 'USER_INFO_URL') - self.keycloak_settings.VERIFY_SSL = config.getboolean('KeycloakServer', 'VERIFY_SSL') + self.settings.CLIENT_ID = config.get('KeycloakServer', 'CLIENT_ID') + self.settings.CLIENT_SECRET = config.get('KeycloakServer', 'CLIENT_SECRET') + self.settings.TOKEN_URL = config.get('KeycloakServer', 'TOKEN_URL') + self.settings.USER_INFO_URL = config.get('KeycloakServer', 'USER_INFO_URL') + self.settings.VERIFY_SSL = config.getboolean('KeycloakServer', 'VERIFY_SSL') diff --git a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/sftp_file_handling_client.py b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/sftp_file_handling_client.py index c8c306fe63..dac23117d5 100644 --- a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/sftp_file_handling_client.py +++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/sftp_file_handling_client.py @@ -43,7 +43,7 @@ class SFTPConnector(object): ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - def upload_files(self, local_path, project_name, exprement_id): + def upload_files(self, local_path: str, project_name: str, exprement_id: str): project_name = project_name.replace(" ", "_") time = datetime.now().strftime('%Y-%m-%d %H:%M:%S').replace(" ", "_") time = time.replace(":", "_") @@ -72,8 +72,7 @@ class SFTPConnector(object): transport.close() return pathsuffix - def download_files(self, local_path, remote_path): - + def download_files(self, local_path: str, remote_path: str): self.ssh.connect(self.host, self.port, self.username, password = self.password) transport = self.ssh.get_transport() assert transport is not None @@ -82,6 +81,5 @@ class SFTPConnector(object): self.ssh.close() @staticmethod - def uploading_info(uploaded_file_size, total_file_size): - logging.info('uploaded_file_size : {} total_file_size : {}'. - format(uploaded_file_size, total_file_size)) \ No newline at end of file + def uploading_info(uploaded_file_size: str, total_file_size: str): + logging.info('uploaded_file_size : {} total_file_size : {}'.format(uploaded_file_size, total_file_size)) \ No newline at end of file diff --git a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/sharing_registry_client.py b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/sharing_registry_client.py index 120b323a1a..e098fefcbc 100644 --- a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/sharing_registry_client.py +++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/sharing_registry_client.py @@ -16,8 +16,8 @@ import configparser import logging +from typing import Optional -from airavata.api.sharing.SharingRegistryService import Client from airavata_sdk.transport import utils from airavata_sdk.transport.settings import SharingServerSettings @@ -35,13 +35,13 @@ logger.addHandler(handler) class SharingRegistryClient(object): - def __init__(self, configuration_file_location=None): - self.sharing_registry_client_settings = SharingServerSettings(configuration_file_location) + def __init__(self, configuration_file_location: Optional[str] = None): + self.settings = SharingServerSettings(configuration_file_location) self._load_settings(configuration_file_location) - self.client: Client = utils.initialize_sharing_registry_client( - self.sharing_registry_client_settings.SHARING_API_HOST, - self.sharing_registry_client_settings.SHARING_API_PORT, - self.sharing_registry_client_settings.SHARING_API_SECURE, + self.client = utils.initialize_sharing_registry_client( + self.settings.SHARING_API_HOST, + self.settings.SHARING_API_PORT, + self.settings.SHARING_API_SECURE, ) # expose the needed functions self.create_domain = self.client.createDomain @@ -103,11 +103,10 @@ class SharingRegistryClient(object): self.user_has_access = self.client.userHasAccess - def _load_settings(self, configuration_file_location): + def _load_settings(self, configuration_file_location: Optional[str]): if configuration_file_location is not None: config = configparser.ConfigParser() config.read(configuration_file_location) - self.sharing_registry_client_settings.SHARING_API_HOST = config.get('SharingServer', 'SHARING_API_HOST') - self.sharing_registry_client_settings.SHARING_API_PORT = config.getint('SharingServer', 'SHARING_API_PORT') - self.sharing_registry_client_settings.SHARING_API_SECURE = config.getboolean('SharingServer', - 'SHARING_API_SECURE') + self.settings.SHARING_API_HOST = config.get('SharingServer', 'SHARING_API_HOST') + self.settings.SHARING_API_PORT = config.getint('SharingServer', 'SHARING_API_PORT') + self.settings.SHARING_API_SECURE = config.getboolean('SharingServer', 'SHARING_API_SECURE') diff --git a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/tenant_profile_client.py b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/tenant_profile_client.py index e180492388..dd0f920403 100644 --- a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/tenant_profile_client.py +++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/tenant_profile_client.py @@ -16,8 +16,8 @@ import configparser import logging +from typing import Optional -from airavata.service.profile.tenant.cpi.TenantProfileService import Client from airavata_sdk.transport import utils from airavata_sdk.transport.settings import ProfileServerSettings @@ -35,13 +35,13 @@ logger.addHandler(handler) class TenantProfileClient(object): - def __init__(self, configuration_file_location=None): - self.tenant_profile_settings = ProfileServerSettings(configuration_file_location) + def __init__(self, configuration_file_location: Optional[str] = None): + self.settings = ProfileServerSettings(configuration_file_location) self._load_settings(configuration_file_location) - self.client: Client = utils.initialize_tenant_profile_client( - self.tenant_profile_settings.PROFILE_SERVICE_HOST, - self.tenant_profile_settings.PROFILE_SERVICE_PORT, - self.tenant_profile_settings.PROFILE_SERVICE_SECURE, + self.client = utils.initialize_tenant_profile_client( + self.settings.PROFILE_SERVICE_HOST, + self.settings.PROFILE_SERVICE_PORT, + self.settings.PROFILE_SERVICE_SECURE, ) # expose the needed functions self.add_gateway = self.client.addGateway @@ -53,11 +53,10 @@ class TenantProfileClient(object): self.get_all_gateways_for_user = self.client.getAllGatewaysForUser - def _load_settings(self, configuration_file_location): + def _load_settings(self, configuration_file_location: Optional[str]): if configuration_file_location is not None: config = configparser.ConfigParser() config.read(configuration_file_location) - settings = config['ProfileServer'] - self.tenant_profile_settings.PROFILE_SERVICE_HOST = config.get('ProfileServer', 'PROFILE_SERVICE_HOST') - self.tenant_profile_settings.PROFILE_SERVICE_PORT = config.getint('ProfileServer', 'PROFILE_SERVICE_PORT') - self.tenant_profile_settings.PROFILE_SERVICE_SECURE = config.getboolean('ProfileServer', 'PROFILE_SERVICE_SECURE') + self.settings.PROFILE_SERVICE_HOST = config.get('ProfileServer', 'PROFILE_SERVICE_HOST') + self.settings.PROFILE_SERVICE_PORT = config.getint('ProfileServer', 'PROFILE_SERVICE_PORT') + self.settings.PROFILE_SERVICE_SECURE = config.getboolean('ProfileServer', 'PROFILE_SERVICE_SECURE') diff --git a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/user_profile_client.py b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/user_profile_client.py index b90b369323..f42e91cac1 100644 --- a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/user_profile_client.py +++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/user_profile_client.py @@ -16,9 +16,8 @@ import configparser import logging +from typing import Optional -from airavata.service.profile.user.cpi.UserProfileService import Client -from airavata.api.error.ttypes import TException from airavata_sdk.transport import utils from airavata_sdk.transport.settings import ProfileServerSettings @@ -36,13 +35,14 @@ logger.addHandler(handler) class UserProfileClient(object): - def __init__(self, configuration_file_location=None): - self.user_profile_client_settings = ProfileServerSettings(configuration_file_location) + def __init__(self, configuration_file_location: Optional[str] = None): + self.settings = ProfileServerSettings(configuration_file_location) self._load_settings(configuration_file_location) self.client = utils.initialize_user_profile_client( - self.user_profile_client_settings.PROFILE_SERVICE_HOST, - self.user_profile_client_settings.PROFILE_SERVICE_PORT, - self.user_profile_client_settings.PROFILE_SERVICE_SECURE) + self.settings.PROFILE_SERVICE_HOST, + self.settings.PROFILE_SERVICE_PORT, + self.settings.PROFILE_SERVICE_SECURE, + ) # expose the needed functions self.get_api_version = self.client.getAPIVersion self.initialize_user_profile = self.client.initializeUserProfile @@ -54,10 +54,10 @@ class UserProfileClient(object): self.does_user_exist = self.client.doesUserExist - def _load_settings(self, configuration_file_location): + def _load_settings(self, configuration_file_location: Optional[str]): if configuration_file_location is not None: config = configparser.ConfigParser() config.read(configuration_file_location) - self.user_profile_client_settings.PROFILE_SERVICE_HOST = config.get('ProfileServer', 'PROFILE_SERVICE_HOST') - self.user_profile_client_settings.PROFILE_SERVICE_PORT = config.getint('ProfileServer', 'PROFILE_SERVICE_PORT') - self.user_profile_client_settings.PROFILE_SERVICE_SECURE = config.getboolean('ProfileServer', 'PROFILE_SERVICE_SECURE') + self.settings.PROFILE_SERVICE_HOST = config.get('ProfileServer', 'PROFILE_SERVICE_HOST') + self.settings.PROFILE_SERVICE_PORT = config.getint('ProfileServer', 'PROFILE_SERVICE_PORT') + self.settings.PROFILE_SERVICE_SECURE = config.getboolean('ProfileServer', 'PROFILE_SERVICE_SECURE') diff --git a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/utils/api_server_client_util.py b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/utils/api_server_client_util.py index f1cb87cc03..957e3f3a84 100644 --- a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/utils/api_server_client_util.py +++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/utils/api_server_client_util.py @@ -26,59 +26,65 @@ logger.setLevel(logging.INFO) class APIServerClientUtil(object): - def __init__(self, configuration_file_location, username, password, gateway_id, access_token): + def __init__(self, configuration_file_location: Optional[str], gateway_id: str, username: str, password: Optional[str], access_token: Optional[str] = None): self.authenticator = Authenticator(configuration_file_location) if access_token: - self.token = self.authenticator.get_airavata_authz_token(username=username, - token=access_token, - gateway_id=gateway_id) + self.token = self.authenticator.get_airavata_authz_token( + gateway_id=gateway_id, + username=username, + token=access_token, + ) else: - self.token = self.authenticator.get_token_and_user_info_password_flow(username=username, - password=password, - gateway_id=gateway_id) + assert password is not None + self.token = self.authenticator.get_token_and_user_info_password_flow( + gateway_id=gateway_id, + username=username, + password=password, + ) self.gateway_id = gateway_id self.username = username self.api_server_client = APIServerClient(configuration_file_location) - def get_project_id(self, project_name) -> Optional[str]: + def get_project_id(self, project_name: str) -> Optional[str]: response = self.api_server_client.get_user_projects(self.token, self.gateway_id, self.username, 10, 0) for project in response: if project.name == project_name and project.owner == self.username: return project.projectID return None - def get_execution_id(self, application_name): + def get_execution_id(self, application_name: str): response = self.api_server_client.get_all_application_interfaces(self.token, self.gateway_id) for app in response: if app.applicationName == application_name: return app.applicationInterfaceId return None - def get_resource_host_id(self, resource_name): + def get_resource_host_id(self, resource_name: str): response = self.api_server_client.get_all_compute_resource_names(self.token) for k in response.keys(): if response[k] == resource_name: return k return None - def get_group_resource_profile_id(self, group_resource_profile_name): + def get_group_resource_profile_id(self, group_resource_profile_name: str): response = self.api_server_client.get_group_resource_list(self.token, self.gateway_id) for x in response: if x.groupResourceProfileName == group_resource_profile_name: return x.groupResourceProfileId return None - def get_storage_resource_id(self, storage_name): + def get_storage_resource_id(self, storage_name: str): response = self.api_server_client.get_all_storage_resource_names(self.token) for k in response.keys(): if response[k] == storage_name: return k return None - def get_queue_names(self, resource_host_id): + def get_queue_names(self, resource_host_id: str): resource = self.api_server_client.get_compute_resource(self.token, resource_host_id) batchqueues = resource.batchQueues - allowed_queue_names = [] + assert batchqueues is not None + allowed_queue_names = list[str]() for queue in batchqueues: allowed_queue_names.append(queue.queueName) return allowed_queue_names diff --git a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/utils/data_model_creation_util.py b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/utils/data_model_creation_util.py index 9f4de1c240..14bfbad073 100644 --- a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/utils/data_model_creation_util.py +++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/utils/data_model_creation_util.py @@ -15,6 +15,7 @@ # import logging +from typing import Optional from airavata.model.application.io.ttypes import InputDataObjectType from airavata.model.data.replica.ttypes import DataProductModel, DataProductType, DataReplicaLocationModel, ReplicaLocationCategory @@ -30,25 +31,34 @@ logger.setLevel(logging.DEBUG) class DataModelCreationUtil(object): - def __init__(self, configuration_file_location, username, password, gateway_id, access_token): + def __init__(self, configuration_file_location: Optional[str], gateway_id: str, username: str, password: Optional[str], access_token: Optional[str] = None): self.authenticator = Authenticator(configuration_file_location) if access_token: - self.token = self.authenticator.get_airavata_authz_token(username=username, - token=access_token, - gateway_id=gateway_id) + self.token = self.authenticator.get_airavata_authz_token( + gateway_id=gateway_id, + username=username, + token=access_token, + ) else: - self.token = self.authenticator.get_token_and_user_info_password_flow(username=username, - password=password, - gateway_id=gateway_id) + assert password is not None + self.token = self.authenticator.get_token_and_user_info_password_flow( + gateway_id=gateway_id, + username=username, + password=password, + ) self.gateway_id = gateway_id self.username = username self.password = password self.api_server_client = APIServerClient(configuration_file_location) - self.airavata_util = APIServerClientUtil(configuration_file_location, self.username, self.password, - self.gateway_id, access_token) - - def get_experiment_data_model_for_single_application(self, project_name, application_name, experiment_name, - description): + self.airavata_util = APIServerClientUtil( + configuration_file_location, + self.gateway_id, + self.username, + self.password, + access_token, + ) + + def get_experiment_data_model_for_single_application(self, project_name: str, application_name: str, experiment_name: str, description: str): execution_id = self.airavata_util.get_execution_id(application_name) project_id = self.airavata_util.get_project_id(project_name) assert project_id is not None @@ -62,12 +72,19 @@ class DataModelCreationUtil(object): experiment.executionId = execution_id return experiment - def configure_computation_resource_scheduling(self, - experiment_model, computation_resource_name, - group_resource_profile_name, - storageId, - node_count, total_cpu_count, queue_name, wall_time_limit, - experiment_dir_path, auto_schedule=False): + def configure_computation_resource_scheduling( + self, + experiment_model: ExperimentModel, + computation_resource_name: str, + group_resource_profile_name: str, + storageId: str, + node_count: int, + total_cpu_count: int, + queue_name: str, + wall_time_limit: int, + experiment_dir_path: str, + auto_schedule: bool = False, + ): resource_host_id = self.airavata_util.get_resource_host_id(computation_resource_name) groupResourceProfileId = self.airavata_util.get_group_resource_profile_id(group_resource_profile_name) computRes = ComputationalResourceSchedulingModel() @@ -89,7 +106,14 @@ class DataModelCreationUtil(object): return experiment_model - def register_input_file(self, file_identifier, storage_name, storageId, input_file_name, uploaded_storage_path): + def register_input_file( + self, + file_identifier: str, + storage_name: str, + storageId: str, + input_file_name: str, + uploaded_storage_path: str, + ): dataProductModel = DataProductModel() dataProductModel.gatewayId = self.gateway_id dataProductModel.ownerName = self.username @@ -100,15 +124,20 @@ class DataModelCreationUtil(object): replicaLocation.storageResourceId = storageId replicaLocation.replicaName = "{} gateway data store copy".format(input_file_name) replicaLocation.replicaLocationCategory = ReplicaLocationCategory.GATEWAY_DATA_STORE - replicaLocation.filePath = "file://{}:{}".format(storage_name, - uploaded_storage_path + input_file_name) + replicaLocation.filePath = "file://{}:{}".format(storage_name, uploaded_storage_path + input_file_name) dataProductModel.replicaLocations = [replicaLocation] return self.api_server_client.register_data_product(self.token, dataProductModel) - def configure_input_and_outputs(self, experiment_model, input_files, application_name, file_mapping={}): + def configure_input_and_outputs( + self, + experiment_model: ExperimentModel, + input_files: list[str], + application_name: str, + file_mapping: dict[str, str]={}, + ): execution_id = self.airavata_util.get_execution_id(application_name) - + assert execution_id is not None inputs = self.api_server_client.get_application_inputs(self.token, execution_id) configured_inputs = [] diff --git a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/utils/experiment_handler_util.py b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/utils/experiment_handler_util.py index 545d211181..389f3dd9d8 100644 --- a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/utils/experiment_handler_util.py +++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/clients/utils/experiment_handler_util.py @@ -13,32 +13,27 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import getpass import logging -import time -import logging -import airavata_sdk.samples.file_utils as fb import os -import jwt - -import getpass +import time +from typing import Optional -from airavata_sdk.clients.keycloak_token_fetcher import Authenticator +import jwt from airavata_sdk.clients.api_server_client import APIServerClient +from airavata_sdk.clients.keycloak_token_fetcher import Authenticator +from airavata_sdk.clients.sftp_file_handling_client import SFTPConnector from airavata_sdk.clients.utils.api_server_client_util import APIServerClientUtil - from airavata_sdk.clients.utils.data_model_creation_util import DataModelCreationUtil - -from airavata_sdk.transport.settings import GatewaySettings, ExperimentSettings, KeycloakServerSettings - -from airavata_sdk.clients.sftp_file_handling_client import SFTPConnector +from airavata_sdk.transport.settings import ExperimentSettings, GatewaySettings logger = logging.getLogger('airavata_sdk.clients') logger.setLevel(logging.INFO) class ExperimentHandlerUtil(object): - def __init__(self, configuration_file_location=None, access_token=None): + def __init__(self, configuration_file_location: Optional[str] = None, access_token: Optional[str] = None): self.configuration_file = configuration_file_location self.gateway_conf = GatewaySettings(configuration_file_location) self.experiment_conf = ExperimentSettings(configuration_file_location) @@ -49,10 +44,18 @@ class ExperimentHandlerUtil(object): self.access_token = access_token decode = jwt.decode(access_token, options={"verify_signature": False}) self.user_id = decode['preferred_username'] - self.airavata_token = self.authenticator.get_airavata_authz_token(self.user_id, access_token, - self.gateway_conf.GATEWAY_ID) - self.airavata_util = APIServerClientUtil(self.configuration_file, username=self.user_id, password=None, - gateway_id=self.gateway_conf.GATEWAY_ID, access_token=access_token) + self.airavata_token = self.authenticator.get_airavata_authz_token( + gateway_id=self.gateway_conf.GATEWAY_ID, + username=self.user_id, + token=access_token, + ) + self.airavata_util = APIServerClientUtil( + self.configuration_file, + gateway_id=self.gateway_conf.GATEWAY_ID, + username=self.user_id, + password=None, + access_token=access_token, + ) self.data_model_client = DataModelCreationUtil(self.configuration_file, username=self.user_id, @@ -62,32 +65,45 @@ class ExperimentHandlerUtil(object): self.api_server_client = APIServerClient(self.configuration_file) - def queue_names(self, computation_resource_name): + def queue_names(self, computation_resource_name: str): resource_id = self.airavata_util.get_resource_host_id(computation_resource_name) + assert resource_id is not None return self.airavata_util.get_queue_names(resource_id) - def launch_experiment(self, experiment_name="default_exp", description="this is default exp", - local_input_path="/tmp", input_file_mapping={}, computation_resource_name=None, - queue_name=None, node_count=1, cpu_count=1, walltime=30, auto_schedule=False, output_path='.'): + def launch_experiment( + self, + experiment_name: str = "default_exp", + description: str = "this is default exp", + local_input_path: str = "/tmp", + input_file_mapping: dict[str, str] = {}, + computation_resource_name: Optional[str] = None, + queue_name: Optional[str] = None, + node_count: int = 1, + cpu_count: int = 1, + walltime: int = 30, + auto_schedule: bool = False, + output_path: str = '.', + ): execution_id = self.airavata_util.get_execution_id(self.experiment_conf.APPLICATION_NAME) + assert execution_id is not None project_id = self.airavata_util.get_project_id(self.experiment_conf.PROJECT_NAME) hosts = self.experiment_conf.COMPUTE_HOST_DOMAIN.split(",") computation_resource_name = computation_resource_name if computation_resource_name is not None else hosts[0] resource_host_id = self.airavata_util.get_resource_host_id(computation_resource_name) - group_resource_profile_id = self.airavata_util.get_group_resource_profile_id( - self.experiment_conf.GROUP_RESOURCE_PROFILE_NAME) + group_resource_profile_id = self.airavata_util.get_group_resource_profile_id(self.experiment_conf.GROUP_RESOURCE_PROFILE_NAME) storage_id = self.airavata_util.get_storage_resource_id(self.experiment_conf.STORAGE_RESOURCE_HOST) + assert storage_id is not None logger.info("creating experiment %s", experiment_name) experiment = self.data_model_client.get_experiment_data_model_for_single_application( project_name=self.experiment_conf.PROJECT_NAME, application_name=self.experiment_conf.APPLICATION_NAME, experiment_name=experiment_name, - description=description) + description=description, + ) - logger.info("connnecting to file upload endpoint %s : %s", self.experiment_conf.STORAGE_RESOURCE_HOST, - self.experiment_conf.SFTP_PORT) + logger.info("connnecting to file upload endpoint %s : %s", self.experiment_conf.STORAGE_RESOURCE_HOST, self.experiment_conf.SFTP_PORT) sftp_connector = SFTPConnector(host=self.experiment_conf.STORAGE_RESOURCE_HOST, port=self.experiment_conf.SFTP_PORT, username=self.user_id, @@ -142,7 +158,7 @@ class ExperimentHandlerUtil(object): input_file_name=x, uploaded_storage_path=path) new_file_mapping[key] = data_uri - experiment = self.data_model_client.configure_input_and_outputs(experiment, input_files=None, + experiment = self.data_model_client.configure_input_and_outputs(experiment, input_files=[], application_name=self.experiment_conf.APPLICATION_NAME, file_mapping=new_file_mapping) else: diff --git a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/api_server_client_samples.py b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/api_server_client_samples.py index 37dd59b3ca..33191ce2f9 100644 --- a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/api_server_client_samples.py +++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/api_server_client_samples.py @@ -14,16 +14,13 @@ # limitations under the License. # import logging -from airavata_sdk.clients.api_server_client import APIServerClient - -from airavata_sdk.clients.keycloak_token_fetcher import Authenticator -from airavata.model.workspace.ttypes import Gateway, Notification, Project -from airavata.model.experiment.ttypes import ExperimentModel, ExperimentType, UserConfigurationDataModel +from airavata.api.error.ttypes import AiravataClientException, AiravataSystemException, AuthorizationException, InvalidRequestException from airavata.model.appcatalog.groupresourceprofile.ttypes import GroupResourceProfile - -from airavata.api.error.ttypes import TException, InvalidRequestException, AiravataSystemException, \ - AiravataClientException, AuthorizationException +from airavata.model.experiment.ttypes import ExperimentModel, ExperimentType, ProjectSearchFields, UserConfigurationDataModel +from airavata.model.workspace.ttypes import Gateway, GatewayApprovalStatus, Notification, Project +from airavata_sdk.clients.api_server_client import APIServerClient +from airavata_sdk.clients.keycloak_token_fetcher import Authenticator logger = logging.getLogger(__name__) @@ -72,7 +69,7 @@ def add_gateway(): gateway.gatewayAdminFirstName = "isuru" gateway.gatewayAdminLastName = "ranawaka" gateway.gatewayName = "test-gw" - gateway.gatewayApprovalStatus = 0 + gateway.gatewayApprovalStatus = GatewayApprovalStatus.REQUESTED gateway_id = client.add_gateway(token, gateway) print("Gateway Id :" + gateway_id) except (InvalidRequestException, AiravataClientException, AuthorizationException, AiravataSystemException): @@ -134,7 +131,7 @@ def create_project(): def search_projects(): try: - filter = {1: 'defaultProject'} + filter = {ProjectSearchFields.PROJECT_DESCRIPTION: 'defaultProject'} projects = client.search_projects(token, "default-gateway", "default-admin", filter, limit=0, offset=10) print(projects) except (InvalidRequestException, AiravataClientException, AuthorizationException, AiravataSystemException): diff --git a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/create_launch_echo_experiment.py b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/create_launch_echo_experiment.py index fd08823a94..65e063adfd 100644 --- a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/create_launch_echo_experiment.py +++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/create_launch_echo_experiment.py @@ -1,52 +1,45 @@ import logging import time -import json -import airavata_sdk.samples.file_utils as fb - -from airavata_sdk.clients.keycloak_token_fetcher import Authenticator from airavata_sdk.clients.api_server_client import APIServerClient - from airavata_sdk.clients.credential_store_client import CredentialStoreClient - -from airavata.model.experiment.ttypes import ExperimentModel, ExperimentType, UserConfigurationDataModel -from airavata.model.scheduling.ttypes import ComputationalResourceSchedulingModel - -from airavata_sdk.clients.utils.data_model_creation_util import DataModelCreationUtil - -from airavata_sdk.clients.utils.api_server_client_util import APIServerClientUtil - +from airavata_sdk.clients.keycloak_token_fetcher import Authenticator from airavata_sdk.clients.sftp_file_handling_client import SFTPConnector - +from airavata_sdk.clients.utils.api_server_client_util import APIServerClientUtil +from airavata_sdk.clients.utils.data_model_creation_util import DataModelCreationUtil from airavata_sdk.transport.settings import GatewaySettings logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) -configFile = "/Users/isururanawaka/Documents/Airavata_Repository/airavata/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/transport/settings.ini" +configFile: str = "/Users/isururanawaka/Documents/Airavata_Repository/airavata/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/transport/settings.ini" authenticator = Authenticator(configFile) -username = "username" -password = "password" -gateway_id = "cyberwater" +username: str = "username" +password: str = "password" +gateway_id: str = "cyberwater" token = authenticator.get_token_and_user_info_password_flow(username=username, password=password, gateway_id=gateway_id) api_server_client = APIServerClient(configFile) data_model_client = DataModelCreationUtil(configFile, + gateway_id=gateway_id, username=username, password=password, - gateway_id=gateway_id) + access_token=token.accessToken) credential_store_client = CredentialStoreClient(configFile) -airavata_util = APIServerClientUtil(configFile, - username=username, - password=password, - gateway_id=gateway_id) +airavata_util = APIServerClientUtil( + configFile, + gateway_id=gateway_id, + username=username, + password=password, +) executionId = airavata_util.get_execution_id("Echo") +assert executionId is not None projectId = airavata_util.get_project_id("Default Project") @@ -82,7 +75,7 @@ path = gateway_settings.GATEWAY_DATA_STORE_DIR + path_suffix experiment = data_model_client.configure_computation_resource_scheduling(experiment_model=experiment, computation_resource_name="karst.uits.iu.edu", group_resource_profile_name="Default Gateway Profile", - storage_name="pgadev.scigap.org", + storageId="pgadev.scigap.org", node_count=1, total_cpu_count=16, wall_time_limit=15, @@ -116,4 +109,4 @@ while status.state <= 6: print("Completed") -sftp_connector.download_files(".", "Default_Project", experiment.experimentName) +sftp_connector.download_files(".", f"Default_Project/{experiment.experimentName}") diff --git a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/create_launch_gaussian_experiment.py b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/create_launch_gaussian_experiment.py index 557416ecd0..e0b8e4920f 100644 --- a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/create_launch_gaussian_experiment.py +++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/create_launch_gaussian_experiment.py @@ -16,32 +16,14 @@ import logging import time -import json -import airavata_sdk.samples.file_utils as fb - -from airavata_sdk.clients.keycloak_token_fetcher import Authenticator +import airavata_sdk.samples.file_utils as fb from airavata_sdk.clients.api_server_client import APIServerClient - -from airavata_sdk.clients.utils.api_server_client_util import APIServerClientUtil - from airavata_sdk.clients.credential_store_client import CredentialStoreClient - +from airavata_sdk.clients.keycloak_token_fetcher import Authenticator +from airavata_sdk.clients.utils.api_server_client_util import APIServerClientUtil from airavata_sdk.clients.utils.data_model_creation_util import DataModelCreationUtil -from airavata.model.workspace.ttypes import Gateway, Notification, Project -from airavata.model.experiment.ttypes import ExperimentModel, ExperimentType, UserConfigurationDataModel -from airavata.model.scheduling.ttypes import ComputationalResourceSchedulingModel -from airavata.model.data.replica.ttypes import DataProductModel, DataProductType, DataReplicaLocationModel, \ - ReplicaLocationCategory, ReplicaPersistentType - -from airavata.model.application.io.ttypes import InputDataObjectType - -from airavata.model.appcatalog.groupresourceprofile.ttypes import GroupResourceProfile - -from airavata.api.error.ttypes import TException, InvalidRequestException, AiravataSystemException, \ - AiravataClientException, AuthorizationException - logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -54,16 +36,26 @@ user_name = "username" password = "password" gateway_id = "cyberwater" -token = authenticator.get_token_and_user_info_password_flow(username=user_name, password=password, - gateway_id=gateway_id) +token = authenticator.get_token_and_user_info_password_flow( + username=user_name, + password=password, + gateway_id=gateway_id, +) api_server_client = APIServerClient(configFile) -airavata_util = APIServerClientUtil(configFile, username=user_name, password=password, gateway_id=gateway_id) -data_model_client = DataModelCreationUtil(configFile, - username=user_name, - password=password, - gateway_id=gateway_id) +airavata_util = APIServerClientUtil( + configFile, + gateway_id=gateway_id, + username=user_name, + password=password, +) +data_model_client = DataModelCreationUtil( + configFile, + gateway_id=gateway_id, + username=user_name, + password=password, +) credential_store_client = CredentialStoreClient(configFile) @@ -94,7 +86,7 @@ path = fb.upload_files(api_server_client, credential_store_client, token, gatewa experiment = data_model_client.configure_computation_resource_scheduling(experiment_model=experiment, computation_resource_name="karst.uits.iu.edu", group_resource_profile_name="Default Gateway Profile", - storage_name="pgadev.scigap.org", + storageId="pgadev.scigap.org", node_count=1, total_cpu_count=16, wall_time_limit=15, diff --git a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/file_utils.py b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/file_utils.py index 3d0374931f..fd3fa6a731 100644 --- a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/file_utils.py +++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/file_utils.py @@ -1,4 +1,5 @@ import io + import paramiko from airavata_sdk.clients.file_handling_client import FileHandler diff --git a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/group_manager_client_samples.py b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/group_manager_client_samples.py index 18b1e1becb..a568ab3189 100644 --- a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/group_manager_client_samples.py +++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/group_manager_client_samples.py @@ -15,13 +15,11 @@ # import logging -from airavata_sdk.clients.group_manager_client import GroupManagerClient - -from airavata_sdk.clients.keycloak_token_fetcher import Authenticator from airavata.api.error.ttypes import TException - from airavata.model.group.ttypes import GroupModel +from airavata_sdk.clients.group_manager_client import GroupManagerClient +from airavata_sdk.clients.keycloak_token_fetcher import Authenticator logger = logging.getLogger(__name__) @@ -70,7 +68,7 @@ def get_groups(): def add_group_admin(): try: - created_group = client.add_group_admins(token, "testing_group", "default-admin") + created_group = client.add_group_admins(token, "testing_group", ["default-admin"]) print("Groups :", created_group) except TException: logger.exception("Exception occurred") diff --git a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/iam_admin_client_samples.py b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/iam_admin_client_samples.py index 776f7e255e..6b4465ea7b 100644 --- a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/iam_admin_client_samples.py +++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/iam_admin_client_samples.py @@ -15,11 +15,10 @@ # import logging -from airavata_sdk.clients.iam_admin_client import IAMAdminClient - -from airavata_sdk.clients.keycloak_token_fetcher import Authenticator from airavata.api.error.ttypes import TException +from airavata_sdk.clients.iam_admin_client import IAMAdminClient +from airavata_sdk.clients.keycloak_token_fetcher import Authenticator logger = logging.getLogger(__name__) diff --git a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/metadata_fetcher.py b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/metadata_fetcher.py index cb2e63c7b8..82da39e1b9 100644 --- a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/metadata_fetcher.py +++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/metadata_fetcher.py @@ -16,22 +16,8 @@ import logging -from airavata_sdk.clients.keycloak_token_fetcher import Authenticator - from airavata_sdk.clients.api_server_client import APIServerClient - -from airavata.model.workspace.ttypes import Gateway, Notification, Project -from airavata.model.experiment.ttypes import ExperimentModel, ExperimentType, UserConfigurationDataModel -from airavata.model.scheduling.ttypes import ComputationalResourceSchedulingModel -from airavata.model.data.replica.ttypes import DataProductModel, DataProductType, DataReplicaLocationModel, \ - ReplicaLocationCategory, ReplicaPersistentType - -from airavata.model.application.io.ttypes import InputDataObjectType - -from airavata.model.appcatalog.groupresourceprofile.ttypes import GroupResourceProfile - -from airavata.api.error.ttypes import TException, InvalidRequestException, AiravataSystemException, \ - AiravataClientException, AuthorizationException +from airavata_sdk.clients.keycloak_token_fetcher import Authenticator logger = logging.getLogger(__name__) diff --git a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/sharing_registry_client_samples.py b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/sharing_registry_client_samples.py index 408441d089..2e543d9e4a 100644 --- a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/sharing_registry_client_samples.py +++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/sharing_registry_client_samples.py @@ -16,13 +16,11 @@ import logging -from airavata_sdk.clients.sharing_registry_client import SharingRegistryClient - -from airavata_sdk.clients.keycloak_token_fetcher import Authenticator from airavata.api.error.ttypes import TException - from airavata.model.sharing.ttypes import Domain, Entity, EntityType +from airavata_sdk.clients.keycloak_token_fetcher import Authenticator +from airavata_sdk.clients.sharing_registry_client import SharingRegistryClient logger = logging.getLogger(__name__) diff --git a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/tenant_profile_client_samples.py b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/tenant_profile_client_samples.py index a1b56128d6..5bca7bb10b 100644 --- a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/tenant_profile_client_samples.py +++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/tenant_profile_client_samples.py @@ -15,11 +15,10 @@ # import logging -from airavata_sdk.clients.tenant_profile_client import TenantProfileClient - -from airavata_sdk.clients.keycloak_token_fetcher import Authenticator from airavata.api.error.ttypes import TException +from airavata_sdk.clients.keycloak_token_fetcher import Authenticator +from airavata_sdk.clients.tenant_profile_client import TenantProfileClient logger = logging.getLogger(__name__) diff --git a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/user_profile_client_samples.py b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/user_profile_client_samples.py index a99cb36c43..ef9054ed1a 100644 --- a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/user_profile_client_samples.py +++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/samples/user_profile_client_samples.py @@ -15,13 +15,11 @@ # import logging -from airavata_sdk.clients.user_profile_client import UserProfileClient - -from airavata_sdk.clients.keycloak_token_fetcher import Authenticator from airavata.api.error.ttypes import TException - -from airavata.model.user.ttypes import UserProfile, Status +from airavata.model.user.ttypes import Status, UserProfile +from airavata_sdk.clients.keycloak_token_fetcher import Authenticator +from airavata_sdk.clients.user_profile_client import UserProfileClient logger = logging.getLogger(__name__) diff --git a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/transport/settings.py b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/transport/settings.py index 779d7c2338..a127af61cb 100644 --- a/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/transport/settings.py +++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/airavata_sdk/transport/settings.py @@ -16,6 +16,7 @@ import configparser import os +from typing import Optional config = configparser.ConfigParser() @@ -25,7 +26,7 @@ config.read(defaultSettings) class APIServerSettings(object): - def __init__(self, configFileLocation=None): + def __init__(self, configFileLocation: Optional[str] = None): if configFileLocation is not None: config.read(configFileLocation) self.API_SERVER_HOST = config.get('APIServer', 'API_HOST') @@ -34,7 +35,7 @@ class APIServerSettings(object): class ProfileServerSettings(object): - def __init__(self, configFileLocation=None): + def __init__(self, configFileLocation: Optional[str] = None): if configFileLocation is not None: config.read(configFileLocation) self.PROFILE_SERVICE_HOST = config.get('ProfileServer', 'PROFILE_SERVICE_HOST') @@ -43,7 +44,7 @@ class ProfileServerSettings(object): class SharingServerSettings(object): - def __init__(self, configFileLocation=None): + def __init__(self, configFileLocation: Optional[str] = None): if configFileLocation is not None: config.read(configFileLocation) self.SHARING_API_HOST = config.get('SharingServer', 'SHARING_API_HOST') @@ -52,7 +53,7 @@ class SharingServerSettings(object): class CredentialStoreServerSettings(object): - def __init__(self, configFileLocation=None): + def __init__(self, configFileLocation: Optional[str] = None): if configFileLocation is not None: config.read(configFileLocation) self.CREDENTIAL_STORE_API_HOST = config.get('CredentialStoreServer', 'CREDENTIAL_STORE_API_HOST') @@ -61,14 +62,14 @@ class CredentialStoreServerSettings(object): class ThriftSettings(object): - def __init__(self, configFileLocation=None): + def __init__(self, configFileLocation: Optional[str] = None): if configFileLocation is not None: config.read(configFileLocation) self.THRIFT_CLIENT_POOL_KEEPALIVE = config.getfloat('Thrift', 'THRIFT_CLIENT_POOL_KEEPALIVE') class KeycloakServerSettings(object): - def __init__(self, configFileLocation=None): + def __init__(self, configFileLocation: Optional[str] = None): if configFileLocation is not None: config.read(configFileLocation) # self.KEYCLOAK_CA_CERTIFICATE = os.path.join(BASE_DIR, "samples", "resources", "incommon_rsa_server_ca.pem") @@ -81,7 +82,7 @@ class KeycloakServerSettings(object): class GatewaySettings(object): - def __init__(self, configFileLocation=None): + def __init__(self, configFileLocation: Optional[str] = None): if configFileLocation is not None: config.read(configFileLocation) self.GATEWAY_ID = config.get('Gateway', 'GATEWAY_ID') @@ -93,7 +94,7 @@ class GatewaySettings(object): class ExperimentSettings(object): - def __init__(self, configFileLocation=None): + def __init__(self, configFileLocation: Optional[str] = None): if configFileLocation is not None: config.read(configFileLocation) self.APPLICATION_NAME = config.get('ExperimentConf', 'APPLICATION_NAME')
