This is an automated email from the ASF dual-hosted git repository. lfrolov pushed a commit to branch DATALAB-1408 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit a3b6d915e175325064ebcea8583a98547d7c792c Author: leonidfrolov <[email protected]> AuthorDate: Tue Sep 6 16:08:25 2022 +0300 [DATALAB-1408]: fixed shapes --- .../files/azure/dataengine-service_description.json | 18 +++++------------- .../src/general/lib/azure/actions_lib.py | 1 + .../general/scripts/azure/dataengine-service_create.py | 4 ++-- .../scripts/azure/dataengine-service_prepare.py | 7 ++++++- 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/infrastructure-provisioning/src/general/files/azure/dataengine-service_description.json b/infrastructure-provisioning/src/general/files/azure/dataengine-service_description.json index c41965004..faa7d1767 100644 --- a/infrastructure-provisioning/src/general/files/azure/dataengine-service_description.json +++ b/infrastructure-provisioning/src/general/files/azure/dataengine-service_description.json @@ -5,24 +5,16 @@ "computation_resources_shapes": { "For testing" : [ - {"Size": "S", "Description": "Standard_F4s", "Type": "Standard_F4s","Ram": "8.0 GB","Cpu": "4"} + {"Size": "S", "Description": "Standard_D12_v2", "Type": "Standard_D12_v2","Ram": "28.0 GB","Cpu": "4"} ], "Memory optimized" : [ - {"Size": "S", "Description": "Standard_E4s_v3", "Type": "Standard_E4s_v3","Ram": "32 GB","Cpu": "4"}, - {"Size": "M", "Description": "Standard_E16s_v3", "Type": "Standard_E16s_v3","Ram": "128 GB","Cpu": "16"}, - {"Size": "L", "Description": "Standard_E32s_v3", "Type": "Standard_E32s_v3","Ram": "256 GB","Cpu": "32"} - ], - "Compute optimized": [ - {"Size": "S", "Description": "Standard_F4s", "Type": "Standard_F4s","Ram": "8.0 GB","Cpu": "4"}, - {"Size": "M", "Description": "Standard_F8s", "Type": "Standard_F8s","Ram": "16.0 GB","Cpu": "8"}, - {"Size": "L", "Description": "Standard_F16s", "Type": "Standard_F16s","Ram": "32.0 GB","Cpu": "16"} - ], - "GPU optimized": [ - {"Size": "S", "Description": "Standard_NC6", "Type": "Standard_NC6","Ram": "56.0 GB","Cpu": "6"} + {"Size": "S", "Description": "Standard_E4_v3", "Type": "Standard_E4_v3","Ram": "32 GB","Cpu": "4"}, + {"Size": "M", "Description": "Standard_E16_v3", "Type": "Standard_E16_v3","Ram": "128 GB","Cpu": "16"}, + {"Size": "L", "Description": "Standard_E32_v3", "Type": "Standard_E32_v3","Ram": "256 GB","Cpu": "32"} ] }, "templates": [ - {"version":"4.0", "applications": [{"Name":"Spark", "Version": "x.x.x"}]} + {"version":"4.0", "applications": [{"Name":"Spark", "Version": "2.4"}]} ] } \ No newline at end of file diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py index db9088aa6..1b3086e69 100644 --- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py +++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py @@ -509,6 +509,7 @@ class AzureActions: resource_group_name, account_name ) + logging.info("Storage account {} was removed.".format(account_name)) return result except Exception as err: logging.info( diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py index 741424eed..8684a329b 100644 --- a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py +++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py @@ -52,7 +52,7 @@ args = parser.parse_args() def build_hdinsight_cluster(resource_group_name, cluster_name, params): - logging.info("Will be created cluster: {}".format(cluster_name)) + logging.info("{} cluster creation".format(cluster_name)) return datalab.actions_lib.AzureActions().create_hdinsight_cluster(resource_group_name, cluster_name, params) @@ -122,7 +122,7 @@ def create_cluster_parameters(location, tags, cluster_version, cluster_login_use Role( name="zookeepernode", target_instance_count=3, - hardware_profile=HardwareProfile(vm_size=Small), + hardware_profile=HardwareProfile(vm_size="Standard_A4_v2"), os_profile=OsProfile( linux_operating_system_profile=LinuxOperatingSystemProfile( username=cluster_login_username, diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py index 49f636566..af106da59 100644 --- a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py +++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py @@ -23,6 +23,7 @@ import datalab.fab import datalab.meta_lib +import datalab.actions_lib import json import multiprocessing import os @@ -35,6 +36,7 @@ from fabric import * if __name__ == "__main__": try: + AzureActions = datalab.actions_lib.AzureActions() AzureMeta = datalab.meta_lib.AzureMeta() logging.info('Generating infrastructure names and tags') hdinsight_conf = dict() @@ -99,6 +101,9 @@ if __name__ == "__main__": hdinsight_conf['vpc_name'], hdinsight_conf['subnet_name']).id + hdinsight_conf['hdinsight_master_instance_type'] = os.environ['hdinsight_master_instance_type'] + hdinsight_conf['hdinsight_slave_instance_type'] = os.environ['hdinsight_slave_instance_type'] + except Exception as err: datalab.fab.append_result("Failed to generate variables dictionary. Exception:" + str(err)) sys.exit(1) @@ -137,7 +142,7 @@ if __name__ == "__main__": "--tags '{}' --public_key '{}' --vpc_id {} --subnet {}"\ .format(hdinsight_conf['resource_group_name'], hdinsight_conf['cluster_name'], hdinsight_conf['release_label'], hdinsight_conf['region'], - os.environ['hdinsight_master_instance_type'], os.environ['hdinsight_slave_instance_type'], + hdinsight_conf['hdinsight_master_instance_type'], hdinsight_conf['hdinsight_slave_instance_type'], hdinsight_conf['hdinsight_worker_count'], hdinsight_conf['storage_account_name'], hdinsight_conf['storage_account_key'], hdinsight_conf['container_name'], json.dumps(hdinsight_conf['cluster_tags']), ssh_admin_pubkey, hdinsight_conf['vpc_id'], --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
