This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a commit to branch odahu-integration
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/odahu-integration by this push:
new 9f0e561 [odahu-integration] Bugfixing
new 3882225 Merge remote-tracking branch 'origin/odahu-integration' into
odahu-integration
9f0e561 is described below
commit 9f0e561721ff0cbbc2038d33d80f7f67cac19056
Author: Demyan Mysakovets <[email protected]>
AuthorDate: Fri Apr 17 16:09:51 2020 +0300
[odahu-integration] Bugfixing
---
.../src/general/lib/aws/actions_lib.py | 15 ++++++++++++++
.../src/general/scripts/aws/odahu_deploy.py | 23 ++++------------------
.../src/general/scripts/aws/odahu_prepare.py | 5 ++++-
.../src/ssn/scripts/configure_docker.py | 6 +++---
4 files changed, 26 insertions(+), 23 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
index cb0a7ad..f9efa6d 100644
--- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
@@ -758,6 +758,21 @@ def disassociate_elastic_ip(association_id):
"error_message": str(err) + "\n Traceback: " +
traceback.print_exc(file=sys.stdout)}))
traceback.print_exc(file=sys.stdout)
+def create_nat_gateway(allocation_id, subnet_id, project_name):
+ try:
+ client = boto3.client('ec2')
+ client.create_nat_gateway(AllocationId=allocation_id,
SubnetId=subnet_id, TagSpecifications=[
+ {
+ 'ResourceType': 'natgateway',
+ 'Tags': [{'Key': 'Name', 'Value':
'{}-nat'.format(project_name)}]
+ }])
+ except Exception as err:
+ logging.info("Unable to create NAT Gateway: " + str(err) + "\n
Traceback: " + traceback.print_exc(
+ file=sys.stdout))
+ append_result(str({"error": "Unable to create NAT Gateway",
+ "error_message": str(err) + "\n Traceback: " +
traceback.print_exc(file=sys.stdout)}))
+ traceback.print_exc(file=sys.stdout)
+
def remove_ec2(tag_name, tag_value):
try:
diff --git
a/infrastructure-provisioning/src/general/scripts/aws/odahu_deploy.py
b/infrastructure-provisioning/src/general/scripts/aws/odahu_deploy.py
index e7e0363..1324b4b 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/odahu_deploy.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/odahu_deploy.py
@@ -45,17 +45,6 @@ if __name__ == "__main__":
odahu_conf['region'] = (os.environ['aws_region'])
odahu_conf['zone'] = (os.environ['aws_zone'])
odahu_conf['edge_user_name'] = os.environ['edge_user_name']
- if os.environ['aws_region'] == 'us-east':
- odahu_conf['az_list'] = ["us-east-1", "us-east-2"]
- elif os.environ['aws_region'] == 'ap-south':
- odahu_conf['az_list'] = ["ap-south-1"]
- elif os.environ['aws_region'] == 'us-west':
- odahu_conf['az_list'] = ["us-west-1", "us-west-2"]
- elif os.environ['aws_region'] == 'eu-west':
- odahu_conf['az_list'] = ["eu-west-1", "eu-west-2", "eu-west-3"]
- elif os.environ['aws_region'] == 'ca-central':
- odahu_conf['az_list'] = ["ca-central-1"]
- odahu_conf['az_list'] = GCPMeta().get_available_zones()
odahu_conf['dns_zone_name'] = os.environ['odahu_dns_zone_name']
odahu_conf['docker_repo'] = os.environ['odahu_docker_repo']
odahu_conf['cidr'] = os.environ['odahu_cidr']
@@ -64,7 +53,6 @@ if __name__ == "__main__":
odahu_conf['cluster_name'] =
"{}-{}".format((os.environ['conf_service_base_name']).lower().replace('_', '-'),
(os.environ['odahu_cluster_name']).lower().replace('_', '-'))
odahu_conf['bucket_name'] = "{}-tfstate".format(odahu_conf['cluster_name'])
- odahu_conf['static_address_name'] =
"{}-nat-gw".format(odahu_conf['cluster_name'])
try:
if os.environ['gcp_vpc_name'] == '':
raise KeyError
@@ -73,8 +61,9 @@ if __name__ == "__main__":
except KeyError:
odahu_conf['vpc_name'] = odahu_conf['service_base_name'] + '-ssn-vpc'
odahu_conf['vpc_cidr'] = os.environ['conf_vpc_cidr']
- odahu_conf['private_subnet_name'] =
'{0}-{1}-subnet'.format(odahu_conf['service_base_name'],
-
odahu_conf['project_name'])
+ tag = {"Key": '{}-Tag'.format(odahu_conf['service_base_name']),
+ "Value": "{}-{}-subnet".format(odahu_conf['service_base_name'],
odahu_conf['project_name'])}
+ odahu_conf['private_subnet_cidr'] = get_subnet_by_tag(tag)
odahu_conf['grafana_admin'] = os.environ['odahu_grafana_admin']
odahu_conf['grafana_pass'] = id_generator()
odahu_conf['docker_password'] =
base64.b64decode(os.environ['odahu_docker_password'] + "==")
@@ -94,8 +83,6 @@ if __name__ == "__main__":
odahu_conf['mlflow_toolchain_version'] =
os.environ['odahu_mlflow_toolchain_version']
odahu_conf['jupyterlab_version'] = os.environ['odahu_jupyterlab_version']
odahu_conf['packager_version'] = os.environ['odahu_packager_version']
- odahu_conf['private_subnet_cidrs'] =
os.environ['odahu_private_subnet_cidrs'].split(',')
- odahu_conf['public_subnet_cidrs'] =
os.environ['odahu_public_subnet_cidrs'].split(',')
odahu_conf['infra_cidr'] = os.environ['odahu_pods_cidr']
odahu_conf['root_domain'] = os.environ['odahu_root_domain']
odahu_conf['service_cidr'] = os.environ['odahu_service_cidr']
@@ -127,7 +114,7 @@ if __name__ == "__main__":
"authz_dry_run": "false",
"cloud": {
"aws": {
- "az_list": odahu_conf['az_list'],
+ "az_list": ["{}".format(odahu_conf['zone'])],
"zone": "{}".format(odahu_conf['zone']),
},
"region": "{}".format(odahu_conf['region']),
@@ -235,8 +222,6 @@ if __name__ == "__main__":
"odahuflow_version":
"{}".format(odahu_conf['odahuflow_version']),
"opa_policies": {},
"packager_version":
"{}".format(odahu_conf['packager_version']),
- "private_subnet_cidrs": odahu_conf['private_subnet_cidrs'],
- "public_subnet_cidrs": odahu_conf['public_subnet_cidrs'],
"service_accounts": {
"airflow": {
"client_id": "sa-airflow",
diff --git
a/infrastructure-provisioning/src/general/scripts/aws/odahu_prepare.py
b/infrastructure-provisioning/src/general/scripts/aws/odahu_prepare.py
index d4168ed..72e38d4 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/odahu_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/odahu_prepare.py
@@ -49,6 +49,7 @@ if __name__ == "__main__":
odahu_conf['endpoint_tag'] =
(os.environ['endpoint_name']).lower().replace('_', '-')
odahu_conf['project_tag'] =
(os.environ['project_name']).lower().replace('_', '-')
odahu_conf['region'] = os.environ['gcp_region']
+ odahu_conf['ssn_subnet_id'] = os.environ['ssn_subnet_id']
odahu_conf['bucket_name'] = "{}-tfstate".format(odahu_conf['cluster_name'])
odahu_conf['static_address_name'] =
"{}-nat-gw".format(odahu_conf['cluster_name'])
odahu_conf['keycloak_auth_server_url'] =
os.environ['keycloak_auth_server_url']
@@ -99,9 +100,11 @@ if __name__ == "__main__":
tag_name = {"Key": "Name", "Value": odahu_conf['static_address_name']}
create_tag(allocation_id, tag)
create_tag(allocation_id, tag_name)
+ print("Creating NAT")
+ create_nat_gatway(allocation_id, odahu_conf['ssn_subnet_id'],
odahu_conf['project_tag'])
except Exception as err:
print('Error: {0}'.format(err))
- append_result("Unable to reserve static ip.", str(err))
+ append_result("Unable to Unable to create NAT Gateway.", str(err))
remove_s3(bucket_type='odahu')
sys.exit(1)
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
index 86e9e65..e2f617c 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
@@ -81,7 +81,7 @@ def add_china_repository(dlab_path):
sudo('sed -i "/pip install/s/jupyter/ipython==5.0.0 jupyter==1.0.0/g"
Dockerfile')
sudo('sed -i "22i COPY general/files/os/debian/sources.list
/etc/apt/sources.list" Dockerfile')
-def login_in_gcr(os_user, gcr_creds, odahu_image, dlab_path, os_family):
+def login_in_gcr(os_user, gcr_creds, odahu_image, dlab_path, cloud_provider):
if os.environ['conf_cloud_provider'] != 'gcp':
try:
sudo('echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg]
https://packages.cloud.google.com/apt '
@@ -101,7 +101,7 @@ def login_in_gcr(os_user, gcr_creds, odahu_image,
dlab_path, os_family):
sudo('mkdir /home/{}/.docker'.format(os_user))
sudo('cp /tmp/config /home/{}/.docker/config.json'.format(os_user))
sudo('sed -i "s|ODAHU_IMAGE|{}|"
{}sources/infrastructure-provisioning/src/general/files/{}/odahu_Dockerfile'
- .format(odahu_image, dlab_path, os_family))
+ .format(odahu_image, dlab_path, cloud_provider))
except Exception as err:
traceback.print_exc()
print('Failed to prepare odahu image: ', str(err))
@@ -205,7 +205,7 @@ if __name__ == "__main__":
sys.exit(1)
print("Login in Google Container Registry")
- login_in_gcr(args.os_user, args.gcr_creds, args.odahu_image,
args.dlab_path, args.os_family)
+ login_in_gcr(args.os_user, args.gcr_creds, args.odahu_image,
args.dlab_path, args.cloud_provider)
print("Building dlab images")
count = 0
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]