This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DLAB-1404
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit ebfa22c9eedfd6eb4b68abf73c4e1dd14019b4bd
Author: Mykola_Bodnar1 <bodnarmyk...@gmail.com>
AuthorDate: Thu Jan 9 12:14:17 2020 +0200
[DLAB-1404] - [GCP] Simultaneously Data Engine Service creation fixed
---
infrastructure-provisioning/src/general/lib/gcp/meta_lib.py | 11 +++++++++++
.../src/general/scripts/gcp/dataengine-service_prepare.py | 9 ++++++++-
2 files changed, 19 insertions(+), 1 deletion(-)
diff --git a/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py
b/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py
index 5e7a96b..e961ff2 100644
--- a/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py
@@ -650,6 +650,17 @@ class GCPMeta:
traceback.print_exc(file=sys.stdout)
return ''
+ def dataproc_waiter(self, labels):
+ if os.path.exists(
+ '/response/.emr_creating_' + os.environ['exploratory_name'])
or GCPMeta().get_not_configured_dataproc(
+ os.environ['notebook_instance_name']):
+ with hide('stderr', 'running', 'warnings'):
+ local("echo 'Some Dataproc cluster is still being
created/terminated, waiting..'")
+ time.sleep(60)
+ self.dataproc_waiter(labels)
+ else:
+ return True
+
def get_dataproc_jobs(self):
jobs = []
try:
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py
b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py
index d6bb17a..91a8535 100644
---
a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py
+++
b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py
@@ -108,7 +108,14 @@ if __name__ == "__main__":
print("Will create exploratory environment with edge node as access point
as following: ".format(json.dumps(dataproc_conf, sort_keys=True, indent=4,
separators=(',', ': '))))
logging.info(json.dumps(dataproc_conf))
- local('touch
/response/.dataproc_creating_{}'.format(os.environ['exploratory_name']))
+ try:
+ meta_lib.GCPMeta().dataproc_waiter(dataproc_conf['cluster_labels'])
+ local('touch
/response/.dataproc_creating_{}'.format(os.environ['exploratory_name']))
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Dataproc waiter fail.", str(err))
+ sys.exit(1)
+
local("echo Waiting for changes to propagate; sleep 10")
dataproc_cluster =
json.loads(open('/root/templates/dataengine-service_cluster.json').read().decode('utf-8-sig'))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@dlab.apache.org
For additional commands, e-mail: commits-h...@dlab.apache.org
