This is an automated email from the ASF dual-hosted git repository.
omartushevskyi pushed a commit to branch inactivity_integration
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/inactivity_integration by this
push:
new ef7c735 [DLAB-402] Update last time inactivity after restarting by
starting time [DLAB-405] Changes in names of idle_check parameters
new 7152eb7 Merge pull request #6 from ioleksandr/inactivity_integration
ef7c735 is described below
commit ef7c735e0dc89aa5513c0756d12d743accd8e048
Author: Oleksandr_Isniuk <[email protected]>
AuthorDate: Tue Feb 19 15:01:03 2019 +0200
[DLAB-402] Update last time inactivity after restarting by starting time
[DLAB-405] Changes in names of idle_check parameters
---
.../src/general/files/aws/dataengine_Dockerfile | 1 +
.../src/general/files/azure/dataengine_Dockerfile | 1 +
.../src/general/files/gcp/dataengine_Dockerfile | 1 +
.../general/scripts/aws/common_start_notebook.py | 14 ++++++
.../src/general/scripts/aws/dataengine_start.py | 52 ++++++++++++++++------
.../general/scripts/azure/common_start_notebook.py | 14 ++++++
.../src/general/scripts/azure/dataengine_start.py | 23 ++++++++++
.../general/scripts/gcp/common_start_notebook.py | 14 ++++++
.../src/general/scripts/gcp/dataengine_start.py | 22 +++++++++
.../scripts/os/notebook_inactivity_check.py | 2 +-
.../scripts/os/update_inactivity_on_start.py | 48 ++++++++++++++++++++
.../src/general/templates/os/inactive.service | 18 ++++++++
.../src/general/templates/os/inactive.timer | 18 ++++++++
13 files changed, 213 insertions(+), 15 deletions(-)
diff --git
a/infrastructure-provisioning/src/general/files/aws/dataengine_Dockerfile
b/infrastructure-provisioning/src/general/files/aws/dataengine_Dockerfile
index b9ecf0d..cbc0680 100644
--- a/infrastructure-provisioning/src/general/files/aws/dataengine_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/dataengine_Dockerfile
@@ -22,6 +22,7 @@ ARG OS
COPY dataengine/ /root/
COPY general/scripts/os/dataengine_* /root/scripts/
+COPY general/scripts/os/update_inactivity_on_start.py /root/scripts/
COPY general/scripts/os/reconfigure_spark.py /root/scripts/
COPY general/scripts/os/install_additional_libs.py
/root/scripts/install_additional_libs.py
COPY general/scripts/os/get_list_available_pkgs.py
/root/scripts/get_list_available_pkgs.py
diff --git
a/infrastructure-provisioning/src/general/files/azure/dataengine_Dockerfile
b/infrastructure-provisioning/src/general/files/azure/dataengine_Dockerfile
index 668c8c7..9b5a323 100644
--- a/infrastructure-provisioning/src/general/files/azure/dataengine_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/azure/dataengine_Dockerfile
@@ -23,6 +23,7 @@ ARG OS
COPY dataengine/ /root/
COPY general/lib/os/${OS}/notebook_lib.py
/usr/lib/python2.7/dlab/notebook_lib.py
COPY general/scripts/os/dataengine_* /root/scripts/
+COPY general/scripts/os/update_inactivity_on_start.py /root/scripts/
COPY general/scripts/os/reconfigure_spark.py /root/scripts/
COPY general/scripts/os/install_additional_libs.py
/root/scripts/install_additional_libs.py
COPY general/scripts/os/get_list_available_pkgs.py
/root/scripts/get_list_available_pkgs.py
diff --git
a/infrastructure-provisioning/src/general/files/gcp/dataengine_Dockerfile
b/infrastructure-provisioning/src/general/files/gcp/dataengine_Dockerfile
index 7630cd5..69a21aa 100644
--- a/infrastructure-provisioning/src/general/files/gcp/dataengine_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/dataengine_Dockerfile
@@ -22,6 +22,7 @@ ARG OS
COPY dataengine/ /root/
COPY general/scripts/os/dataengine_* /root/scripts/
+COPY general/scripts/os/update_inactivity_on_start.py /root/scripts/
COPY general/scripts/os/reconfigure_spark.py /root/scripts/
COPY general/scripts/os/install_additional_libs.py
/root/scripts/install_additional_libs.py
COPY general/scripts/os/get_list_available_pkgs.py
/root/scripts/get_list_available_pkgs.py
diff --git
a/infrastructure-provisioning/src/general/scripts/aws/common_start_notebook.py
b/infrastructure-provisioning/src/general/scripts/aws/common_start_notebook.py
index 653eda9..e167c98 100644
---
a/infrastructure-provisioning/src/general/scripts/aws/common_start_notebook.py
+++
b/infrastructure-provisioning/src/general/scripts/aws/common_start_notebook.py
@@ -75,6 +75,20 @@ if __name__ == "__main__":
except:
sys.exit(1)
+ try:
+ logging.info('[UPDATE LAST ACTIVITY TIME]')
+ print('[UPDATE LAST ACTIVITY TIME]')
+ params = '--os_user {} --notebook_ip {} --keyfile "{}"' \
+ .format(os.environ['conf_os_user'],
notebook_config['notebook_ip'], notebook_config['keyfile'])
+ try:
+ local("~/scripts/{}.py {}".format('update_inactivity_on_start',
params))
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to update last activity time.", str(err))
+ raise Exception
+ except:
+ sys.exit(1)
+
try:
ip_address = get_instance_ip_address(notebook_config['tag_name'],
notebook_config['notebook_name']).get('Private')
diff --git
a/infrastructure-provisioning/src/general/scripts/aws/dataengine_start.py
b/infrastructure-provisioning/src/general/scripts/aws/dataengine_start.py
index 98c74f6..e659d21 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine_start.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine_start.py
@@ -46,37 +46,61 @@ if __name__ == "__main__":
# generating variables dictionary
create_aws_config_files()
print('Generating infrastructure names and tags')
- data_engine_config = dict()
+ data_engine = dict()
try:
- data_engine_config['exploratory_name'] = os.environ['exploratory_name']
+ data_engine['exploratory_name'] = os.environ['exploratory_name']
except:
- data_engine_config['exploratory_name'] = ''
+ data_engine['exploratory_name'] = ''
try:
- data_engine_config['computational_name'] =
os.environ['computational_name']
+ data_engine['computational_name'] = os.environ['computational_name']
except:
- data_engine_config['computational_name'] = ''
- data_engine_config['service_base_name'] =
os.environ['conf_service_base_name']
- data_engine_config['user_name'] = os.environ['edge_user_name']
- data_engine_config['cluster_name'] = \
- data_engine_config['service_base_name'] + '-' + \
- data_engine_config['user_name'] + '-de-' + \
- data_engine_config['exploratory_name'] + '-' + \
- data_engine_config['computational_name']
+ data_engine['computational_name'] = ''
+ data_engine['service_base_name'] = os.environ['conf_service_base_name']
+ data_engine['user_name'] = os.environ['edge_user_name']
+ data_engine['cluster_name'] = \
+ data_engine['service_base_name'] + '-' + \
+ data_engine['user_name'] + '-de-' + \
+ data_engine['exploratory_name'] + '-' + \
+ data_engine['computational_name']
+
logging.info('[START DATA ENGINE CLUSTER]')
print('[START DATA ENGINE CLUSTER]')
try:
start_data_engine("{}:{}".format(os.environ['conf_service_base_name'],
- data_engine_config['cluster_name']))
+ data_engine['cluster_name']))
except Exception as err:
print('Error: {0}'.format(err))
append_result("Failed to start Data Engine.", str(err))
sys.exit(1)
try:
+ logging.info('[UPDATE LAST ACTIVITY TIME]')
+ print('[UPDATE LAST ACTIVITY TIME]')
+ data_engine['computational_id'] = data_engine['cluster_name'] + '-m'
+ data_engine['tag_name'] = data_engine['service_base_name'] + '-Tag'
+ data_engine['notebook_ip'] =
get_instance_ip_address(data_engine['tag_name'],
+
os.environ['notebook_instance_name']).get('Private')
+ data_engine['computational_ip'] =
get_instance_ip_address(data_engine['tag_name'],
+
data_engine['computational_id']).get(
+ 'Private')
+ data_engine['keyfile'] = '{}{}.pem'.format(os.environ['conf_key_dir'],
os.environ['conf_key_name'])
+ params = '--os_user {0} --notebook_ip {1} --keyfile "{2}" --cluster_ip
{3}' \
+ .format(os.environ['conf_os_user'], data_engine['notebook_ip'],
data_engine['keyfile'],
+ data_engine['computational_ip'])
+ try:
+ local("~/scripts/{}.py {}".format('update_inactivity_on_start',
params))
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to update last activity time.", str(err))
+ raise Exception
+ except:
+ sys.exit(1)
+
+ try:
with open("/root/result.json", 'w') as result:
- res = {"service_base_name":
data_engine_config['service_base_name'],
+ res = {"service_base_name": data_engine['service_base_name'],
"Action": "Start Data Engine"}
print(json.dumps(res))
result.write(json.dumps(res))
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/common_start_notebook.py
b/infrastructure-provisioning/src/general/scripts/azure/common_start_notebook.py
index 0b206fe..5ecbb20 100644
---
a/infrastructure-provisioning/src/general/scripts/azure/common_start_notebook.py
+++
b/infrastructure-provisioning/src/general/scripts/azure/common_start_notebook.py
@@ -97,6 +97,20 @@ if __name__ == "__main__":
sys.exit(1)
try:
+ logging.info('[UPDATE LAST ACTIVITY TIME]')
+ print('[UPDATE LAST ACTIVITY TIME]')
+ params = '--os_user {} --notebook_ip {} --keyfile "{}"' \
+ .format(os.environ['conf_os_user'],
notebook_config['notebook_ip'], notebook_config['keyfile'])
+ try:
+ local("~/scripts/{}.py {}".format('update_inactivity_on_start',
params))
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to update last activity time.", str(err))
+ raise Exception
+ except:
+ sys.exit(1)
+
+ try:
ip_address =
AzureMeta().get_private_ip_address(notebook_config['resource_group_name'],
notebook_config['notebook_name'])
print('[SUMMARY]')
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/dataengine_start.py
b/infrastructure-provisioning/src/general/scripts/azure/dataengine_start.py
index bb0290d..af42158 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine_start.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine_start.py
@@ -79,6 +79,29 @@ if __name__ == "__main__":
sys.exit(1)
try:
+ logging.info('[UPDATE LAST ACTIVITY TIME]')
+ print('[UPDATE LAST ACTIVITY TIME]')
+ data_engine['computational_id'] = data_engine['cluster_name'] + '-m'
+ data_engine['tag_name'] = data_engine['service_base_name'] + '-Tag'
+ data_engine['notebook_ip'] =
AzureMeta().get_private_ip_address(data_engine['resource_group_name'],
os.environ['notebook_instance_name'])
+ data_engine['computational_ip'] =
AzureMeta().get_private_ip_address(data_engine['tag_name'],
+
data_engine['computational_id']).get(
+ 'Private')
+ data_engine['keyfile'] = '{}{}.pem'.format(os.environ['conf_key_dir'],
os.environ['conf_key_name'])
+ params = '--os_user {0} --notebook_ip {1} --keyfile "{2}" --cluster_ip
{3}' \
+ .format(os.environ['conf_os_user'], data_engine['notebook_ip'],
data_engine['keyfile'],
+ data_engine['computational_ip'])
+ try:
+ local("~/scripts/{}.py {}".format('update_inactivity_on_start',
params))
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to update last activity time.", str(err))
+ raise Exception
+ except:
+ sys.exit(1)
+
+
+ try:
with open("/root/result.json", 'w') as result:
res = {"service_base_name": data_engine['service_base_name'],
"Action": "Start Data Engine"}
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/common_start_notebook.py
b/infrastructure-provisioning/src/general/scripts/gcp/common_start_notebook.py
index 65795d7..6a9a020 100644
---
a/infrastructure-provisioning/src/general/scripts/gcp/common_start_notebook.py
+++
b/infrastructure-provisioning/src/general/scripts/gcp/common_start_notebook.py
@@ -73,6 +73,20 @@ if __name__ == "__main__":
sys.exit(1)
try:
+ logging.info('[UPDATE LAST ACTIVITY TIME]')
+ print('[UPDATE LAST ACTIVITY TIME]')
+ params = '--os_user {} --notebook_ip {} --keyfile "{}"' \
+ .format(os.environ['conf_os_user'],
notebook_config['notebook_ip'], notebook_config['keyfile'])
+ try:
+ local("~/scripts/{}.py {}".format('update_inactivity_on_start',
params))
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to update last activity time.", str(err))
+ raise Exception
+ except:
+ sys.exit(1)
+
+ try:
print('[SUMMARY]')
logging.info('[SUMMARY]')
print("Instance name: {}".format(notebook_config['notebook_name']))
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_start.py
b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_start.py
index d68340a..5f9d874 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_start.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_start.py
@@ -75,6 +75,28 @@ if __name__ == "__main__":
raise Exception
except:
sys.exit(1)
+
+ try:
+ logging.info('[UPDATE LAST ACTIVITY TIME]')
+ print('[UPDATE LAST ACTIVITY TIME]')
+ data_engine['computational_id'] = data_engine['cluster_name'] + '-m'
+ data_engine['tag_name'] = data_engine['service_base_name'] + '-Tag'
+ data_engine['notebook_ip'] =
GCPMeta().get_private_ip_address(os.environ['notebook_instance_name'])
+ data_engine['computational_ip'] =
GCPMeta().get_private_ip_address(data_engine['computational_id'])
+ data_engine['keyfile'] = '{}{}.pem'.format(os.environ['conf_key_dir'],
os.environ['conf_key_name'])
+ params = '--os_user {0} --notebook_ip {1} --keyfile "{2}" --cluster_ip
{3}' \
+ .format(os.environ['conf_os_user'], data_engine['notebook_ip'],
data_engine['keyfile'],
+ data_engine['computational_ip'])
+ try:
+ local("~/scripts/{}.py {}".format('update_inactivity_on_start',
params))
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to update last activity time.", str(err))
+ raise Exception
+ except:
+ sys.exit(1)
+
+
try:
with open("/root/result.json", 'w') as result:
res = {"service_base_name": data_engine['service_base_name'],
diff --git
a/infrastructure-provisioning/src/general/scripts/os/notebook_inactivity_check.py
b/infrastructure-provisioning/src/general/scripts/os/notebook_inactivity_check.py
index 29c3535..a231f07 100644
---
a/infrastructure-provisioning/src/general/scripts/os/notebook_inactivity_check.py
+++
b/infrastructure-provisioning/src/general/scripts/os/notebook_inactivity_check.py
@@ -47,7 +47,7 @@ if __name__ == "__main__":
notebook_config['tag_name'], notebook_config['notebook_name'])
notebook_config['keyfile'] =
'{}{}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
if notebook_config['resource_type'] == 'dataengine':
- notebook_config['dataengine_name'] =
'{}-m'.format(os.environ['conf_computational_id'])
+ notebook_config['dataengine_name'] =
'{}-m'.format(os.environ['computational_id'])
notebook_config['dataengine_ip'] =
get_instance_private_ip_address(
notebook_config['tag_name'],
notebook_config['dataengine_name'])
else:
diff --git
a/infrastructure-provisioning/src/general/scripts/os/update_inactivity_on_start.py
b/infrastructure-provisioning/src/general/scripts/os/update_inactivity_on_start.py
new file mode 100644
index 0000000..81043a5
--- /dev/null
+++
b/infrastructure-provisioning/src/general/scripts/os/update_inactivity_on_start.py
@@ -0,0 +1,48 @@
+#!/usr/bin/python
+
+# *****************************************************************************
+#
+# Copyright (c) 2016, EPAM SYSTEMS INC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
******************************************************************************
+
+import argparse
+from dlab.notebook_lib import *
+from dlab.actions_lib import *
+from dlab.fab import *
+from fabric.api import *
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--keyfile', type=str, default='')
+parser.add_argument('--notebook_ip', type=str, default='')
+parser.add_argument('--os_user', type=str, default='')
+parser.add_argument('--cluster_ip', type=str, default='none')
+args = parser.parse_args()
+
+
+if __name__ == "__main__":
+ env.hosts = "{}".format(args.notebook_ip)
+ env['connection_attempts'] = 100
+ env.user = args.os_user
+ env.key_filename = "{}".format(args.keyfile)
+ env.host_string = env.user + "@" + env.hosts
+
+ if args.cluster_ip == "none":
+ kernel = 'local'
+ else:
+ kernel = args.cluster_ip.replace('.', '-')
+
+ sudo("date +%s > /opt/inactivity/{}_inactivity".format(kernel))
diff --git
a/infrastructure-provisioning/src/general/templates/os/inactive.service
b/infrastructure-provisioning/src/general/templates/os/inactive.service
index fde297f..0c8f244 100644
--- a/infrastructure-provisioning/src/general/templates/os/inactive.service
+++ b/infrastructure-provisioning/src/general/templates/os/inactive.service
@@ -1,3 +1,21 @@
+# *****************************************************************************
+#
+# Copyright (c) 2016, EPAM SYSTEMS INC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
******************************************************************************
+
[Unit]
Description=inactivity check
diff --git
a/infrastructure-provisioning/src/general/templates/os/inactive.timer
b/infrastructure-provisioning/src/general/templates/os/inactive.timer
index cef5375..63acd6f 100644
--- a/infrastructure-provisioning/src/general/templates/os/inactive.timer
+++ b/infrastructure-provisioning/src/general/templates/os/inactive.timer
@@ -1,3 +1,21 @@
+# *****************************************************************************
+#
+# Copyright (c) 2016, EPAM SYSTEMS INC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
******************************************************************************
+
[Unit]
Description=inactive service
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
