AMBARI-19577. Add Livy session recovery configurations in Ambari (Saisai Shao via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a0ba7fbe Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a0ba7fbe Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a0ba7fbe Branch: refs/heads/branch-feature-AMBARI-12556 Commit: a0ba7fbe7f80332bcff1cab9ea7dbe27ca59a003 Parents: eecb5a3 Author: Sumit Mohanty <[email protected]> Authored: Fri Feb 10 08:36:58 2017 -0800 Committer: Sumit Mohanty <[email protected]> Committed: Fri Feb 10 08:42:42 2017 -0800 ---------------------------------------------------------------------- .../SPARK/1.2.1/package/scripts/params.py | 1 + .../SPARK/1.2.1/package/scripts/setup_livy.py | 8 +++++ .../SPARK2/2.0.0/package/scripts/params.py | 1 + .../SPARK2/2.0.0/package/scripts/setup_livy2.py | 8 +++++ .../services/SPARK/configuration/livy-conf.xml | 24 +++++++++++++++ .../SPARK2/configuration/livy2-conf.xml | 24 +++++++++++++++ .../python/stacks/2.5/SPARK/test_spark_livy.py | 31 ++++++++++++++++++++ .../stacks/2.6/SPARK2/test_spark_livy2.py | 31 ++++++++++++++++++++ 8 files changed, 128 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/a0ba7fbe/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py index 82cd0b2..6a59caf 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py @@ -222,6 +222,7 @@ if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY, stac user_group = status_params.user_group livy_hdfs_user_dir = format("/user/{livy_user}") livy_server_pid_file = status_params.livy_server_pid_file + livy_recovery_dir = default("/configurations/livy-conf/livy.server.recovery.state-store.url", "/livy-recovery") livy_server_start = format("{livy_home}/bin/livy-server start") livy_server_stop = format("{livy_home}/bin/livy-server stop") http://git-wip-us.apache.org/repos/asf/ambari/blob/a0ba7fbe/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_livy.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_livy.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_livy.py index 07b704f..32615c3 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_livy.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_livy.py @@ -40,6 +40,14 @@ def setup_livy(env, type, upgrade_type = None, action = None): ) params.HdfsResource(None, action="execute") + params.HdfsResource(params.livy_recovery_dir, + type="directory", + action="create_on_execute", + owner=params.livy_user, + mode=0775 + ) + params.HdfsResource(None, action="execute") + # create livy-env.sh in etc/conf dir File(os.path.join(params.livy_conf, 'livy-env.sh'), owner=params.livy_user, http://git-wip-us.apache.org/repos/asf/ambari/blob/a0ba7fbe/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py index c19f88c..b6889e4 100755 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py @@ -207,6 +207,7 @@ if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY2, sta user_group = status_params.user_group livy2_hdfs_user_dir = format("/user/{livy2_user}") livy2_server_pid_file = status_params.livy2_server_pid_file + livy2_recovery_dir = default("/configurations/livy2-conf/livy.server.recovery.state-store.url", "/livy2-recovery") livy2_server_start = format("{livy2_home}/bin/livy-server start") livy2_server_stop = format("{livy2_home}/bin/livy-server stop") http://git-wip-us.apache.org/repos/asf/ambari/blob/a0ba7fbe/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py index f2178f7..2e92509 100644 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py @@ -40,6 +40,14 @@ def setup_livy(env, type, upgrade_type = None, action = None): ) params.HdfsResource(None, action="execute") + params.HdfsResource(params.livy2_recovery_dir, + type="directory", + action="create_on_execute", + owner=params.livy2_user, + mode=0775 + ) + params.HdfsResource(None, action="execute") + # create livy-env.sh in etc/conf dir File(os.path.join(params.livy2_conf, 'livy-env.sh'), owner=params.livy2_user, http://git-wip-us.apache.org/repos/asf/ambari/blob/a0ba7fbe/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/livy-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/livy-conf.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/livy-conf.xml index 23583c0..28eeab5 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/livy-conf.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/livy-conf.xml @@ -78,4 +78,28 @@ </description> <on-ambari-upgrade add="true"/> </property> + <property> + <name>livy.server.recovery.mode</name> + <value>recovery</value> + <description> + Recovery mode for livy, either be "off" or "recovery". + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>livy.server.recovery.state-store</name> + <value>filesystem</value> + <description> + Where Livy should store state for recovery. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>livy.server.recovery.state-store.url</name> + <value>/livy-recovery</value> + <description> + Where Livy should store state for recovery. + </description> + <on-ambari-upgrade add="false"/> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/a0ba7fbe/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml index 231ef70..c8a65bd 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml @@ -85,6 +85,30 @@ Whether to enable HiveContext in livy interpreter </description> <on-ambari-upgrade add="false"/> + <property> + </property> + <name>livy.server.recovery.mode</name> + <value>recovery</value> + <description> + Recovery mode for livy, either be "off" or "recovery". + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>livy.server.recovery.state-store</name> + <value>filesystem</value> + <description> + Where Livy should store state for recovery. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>livy.server.recovery.state-store.url</name> + <value>/livy2-recovery</value> + <description> + Where Livy should store state for recovery. + </description> + <on-ambari-upgrade add="false"/> </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/a0ba7fbe/ambari-server/src/test/python/stacks/2.5/SPARK/test_spark_livy.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/SPARK/test_spark_livy.py b/ambari-server/src/test/python/stacks/2.5/SPARK/test_spark_livy.py index b56474a..b9199c7 100644 --- a/ambari-server/src/test/python/stacks/2.5/SPARK/test_spark_livy.py +++ b/ambari-server/src/test/python/stacks/2.5/SPARK/test_spark_livy.py @@ -84,6 +84,37 @@ class TestSparkClient(RMFTestCase): dfs_type = '', hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', ) + self.assertResourceCalled('HdfsResource', '/livy-recovery', + immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, + security_enabled = False, + hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin', + keytab = UnknownConfigurationMock(), + default_fs = 'hdfs://c6401.ambari.apache.org:8020', + hdfs_site = {u'a': u'b'}, + kinit_path_local = '/usr/bin/kinit', + principal_name = UnknownConfigurationMock(), + user = 'hdfs', + owner = 'livy', + hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', + type = 'directory', + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', + dfs_type = '', + mode = 0775, + ) + self.assertResourceCalled('HdfsResource', None, + immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, + security_enabled = False, + hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin', + keytab = UnknownConfigurationMock(), + default_fs = 'hdfs://c6401.ambari.apache.org:8020', + hdfs_site = {u'a': u'b'}, + kinit_path_local = '/usr/bin/kinit', + principal_name = UnknownConfigurationMock(), + user = 'hdfs', + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', + dfs_type = '', + hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', + ) self.assertResourceCalled('File', '/usr/hdp/current/livy-server/conf/livy-env.sh', content = InlineTemplate(self.getConfig()['configurations']['livy-env']['content']), owner = 'livy', http://git-wip-us.apache.org/repos/asf/ambari/blob/a0ba7fbe/ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py b/ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py index 6244d87..75aec84 100644 --- a/ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py +++ b/ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py @@ -84,6 +84,37 @@ class TestSparkClient(RMFTestCase): dfs_type = '', hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', ) + self.assertResourceCalled('HdfsResource', '/livy2-recovery', + immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, + security_enabled = False, + hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin', + keytab = UnknownConfigurationMock(), + default_fs = 'hdfs://c6401.ambari.apache.org:8020', + hdfs_site = {u'a': u'b'}, + kinit_path_local = '/usr/bin/kinit', + principal_name = UnknownConfigurationMock(), + user = 'hdfs', + owner = 'livy', + hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', + type = 'directory', + action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', + dfs_type = '', + mode = 0775, + ) + self.assertResourceCalled('HdfsResource', None, + immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, + security_enabled = False, + hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin', + keytab = UnknownConfigurationMock(), + default_fs = 'hdfs://c6401.ambari.apache.org:8020', + hdfs_site = {u'a': u'b'}, + kinit_path_local = '/usr/bin/kinit', + principal_name = UnknownConfigurationMock(), + user = 'hdfs', + action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', + dfs_type = '', + hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', + ) self.assertResourceCalled('File', '/usr/hdp/current/livy2-server/conf/livy-env.sh', content = InlineTemplate(self.getConfig()['configurations']['livy2-env']['content']), owner = 'livy',
