Repository: ambari Updated Branches: refs/heads/trunk b69ef96b0 -> eca4e54ee
AMBARI-19297 Decrease time it takes to run Oozie Service Check (dsen) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/eca4e54e Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/eca4e54e Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/eca4e54e Branch: refs/heads/trunk Commit: eca4e54ee88bccf2ea8342d421dac5362fc6c3a3 Parents: b69ef96 Author: Dmytro Sen <[email protected]> Authored: Tue Dec 27 12:42:31 2016 +0200 Committer: Dmytro Sen <[email protected]> Committed: Tue Dec 27 12:42:41 2016 +0200 ---------------------------------------------------------------------- .../OOZIE/4.0.0.2.0/configuration/oozie-env.xml | 8 ++++++++ .../OOZIE/4.0.0.2.0/package/files/oozieSmoke2.sh | 11 ++++++----- .../package/files/prepareOozieHdfsDirectories.sh | 15 ++++++++------- .../OOZIE/4.0.0.2.0/package/scripts/params_linux.py | 2 ++ .../OOZIE/4.0.0.2.0/package/scripts/service_check.py | 8 ++++---- .../stacks/2.0.6/OOZIE/test_oozie_service_check.py | 3 ++- .../python/stacks/2.0.6/OOZIE/test_service_check.py | 4 ++-- 7 files changed, 32 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/eca4e54e/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-env.xml b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-env.xml index dbc781f..7748176 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-env.xml +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-env.xml @@ -223,4 +223,12 @@ export JAVA_LIBRARY_PATH={{hadoop_lib_home}}/native/Linux-{{architecture}}-64 </value-attributes> <on-ambari-upgrade add="true"/> </property> + <property> + <name>service_check_job_name</name> + <value>no-op</value> + <description> + Job name from Oozie examples that will be executed at each Oozie service check action. + </description> + <on-ambari-upgrade add="true"/> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/eca4e54e/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/files/oozieSmoke2.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/files/oozieSmoke2.sh b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/files/oozieSmoke2.sh index 71a0c85..60716ae 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/files/oozieSmoke2.sh +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/files/oozieSmoke2.sh @@ -29,10 +29,11 @@ export oozie_examples_dir=$6 export hadoop_conf_dir=$7 export hadoop_bin_dir=$8 export smoke_test_user=$9 -export security_enabled=${10} -export smoke_user_keytab=${11} -export kinit_path_local=${12} -export smokeuser_principal=${13} +export job_name=${10} +export security_enabled=${11} +export smoke_user_keytab=${12} +export kinit_path_local=${13} +export smokeuser_principal=${14} function checkOozieJobStatus { local job_id=$1 @@ -74,7 +75,7 @@ else kinitcmd="" fi -cmd="${kinitcmd}source ${oozie_conf_dir}/oozie-env.sh ; ${oozie_bin_dir}/oozie -Doozie.auth.token.cache=false job -oozie $OOZIE_SERVER -config $oozie_examples_dir/examples/apps/map-reduce/job.properties -run" +cmd="${kinitcmd}source ${oozie_conf_dir}/oozie-env.sh ; ${oozie_bin_dir}/oozie -Doozie.auth.token.cache=false job -oozie $OOZIE_SERVER -config $oozie_examples_dir/examples/apps/${job_name}/job.properties -run" echo $cmd job_info=`/var/lib/ambari-agent/ambari-sudo.sh su ${smoke_test_user} -s /bin/bash - -c "$cmd" | grep "job:"` job_id="`echo $job_info | cut -d':' -f2`" http://git-wip-us.apache.org/repos/asf/ambari/blob/eca4e54e/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/files/prepareOozieHdfsDirectories.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/files/prepareOozieHdfsDirectories.sh b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/files/prepareOozieHdfsDirectories.sh index 722c047..f2bee2d 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/files/prepareOozieHdfsDirectories.sh +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/files/prepareOozieHdfsDirectories.sh @@ -26,16 +26,17 @@ export hadoop_conf_dir=$3 export JOBTRACKER=$4 export NAMENODE=$5 export QUEUE=$6 +export JOB_NAME=$7 cd $oozie_examples_dir /var/lib/ambari-agent/ambari-sudo.sh tar -zxf oozie-examples.tar.gz /var/lib/ambari-agent/ambari-sudo.sh chmod -R o+rx examples -/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|nameNode=hdfs://localhost:8020|nameNode=$NAMENODE|g" examples/apps/map-reduce/job.properties -/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|nameNode=hdfs://localhost:9000|nameNode=$NAMENODE|g" examples/apps/map-reduce/job.properties -/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|jobTracker=localhost:8021|jobTracker=$JOBTRACKER|g" examples/apps/map-reduce/job.properties -/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|jobTracker=localhost:9001|jobTracker=$JOBTRACKER|g" examples/apps/map-reduce/job.properties -/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|jobTracker=localhost:8032|jobTracker=$JOBTRACKER|g" examples/apps/map-reduce/job.properties -/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|queueName=default|queueName=$QUEUE|g" examples/apps/map-reduce/job.properties -/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|oozie.wf.application.path=hdfs://localhost:9000|oozie.wf.application.path=$NAMENODE|g" examples/apps/map-reduce/job.properties +/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|nameNode=hdfs://localhost:8020|nameNode=$NAMENODE|g" examples/apps/$JOB_NAME/job.properties +/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|nameNode=hdfs://localhost:9000|nameNode=$NAMENODE|g" examples/apps/$JOB_NAME/job.properties +/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|jobTracker=localhost:8021|jobTracker=$JOBTRACKER|g" examples/apps/$JOB_NAME/job.properties +/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|jobTracker=localhost:9001|jobTracker=$JOBTRACKER|g" examples/apps/$JOB_NAME/job.properties +/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|jobTracker=localhost:8032|jobTracker=$JOBTRACKER|g" examples/apps/$JOB_NAME/job.properties +/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|queueName=default|queueName=$QUEUE|g" examples/apps/$JOB_NAME/job.properties +/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|oozie.wf.application.path=hdfs://localhost:9000|oozie.wf.application.path=$NAMENODE|g" examples/apps/$JOB_NAME/job.properties http://git-wip-us.apache.org/repos/asf/ambari/blob/eca4e54e/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py index 284794d..dd4e373 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py @@ -208,6 +208,8 @@ if https_port is None and 'oozie.https.port' in config['configurations']['oozie- oozie_base_url = config['configurations']['oozie-site']['oozie.base.url'] +service_check_job_name = default("/configurations/oozie-env/service_check_job_name", "no-op") + # construct proper url for https if https_port is not None: parsed_url = urlparse(oozie_base_url) http://git-wip-us.apache.org/repos/asf/ambari/blob/eca4e54e/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/service_check.py index 1a3ab08..ae7cb21 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/service_check.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/service_check.py @@ -70,7 +70,7 @@ class OozieServiceCheckDefault(OozieServiceCheck): raise Fail(format(NO_DOCS_FOLDER_MESSAGE)) oozie_examples_dir = oozie_examples_dir_regex_matches[0] - Execute((format("{tmp_dir}/{prepare_hdfs_file_name}"), params.conf_dir, oozie_examples_dir, params.hadoop_conf_dir, params.yarn_resourcemanager_address, params.fs_root, params.service_check_queue_name), + Execute((format("{tmp_dir}/{prepare_hdfs_file_name}"), params.conf_dir, oozie_examples_dir, params.hadoop_conf_dir, params.yarn_resourcemanager_address, params.fs_root, params.service_check_queue_name, params.service_check_job_name), tries=3, try_sleep=5, logoutput=True @@ -112,10 +112,10 @@ class OozieServiceCheckDefault(OozieServiceCheck): if params.security_enabled: sh_cmd = format( - "{tmp_dir}/{file_name} {os_family} {oozie_lib_dir} {conf_dir} {oozie_bin_dir} {oozie_base_url} {oozie_examples_dir} {hadoop_conf_dir} {hadoop_bin_dir} {smokeuser} {security_enabled} {smokeuser_keytab} {kinit_path_local} {smokeuser_principal}") + "{tmp_dir}/{file_name} {os_family} {oozie_lib_dir} {conf_dir} {oozie_bin_dir} {oozie_base_url} {oozie_examples_dir} {hadoop_conf_dir} {hadoop_bin_dir} {smokeuser} {service_check_job_name} {security_enabled} {smokeuser_keytab} {kinit_path_local} {smokeuser_principal}") else: sh_cmd = format( - "{tmp_dir}/{file_name} {os_family} {oozie_lib_dir} {conf_dir} {oozie_bin_dir} {oozie_base_url} {oozie_examples_dir} {hadoop_conf_dir} {hadoop_bin_dir} {smokeuser} {security_enabled}") + "{tmp_dir}/{file_name} {os_family} {oozie_lib_dir} {conf_dir} {oozie_bin_dir} {oozie_base_url} {oozie_examples_dir} {hadoop_conf_dir} {hadoop_bin_dir} {smokeuser} {service_check_job_name} {security_enabled}") Execute(sh_cmd, path=params.execute_path, @@ -137,4 +137,4 @@ class OozieServiceCheckWindows(OozieServiceCheck): if __name__ == "__main__": OozieServiceCheck().execute() - + http://git-wip-us.apache.org/repos/asf/ambari/blob/eca4e54e/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_service_check.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_service_check.py index c3b4ce6..468f602 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_service_check.py +++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_service_check.py @@ -40,6 +40,7 @@ class TestOozieServiceCheck(RMFTestCase): json_content['commandParams']['version'] = version json_content['hostLevelParams']['stack_name'] = 'HDP' json_content['hostLevelParams']['stack_version'] = '2.2' + json_content['configurations']['oozie-env']['service_check_job_name'] = 'map-reduce' mocks_dict = {} self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/service_check.py", @@ -64,7 +65,7 @@ class TestOozieServiceCheck(RMFTestCase): mode = 0755) self.assertResourceCalled('Execute', - ('/tmp/prepareOozieHdfsDirectories.sh', '/usr/hdp/current/oozie-client/conf', 'examples-dir', '/usr/hdp/current/hadoop-client/conf', 'c6402.ambari.apache.org:8050', 'hdfs://c6401.ambari.apache.org:8020', 'default'), + ('/tmp/prepareOozieHdfsDirectories.sh', '/usr/hdp/current/oozie-client/conf', 'examples-dir', '/usr/hdp/current/hadoop-client/conf', 'c6402.ambari.apache.org:8050', 'hdfs://c6401.ambari.apache.org:8020', 'default', 'map-reduce'), tries = 3, try_sleep = 5, logoutput = True) http://git-wip-us.apache.org/repos/asf/ambari/blob/eca4e54e/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_service_check.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_service_check.py index 2626578..bfa7c01 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_service_check.py +++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_service_check.py @@ -62,7 +62,7 @@ class TestServiceCheck(RMFTestCase): content = StaticFile('prepareOozieHdfsDirectories.sh'), mode = 0755, ) - self.assertResourceCalled('Execute', ('/tmp/prepareOozieHdfsDirectories.sh', '/etc/oozie/conf', '/', '/etc/hadoop/conf', 'c6402.ambari.apache.org:8050', 'hdfs://c6401.ambari.apache.org:8020', 'default'), + self.assertResourceCalled('Execute', ('/tmp/prepareOozieHdfsDirectories.sh', '/etc/oozie/conf', '/', '/etc/hadoop/conf', 'c6402.ambari.apache.org:8050', 'hdfs://c6401.ambari.apache.org:8020', 'default', 'no-op'), logoutput = True, tries = 3, try_sleep = 5, @@ -146,7 +146,7 @@ class TestServiceCheck(RMFTestCase): action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', hadoop_conf_dir = '/etc/hadoop/conf', ) - self.assertResourceCalled('Execute', '/tmp/oozieSmoke2.sh suse /var/lib/oozie /etc/oozie/conf /usr/bin http://c6402.ambari.apache.org:11000/oozie / /etc/hadoop/conf /usr/bin ambari-qa False', + self.assertResourceCalled('Execute', '/tmp/oozieSmoke2.sh suse /var/lib/oozie /etc/oozie/conf /usr/bin http://c6402.ambari.apache.org:11000/oozie / /etc/hadoop/conf /usr/bin ambari-qa no-op False', logoutput = True, path = ['/usr/bin:/usr/bin'], tries = 3,
