Updated Branches: refs/heads/trunk bf0aee7f6 -> 13ac36e28
AMBARI-4222. Add historyserver to HDP1 on backend (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/13ac36e2 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/13ac36e2 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/13ac36e2 Branch: refs/heads/trunk Commit: 13ac36e28bfec6c40f627001aca421ceca16f627 Parents: bf0aee7 Author: Andrew Onischuk <[email protected]> Authored: Fri Jan 3 07:24:17 2014 -0800 Committer: Andrew Onischuk <[email protected]> Committed: Fri Jan 3 07:25:52 2014 -0800 ---------------------------------------------------------------------- .../HDFS/package/scripts/hdfs_namenode.py | 11 ++------- .../services/HDFS/package/scripts/params.py | 2 -- .../MAPREDUCE/configuration/mapred-site.xml | 22 ++++++++++++++++++ .../HDP/1.3.4/services/MAPREDUCE/metainfo.xml | 11 +++++++++ .../MAPREDUCE/package/scripts/historyserver.py | 24 ++++++++++---------- .../MAPREDUCE/package/scripts/jobtracker.py | 9 ++------ .../HDFS/package/scripts/hdfs_namenode.py | 4 ++-- 7 files changed, 51 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/13ac36e2/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/HDFS/package/scripts/hdfs_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/HDFS/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/HDFS/package/scripts/hdfs_namenode.py index 1132cd7..9830b63 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/HDFS/package/scripts/hdfs_namenode.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/HDFS/package/scripts/hdfs_namenode.py @@ -101,23 +101,16 @@ def create_app_directories(): mode="755" ) if len(params.hs_host) != 0: - if params.yarn_log_aggregation_enabled: - hdfs_directory(name=params.yarn_nm_app_log_dir, - owner=params.yarn_user, - group=params.user_group, - mode="1777", - recursive_chmod=True - ) hdfs_directory(name=params.mapreduce_jobhistory_intermediate_done_dir, owner=params.mapred_user, group=params.user_group, - mode="1777" + mode="777" ) hdfs_directory(name=params.mapreduce_jobhistory_done_dir, owner=params.mapred_user, group=params.user_group, - mode="1777" + mode="777" ) pass http://git-wip-us.apache.org/repos/asf/ambari/blob/13ac36e2/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/HDFS/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/HDFS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/HDFS/package/scripts/params.py index a727aec..e93c50a 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/HDFS/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/HDFS/package/scripts/params.py @@ -127,8 +127,6 @@ hbase_hdfs_root_dir = config['configurations']['hbase-site']['hbase.rootdir']#", hbase_staging_dir = "/apps/hbase/staging" hive_apps_whs_dir = config['configurations']['hive-site']["hive.metastore.warehouse.dir"] #, "/apps/hive/warehouse") webhcat_apps_dir = "/apps/webhcat" -yarn_log_aggregation_enabled = config['configurations']['yarn-site']['yarn.log-aggregation-enable']#","true") -yarn_nm_app_log_dir = config['configurations']['yarn-site']['yarn.nodemanager.remote-app-log-dir']#","/app-logs") mapreduce_jobhistory_intermediate_done_dir = config['configurations']['mapred-site']['mapreduce.jobhistory.intermediate-done-dir']#","/app-logs") mapreduce_jobhistory_done_dir = config['configurations']['mapred-site']['mapreduce.jobhistory.done-dir']#","/mr-history/done") http://git-wip-us.apache.org/repos/asf/ambari/blob/13ac36e2/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/configuration/mapred-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/configuration/mapred-site.xml b/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/configuration/mapred-site.xml index da3545c..8367581 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/configuration/mapred-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/configuration/mapred-site.xml @@ -555,6 +555,28 @@ </property> <property> + <name>mapreduce.jobhistory.intermediate-done-dir</name> + <value>/mr-history/tmp</value> + <description> + Directory where history files are written by MapReduce jobs. + </description> + </property> + + <property> + <name>mapreduce.jobhistory.done-dir</name> + <value>/mr-history/done</value> + <description> + Directory where history files are managed by the MR JobHistory Server. + </description> + </property> + + <property> Â Â Â Â + <name>mapreduce.jobhistory.webapp.address</name> + <value>localhost:19888</value> + <description>Enter your JobHistoryServer hostname.</description> + </property> + + <property> <name>mapred.jobtracker.blacklist.fault-timeout-window</name> <value>180</value> <description> http://git-wip-us.apache.org/repos/asf/ambari/blob/13ac36e2/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/metainfo.xml index a1ca7ba..1e44d2c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/metainfo.xml @@ -54,6 +54,17 @@ <timeout>600</timeout> </commandScript> </component> + + + <component> + <name>HISTORYSERVER</name> + <category>MASTER</category> + <commandScript> + <script>scripts/historyserver.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> </components> http://git-wip-us.apache.org/repos/asf/ambari/blob/13ac36e2/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/package/scripts/historyserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/package/scripts/historyserver.py b/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/package/scripts/historyserver.py index c582a02..8eb2089 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/package/scripts/historyserver.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/package/scripts/historyserver.py @@ -26,14 +26,14 @@ from mapreduce import mapreduce from service import service class Historyserver(Script): - # def install(self, env): - # self.install_packages(env) - # self.configure(env) - # - # def configure(self, env): - # import params - # env.set_params(params) - # mapreduce() + def install(self, env): + self.install_packages(env) + self.configure(env) + + def configure(self, env): + import params + env.set_params(params) + mapreduce() def start(self, env): import params @@ -50,10 +50,10 @@ class Historyserver(Script): action='stop' ) - # def status(self, env): - # import status_params - # env.set_params(status_params) - # check_process_status(status_params.historyserver_pid_file) + def status(self, env): + import status_params + env.set_params(status_params) + check_process_status(status_params.historyserver_pid_file) if __name__ == "__main__": Historyserver().execute() \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/13ac36e2/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/package/scripts/jobtracker.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/package/scripts/jobtracker.py b/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/package/scripts/jobtracker.py index ab33db7..a48cc23 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/package/scripts/jobtracker.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/package/scripts/jobtracker.py @@ -46,7 +46,6 @@ from resource_management import * from mapreduce import mapreduce from service import service -from historyserver import Historyserver class Jobtracker(Script): def install(self, env): @@ -65,9 +64,7 @@ class Jobtracker(Script): service('jobtracker', action='start' ) - - Historyserver().start(env) #TODO HS will be separate component - + def stop(self, env): import params env.set_params(params) @@ -75,9 +72,7 @@ class Jobtracker(Script): service('jobtracker', action='stop' ) - - Historyserver().stop(env) #TODO HS will be separate component - + def status(self, env): import status_params env.set_params(status_params) http://git-wip-us.apache.org/repos/asf/ambari/blob/13ac36e2/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/HDFS/package/scripts/hdfs_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/HDFS/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/HDFS/package/scripts/hdfs_namenode.py index aafaf72..ecda794 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/HDFS/package/scripts/hdfs_namenode.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/HDFS/package/scripts/hdfs_namenode.py @@ -106,13 +106,13 @@ def create_app_directories(): hdfs_directory(name=params.yarn_nm_app_log_dir, owner=params.yarn_user, group=params.user_group, - mode="1777", + mode="777", recursive_chmod=True ) hdfs_directory(name=params.mapreduce_jobhistory_intermediate_done_dir, owner=params.mapred_user, group=params.user_group, - mode="1777" + mode="777" ) hdfs_directory(name=params.mapreduce_jobhistory_done_dir,
