This is an automated email from the ASF dual-hosted git repository.
arvindsh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/fluo-muchos.git
The following commit(s) were added to refs/heads/master by this push:
new 0f2ba91 Configure Job history server (#327)
0f2ba91 is described below
commit 0f2ba916628b93d5509fbdc9f69029231dcf049d
Author: Karthick Narendran <[email protected]>
AuthorDate: Fri Mar 6 19:44:46 2020 +0000
Configure Job history server (#327)
* Configure Job history server
* Removed 'grep -v grep' from jps
---
ansible/hadoop.yml | 1 +
.../tasks/start-jhs.yml} | 21 ++++++----
ansible/roles/hadoop/templates/mapred-site.xml | 32 +++++++++++++++
ansible/roles/hadoop/templates/yarn-site.xml | 48 ++++++++++++++++++++++
ansible/roles/zookeeper/tasks/start-zookeeper.yml | 6 +++
5 files changed, 100 insertions(+), 8 deletions(-)
diff --git a/ansible/hadoop.yml b/ansible/hadoop.yml
index 3c843b8..c4832ed 100644
--- a/ansible/hadoop.yml
+++ b/ansible/hadoop.yml
@@ -46,3 +46,4 @@
- hosts: resourcemanager
tasks:
- import_tasks: roles/hadoop/tasks/start-yarn.yml
+ - import_tasks: roles/hadoop/tasks/start-jhs.yml
diff --git a/ansible/roles/zookeeper/tasks/start-zookeeper.yml
b/ansible/roles/hadoop/tasks/start-jhs.yml
similarity index 59%
copy from ansible/roles/zookeeper/tasks/start-zookeeper.yml
copy to ansible/roles/hadoop/tasks/start-jhs.yml
index 2bbf0b3..f701f4e 100644
--- a/ansible/roles/zookeeper/tasks/start-zookeeper.yml
+++ b/ansible/roles/hadoop/tasks/start-jhs.yml
@@ -15,11 +15,16 @@
# limitations under the License.
#
-- name: "ensure zookeeper data dir exists"
- file: path={{ default_data_dirs[0] }}/zookeeper state=directory
-- name: "ensure myid file is set"
- template: src=roles/zookeeper/templates/myid dest={{ default_data_dirs[0]
}}/zookeeper/myid
-- name: "start zookeeper"
- shell: "{{ zookeeper_home }}/bin/zkServer.sh start"
- register: zk_start
- changed_when: "'STARTED' in zk_start.stdout"
+- name: Check if JobHistoryServer is running
+ shell: jps | grep "JobHistoryServer"
+ ignore_errors: yes
+ changed_when: false
+ register: jhs_status
+
+- name: Start Job History Server on Hadoop 3.x
+ command: "nohup {{ hadoop_home }}/bin/mapred --daemon start historyserver"
+ when: jhs_status.rc == 1 and hadoop_major_version == '3'
+
+- name: Start Job History Server on Hadoop 2.x
+ command: "nohup {{ hadoop_home }}/sbin/mr-jobhistory-daemon.sh start
historyserver"
+ when: jhs_status.rc == 1 and hadoop_major_version == '2'
diff --git a/ansible/roles/hadoop/templates/mapred-site.xml
b/ansible/roles/hadoop/templates/mapred-site.xml
index 04c9219..2088ce3 100644
--- a/ansible/roles/hadoop/templates/mapred-site.xml
+++ b/ansible/roles/hadoop/templates/mapred-site.xml
@@ -60,4 +60,36 @@
<value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*,$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*,$HADOOP_MAPRED_HOME/share/hadoop/common/*,$HADOOP_MAPRED_HOME/share/hadoop/common/lib/*,$HADOOP_MAPRED_HOME/share/hadoop/yarn/*,$HADOOP_MAPRED_HOME/share/hadoop/yarn/lib/*,$HADOOP_MAPRED_HOME/share/hadoop/hdfs/*,$HADOOP_MAPRED_HOME/share/hadoop/hdfs/lib/*,$HADOOP_MAPRED_HOME/share/hadoop/tools/lib/*,${HADOOP_HOME}/share/hadoop/client/*</value>
</property>
{% endif %}
+ <property>
+ <name>mapreduce.jobhistory.address</name>
+ <value>{{ groups['resourcemanager'][0] }}:10020</value>
+ </property>
+ <property>
+ <name>mapreduce.jobhistory.webapp.address</name>
+ <value>{{ groups['resourcemanager'][0] }}:19888</value>
+ </property>
+ <property>
+ <name>mapreduce.jobhistory.done-dir</name>
+ <value>/mr-history/done</value>
+ </property>
+ <property>
+ <name>mapreduce.jobhistory.http.policy</name>
+ <value>HTTP_ONLY</value>
+ </property>
+ <property>
+ <name>mapreduce.jobhistory.intermediate-done-dir</name>
+ <value>/mr-history/tmp</value>
+ </property>
+ <property>
+ <name>mapreduce.jobhistory.recovery.enable</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>mapreduce.jobhistory.recovery.store.class</name>
+
<value>org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService</value>
+ </property>
+ <property>
+ <name>mapreduce.jobhistory.recovery.store.leveldb.path</name>
+ <value>{{ worker_data_dirs[0] }}/hadoop/mapreduce/jhs</value>
+ </property>
</configuration>
diff --git a/ansible/roles/hadoop/templates/yarn-site.xml
b/ansible/roles/hadoop/templates/yarn-site.xml
index 5d9c4d0..b4e03c4 100644
--- a/ansible/roles/hadoop/templates/yarn-site.xml
+++ b/ansible/roles/hadoop/templates/yarn-site.xml
@@ -86,4 +86,52 @@
<value>${HADOOP_HOME}/share/hadoop/tools/lib/*,${HADOOP_HOME}/share/hadoop/hdfs/lib/*,${HADOOP_HOME}/share/hadoop/common/lib/*,${HADOOP_HOME}/share/hadoop/yarn/*,${HADOOP_HOME}/share/hadoop/yarn/lib/*,${HADOOP_HOME}/share/hadoop/hdfs/*,${HADOOP_HOME}/share/hadoop/common/*,${HADOOP_HOME}/share/hadoop/mapreduce/*,${HADOOP_HOME}/share/hadoop/mapreduce/lib/*,${HADOOP_HOME}/share/hadoop/client/*</value>
</property>
{% endif %}
+ <property>
+ <name>yarn.resourcemanager.webapp.address</name>
+ <value>{{ groups['resourcemanager'][0] }}:8088</value>
+ </property>
+ <property>
+ <name>yarn.log.server.url</name>
+ <value>http://{{ groups['resourcemanager'][0]
}}:19888/jobhistory/logs</value>
+ </property>
+ <property>
+ <name>yarn.log-aggregation-enable</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>yarn.log-aggregation.retain-seconds</name>
+ <value>2592000</value>
+ </property>
+ <property>
+ <name>yarn.log.server.web-service.url</name>
+ <value>http://{{ groups['resourcemanager'][0]
}}:8188/ws/v1/applicationhistory</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.log-aggregation.compression-type</name>
+ <value>gz</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.log-aggregation.debug-enabled</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.log-aggregation.num-log-files-per-app</name>
+ <value>30</value>
+ </property>
+ <property>
+
<name>yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</name>
+ <value>3600</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.log.retain-seconds</name>
+ <value>604800</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.remote-app-log-dir</name>
+ <value>/app-logs</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.remote-app-log-dir-suffix</name>
+ <value>logs</value>
+ </property>
</configuration>
diff --git a/ansible/roles/zookeeper/tasks/start-zookeeper.yml
b/ansible/roles/zookeeper/tasks/start-zookeeper.yml
index 2bbf0b3..cb2f9dc 100644
--- a/ansible/roles/zookeeper/tasks/start-zookeeper.yml
+++ b/ansible/roles/zookeeper/tasks/start-zookeeper.yml
@@ -19,7 +19,13 @@
file: path={{ default_data_dirs[0] }}/zookeeper state=directory
- name: "ensure myid file is set"
template: src=roles/zookeeper/templates/myid dest={{ default_data_dirs[0]
}}/zookeeper/myid
+- name: Check if Zookeeper is running
+ shell: jps | grep "QuorumPeerMain"
+ ignore_errors: yes
+ changed_when: false
+ register: zookeeper_status
- name: "start zookeeper"
shell: "{{ zookeeper_home }}/bin/zkServer.sh start"
register: zk_start
changed_when: "'STARTED' in zk_start.stdout"
+ when: zookeeper_status.rc == 1
