Repository: bigtop Updated Branches: refs/heads/master 25fee1284 -> defa03814
BIGTOP-1700: Package YARN Timeline Server service for Apache Hadoop Signed-off-by: Mark Grover <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/defa0381 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/defa0381 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/defa0381 Branch: refs/heads/master Commit: defa03814385cebd707aad6553a6a3d6d29b1164 Parents: 25fee12 Author: Youngwoo Kim <[email protected]> Authored: Mon Feb 23 15:47:07 2015 +0900 Committer: Mark Grover <[email protected]> Committed: Tue Mar 3 17:46:52 2015 -0800 ---------------------------------------------------------------------- .../hadoop/hadoop-yarn-timelineserver.svc | 24 +++++++++++ bigtop-packages/src/deb/hadoop/control | 8 ++++ bigtop-packages/src/deb/hadoop/rules | 2 +- .../src/rpm/hadoop/SPECS/hadoop.spec | 42 +++++++++++++------- 4 files changed, 60 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/defa0381/bigtop-packages/src/common/hadoop/hadoop-yarn-timelineserver.svc ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/hadoop/hadoop-yarn-timelineserver.svc b/bigtop-packages/src/common/hadoop/hadoop-yarn-timelineserver.svc new file mode 100644 index 0000000..11481cc --- /dev/null +++ b/bigtop-packages/src/common/hadoop/hadoop-yarn-timelineserver.svc @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +HADOOP_NAME="timelineserver" +DAEMON="hadoop-yarn-$HADOOP_NAME" +DESC="Hadoop $HADOOP_NAME" +EXEC_PATH="/usr/lib/hadoop-yarn/sbin/yarn-daemon.sh" +SVC_USER="yarn" +WORKING_DIR="/var/lib/hadoop-yarn" +DAEMON_FLAGS="$HADOOP_NAME" +CONF_DIR="/etc/hadoop/conf" +PIDFILE="/var/run/hadoop-yarn/yarn-$SVC_USER-$HADOOP_NAME.pid" +CHKCONFIG="2345 95 15" http://git-wip-us.apache.org/repos/asf/bigtop/blob/defa0381/bigtop-packages/src/deb/hadoop/control ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/hadoop/control b/bigtop-packages/src/deb/hadoop/control index 5234bb5..c6655bd 100644 --- a/bigtop-packages/src/deb/hadoop/control +++ b/bigtop-packages/src/deb/hadoop/control @@ -129,6 +129,14 @@ Depends: hadoop-yarn (= ${binary:Version}) Description: YARN Web Proxy The web proxy server sits in front of the YARN application master web UI. +Package: hadoop-yarn-timelineserver +Architecture: any +Depends: hadoop-yarn (= ${binary:Version}) +Description: YARN Timeline Server + Storage and retrieval of applications' current as well as historic + information in a generic fashion is solved in YARN through the + Timeline Server. + Package: hadoop-hdfs-namenode Architecture: any Depends: hadoop-hdfs (= ${binary:Version}) http://git-wip-us.apache.org/repos/asf/bigtop/blob/defa0381/bigtop-packages/src/deb/hadoop/rules ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/hadoop/rules b/bigtop-packages/src/deb/hadoop/rules index 8c2512c..58d4edc 100755 --- a/bigtop-packages/src/deb/hadoop/rules +++ b/bigtop-packages/src/deb/hadoop/rules @@ -40,7 +40,7 @@ override_dh_auto_build: bash debian/do-component-build -Divy.home=${HOME}/.ivy2 hadoop_svcs=hdfs-namenode hdfs-secondarynamenode hdfs-datanode hdfs-zkfc hdfs-journalnode \ - yarn-resourcemanager yarn-nodemanager yarn-proxyserver \ + yarn-resourcemanager yarn-nodemanager yarn-proxyserver yarn-timelineserver\ mapreduce-historyserver httpfs $(hadoop_svcs): debian/init.d.tmpl http://git-wip-us.apache.org/repos/asf/bigtop/blob/defa0381/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec b/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec index 5300a8b..d21c38f 100644 --- a/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec +++ b/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec @@ -59,7 +59,7 @@ %define httpfs_services httpfs %define mapreduce_services mapreduce-historyserver %define hdfs_services hdfs-namenode hdfs-secondarynamenode hdfs-datanode hdfs-zkfc hdfs-journalnode -%define yarn_services yarn-resourcemanager yarn-nodemanager yarn-proxyserver +%define yarn_services yarn-resourcemanager yarn-nodemanager yarn-proxyserver yarn-timelineserver %define hadoop_services %{hdfs_services} %{mapreduce_services} %{yarn_services} %{httpfs_services} # Hadoop outputs built binaries into %{hadoop_build} %define hadoop_build_path build @@ -150,7 +150,7 @@ Source7: hadoop-fuse-dfs.1 Source8: hdfs.conf Source9: yarn.conf Source10: mapreduce.conf -Source11: init.d.tmpl +Source11: init.d.tmpl Source12: hadoop-hdfs-namenode.svc Source13: hadoop-hdfs-datanode.svc Source14: hadoop-hdfs-secondarynamenode.svc @@ -168,6 +168,7 @@ Source25: httpfs-tomcat-deployment.sh Source26: yarn.1 Source27: hdfs.1 Source28: mapred.1 +Source29: hadoop-yarn-timelineserver.svc Buildroot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id} -u -n) BuildRequires: fuse-devel, fuse, cmake Requires: coreutils, /usr/sbin/useradd, /usr/sbin/usermod, /sbin/chkconfig, /sbin/service, bigtop-utils >= 0.7, zookeeper >= 3.4.0 @@ -224,9 +225,9 @@ Group: System/Daemons Requires: %{name} = %{version}-%{release}, bigtop-jsvc %description hdfs -Hadoop Distributed File System (HDFS) is the primary storage system used by -Hadoop applications. HDFS creates multiple replicas of data blocks and distributes -them on compute nodes throughout a cluster to enable reliable, extremely rapid +Hadoop Distributed File System (HDFS) is the primary storage system used by +Hadoop applications. HDFS creates multiple replicas of data blocks and distributes +them on compute nodes throughout a cluster to enable reliable, extremely rapid computations. %package yarn @@ -236,18 +237,18 @@ Requires: %{name} = %{version}-%{release} %description yarn YARN (Hadoop NextGen MapReduce) is a general purpose data-computation framework. -The fundamental idea of YARN is to split up the two major functionalities of the +The fundamental idea of YARN is to split up the two major functionalities of the JobTracker, resource management and job scheduling/monitoring, into separate daemons: ResourceManager and NodeManager. -The ResourceManager is the ultimate authority that arbitrates resources among all +The ResourceManager is the ultimate authority that arbitrates resources among all the applications in the system. The NodeManager is a per-node slave managing allocation -of computational resources on a single node. Both work in support of per-application +of computational resources on a single node. Both work in support of per-application ApplicationMaster (AM). -An ApplicationMaster is, in effect, a framework specific library and is tasked with -negotiating resources from the ResourceManager and working with the NodeManager(s) to -execute and monitor the tasks. +An ApplicationMaster is, in effect, a framework specific library and is tasked with +negotiating resources from the ResourceManager and working with the NodeManager(s) to +execute and monitor the tasks. %package mapreduce @@ -256,7 +257,7 @@ Group: System/Daemons Requires: %{name}-yarn = %{version}-%{release} %description mapreduce -Hadoop MapReduce is a programming model and software framework for writing applications +Hadoop MapReduce is a programming model and software framework for writing applications that rapidly process vast amounts of data in parallel on large clusters of compute nodes. @@ -305,8 +306,8 @@ Requires: %{name}-hdfs = %{version}-%{release} Requires(pre): %{name} = %{version}-%{release} %description hdfs-journalnode -The HDFS JournalNode is responsible for persisting NameNode edit logs. -In a typical deployment the JournalNode daemon runs on at least three +The HDFS JournalNode is responsible for persisting NameNode edit logs. +In a typical deployment the JournalNode daemon runs on at least three separate machines in the cluster. %package hdfs-datanode @@ -364,6 +365,16 @@ Requires(pre): %{name}-yarn = %{version}-%{release} %description yarn-proxyserver The web proxy server sits in front of the YARN application master web UI. +%package yarn-timelineserver +Summary: YARN Timeline Server +Group: System/Daemons +Requires: %{name}-yarn = %{version}-%{release} +Requires(pre): %{name} = %{version}-%{release} +Requires(pre): %{name}-yarn = %{version}-%{release} + +%description yarn-timelineserver +Storage and retrieval of applications' current as well as historic information in a generic fashion is solved in YARN through the Timeline Server. + %package mapreduce-historyserver Summary: MapReduce History Server Group: System/Daemons @@ -538,7 +549,7 @@ getent group hadoop >/dev/null || groupadd -r hadoop getent group hdfs >/dev/null || groupadd -r hdfs getent passwd hdfs >/dev/null || /usr/sbin/useradd --comment "Hadoop HDFS" --shell /bin/bash -M -r -g hdfs -G hadoop --home %{state_hdfs} hdfs -%pre httpfs +%pre httpfs getent group httpfs >/dev/null || groupadd -r httpfs getent passwd httpfs >/dev/null || /usr/sbin/useradd --comment "Hadoop HTTPFS" --shell /bin/bash -M -r -g httpfs -G httpfs --home %{run_httpfs} httpfs @@ -697,6 +708,7 @@ fi %service_macro yarn-resourcemanager %service_macro yarn-nodemanager %service_macro yarn-proxyserver +%service_macro yarn-timelineserver %service_macro mapreduce-historyserver # Pseudo-distributed Hadoop installation
