Repository: reef Updated Branches: refs/heads/master 1f22f2d94 -> 630124476
[REEF-1329] Add dockerfiles for HDP 2.4 HDP 2.4 was released on last month and HDInsight 3.4 also uses HDP 2.4. This PR aims to provide HDP 2.4 for REEF integration test. JIRA: [REEF-1329](https://issues.apache.org/jira/browse/REEF-1329) Pull Request: This closes #942 Project: http://git-wip-us.apache.org/repos/asf/reef/repo Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/63012447 Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/63012447 Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/63012447 Branch: refs/heads/master Commit: 630124476b5c1a41c2e043a672ec2db79ac16da0 Parents: 1f22f2d Author: Dongjoon Hyun <[email protected]> Authored: Mon Apr 11 12:35:25 2016 -0700 Committer: John Yang <[email protected]> Committed: Fri Apr 15 10:48:32 2016 +0900 ---------------------------------------------------------------------- dev/docker/README.md | 3 +- dev/docker/ubuntu12.04-jdk7-hdp2.4/Dockerfile | 49 ++++++++++++ .../ubuntu12.04-jdk7-hdp2.4/core-site.xml | 27 +++++++ .../ubuntu12.04-jdk7-hdp2.4/hdfs-site.xml | 27 +++++++ dev/docker/ubuntu12.04-jdk7-hdp2.4/init-nn.sh | 40 ++++++++++ .../ubuntu12.04-jdk7-hdp2.4/mapred-site.xml | 27 +++++++ .../ubuntu12.04-jdk7-hdp2.4/yarn-site.xml | 83 ++++++++++++++++++++ 7 files changed, 255 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/reef/blob/63012447/dev/docker/README.md ---------------------------------------------------------------------- diff --git a/dev/docker/README.md b/dev/docker/README.md index e85ff8b..447530c 100644 --- a/dev/docker/README.md +++ b/dev/docker/README.md @@ -15,6 +15,7 @@ Docker-based Test Cluster | YARN | Ubuntu 12.04 | HDP 2.2.7 (Hadoop 2.6.0) | HDInsight 3.2 | hdi3.2 | | YARN | Ubuntu 12.04 | HDP 2.2.8 (Hadoop 2.6.0) | | hdp2.2 | | YARN | Ubuntu 12.04 | HDP 2.3.2 (Hadoop 2.7.1) | | hdp2.3 | +| YARN | Ubuntu 12.04 | HDP 2.4 (Hadoop 2.7.1) | | hdp2.4 | | YARN | Ubuntu 12.04 | Apache Hadoop 2.7.2 | | apache2.7 | | MESOS | Ubuntu 12.04 | Apache Mesos 0.24~0.28 | | mesos0.24 ~ 0.28 | @@ -55,7 +56,7 @@ $ root@hnn-001-01:~# Test Apache REEF on a docker-based YARN cluster ----------------------------------------------- -Among reefrt/hdi3.1, reefrt/hdi3.2, reefrt/hdp2.2, reefrt/hdp2.3, and reefrt/apache2.7, +Among reefrt/hdi3.1, reefrt/hdi3.2, reefrt/hdp2.2 ~ 2.4, and reefrt/apache2.7, choose one and run it as described in section _Run a docker-based cluster_. If you want to test on Hadoop 2.7.2, choose reefrt/apache2.7. To simulate HDInsight 3.2, choose reefrt/hdi3.2. http://git-wip-us.apache.org/repos/asf/reef/blob/63012447/dev/docker/ubuntu12.04-jdk7-hdp2.4/Dockerfile ---------------------------------------------------------------------- diff --git a/dev/docker/ubuntu12.04-jdk7-hdp2.4/Dockerfile b/dev/docker/ubuntu12.04-jdk7-hdp2.4/Dockerfile new file mode 100644 index 0000000..e1876b3 --- /dev/null +++ b/dev/docker/ubuntu12.04-jdk7-hdp2.4/Dockerfile @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM reefrt/ubuntu12.04-jdk7 +MAINTAINER Apache REEF <[email protected]> + +# HDP 2.4.0 +RUN \ + wget http://public-repo-1.hortonworks.com/HDP/ubuntu12/2.x/updates/2.4.0.0/hdp.list -O /etc/apt/sources.list.d/hdp.list && \ + gpg --keyserver pgp.mit.edu --recv-keys B9733A7A07513CAD && \ + gpg -a --export 07513CAD | apt-key add - && \ + apt-get update && \ + apt-get install -y hadoop-2-4-0* && \ + apt-get clean +ENV HADOOP_PREFIX /usr/hdp/2.4.0.0-169/hadoop + +ENV YARN_CONF_DIR $HADOOP_PREFIX/etc/hadoop +RUN \ + echo 'HADOOP_PREFIX="/usr/hdp/2.4.0.0-169/hadoop"' >> /etc/environment && \ + echo 'HADOOP_COMMON_HOME="/usr/hdp/2.4.0.0-169/hadoop"' >> /etc/environment && \ + echo 'HADOOP_HDFS_HOME="/usr/hdp/2.4.0.0-169/hadoop"' >> /etc/environment && \ + echo 'HADOOP_MAPRED_HOME="/usr/hdp/2.4.0.0-169/hadoop"' >> /etc/environment && \ + echo 'HADOOP_YARN_HOME="/usr/hdp/2.4.0.0-169/hadoop-yarn"' >> /etc/environment && \ + echo 'HADOOP_CONF_DIR="/usr/hdp/2.4.0.0-169/hadoop/etc/hadoop"' >> /etc/environment && \ + echo 'YARN_CONF_DIR="/usr/hdp/2.4.0.0-169/hadoop/etc/hadoop"' >> /etc/environment +ENV PATH $PATH:$HADOOP_PREFIX/bin:$HADOOP_PREFIX/sbin:$HADOOP_PREFIX-yarn/bin:$HADOOP_PREFIX-yarn/sbin:$HADOOP_PREFIX-hdfs/bin:$HADOOP_PREFIX-hdfs/sbin + +COPY core-site.xml $HADOOP_PREFIX/etc/hadoop/ +COPY hdfs-site.xml $HADOOP_PREFIX/etc/hadoop/ +COPY mapred-site.xml $HADOOP_PREFIX/etc/hadoop/ +COPY yarn-site.xml $HADOOP_PREFIX/etc/hadoop/ + +COPY init-nn.sh /root/ + +EXPOSE 22 7077 8020 8030 8031 8032 8033 8040 8042 8080 8088 10000 50010 50020 50060 50070 50075 50090 http://git-wip-us.apache.org/repos/asf/reef/blob/63012447/dev/docker/ubuntu12.04-jdk7-hdp2.4/core-site.xml ---------------------------------------------------------------------- diff --git a/dev/docker/ubuntu12.04-jdk7-hdp2.4/core-site.xml b/dev/docker/ubuntu12.04-jdk7-hdp2.4/core-site.xml new file mode 100644 index 0000000..65a4e52 --- /dev/null +++ b/dev/docker/ubuntu12.04-jdk7-hdp2.4/core-site.xml @@ -0,0 +1,27 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +<configuration> + <property> + <name>fs.defaultFS</name> + <value>hdfs://hnn-001-01:9000</value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/reef/blob/63012447/dev/docker/ubuntu12.04-jdk7-hdp2.4/hdfs-site.xml ---------------------------------------------------------------------- diff --git a/dev/docker/ubuntu12.04-jdk7-hdp2.4/hdfs-site.xml b/dev/docker/ubuntu12.04-jdk7-hdp2.4/hdfs-site.xml new file mode 100644 index 0000000..33d6f22 --- /dev/null +++ b/dev/docker/ubuntu12.04-jdk7-hdp2.4/hdfs-site.xml @@ -0,0 +1,27 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +<configuration> + <property> + <name>dfs.replication</name> + <value>1</value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/reef/blob/63012447/dev/docker/ubuntu12.04-jdk7-hdp2.4/init-nn.sh ---------------------------------------------------------------------- diff --git a/dev/docker/ubuntu12.04-jdk7-hdp2.4/init-nn.sh b/dev/docker/ubuntu12.04-jdk7-hdp2.4/init-nn.sh new file mode 100755 index 0000000..d50b070 --- /dev/null +++ b/dev/docker/ubuntu12.04-jdk7-hdp2.4/init-nn.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +/usr/sbin/sshd + +/root/sync-hosts.sh + +grep hdn /etc/hosts | awk '{print $1}' | sort | uniq > $HADOOP_PREFIX/etc/hadoop/slaves +for host in `cat $HADOOP_PREFIX/etc/hadoop/slaves` +do + scp /etc/hosts $host:/etc/hosts + scp $HADOOP_PREFIX/etc/hadoop/slaves $host:$HADOOP_PREFIX/etc/hadoop/slaves +done + +hdfs namenode -format + +hadoop-daemon.sh --script hdfs start namenode +slaves.sh /usr/hdp/2.4.0.0-169/hadoop/sbin/hadoop-daemon.sh --script hdfs start datanode + +yarn-daemon.sh start resourcemanager +slaves.sh /usr/hdp/2.4.0.0-169/hadoop-yarn/sbin/yarn-daemon.sh start nodemanager + +cd ~ && /bin/bash http://git-wip-us.apache.org/repos/asf/reef/blob/63012447/dev/docker/ubuntu12.04-jdk7-hdp2.4/mapred-site.xml ---------------------------------------------------------------------- diff --git a/dev/docker/ubuntu12.04-jdk7-hdp2.4/mapred-site.xml b/dev/docker/ubuntu12.04-jdk7-hdp2.4/mapred-site.xml new file mode 100644 index 0000000..586b3eb --- /dev/null +++ b/dev/docker/ubuntu12.04-jdk7-hdp2.4/mapred-site.xml @@ -0,0 +1,27 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +<configuration> + <property> + <name>mapreduce.framework.name</name> + <value>yarn</value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/reef/blob/63012447/dev/docker/ubuntu12.04-jdk7-hdp2.4/yarn-site.xml ---------------------------------------------------------------------- diff --git a/dev/docker/ubuntu12.04-jdk7-hdp2.4/yarn-site.xml b/dev/docker/ubuntu12.04-jdk7-hdp2.4/yarn-site.xml new file mode 100644 index 0000000..41e8096 --- /dev/null +++ b/dev/docker/ubuntu12.04-jdk7-hdp2.4/yarn-site.xml @@ -0,0 +1,83 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +<configuration> + <property> + <name>yarn.resourcemanager.scheduler.address</name> + <value>hnn-001-01:8030</value> + </property> + + <property> + <name>yarn.resourcemanager.resource-tracker.address</name> + <value>hnn-001-01:8025</value> + </property> + + <property> + <name>yarn.resourcemanager.address</name> + <value>hnn-001-01:8050</value> + </property> + + <property> + <name>yarn.resourcemanager.admin.address</name> + <value>hnn-001-01:8141</value> + </property> + + <property> + <name>yarn.resourcemanager.webapp.address</name> + <value>hnn-001-01:8088</value> + </property> + + <property> + <name>yarn.log.server.url</name> + <value>http://hnn-001-01:19888/jobhistory/logs/</value> + <description>URL for job history server</description> + </property> + + <property> + <name>yarn.application.classpath</name> + <value>/usr/hdp/2.4.0.0-169/hadoop/etc/hadoop, /usr/hdp/2.4.0.0-169/hadoop/*, /usr/hdp/2.4.0.0-169/hadoop/lib/*, /usr/hdp/2.4.0.0-169/hadoop-hdfs/*, /usr/hdp/2.4.0.0-169/hadoop-hdfs/lib/*, /usr/hdp/2.4.0.0-169/hadoop-mapreduce/*, /usr/hdp/2.4.0.0-169/hadoop-mapreduce/lib/*, /usr/hdp/2.4.0.0-169/hadoop-yarn/*, /usr/hdp/2.4.0.0-169/hadoop-yarn/lib/*</value> + </property> + + <property> + <name>yarn.scheduler.minimum-allocation-mb</name> + <value>1024</value> + </property> + + <property> + <name>yarn.nodemanager.aux-services</name> + <value>mapreduce_shuffle</value> + </property> + + <property> + <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name> + <value>org.apache.hadoop.mapred.ShuffleHandler</value> + </property> + + <property> + <name>yarn.resourcemanager.scheduler.class</name> + <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value> + </property> + + <property> + <name>yarn.nodemanager.resource.memory-mb</name> + <value>87040</value> + </property> +</configuration>
