Repository: hadoop Updated Branches: refs/heads/docker-hadoop-2 [created] 2c0182759
HADOOP-15084. Create docker images for latest stable hadoop2 build. Contributed by Elek, Marton. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2c018275 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2c018275 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2c018275 Branch: refs/heads/docker-hadoop-2 Commit: 2c0182759e1102fcc293582a42cfee3ed4e069ce Parents: Author: Anu Engineer <[email protected]> Authored: Tue Mar 6 15:41:53 2018 -0800 Committer: Anu Engineer <[email protected]> Committed: Tue Mar 6 15:50:01 2018 -0800 ---------------------------------------------------------------------- .gitignore | 1 + Dockerfile | 21 +++++++++++++++++++++ README.md | 23 +++++++++++++++++++++++ build.sh | 27 +++++++++++++++++++++++++++ config | 38 ++++++++++++++++++++++++++++++++++++++ docker-compose.yaml | 47 +++++++++++++++++++++++++++++++++++++++++++++++ log4j.properties | 19 +++++++++++++++++++ 7 files changed, 176 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c018275/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bee8a64 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__ http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c018275/Dockerfile ---------------------------------------------------------------------- diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7d3d755 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM apache/hadoop-runner +ARG HADOOP_URL=https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=hadoop/common/hadoop-2.9.0/hadoop-2.9.0.tar.gz +WORKDIR /opt +RUN sudo rm -rf /opt/hadoop && wget $HADOOP_URL -O hadoop.tar.gz && tar zxf hadoop.tar.gz && rm hadoop.tar.gz && mv hadoop* hadoop && rm -rf /opt/hadoop/share/doc +WORKDIR /opt/hadoop +ADD log4j.properties /opt/hadoop/etc/hadoop/log4j.properties http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c018275/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md new file mode 100644 index 0000000..90607ee --- /dev/null +++ b/README.md @@ -0,0 +1,23 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +Please use the included docker-compose.yaml to test it: + +``` +docker-compose build +docker-compose up -d +``` http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c018275/build.sh ---------------------------------------------------------------------- diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..07e3fb2 --- /dev/null +++ b/build.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +set -e +mkdir -p build +if [ ! -d "$DIR/build/apache-rat-0.12" ]; then + wget "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=creadur/apache-rat-0.12/apache-rat-0.12-bin.tar.gz"; -O "$DIR/build/apache-rat.tar.gz" + cd $DIR/build + tar zvxf apache-rat.tar.gz + cd - +fi +java -jar $DIR/build/apache-rat-0.12/apache-rat-0.12.jar $DIR -e public -e apache-rat-0.12 -e .git -e .gitignore +docker build -t apache/hadoop:2 . http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c018275/config ---------------------------------------------------------------------- diff --git a/config b/config new file mode 100644 index 0000000..9071494 --- /dev/null +++ b/config @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CORE-SITE.XML_fs.default.name=hdfs://namenode +CORE-SITE.XML_fs.defaultFS=hdfs://namenode +HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:8020 +HDFS-SITE.XML_dfs.replication=1 +MAPRED-SITE.XML_mapreduce.framework.name=yarn +YARN-SITE.XML_yarn.resourcemanager.hostname=resourcemanager +YARN-SITE.XML_yarn.nodemanager.pmem-check-enabled=false +YARN-SITE.XML_yarn.nodemanager.delete.debug-delay-sec=600 +YARN-SITE.XML_yarn.nodemanager.vmem-check-enabled=false +YARN-SITE.XML_yarn.nodemanager.aux-services=mapreduce_shuffle +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-applications=10000 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-am-resource-percent=0.1 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.queues=default +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.capacity=100 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.user-limit-factor=1 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-capacity=100 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.state=RUNNING +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_submit_applications=* +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_administer_queue=* +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.node-locality-delay=40 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings= +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings-override.enable=false http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c018275/docker-compose.yaml ---------------------------------------------------------------------- diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..a500f83 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "2" +services: + namenode: + build: . + hostname: namenode + command: ["hdfs", "namenode"] + ports: + - 50070:50070 + env_file: + - ./config + environment: + ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name" + datanode: + build: . + command: ["hdfs", "datanode"] + env_file: + - ./config + resourcemanager: + build: . + hostname: resourcemanager + command: ["yarn", "resourcemanager"] + ports: + - 8088:8088 + env_file: + - ./config + volumes: + - ./test.sh:/opt/test.sh + nodemanager: + build: . + command: ["yarn", "nodemanager"] + env_file: + - ./config http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c018275/log4j.properties ---------------------------------------------------------------------- diff --git a/log4j.properties b/log4j.properties new file mode 100644 index 0000000..a481d09 --- /dev/null +++ b/log4j.properties @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +log4j.appender.stdout.layout: org.apache.log4j.PatternLayout +log4j.rootLogger: INFO, stdout +log4j.appender.stdout: org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout.ConversionPattern: %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
