Repository: hadoop Updated Branches: refs/heads/docker-hadoop-3 [created] f309d08bb
HADOOP-15256. Create docker images for latest stable hadoop3 build. Contributed by Elek, Marton. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f309d08b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f309d08b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f309d08b Branch: refs/heads/docker-hadoop-3 Commit: f309d08bb1b25f92a8bc55d263c1534514a79f75 Parents: Author: Anu Engineer <aengin...@apache.org> Authored: Tue Mar 6 15:53:56 2018 -0800 Committer: Anu Engineer <aengin...@apache.org> Committed: Tue Mar 6 15:53:56 2018 -0800 ---------------------------------------------------------------------- .gitignore | 1 + Dockerfile | 21 +++++++++++++++++++++ README.md | 23 +++++++++++++++++++++++ build.sh | 27 +++++++++++++++++++++++++++ config | 38 ++++++++++++++++++++++++++++++++++++++ docker-compose.yaml | 47 +++++++++++++++++++++++++++++++++++++++++++++++ log4j.properties | 19 +++++++++++++++++++ 7 files changed, 176 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f309d08b/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bee8a64 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__ http://git-wip-us.apache.org/repos/asf/hadoop/blob/f309d08b/Dockerfile ---------------------------------------------------------------------- diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..9a69285 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM apache/hadoop-runner +ARG HADOOP_URL=https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=hadoop/common/hadoop-3.0.0/hadoop-3.0.0.tar.gz +WORKDIR /opt +RUN sudo rm -rf /opt/hadoop && wget $HADOOP_URL -O hadoop.tar.gz && tar zxf hadoop.tar.gz && rm hadoop.tar.gz && mv hadoop* hadoop && rm -rf /opt/hadoop/share/doc +WORKDIR /opt/hadoop +ADD log4j.properties /opt/hadoop/etc/hadoop/log4j.properties http://git-wip-us.apache.org/repos/asf/hadoop/blob/f309d08b/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md new file mode 100644 index 0000000..90607ee --- /dev/null +++ b/README.md @@ -0,0 +1,23 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +Please use the included docker-compose.yaml to test it: + +``` +docker-compose build +docker-compose up -d +``` http://git-wip-us.apache.org/repos/asf/hadoop/blob/f309d08b/build.sh ---------------------------------------------------------------------- diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..85aacd5 --- /dev/null +++ b/build.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +set -e +mkdir -p build +if [ ! -d "$DIR/build/apache-rat-0.12" ]; then + wget "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=creadur/apache-rat-0.12/apache-rat-0.12-bin.tar.gz"; -O "$DIR/build/apache-rat.tar.gz" + cd $DIR/build + tar zvxf apache-rat.tar.gz + cd - +fi +java -jar $DIR/build/apache-rat-0.12/apache-rat-0.12.jar $DIR -e public -e apache-rat-0.12 -e .git -e .gitignore +docker build -t apache/hadoop:3 . http://git-wip-us.apache.org/repos/asf/hadoop/blob/f309d08b/config ---------------------------------------------------------------------- diff --git a/config b/config new file mode 100644 index 0000000..9071494 --- /dev/null +++ b/config @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CORE-SITE.XML_fs.default.name=hdfs://namenode +CORE-SITE.XML_fs.defaultFS=hdfs://namenode +HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:8020 +HDFS-SITE.XML_dfs.replication=1 +MAPRED-SITE.XML_mapreduce.framework.name=yarn +YARN-SITE.XML_yarn.resourcemanager.hostname=resourcemanager +YARN-SITE.XML_yarn.nodemanager.pmem-check-enabled=false +YARN-SITE.XML_yarn.nodemanager.delete.debug-delay-sec=600 +YARN-SITE.XML_yarn.nodemanager.vmem-check-enabled=false +YARN-SITE.XML_yarn.nodemanager.aux-services=mapreduce_shuffle +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-applications=10000 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-am-resource-percent=0.1 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.queues=default +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.capacity=100 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.user-limit-factor=1 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-capacity=100 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.state=RUNNING +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_submit_applications=* +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_administer_queue=* +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.node-locality-delay=40 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings= +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings-override.enable=false http://git-wip-us.apache.org/repos/asf/hadoop/blob/f309d08b/docker-compose.yaml ---------------------------------------------------------------------- diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..14c4dc1 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "2" +services: + namenode: + build: . + hostname: namenode + command: ["hdfs", "namenode"] + ports: + - 9870:9870 + env_file: + - ./config + environment: + ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name" + datanode: + build: . + command: ["hdfs", "datanode"] + env_file: + - ./config + resourcemanager: + build: . + hostname: resourcemanager + command: ["yarn", "resourcemanager"] + ports: + - 8088:8088 + env_file: + - ./config + volumes: + - ./test.sh:/opt/test.sh + nodemanager: + build: . + command: ["yarn", "nodemanager"] + env_file: + - ./config http://git-wip-us.apache.org/repos/asf/hadoop/blob/f309d08b/log4j.properties ---------------------------------------------------------------------- diff --git a/log4j.properties b/log4j.properties new file mode 100644 index 0000000..a481d09 --- /dev/null +++ b/log4j.properties @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +log4j.appender.stdout.layout: org.apache.log4j.PatternLayout +log4j.rootLogger: INFO, stdout +log4j.appender.stdout: org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout.ConversionPattern: %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
