HDDS-447. Separate ozone-dist and hadoop-dist projects with real classpath separation. Contributed by Elek Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/62f817d3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/62f817d3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/62f817d3 Branch: refs/heads/trunk Commit: 62f817d32e6304b4cc1f9b909e7b815316b3a021 Parents: d060cbe Author: Bharat Viswanadham <[email protected]> Authored: Mon Sep 24 10:10:11 2018 -0700 Committer: Bharat Viswanadham <[email protected]> Committed: Mon Sep 24 10:10:11 2018 -0700 ---------------------------------------------------------------------- dev-support/bin/ozone-dist-layout-stitching | 181 ------------ dev-support/bin/ozone-dist-tar-stitching | 43 --- hadoop-dist/pom.xml | 118 -------- hadoop-dist/src/main/compose/README.md | 51 ---- hadoop-dist/src/main/compose/ozone-hdfs/.env | 17 -- .../main/compose/ozone-hdfs/docker-compose.yaml | 60 ---- .../src/main/compose/ozone-hdfs/docker-config | 76 ----- hadoop-dist/src/main/compose/ozone/.env | 17 -- .../src/main/compose/ozone/docker-compose.yaml | 49 ---- .../src/main/compose/ozone/docker-config | 75 ----- .../main/compose/ozonefs/docker-compose.yaml | 59 ---- .../src/main/compose/ozonefs/docker-config | 35 --- hadoop-dist/src/main/compose/ozoneperf/.env | 17 -- .../src/main/compose/ozoneperf/README.md | 73 ----- .../src/main/compose/ozoneperf/compose-all.sh | 18 -- .../compose/ozoneperf/docker-compose-freon.yaml | 26 -- .../main/compose/ozoneperf/docker-compose.yaml | 64 ----- .../src/main/compose/ozoneperf/docker-config | 36 --- hadoop-dist/src/main/compose/ozoneperf/init.sh | 21 -- .../src/main/compose/ozoneperf/prometheus.yml | 24 -- .../compose/ozonescripts/.ssh/authorized_keys | 16 -- .../src/main/compose/ozonescripts/.ssh/config | 18 -- .../main/compose/ozonescripts/.ssh/environment | 16 -- .../src/main/compose/ozonescripts/.ssh/id_rsa | 42 --- .../main/compose/ozonescripts/.ssh/id_rsa.pub | 16 -- .../src/main/compose/ozonescripts/Dockerfile | 33 --- .../src/main/compose/ozonescripts/README.md | 38 --- .../compose/ozonescripts/docker-compose.yaml | 42 --- .../src/main/compose/ozonescripts/docker-config | 38 --- hadoop-dist/src/main/compose/ozonescripts/ps.sh | 17 -- .../src/main/compose/ozonescripts/start.sh | 24 -- .../src/main/compose/ozonescripts/stop.sh | 17 -- hadoop-dist/src/main/ozone/README.txt | 51 ---- hadoop-dist/src/main/smoketest/README.md | 30 -- .../src/main/smoketest/basic/basic.robot | 47 --- .../src/main/smoketest/basic/ozone-shell.robot | 82 ------ hadoop-dist/src/main/smoketest/commonlib.robot | 24 -- .../src/main/smoketest/ozonefs/ozonefs.robot | 35 --- hadoop-dist/src/main/smoketest/test.sh | 101 ------- hadoop-hdds/client/pom.xml | 5 - hadoop-hdds/common/pom.xml | 2 - hadoop-hdds/container-service/pom.xml | 5 - hadoop-hdds/framework/pom.xml | 5 - hadoop-hdds/pom.xml | 30 ++ hadoop-hdds/server-scm/pom.xml | 50 +--- .../src/main/resources/webapps/scm/index.html | 76 +++++ .../src/main/resources/webapps/scm/main.html | 20 ++ .../resources/webapps/scm/scm-overview.html | 60 ++++ .../src/main/resources/webapps/scm/scm.js | 54 ++++ .../server-scm/src/main/webapps/scm/index.html | 76 ----- .../server-scm/src/main/webapps/scm/main.html | 20 -- .../src/main/webapps/scm/scm-overview.html | 60 ---- .../server-scm/src/main/webapps/scm/scm.js | 54 ---- hadoop-hdds/tools/pom.xml | 8 - hadoop-ozone/client/pom.xml | 6 - hadoop-ozone/common/pom.xml | 5 - hadoop-ozone/common/src/main/bin/ozone | 50 +++- .../src/main/shellprofile.d/hadoop-ozone.sh | 24 -- hadoop-ozone/datanode/pom.xml | 49 ++++ .../dist/dev-support/bin/dist-layout-stitching | 113 ++++++++ .../dist/dev-support/bin/dist-tar-stitching | 45 +++ hadoop-ozone/dist/pom.xml | 184 ++++++++++++ hadoop-ozone/dist/src/main/compose/README.md | 51 ++++ .../dist/src/main/compose/ozone-hdfs/.env | 17 ++ .../main/compose/ozone-hdfs/docker-compose.yaml | 60 ++++ .../src/main/compose/ozone-hdfs/docker-config | 76 +++++ hadoop-ozone/dist/src/main/compose/ozone/.env | 17 ++ .../src/main/compose/ozone/docker-compose.yaml | 49 ++++ .../dist/src/main/compose/ozone/docker-config | 75 +++++ .../main/compose/ozonefs/docker-compose.yaml | 59 ++++ .../dist/src/main/compose/ozonefs/docker-config | 35 +++ .../dist/src/main/compose/ozoneperf/.env | 17 ++ .../dist/src/main/compose/ozoneperf/README.md | 73 +++++ .../src/main/compose/ozoneperf/compose-all.sh | 18 ++ .../compose/ozoneperf/docker-compose-freon.yaml | 26 ++ .../main/compose/ozoneperf/docker-compose.yaml | 64 +++++ .../src/main/compose/ozoneperf/docker-config | 35 +++ .../dist/src/main/compose/ozoneperf/init.sh | 21 ++ .../src/main/compose/ozoneperf/prometheus.yml | 24 ++ .../compose/ozonescripts/.ssh/authorized_keys | 16 ++ .../src/main/compose/ozonescripts/.ssh/config | 18 ++ .../main/compose/ozonescripts/.ssh/environment | 16 ++ .../src/main/compose/ozonescripts/.ssh/id_rsa | 42 +++ .../main/compose/ozonescripts/.ssh/id_rsa.pub | 16 ++ .../src/main/compose/ozonescripts/Dockerfile | 33 +++ .../src/main/compose/ozonescripts/README.md | 38 +++ .../compose/ozonescripts/docker-compose.yaml | 42 +++ .../src/main/compose/ozonescripts/docker-config | 38 +++ .../dist/src/main/compose/ozonescripts/ps.sh | 17 ++ .../dist/src/main/compose/ozonescripts/start.sh | 24 ++ .../dist/src/main/compose/ozonescripts/stop.sh | 17 ++ hadoop-ozone/dist/src/main/ozone/README.txt | 51 ++++ hadoop-ozone/dist/src/main/smoketest/README.md | 30 ++ .../dist/src/main/smoketest/basic/basic.robot | 47 +++ .../src/main/smoketest/basic/ozone-shell.robot | 82 ++++++ .../dist/src/main/smoketest/commonlib.robot | 24 ++ .../src/main/smoketest/ozonefs/ozonefs.robot | 35 +++ hadoop-ozone/dist/src/main/smoketest/test.sh | 101 +++++++ hadoop-ozone/docs/pom.xml | 5 - hadoop-ozone/integration-test/pom.xml | 14 - .../hadoop/ozone/freon/TestDataValidate.java | 115 -------- .../ozone/freon/TestRandomKeyGenerator.java | 106 ------- .../apache/hadoop/ozone/om/TestOmSQLCli.java | 285 ------------------- .../hadoop/ozone/scm/TestContainerSQLCli.java | 253 ---------------- hadoop-ozone/objectstore-service/pom.xml | 5 - hadoop-ozone/ozone-manager/pom.xml | 50 +--- .../resources/webapps/ozoneManager/index.html | 70 +++++ .../resources/webapps/ozoneManager/main.css | 23 ++ .../resources/webapps/ozoneManager/main.html | 18 ++ .../webapps/ozoneManager/om-metrics.html | 44 +++ .../webapps/ozoneManager/ozoneManager.js | 110 +++++++ .../src/main/webapps/ozoneManager/index.html | 70 ----- .../src/main/webapps/ozoneManager/main.css | 23 -- .../src/main/webapps/ozoneManager/main.html | 18 -- .../main/webapps/ozoneManager/om-metrics.html | 44 --- .../main/webapps/ozoneManager/ozoneManager.js | 110 ------- hadoop-ozone/ozonefs/pom.xml | 2 - hadoop-ozone/pom.xml | 25 +- hadoop-ozone/tools/pom.xml | 32 ++- .../hadoop/ozone/freon/TestDataValidate.java | 115 ++++++++ .../ozone/freon/TestRandomKeyGenerator.java | 106 +++++++ .../apache/hadoop/ozone/om/TestOmSQLCli.java | 285 +++++++++++++++++++ .../hadoop/ozone/scm/TestContainerSQLCli.java | 253 ++++++++++++++++ hadoop-project/pom.xml | 5 + 124 files changed, 3183 insertions(+), 3155 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/dev-support/bin/ozone-dist-layout-stitching ---------------------------------------------------------------------- diff --git a/dev-support/bin/ozone-dist-layout-stitching b/dev-support/bin/ozone-dist-layout-stitching deleted file mode 100755 index 00854b4..0000000 --- a/dev-support/bin/ozone-dist-layout-stitching +++ /dev/null @@ -1,181 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# project.version -VERSION=$1 - -# project.build.directory -BASEDIR=$2 - -#hdds.version -HDDS_VERSION=$3 - -function run() -{ - declare res - - echo "\$ ${*}" - "${@}" - res=$? - if [[ ${res} != 0 ]]; then - echo - echo "Failed!" - echo - exit "${res}" - fi -} - -function findfileindir() -{ - declare file="$1" - declare dir="${2:-./share}" - declare count - - count=$(find "${dir}" -iname "${file}" | wc -l) - - #shellcheck disable=SC2086 - echo ${count} -} - -function copyifnotexists() -{ - declare src="$1" - declare dest="$2" - - declare srcname - declare destdir - - declare child - declare childpath - - if [[ -f "${src}" ]]; then - srcname=${src##*/} - if [[ "${srcname}" != *.jar || - $(findfileindir "${srcname}") -eq "0" ]]; then - destdir=$(dirname "${dest}") - mkdir -p "${destdir}" - cp -p "${src}" "${dest}" - fi - else - for childpath in "${src}"/*; do - child="${childpath##*/}" - if [[ "${child}" == "doc" || - "${child}" == "webapps" ]]; then - mkdir -p "${dest}/${child}" - cp -r "${src}/${child}"/* "${dest}/${child}" - continue; - fi - copyifnotexists "${src}/${child}" "${dest}/${child}" - done - fi -} - -#Copy all contents as is except the lib. -#for libs check for existence in share directory, if not exist then only copy. -function copy() -{ - declare src="$1" - declare dest="$2" - - declare child - declare childpath - - if [[ -d "${src}" ]]; then - for childpath in "${src}"/*; do - child="${childpath##*/}" - - if [[ "${child}" == "share" ]]; then - copyifnotexists "${src}/${child}" "${dest}/${child}" - else - if [[ -d "${src}/${child}" ]]; then - mkdir -p "${dest}/${child}" - cp -pr "${src}/${child}"/* "${dest}/${child}" - else - cp -pr "${src}/${child}" "${dest}/${child}" - fi - fi - done - fi -} - -# shellcheck disable=SC2164 -ROOT=$(cd "${BASEDIR}"/../..;pwd) -echo -echo "Current directory $(pwd)" -echo -run rm -rf "ozone-${HDDS_VERSION}" -run mkdir "ozone-${HDDS_VERSION}" -run cd "ozone-${HDDS_VERSION}" -run cp -p "${ROOT}/LICENSE.txt" . -run cp -p "${ROOT}/NOTICE.txt" . - -# Copy hadoop-common first so that it have always have all dependencies. -# Remaining projects will copy only libraries which are not present already in 'share' directory. -run copy "${ROOT}/hadoop-common-project/hadoop-common/target/hadoop-common-${VERSION}" . - - -# HDDS -run copy "${ROOT}/hadoop-hdds/common/target/hadoop-hdds-common-${HDDS_VERSION}" . -run copy "${ROOT}/hadoop-hdds/framework/target/hadoop-hdds-server-framework-${HDDS_VERSION}" . -run copy "${ROOT}/hadoop-hdds/server-scm/target/hadoop-hdds-server-scm-${HDDS_VERSION}" . -run copy "${ROOT}/hadoop-hdds/container-service/target/hadoop-hdds-container-service-${HDDS_VERSION}" . -run copy "${ROOT}/hadoop-hdds/client/target/hadoop-hdds-client-${HDDS_VERSION}" . -run copy "${ROOT}/hadoop-hdds/tools/target/hadoop-hdds-tools-${HDDS_VERSION}" . - -# Ozone -run copy "${ROOT}/hadoop-ozone/common/target/hadoop-ozone-common-${HDDS_VERSION}" . -run copy "${ROOT}/hadoop-ozone/ozone-manager/target/hadoop-ozone-ozone-manager-${HDDS_VERSION}" . -run copy "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}" . -run copy "${ROOT}/hadoop-ozone/client/target/hadoop-ozone-client-${HDDS_VERSION}" . -run copy "${ROOT}/hadoop-ozone/tools/target/hadoop-ozone-tools-${HDDS_VERSION}" . - -#shaded ozonefs -mkdir -p "./share/hadoop/ozonefs" -cp "${ROOT}/hadoop-ozone/ozonefs/target/hadoop-ozone-filesystem-${HDDS_VERSION}.jar" "./share/hadoop/ozonefs/hadoop-ozone-filesystem-${HDDS_VERSION}.jar" - -#shaded datanode service -mkdir -p "./share/hadoop/ozoneplugin" -cp "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}-plugin.jar" "./share/hadoop/ozoneplugin/hadoop-ozone-datanode-plugin-${HDDS_VERSION}.jar" - - -# Optional documentation, could be missing -cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/ozone/webapps/ozoneManager/ -cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/hdds/webapps/scm/ -cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./ - - -rm sbin/*all.sh -rm sbin/*all.cmd - -#remove test and java sources -find . -name "*tests.jar" | xargs rm -find . -name "*sources.jar" | xargs rm -find . -name jdiff -type d | xargs rm -rf - -#add ozone specific readme - -run cp "${ROOT}/hadoop-dist/src/main/ozone/README.txt" README.txt -#Copy docker compose files and robot tests -run cp -p -r "${ROOT}/hadoop-dist/src/main/compose" . -run cp -p -r "${ROOT}/hadoop-dist/src/main/smoketest" . - -mkdir -p ./share/hadoop/mapreduce -mkdir -p ./share/hadoop/yarn -mkdir -p ./share/hadoop/hdfs -echo -echo "Hadoop Ozone dist layout available at: ${BASEDIR}/ozone-${HDDS_VERSION}" -echo http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/dev-support/bin/ozone-dist-tar-stitching ---------------------------------------------------------------------- diff --git a/dev-support/bin/ozone-dist-tar-stitching b/dev-support/bin/ozone-dist-tar-stitching deleted file mode 100755 index 93d0525..0000000 --- a/dev-support/bin/ozone-dist-tar-stitching +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# project.version -VERSION=$1 - -# project.build.directory -BASEDIR=$2 - -function run() -{ - declare res - - echo "\$ ${*}" - "${@}" - res=$? - if [[ ${res} != 0 ]]; then - echo - echo "Failed!" - echo - exit "${res}" - fi -} - -run tar -c -f "ozone-${VERSION}.tar" "ozone-${VERSION}" -run gzip -f "ozone-${VERSION}.tar" -echo -echo "Ozone dist tar available at: ${BASEDIR}/ozone-${VERSION}.tar.gz" -echo http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml index da05015..5447b6a 100644 --- a/hadoop-dist/pom.xml +++ b/hadoop-dist/pom.xml @@ -176,7 +176,6 @@ </argument> <argument>${project.version}</argument> <argument>${project.build.directory}</argument> - <argument>${hdds.version}</argument> </arguments> </configuration> </execution> @@ -224,123 +223,6 @@ </plugins> </build> </profile> - <profile> - <id>hdds</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-ozone-ozone-manager</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-hdds-server-scm</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-hdds-tools</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-hdds-container-service</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-ozone-objectstore-service</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-ozone-tools</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-ozone-docs</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-ozone-filesystem</artifactId> - <scope>provided</scope> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <artifactId>maven-resources-plugin</artifactId> - <executions> - <execution> - <id>copy-dockerfile</id> - <goals> - <goal>copy-resources</goal> - </goals> - <phase>prepare-package</phase> - <configuration> - <outputDirectory>${project.build.directory}</outputDirectory> - <resources> - <resource> - <directory>src/main/docker</directory> - <filtering>true</filtering> - </resource> - </resources> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>exec-maven-plugin</artifactId> - <executions> - <execution> - <id>dist-ozone</id> - <phase>prepare-package</phase> - <goals> - <goal>exec</goal> - </goals> - <configuration> - <executable>${shell-executable}</executable> - <workingDirectory>${project.build.directory} - </workingDirectory> - <arguments> - <argument> - ${basedir}/../dev-support/bin/ozone-dist-layout-stitching - </argument> - <argument>${project.version}</argument> - <argument>${project.build.directory}</argument> - <argument>${hdds.version}</argument> - </arguments> - </configuration> - </execution> - <execution> - <id>tar-ozone</id> - <phase>package</phase> - <goals> - <goal>exec</goal> - </goals> - <configuration> - <executable>${shell-executable}</executable> - <workingDirectory>${project.build.directory} - </workingDirectory> - <arguments> - <argument>${basedir}/../dev-support/bin/ozone-dist-tar-stitching - </argument> - <argument>${hdds.version}</argument> - <argument>${project.build.directory}</argument> - </arguments> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> </profiles> </project> http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/README.md ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/README.md b/hadoop-dist/src/main/compose/README.md deleted file mode 100644 index 8189d2c..0000000 --- a/hadoop-dist/src/main/compose/README.md +++ /dev/null @@ -1,51 +0,0 @@ -<!--- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. ---> - -# Docker cluster definitions - -This directory contains multiple docker cluster definitions to start local pseudo cluster with different configuration. - -It helps to start local (multi-node like) pseudo cluster with docker and docker-compose and obviously it's not for production. - -You may find more information in the specific subdirectories but in generic you can use the following commands: - -## Usage - -To start a cluster go to a subdirectory and start the cluster: - -``` -docker-compose up -d -``` - -You can check the logs of all the components with: - -``` -docker-compose logs -``` - -In case of a problem you can destroy the cluster an delete all the local state with: - -``` -docker-compose down -``` - -(Note: a simple docker-compose stop may not delete all the local data). - -You can scale up and down the components: - -``` -docker-compose scale datanode=5 -``` - -Usually the key webui ports are published on the docker host. http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozone-hdfs/.env ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozone-hdfs/.env b/hadoop-dist/src/main/compose/ozone-hdfs/.env deleted file mode 100644 index c437513..0000000 --- a/hadoop-dist/src/main/compose/ozone-hdfs/.env +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -HADOOP_VERSION=3.1.0 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml b/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml deleted file mode 100644 index b89052d..0000000 --- a/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml +++ /dev/null @@ -1,60 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: "3" -services: - namenode: - image: flokkr/hadoop:${HADOOP_VERSION} - ports: - - 9870:9870 - env_file: - - ./docker-config - environment: - ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name" - command: ["hdfs", "namenode"] - datanode: - image: flokkr/hadoop:${HADOOP_VERSION} - ports: - - 9864 - volumes: - - ../..:/opt/ozone - command: ["hdfs","datanode"] - environment: - HADOOP_CLASSPATH: /opt/ozone/share/hadoop/ozoneplugin/*.jar - env_file: - - ./docker-config - ozoneManager: - image: apache/hadoop-runner - volumes: - - ../..:/opt/hadoop - ports: - - 9874:9874 - environment: - ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION - env_file: - - ./docker-config - command: ["ozone","om"] - scm: - image: apache/hadoop-runner - volumes: - - ../..:/opt/hadoop - ports: - - 9876:9876 - env_file: - - ./docker-config - environment: - ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION - command: ["ozone","scm"] http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozone-hdfs/docker-config ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozone-hdfs/docker-config b/hadoop-dist/src/main/compose/ozone-hdfs/docker-config deleted file mode 100644 index 3b2819f..0000000 --- a/hadoop-dist/src/main/compose/ozone-hdfs/docker-config +++ /dev/null @@ -1,76 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -OZONE-SITE.XML_ozone.om.address=ozoneManager -OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874 -OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.enabled=true -OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id -OZONE-SITE.XML_ozone.scm.block.client.address=scm -OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata -OZONE-SITE.XML_ozone.scm.client.address=scm -OZONE-SITE.XML_ozone.replication=1 -OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService - -HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService -HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000 -HDFS-SITE.XML_rpc.metrics.quantile.enable=true -HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 - -LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout -LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender -LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n - -#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation. -#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm - -#LOG4J2.PROPERTIES_* are for Ozone Audit Logging -LOG4J2.PROPERTIES_monitorInterval=30 -LOG4J2.PROPERTIES_filter=read,write -LOG4J2.PROPERTIES_filter.read.type=MarkerFilter -LOG4J2.PROPERTIES_filter.read.marker=READ -LOG4J2.PROPERTIES_filter.read.onMatch=DENY -LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL -LOG4J2.PROPERTIES_filter.write.type=MarkerFilter -LOG4J2.PROPERTIES_filter.write.marker=WRITE -LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL -LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL -LOG4J2.PROPERTIES_appenders=console, rolling -LOG4J2.PROPERTIES_appender.console.type=Console -LOG4J2.PROPERTIES_appender.console.name=STDOUT -LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout -LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n -LOG4J2.PROPERTIES_appender.rolling.type=RollingFile -LOG4J2.PROPERTIES_appender.rolling.name=RollingFile -LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log -LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz -LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout -LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n -LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies -LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy -LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400 -LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy -LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB -LOG4J2.PROPERTIES_loggers=audit -LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger -LOG4J2.PROPERTIES_logger.audit.name=OMAudit -LOG4J2.PROPERTIES_logger.audit.level=INFO -LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling -LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile -LOG4J2.PROPERTIES_rootLogger.level=INFO -LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout -LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozone/.env ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozone/.env b/hadoop-dist/src/main/compose/ozone/.env deleted file mode 100644 index 67eed25..0000000 --- a/hadoop-dist/src/main/compose/ozone/.env +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -HDDS_VERSION=${hdds.version} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozone/docker-compose.yaml ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-dist/src/main/compose/ozone/docker-compose.yaml deleted file mode 100644 index 0a6a9d8..0000000 --- a/hadoop-dist/src/main/compose/ozone/docker-compose.yaml +++ /dev/null @@ -1,49 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: "3" -services: - datanode: - image: apache/hadoop-runner - volumes: - - ../..:/opt/hadoop - ports: - - 9864 - command: ["/opt/hadoop/bin/ozone","datanode"] - env_file: - - ./docker-config - ozoneManager: - image: apache/hadoop-runner - volumes: - - ../..:/opt/hadoop - ports: - - 9874:9874 - environment: - ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION - env_file: - - ./docker-config - command: ["/opt/hadoop/bin/ozone","om"] - scm: - image: apache/hadoop-runner - volumes: - - ../..:/opt/hadoop - ports: - - 9876:9876 - env_file: - - ./docker-config - environment: - ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION - command: ["/opt/hadoop/bin/ozone","scm"] http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozone/docker-config ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozone/docker-config b/hadoop-dist/src/main/compose/ozone/docker-config deleted file mode 100644 index f2c8db1..0000000 --- a/hadoop-dist/src/main/compose/ozone/docker-config +++ /dev/null @@ -1,75 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -OZONE-SITE.XML_ozone.om.address=ozoneManager -OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874 -OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.enabled=True -OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id -OZONE-SITE.XML_ozone.scm.block.client.address=scm -OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata -OZONE-SITE.XML_ozone.handler.type=distributed -OZONE-SITE.XML_ozone.scm.client.address=scm -OZONE-SITE.XML_ozone.replication=1 -HDFS-SITE.XML_rpc.metrics.quantile.enable=true -HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 -LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout -LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender -LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR -LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN -LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR - -#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation. -#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm - -#LOG4J2.PROPERTIES_* are for Ozone Audit Logging -LOG4J2.PROPERTIES_monitorInterval=30 -LOG4J2.PROPERTIES_filter=read,write -LOG4J2.PROPERTIES_filter.read.type=MarkerFilter -LOG4J2.PROPERTIES_filter.read.marker=READ -LOG4J2.PROPERTIES_filter.read.onMatch=DENY -LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL -LOG4J2.PROPERTIES_filter.write.type=MarkerFilter -LOG4J2.PROPERTIES_filter.write.marker=WRITE -LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL -LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL -LOG4J2.PROPERTIES_appenders=console, rolling -LOG4J2.PROPERTIES_appender.console.type=Console -LOG4J2.PROPERTIES_appender.console.name=STDOUT -LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout -LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n -LOG4J2.PROPERTIES_appender.rolling.type=RollingFile -LOG4J2.PROPERTIES_appender.rolling.name=RollingFile -LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log -LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz -LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout -LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n -LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies -LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy -LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400 -LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy -LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB -LOG4J2.PROPERTIES_loggers=audit -LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger -LOG4J2.PROPERTIES_logger.audit.name=OMAudit -LOG4J2.PROPERTIES_logger.audit.level=INFO -LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling -LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile -LOG4J2.PROPERTIES_rootLogger.level=INFO -LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout -LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonefs/docker-compose.yaml ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozonefs/docker-compose.yaml b/hadoop-dist/src/main/compose/ozonefs/docker-compose.yaml deleted file mode 100644 index a1e8748..0000000 --- a/hadoop-dist/src/main/compose/ozonefs/docker-compose.yaml +++ /dev/null @@ -1,59 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: "3" -services: - datanode: - image: apache/hadoop-runner - volumes: - - ../..:/opt/hadoop - ports: - - 9864 - command: ["/opt/hadoop/bin/ozone","datanode"] - env_file: - - ./docker-config - ozoneManager: - image: apache/hadoop-runner - hostname: ozoneManager - volumes: - - ../..:/opt/hadoop - ports: - - 9874 - environment: - ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION - env_file: - - ./docker-config - command: ["/opt/hadoop/bin/ozone","om"] - scm: - image: apache/hadoop-runner - volumes: - - ../..:/opt/hadoop - ports: - - 9876 - env_file: - - ./docker-config - environment: - ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION - command: ["/opt/hadoop/bin/ozone","scm"] - hadooplast: - image: flokkr/hadoop:3.1.0 - volumes: - - ../..:/opt/ozone - env_file: - - ./docker-config - environment: - HADOOP_CLASSPATH: /opt/ozone/share/hadoop/ozonefs/*.jar - command: ["watch","-n","100000","ls"] http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonefs/docker-config ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozonefs/docker-config b/hadoop-dist/src/main/compose/ozonefs/docker-config deleted file mode 100644 index 3171f08..0000000 --- a/hadoop-dist/src/main/compose/ozonefs/docker-config +++ /dev/null @@ -1,35 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -CORE-SITE.XML_fs.o3.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem -OZONE-SITE.XML_ozone.om.address=ozoneManager -OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874 -OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.enabled=True -OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id -OZONE-SITE.XML_ozone.scm.block.client.address=scm -OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata -OZONE-SITE.XML_ozone.handler.type=distributed -OZONE-SITE.XML_ozone.scm.client.address=scm -OZONE-SITE.XML_ozone.scm.heartbeat.interval=3s -HDFS-SITE.XML_rpc.metrics.quantile.enable=true -HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 -LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout -LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender -LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR -LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozoneperf/.env ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozoneperf/.env b/hadoop-dist/src/main/compose/ozoneperf/.env deleted file mode 100644 index cac418a..0000000 --- a/hadoop-dist/src/main/compose/ozoneperf/.env +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -HDDS_VERSION=${hdds.version} http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozoneperf/README.md ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozoneperf/README.md b/hadoop-dist/src/main/compose/ozoneperf/README.md deleted file mode 100644 index 527ff41..0000000 --- a/hadoop-dist/src/main/compose/ozoneperf/README.md +++ /dev/null @@ -1,73 +0,0 @@ -<!--- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. ---> - -# Compose files for local performance tests - -This directory contains docker-compose definition for an ozone cluster where -all the metrics are saved to a prometheus instance. - - Prometheus follows a pull based approach where the metrics are published - on a HTTP endpoint. - - Our current approach: - - 1. A Java agent activates a prometheus metrics endpoint in every JVM instance - (use `init.sh` to download the agent) - - 2. The Java agent publishes all the jmx parameters in prometheus format AND - register the endpoint address to the consul. - - 3. Prometheus polls all the endpoints which are registered to consul. - - - -## How to use - -First of all download the required Java agent with running `./init.sh` - -After that you can start the cluster with docker-compose: - -``` -docker-compose up -d -``` - -After a while the cluster will be started. You can check the ozone web ui-s: - -https://localhost:9874 -https://localhost:9876 - -You can also scale up the datanodes: - -``` -docker-compose scale datanode=3 -``` - -Freon (Ozone test generator tool) is not part of docker-compose by default, -you can activate it using `compose-all.sh` instead of `docker-compose`: - -``` -compose-all.sh up -d -``` - -Now Freon is running. Let's try to check the metrics from the local Prometheus: - -http://localhost:9090/graph - -Example queries: - -``` -Hadoop_OzoneManager_NumKeyCommits -rate(Hadoop_OzoneManager_NumKeyCommits[10m]) -rate(Hadoop_Ozone_BYTES_WRITTEN[10m]) -``` http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozoneperf/compose-all.sh ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozoneperf/compose-all.sh b/hadoop-dist/src/main/compose/ozoneperf/compose-all.sh deleted file mode 100755 index 82ab8b3..0000000 --- a/hadoop-dist/src/main/compose/ozoneperf/compose-all.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -docker-compose -f docker-compose.yaml -f docker-compose-freon.yaml "$@" http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozoneperf/docker-compose-freon.yaml ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-compose-freon.yaml b/hadoop-dist/src/main/compose/ozoneperf/docker-compose-freon.yaml deleted file mode 100644 index 60bdc4a..0000000 --- a/hadoop-dist/src/main/compose/ozoneperf/docker-compose-freon.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: "3" -services: - freon: - image: apache/hadoop-runner - volumes: - - ../../ozone:/opt/hadoop - - ./jmxpromo.jar:/opt/jmxpromo.jar - env_file: - - ./docker-config - command: ["/opt/hadoop/bin/ozone","freon"] http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml b/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml deleted file mode 100644 index 3b5cc7b..0000000 --- a/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml +++ /dev/null @@ -1,64 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: "3" -services: - datanode: - image: apache/hadoop-runner - volumes: - - ../..:/opt/hadoop - - ./jmxpromo.jar:/opt/jmxpromo.jar - ports: - - 9864 - command: ["/opt/hadoop/bin/ozone","datanode"] - env_file: - - ./docker-config - ozoneManager: - image: apache/hadoop-runner - volumes: - - ../..:/opt/hadoop - - ./jmxpromo.jar:/opt/jmxpromo.jar - ports: - - 9874:9874 - environment: - ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION - env_file: - - ./docker-config - command: ["/opt/hadoop/bin/ozone","om"] - scm: - image: apache/hadoop-runner - volumes: - - ../..:/opt/hadoop - - ./jmxpromo.jar:/opt/jmxpromo.jar - ports: - - 9876:9876 - env_file: - - ./docker-config - environment: - ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION - command: ["/opt/hadoop/bin/ozone","scm"] - consul: - image: consul - command: ["agent", "-dev", "-ui", "-client", "0.0.0.0"] - ports: - - 8500:8500 - prometheus: - image: prom/prometheus - volumes: - - "./prometheus.yml:/etc/prometheus.yml" - command: ["--config.file","/etc/prometheus.yml"] - ports: - - 9090:9090 http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozoneperf/docker-config ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-config b/hadoop-dist/src/main/compose/ozoneperf/docker-config deleted file mode 100644 index 454601e..0000000 --- a/hadoop-dist/src/main/compose/ozoneperf/docker-config +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -OZONE-SITE.XML_ozone.om.address=ozoneManager -OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874 -OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.enabled=True -OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id -OZONE-SITE.XML_ozone.scm.block.client.address=scm -OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata -OZONE-SITE.XML_ozone.handler.type=distributed -OZONE-SITE.XML_ozone.scm.client.address=scm -HDFS-SITE.XML_rpc.metrics.quantile.enable=true -HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 -LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout -LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender -LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -HADOOP_OPTS=-javaagent:/opt/jmxpromo.jar=port=0:consulHost=consul:consulMode=node -LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR -LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN -LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR - http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozoneperf/init.sh ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozoneperf/init.sh b/hadoop-dist/src/main/compose/ozoneperf/init.sh deleted file mode 100755 index cf25398..0000000 --- a/hadoop-dist/src/main/compose/ozoneperf/init.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -EXPORTER_FILE="$DIR/jmxpromo.jar" -if [ ! -f "$EXPORTER_FILE" ]; then - wget https://github.com/flokkr/jmxpromo/releases/download/0.11/jmx_prometheus_javaagent-0.11.jar -O $EXPORTER_FILE -fi http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozoneperf/prometheus.yml ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozoneperf/prometheus.yml b/hadoop-dist/src/main/compose/ozoneperf/prometheus.yml deleted file mode 100644 index 80aa520..0000000 --- a/hadoop-dist/src/main/compose/ozoneperf/prometheus.yml +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -global: - scrape_interval: 15s # By default, scrape targets every 15 seconds. - -scrape_configs: - - job_name: jmxexporter - consul_sd_configs: - - server: consul:8500 - services: - - jmxexporter http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/.ssh/authorized_keys ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozonescripts/.ssh/authorized_keys b/hadoop-dist/src/main/compose/ozonescripts/.ssh/authorized_keys deleted file mode 100644 index ae39052..0000000 --- a/hadoop-dist/src/main/compose/ozonescripts/.ssh/authorized_keys +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDgEmLpYm4BrWtq1KG9hhZXCZgGrETntu0eNTo21U3VKc9nH9/ot7M6lAawsFcT9uXu4b58PTlnfvwH/TATlCFjC8n0Z7SOx+FU6L3Sn8URh9HaX4L0tF8u87oCAD4dBrUGhhB36eiuH9dBBWly6RKffYJvrjatbc7GxBO/e5OSUMtqk/DSVKksmBhZxutrKivCNjDish9ViGIf8b5yS/MlEGmaVKApik1fJ5iOlloM/GgpB60YV/hbqfCecbWgeiM1gK92gdOcA/Wx1C7fj8BSI5iDSE6eZeF80gM3421lvyPDWyVhFaGbka4rXBX/fb9QSRBA9RTqhRKAEmAIf49H hadoop@cdae967fa87a http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/.ssh/config ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozonescripts/.ssh/config b/hadoop-dist/src/main/compose/ozonescripts/.ssh/config deleted file mode 100644 index 6506916..0000000 --- a/hadoop-dist/src/main/compose/ozonescripts/.ssh/config +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -Host * - UserKnownHostsFile /dev/null - StrictHostKeyChecking no http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/.ssh/environment ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozonescripts/.ssh/environment b/hadoop-dist/src/main/compose/ozonescripts/.ssh/environment deleted file mode 100644 index 5685453..0000000 --- a/hadoop-dist/src/main/compose/ozonescripts/.ssh/environment +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64/ http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa b/hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa deleted file mode 100644 index 6632ce5..0000000 --- a/hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa +++ /dev/null @@ -1,42 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEA4BJi6WJuAa1ratShvYYWVwmYBqxE57btHjU6NtVN1SnPZx/f -6LezOpQGsLBXE/bl7uG+fD05Z378B/0wE5QhYwvJ9Ge0jsfhVOi90p/FEYfR2l+C -9LRfLvO6AgA+HQa1BoYQd+norh/XQQVpcukSn32Cb642rW3OxsQTv3uTklDLapPw -0lSpLJgYWcbrayorwjYw4rIfVYhiH/G+ckvzJRBpmlSgKYpNXyeYjpZaDPxoKQet -GFf4W6nwnnG1oHojNYCvdoHTnAP1sdQu34/AUiOYg0hOnmXhfNIDN+NtZb8jw1sl -YRWhm5GuK1wV/32/UEkQQPUU6oUSgBJgCH+PRwIDAQABAoIBAQDI1TH6ZNKchkck -9XgSWsBjOqIcOQN5fCeDT8nho8WjLVpL3/Hcr+ngsxRcAXHK3xyvw33r9SQic1qJ -/pC8u6RBFivo95qJ7vU0GXcp9TG4yLd6tui1U4WMm784U+dYNM7EDh1snSaECt3v -1V3yNJ0QfnoOh2NShn0zAkOA+M4H8Nx2SudMCsjcbK9+fYxzW3hX+sJpMKdjG1HW -DUz+I7cW7t0EGaVrgVSV+eR58LiXu+14YDNMrySiejB4nD2sKrx93XgiCBECCsBN -GLQGJCztaXoAY+5Kf/aJ9EEf2wBF3GecRk+XIAd87PeDmeahLQAVkAJ/rD1vsKFs -8kWi6CrhAoGBAP7leG/dMBhlfvROpBddIfWm2i8B+oZiAlSjdYGz+/ZhUaByXk18 -pupMGiiMQR1ziPnEg0gNgR2ZkH54qrXPn5WcQa4rlSEtUsZEp5v5WblhfX2QwKzY -G/uhA+mB7wXpQkSmXo0LclfPF2teROQrG1OyfWkWbxFH4i3+em7sL95jAoGBAOEK -v+wscqkMLW7Q8ONbWMCCBlmMHr6baB3VDCYZx25lr+GIF5zmJJFTmF2rq2VSAlts -qx1AGmaUSo78kC5FuJvSNTL6a1Us5ucdthQZM3N8pAz+OAE+QEU+BsdA27yAh3tO -yKDsMFNHKtXcgy5LeB5gzENLlNyw2jgkRv2Ef77NAoGAVH8DHqoHEH9Mx3XuRWR1 -JnaqKx0PzE5fEWmiQV3Fr5XxNivTgQJKXq7dmQVtbHLpPErdbhwz6fkHAjXD+UMb -VsAWscL2y6m3n8wQd87/5EkiDWbXyDRXimGE53pQHviFJDa2bzEVNXCMBeaZFb4I -cAViN1zdcrAOlUqfkXewIpsCgYB8wsXl/DpRB+RENGfn0+OfTjaQ/IKq72NIbq1+ -jfondQ6N/TICFQEe5HZrL9okoNOXteYjoD9CsWGoZdLVJGgVUvOVYImSvgMBDFK+ -T75bfzU/0sxfvBBLkviVDJsFpUf3D5VgybB86s6Po+HCD6r3RHjZshRESXOhflMx -B3z+3QKBgE2Lwo0DuwUGkm6k8psyn3x8EiXNsiNw12cojicFTyKUYLHxMBeVbCLW -3j3pxSggJgRuBLLzixUHbHp91r2ARTy28naK7R/la8yKVqK6ojcikN2mQsCHYtwB -nuFwXr42ytn6G+9Wn4xT64tGjRCqyZn0/v0XsPjVCyrZ6G7EtNHP ------END RSA PRIVATE KEY----- http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa.pub ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa.pub b/hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa.pub deleted file mode 100644 index ae39052..0000000 --- a/hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa.pub +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDgEmLpYm4BrWtq1KG9hhZXCZgGrETntu0eNTo21U3VKc9nH9/ot7M6lAawsFcT9uXu4b58PTlnfvwH/TATlCFjC8n0Z7SOx+FU6L3Sn8URh9HaX4L0tF8u87oCAD4dBrUGhhB36eiuH9dBBWly6RKffYJvrjatbc7GxBO/e5OSUMtqk/DSVKksmBhZxutrKivCNjDish9ViGIf8b5yS/MlEGmaVKApik1fJ5iOlloM/GgpB60YV/hbqfCecbWgeiM1gK92gdOcA/Wx1C7fj8BSI5iDSE6eZeF80gM3421lvyPDWyVhFaGbka4rXBX/fb9QSRBA9RTqhRKAEmAIf49H hadoop@cdae967fa87a http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/Dockerfile ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozonescripts/Dockerfile b/hadoop-dist/src/main/compose/ozonescripts/Dockerfile deleted file mode 100644 index 3bdcb0c..0000000 --- a/hadoop-dist/src/main/compose/ozonescripts/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -FROM apache/hadoop-runner -RUN sudo apt-get update && sudo apt-get install -y openssh-server - -RUN sudo mkdir -p /run/sshd -RUN sudo sed -i "s/.*UsePrivilegeSeparation.*/UsePrivilegeSeparation no/g" /etc/ssh/sshd_config -RUN sudo sed -i "s/.*PermitUserEnvironment.*/PermitUserEnvironment yes/g" /etc/ssh/sshd_config -RUN sudo sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd - -#/opt/hadoop is mounted, we can't use it as a home -RUN sudo usermod -d /opt hadoop -ADD .ssh /opt/.ssh -RUN sudo chown -R hadoop /opt/.ssh -RUN sudo chown hadoop /opt -RUN sudo chmod 600 /opt/.ssh/* -RUN sudo chmod 700 /opt/.ssh - -RUN sudo sh -c 'echo "export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64/" >> /etc/profile' -CMD ["sudo","/usr/sbin/sshd","-D"] http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/README.md ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozonescripts/README.md b/hadoop-dist/src/main/compose/ozonescripts/README.md deleted file mode 100644 index 2531fa4..0000000 --- a/hadoop-dist/src/main/compose/ozonescripts/README.md +++ /dev/null @@ -1,38 +0,0 @@ -<!--- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. ---> - -# start-ozone environment - -This is an example environment to use/test `./sbin/start-ozone.sh` and `./sbin/stop-ozone.sh` scripts. - -There are ssh connections between the containers and the start/stop scripts could handle the start/stop process -similar to a real cluster. - -To use it, first start the cluster: - -``` -docker-copmose up -d -``` - -After a successfull startup (which starts only the ssh daemons) you can start ozone: - -``` -./start.sh -``` - -Check it the java processes are started: - -``` -./ps.sh -``` \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/docker-compose.yaml ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozonescripts/docker-compose.yaml b/hadoop-dist/src/main/compose/ozonescripts/docker-compose.yaml deleted file mode 100644 index 62f1163..0000000 --- a/hadoop-dist/src/main/compose/ozonescripts/docker-compose.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: "3" -services: - datanode: - build: . - volumes: - - ../..:/opt/hadoop - ports: - - 9864 - env_file: - - ./docker-config - om: - build: . - volumes: - - ../..:/opt/hadoop - ports: - - 9874:9874 - env_file: - - ./docker-config - scm: - build: . - volumes: - - ../..:/opt/hadoop - ports: - - 9876:9876 - env_file: - - ./docker-config http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/docker-config ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozonescripts/docker-config b/hadoop-dist/src/main/compose/ozonescripts/docker-config deleted file mode 100644 index 1afec73..0000000 --- a/hadoop-dist/src/main/compose/ozonescripts/docker-config +++ /dev/null @@ -1,38 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64/ -CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000 -OZONE-SITE.XML_ozone.ksm.address=ksm -OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.enabled=true -OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id -OZONE-SITE.XML_ozone.om.address=om -OZONE-SITE.XML_ozone.om.http-address=om:9874 -OZONE-SITE.XML_ozone.scm.block.client.address=scm -OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata -OZONE-SITE.XML_ozone.scm.client.address=scm -OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService -HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000 -HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode -HDFS-SITE.XML_rpc.metrics.quantile.enable=true -HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 -HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService -LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout -LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR, stdout -LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender -LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/ps.sh ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozonescripts/ps.sh b/hadoop-dist/src/main/compose/ozonescripts/ps.sh deleted file mode 100755 index d5e2c38..0000000 --- a/hadoop-dist/src/main/compose/ozonescripts/ps.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -docker-compose ps -q | xargs -n1 -I CONTAINER docker exec CONTAINER ps xa \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/start.sh ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozonescripts/start.sh b/hadoop-dist/src/main/compose/ozonescripts/start.sh deleted file mode 100755 index 3358b07..0000000 --- a/hadoop-dist/src/main/compose/ozonescripts/start.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -docker-compose ps | grep datanode | awk '{print $1}' | xargs -n1 docker inspect --format '{{ .Config.Hostname }}' > ../../etc/hadoop/workers -docker-compose exec scm /opt/hadoop/bin/ozone scm -init -docker-compose exec scm /opt/hadoop/sbin/start-ozone.sh -#We need a running SCM for om objectstore creation -#TODO create a utility to wait for the startup -sleep 10 -docker-compose exec om /opt/hadoop/bin/ozone om -createObjectStore -docker-compose exec scm /opt/hadoop/sbin/start-ozone.sh http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/stop.sh ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozonescripts/stop.sh b/hadoop-dist/src/main/compose/ozonescripts/stop.sh deleted file mode 100755 index a3ce08a..0000000 --- a/hadoop-dist/src/main/compose/ozonescripts/stop.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -docker-compose exec scm /opt/hadoop/sbin/stop-ozone.sh http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/ozone/README.txt ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/ozone/README.txt b/hadoop-dist/src/main/ozone/README.txt deleted file mode 100644 index 6bbd83f..0000000 --- a/hadoop-dist/src/main/ozone/README.txt +++ /dev/null @@ -1,51 +0,0 @@ -<!--- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. ---> - -This is the distribution of Apache Hadoop Ozone. - -Ozone is a submodule of Hadoop with separated release cycle. For more information, check - - http://ozone.hadoop.apache.org - - and - - https://cwiki.apache.org/confluence/display/HADOOP/Ozone+Contributor+Guide - -For more information about Hadoop, check: - - http://hadoop.apache.org - -This distribution includes cryptographic software. The country in -which you currently reside may have restrictions on the import, -possession, use, and/or re-export to another country, of -encryption software. BEFORE using any encryption software, please -check your country's laws, regulations and policies concerning the -import, possession, or use, and re-export of encryption software, to -see if this is permitted. See <http://www.wassenaar.org/> for more -information. - -The U.S. Government Department of Commerce, Bureau of Industry and -Security (BIS), has classified this software as Export Commodity -Control Number (ECCN) 5D002.C.1, which includes information security -software using or performing cryptographic functions with asymmetric -algorithms. The form and manner of this Apache Software Foundation -distribution makes it eligible for export under the License Exception -ENC Technology Software Unrestricted (TSU) exception (see the BIS -Export Administration Regulations, Section 740.13) for both object -code and source code. - -The following provides more details on the included cryptographic -software: - Hadoop Core uses the SSL libraries from the Jetty project written -by mortbay.org. http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/smoketest/README.md ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/smoketest/README.md b/hadoop-dist/src/main/smoketest/README.md deleted file mode 100644 index c521a54..0000000 --- a/hadoop-dist/src/main/smoketest/README.md +++ /dev/null @@ -1,30 +0,0 @@ -<!--- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. ---> - -## Ozone Acceptance Tests - -This directory contains a [robotframework](http://robotframework.org/) based test suite for Ozone to make it easier to check the current state of the package. - -You can run in in any environment after [installing](https://github.com/robotframework/robotframework/blob/master/INSTALL.rst) - -``` -cd $DIRECTORY_OF_OZONE -robot smoketest/basic -``` - -The argument of the `robot` could be any robot file or directory. - -The current configuration in the robot files (hostnames, ports) are adjusted for the docker-based setup but you can easily modify it for any environment. - -The `./test.sh` in this directory can start multiple type of clusters (ozone standalone or ozone + hdfs) and execute the test framework with all of the clusters. http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/smoketest/basic/basic.robot ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/smoketest/basic/basic.robot b/hadoop-dist/src/main/smoketest/basic/basic.robot deleted file mode 100644 index a69450d..0000000 --- a/hadoop-dist/src/main/smoketest/basic/basic.robot +++ /dev/null @@ -1,47 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -*** Settings *** -Documentation Smoketest ozone cluster startup -Library OperatingSystem -Resource ../commonlib.robot - -*** Variables *** -${COMMON_REST_HEADER} -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE root" -${DATANODE_HOST} localhost - - -*** Test Cases *** - -Test rest interface - ${result} = Execute curl -i -X POST ${COMMON_RESTHEADER} "http://${DATANODE_HOST}:9880/volume1"; - Should contain ${result} 201 Created - ${result} = Execute curl -i -X POST ${COMMON_RESTHEADER} "http://${DATANODE_HOST}:9880/volume1/bucket1"; - Should contain ${result} 201 Created - ${result} = Execute curl -i -X DELETE ${COMMON_RESTHEADER} "http://${DATANODE_HOST}:9880/volume1/bucket1"; - Should contain ${result} 200 OK - ${result} = Execute curl -i -X DELETE ${COMMON_RESTHEADER} "http://${DATANODE_HOST}:9880/volume1"; - Should contain ${result} 200 OK - -Check webui static resources - ${result} = Execute curl -s -I http://scm:9876/static/bootstrap-3.3.7/js/bootstrap.min.js - Should contain ${result} 200 - ${result} = Execute curl -s -I http://ozoneManager:9874/static/bootstrap-3.3.7/js/bootstrap.min.js - Should contain ${result} 200 - -Start freon testing - ${result} = Execute ozone freon randomkeys --numOfVolumes 5 --numOfBuckets 5 --numOfKeys 5 --numOfThreads 10 - Wait Until Keyword Succeeds 3min 10sec Should contain ${result} Number of Keys added: 125 - Should Not Contain ${result} ERROR --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
