http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/smoketest/basic/ozone-shell.robot ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/smoketest/basic/ozone-shell.robot b/hadoop-dist/src/main/smoketest/basic/ozone-shell.robot deleted file mode 100644 index 14a5761..0000000 --- a/hadoop-dist/src/main/smoketest/basic/ozone-shell.robot +++ /dev/null @@ -1,82 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -*** Settings *** -Documentation Test ozone shell CLI usage -Library OperatingSystem -Resource ../commonlib.robot -Test Timeout 2 minute - -*** Variables *** - -*** Test Cases *** -RestClient without http port - Test ozone shell http:// ozoneManager restwoport - -RestClient with http port - Test ozone shell http:// ozoneManager:9874 restwport - -RestClient without host name - Test ozone shell http:// ${EMPTY} restwohost - -RpcClient with port - Test ozone shell o3:// ozoneManager:9862 rpcwoport - -RpcClient without host - Test ozone shell o3:// ${EMPTY} rpcwport - -RpcClient without scheme - Test ozone shell ${EMPTY} ${EMPTY} rpcwoscheme - - -*** Keywords *** -Test ozone shell - [arguments] ${protocol} ${server} ${volume} - ${result} = Execute ozone sh volume create ${protocol}${server}/${volume} --user bilbo --quota 100TB --root - Should not contain ${result} Failed - Should contain ${result} Creating Volume: ${volume} - ${result} = Execute ozone sh volume list ${protocol}${server}/ --user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="${volume}")' - Should contain ${result} createdOn - ${result} = Execute ozone sh volume list --user bilbo | grep -Ev 'Removed|DEBUG|ERROR|INFO|TRACE|WARN' | jq -r '.[] | select(.volumeName=="${volume}")' - Should contain ${result} createdOn - Execute ozone sh volume update ${protocol}${server}/${volume} --user bill --quota 10TB - ${result} = Execute ozone sh volume info ${protocol}${server}/${volume} | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="${volume}") | .owner | .name' - Should Be Equal ${result} bill - ${result} = Execute ozone sh volume info ${protocol}${server}/${volume} | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="${volume}") | .quota | .size' - Should Be Equal ${result} 10 - Execute ozone sh bucket create ${protocol}${server}/${volume}/bb1 - ${result} = Execute ozone sh bucket info ${protocol}${server}/${volume}/bb1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.bucketName=="bb1") | .storageType' - Should Be Equal ${result} DISK - ${result} = Execute ozone sh bucket update ${protocol}${server}/${volume}/bb1 --addAcl user:frodo:rw,group:samwise:r | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.bucketName=="bb1") | .acls | .[] | select(.name=="samwise") | .type' - Should Be Equal ${result} GROUP - ${result} = Execute ozone sh bucket update ${protocol}${server}/${volume}/bb1 --removeAcl group:samwise:r | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.bucketName=="bb1") | .acls | .[] | select(.name=="frodo") | .type' - Should Be Equal ${result} USER - ${result} = Execute ozone sh bucket list ${protocol}${server}/${volume}/ | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.bucketName=="bb1") | .volumeName' - Should Be Equal ${result} ${volume} - Run Keyword Test key handling ${protocol} ${server} ${volume} - Execute ozone sh bucket delete ${protocol}${server}/${volume}/bb1 - Execute ozone sh volume delete ${protocol}${server}/${volume} --user bilbo - -Test key handling - [arguments] ${protocol} ${server} ${volume} - Execute ozone sh key put ${protocol}${server}/${volume}/bb1/key1 /opt/hadoop/NOTICE.txt - Execute rm -f NOTICE.txt.1 - Execute ozone sh key get ${protocol}${server}/${volume}/bb1/key1 NOTICE.txt.1 - Execute ls -l NOTICE.txt.1 - ${result} = Execute ozone sh key info ${protocol}${server}/${volume}/bb1/key1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.keyName=="key1")' - Should contain ${result} createdOn - ${result} = Execute ozone sh key list ${protocol}${server}/${volume}/bb1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.keyName=="key1") | .keyName' - Should Be Equal ${result} key1 - Execute ozone sh key delete ${protocol}${server}/${volume}/bb1/key1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/smoketest/commonlib.robot ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/smoketest/commonlib.robot b/hadoop-dist/src/main/smoketest/commonlib.robot deleted file mode 100644 index e2620fa..0000000 --- a/hadoop-dist/src/main/smoketest/commonlib.robot +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -*** Keywords *** - - -Execute - [arguments] ${command} - ${rc} ${output} = Run And Return Rc And Output ${command} - Log ${output} - Should Be Equal As Integers ${rc} 0 - [return] ${output} http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/smoketest/ozonefs/ozonefs.robot ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/smoketest/ozonefs/ozonefs.robot b/hadoop-dist/src/main/smoketest/ozonefs/ozonefs.robot deleted file mode 100644 index fb7b98c..0000000 --- a/hadoop-dist/src/main/smoketest/ozonefs/ozonefs.robot +++ /dev/null @@ -1,35 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -*** Settings *** -Documentation Ozonefs test -Library OperatingSystem -Resource ../commonlib.robot - -*** Variables *** - - -*** Test Cases *** -Create volume and bucket - Execute ozone sh volume create http://ozoneManager/fstest --user bilbo --quota 100TB --root - Execute ozone sh bucket create http://ozoneManager/fstest/bucket1 - -Check volume from ozonefs - ${result} = Execute ozone fs -ls o3://bucket1.fstest/ - -Create directory from ozonefs - Execute ozone fs -mkdir -p o3://bucket1.fstest/testdir/deep - ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' - Should contain ${result} testdir/deep http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/smoketest/test.sh ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/smoketest/test.sh b/hadoop-dist/src/main/smoketest/test.sh deleted file mode 100755 index 534bbb7..0000000 --- a/hadoop-dist/src/main/smoketest/test.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -execute_tests(){ - COMPOSE_FILE=$DIR/../compose/$1/docker-compose.yaml - TESTS=$2 - echo "Executing test ${TESTS[*]} with $COMPOSE_FILE" - docker-compose -f "$COMPOSE_FILE" down - docker-compose -f "$COMPOSE_FILE" up -d - docker-compose -f "$COMPOSE_FILE" exec datanode sudo apt-get update - docker-compose -f "$COMPOSE_FILE" exec datanode sudo apt-get install -y python-pip - docker-compose -f "$COMPOSE_FILE" exec datanode sudo pip install robotframework - for TEST in "${TESTS[@]}"; do - set +e - docker-compose -f "$COMPOSE_FILE" exec datanode python -m robot "smoketest/$TEST" - set -e - done - if [ "$KEEP_RUNNING" = false ]; then - docker-compose -f "$COMPOSE_FILE" down - fi -} -RUN_ALL=true -KEEP_RUNNING=false -POSITIONAL=() -while [[ $# -gt 0 ]] -do -key="$1" - -case $key in - --env) - DOCKERENV="$2" - RUN_ALL=false - shift # past argument - shift # past value - ;; - --keep) - KEEP_RUNNING=true - shift # past argument - ;; - --help|-h|-help) - cat << EOF - - Acceptance test executor for ozone. - - This is a lightweight test executor for ozone. - - You can run it with - - ./test.sh - - Which executes all the tests in all the available environments. - - Or you can run manually one test with - - ./test.sh --keep --env ozone-hdfs basic - - --keep means that docker cluster won't be stopped after the test (optional) - --env defines the subdirectory under the compose dir - The remaining parameters define the test suites under smoketest dir. - Could be any directory or robot file relative to the smoketest dir. -EOF - exit 0 - ;; - *) - POSITIONAL+=("$1") # save it in an array for later - shift # past argument - ;; -esac -done - -if [ "$RUN_ALL" = true ]; then -# -# This is the definition of the ozone acceptance test suite -# -# We select the test suites and execute them on multiple type of clusters -# - DEFAULT_TESTS=("basic") - execute_tests ozone "${DEFAULT_TESTS[@]}" - TESTS=("ozonefs") - execute_tests ozonefs "${TESTS[@]}" - -else - execute_tests "$DOCKERENV" "${POSITIONAL[@]}" -fi http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/client/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdds/client/pom.xml b/hadoop-hdds/client/pom.xml index 6a4cc9d..53d72bb 100644 --- a/hadoop-hdds/client/pom.xml +++ b/hadoop-hdds/client/pom.xml @@ -29,11 +29,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <name>Apache Hadoop HDDS Client</name> <packaging>jar</packaging> - <properties> - <hadoop.component>hdds</hadoop.component> - <is.hadoop.component>true</is.hadoop.component> - </properties> - <dependencies> <dependency> <groupId>org.apache.hadoop</groupId> http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/common/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml index d08a5a9..64ebe9b 100644 --- a/hadoop-hdds/common/pom.xml +++ b/hadoop-hdds/common/pom.xml @@ -30,8 +30,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <properties> <hdds.version>0.3.0-SNAPSHOT</hdds.version> - <hadoop.component>hdds</hadoop.component> - <is.hadoop.component>true</is.hadoop.component> <log4j2.version>2.11.0</log4j2.version> <disruptor.version>3.4.2</disruptor.version> <declared.hdds.version>${hdds.version}</declared.hdds.version> http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/container-service/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/pom.xml b/hadoop-hdds/container-service/pom.xml index 3d4e581..54c5fad 100644 --- a/hadoop-hdds/container-service/pom.xml +++ b/hadoop-hdds/container-service/pom.xml @@ -28,11 +28,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <name>Apache Hadoop HDDS Container Service</name> <packaging>jar</packaging> - <properties> - <hadoop.component>hdds</hadoop.component> - <is.hadoop.component>true</is.hadoop.component> - </properties> - <dependencies> <dependency> <groupId>org.apache.hadoop</groupId> http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/framework/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdds/framework/pom.xml b/hadoop-hdds/framework/pom.xml index cb380e2..511f321 100644 --- a/hadoop-hdds/framework/pom.xml +++ b/hadoop-hdds/framework/pom.xml @@ -28,11 +28,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <name>Apache Hadoop HDDS Server Framework</name> <packaging>jar</packaging> - <properties> - <hadoop.component>hdds</hadoop.component> - <is.hadoop.component>true</is.hadoop.component> - </properties> - <dependencies> <dependency> <groupId>org.apache.hadoop</groupId> http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdds/pom.xml b/hadoop-hdds/pom.xml index ce4f234..59dfa79 100644 --- a/hadoop-hdds/pom.xml +++ b/hadoop-hdds/pom.xml @@ -119,6 +119,36 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <excludeFilterFile combine.self="override"></excludeFilterFile> </configuration> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>add-classpath-descriptor</id> + <phase>package</phase> + <goals> + <goal>build-classpath</goal> + </goals> + <configuration> + <attach>true</attach> + <prefix>$HDDS_LIB_JARS_DIR</prefix> + <outputFilterFile>true</outputFilterFile> + <includeScope>runtime</includeScope> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <executions> + <execution> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> </plugins> </build> http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/server-scm/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/pom.xml b/hadoop-hdds/server-scm/pom.xml index 52f42e1..f34e848 100644 --- a/hadoop-hdds/server-scm/pom.xml +++ b/hadoop-hdds/server-scm/pom.xml @@ -28,11 +28,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <name>Apache Hadoop HDDS SCM Server</name> <packaging>jar</packaging> - <properties> - <hadoop.component>hdds</hadoop.component> - <is.hadoop.component>true</is.hadoop.component> - </properties> - <dependencies> <dependency> <groupId>org.apache.hadoop</groupId> @@ -106,28 +101,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-antrun-plugin</artifactId> - <executions> - <execution> - <id>copy web resources</id> - <phase>compile</phase> - <goals> - <goal>run</goal> - </goals> - <configuration> - <target> - <copy toDir="${project.build.directory}/webapps"> - <fileset dir="${basedir}/src/main/webapps"> - <exclude name="**/proto-web.xml"/> - </fileset> - </copy> - </target> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-dependency-plugin</artifactId> <executions> <execution> @@ -136,19 +109,20 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <goals> <goal>unpack</goal> </goals> + <configuration> + <artifactItems> + <artifactItem> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdds-server-framework</artifactId> + <outputDirectory>${project.build.outputDirectory} + </outputDirectory> + <includes>webapps/static/**/*.*</includes> + </artifactItem> + </artifactItems> + <overWriteSnapshots>true</overWriteSnapshots> + </configuration> </execution> </executions> - <configuration> - <artifactItems> - <artifactItem> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-hdds-server-framework</artifactId> - <outputDirectory>${project.build.directory}/</outputDirectory> - <includes>webapps/static/**/*.*</includes> - </artifactItem> - </artifactItems> - <overWriteSnapshots>true</overWriteSnapshots> - </configuration> </plugin> </plugins> </build> http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/server-scm/src/main/resources/webapps/scm/index.html ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/index.html b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/index.html new file mode 100644 index 0000000..2c943b6 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/index.html @@ -0,0 +1,76 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd";> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<html lang="en"> +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> + <meta name="description" content="HDFS Storage Container Manager"> + + <title>HDFS Storage Container Manager</title> + + <link href="static/bootstrap-3.3.7/css/bootstrap.min.css" rel="stylesheet"> + <link href="static/hadoop.css" rel="stylesheet"> + <link href="static/nvd3-1.8.5.min.css" rel="stylesheet"> + + <link href="static/ozone.css" rel="stylesheet"> + +</head> + +<body ng-app="scm"> + +<header class="navbar navbar-inverse navbar-fixed-top bs-docs-nav"> + <div class="container-fluid"> + <div class="navbar-header"> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" + aria-expanded="false" aria-controls="navbar"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand" href="#">HDFS SCM</a> + </div> + + + <navmenu + metrics="{ 'Rpc metrics' : '#!/metrics/rpc'}"></navmenu> + + + </div> +</header> + +<div class="container-fluid" style="margin: 12pt"> + + <ng-view></ng-view> + +</div><!-- /.container --> + +<script src="static/jquery-3.3.1.min.js"></script> +<script src="static/angular-1.6.4.min.js"></script> +<script src="static/angular-route-1.6.4.min.js"></script> +<script src="static/d3-3.5.17.min.js"></script> +<script src="static/nvd3-1.8.5.min.js"></script> +<script src="static/angular-nvd3-1.0.9.min.js"></script> +<script src="static/ozone.js"></script> +<script src="scm.js"></script> +<script src="static/bootstrap-3.3.7/js/bootstrap.min.js"></script> +</body> +</html> http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/server-scm/src/main/resources/webapps/scm/main.html ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/main.html b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/main.html new file mode 100644 index 0000000..2666f81 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/main.html @@ -0,0 +1,20 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> +<overview> + <scm-overview> + </scm-overview> +</overview> http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html new file mode 100644 index 0000000..fca23ba --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html @@ -0,0 +1,60 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<h2>Node counts</h2> + +<table class="table table-bordered table-striped" class="col-md-6"> + <tbody> + <tr ng-repeat="typestat in $ctrl.nodemanagermetrics.NodeCount | orderBy:'key':false:$ctrl.nodeOrder"> + <td>{{typestat.key}}</td> + <td>{{typestat.value}}</td> + </tr> + </tbody> +</table> + +<h2>Status</h2> +<table class="table table-bordered table-striped" class="col-md-6"> + <tbody> + <tr> + <td>Client Rpc port</td> + <td>{{$ctrl.overview.jmx.ClientRpcPort}}</td> + </tr> + <tr> + <td>Datanode Rpc port</td> + <td>{{$ctrl.overview.jmx.DatanodeRpcPort}}</td> + </tr> + <tr> + <td>Block Manager: Open containers</td> + <td>{{$ctrl.blockmanagermetrics.OpenContainersNo}}</td> + </tr> + <tr> + <td>Node Manager: Minimum chill mode nodes</td> + <td>{{$ctrl.nodemanagermetrics.MinimumChillModeNodes}}</td> + </tr> + <tr> + <td>Node Manager: Out-of-node chill mode</td> + <td>{{$ctrl.nodemanagermetrics.OutOfNodeChillMode}}</td> + </tr> + <tr> + <td>Node Manager: Chill mode status</td> + <td>{{$ctrl.nodemanagermetrics.ChillModeStatus}}</td> + </tr> + <tr> + <td>Node Manager: Manual chill mode</td> + <td>{{$ctrl.nodemanagermetrics.InManualChillMode}}</td> + </tr> + </tbody> +</table> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js new file mode 100644 index 0000000..bcfa8b7 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +(function () { + "use strict"; + angular.module('scm', ['ozone', 'nvd3']); + + angular.module('scm').component('scmOverview', { + templateUrl: 'scm-overview.html', + require: { + overview: "^overview" + }, + controller: function ($http) { + var ctrl = this; + $http.get("jmx?qry=Hadoop:service=BlockManager,name=*") + .then(function (result) { + ctrl.blockmanagermetrics = result.data.beans[0]; + }); + $http.get("jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo") + .then(function (result) { + ctrl.nodemanagermetrics = result.data.beans[0]; + }); + + var statusSortOrder = { + "HEALTHY": "a", + "STALE": "b", + "DEAD": "c", + "UNKNOWN": "z", + "DECOMMISSIONING": "x", + "DECOMMISSIONED": "y" + }; + ctrl.nodeOrder = function (v1, v2) { + //status with non defined sort order will be "undefined" + return ("" + statusSortOrder[v1.value]).localeCompare("" + statusSortOrder[v2.value]) + } + + } + }); + +})(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/server-scm/src/main/webapps/scm/index.html ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/webapps/scm/index.html b/hadoop-hdds/server-scm/src/main/webapps/scm/index.html deleted file mode 100644 index 2c943b6..0000000 --- a/hadoop-hdds/server-scm/src/main/webapps/scm/index.html +++ /dev/null @@ -1,76 +0,0 @@ -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" - "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd";> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<html lang="en"> -<head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> - <meta name="description" content="HDFS Storage Container Manager"> - - <title>HDFS Storage Container Manager</title> - - <link href="static/bootstrap-3.3.7/css/bootstrap.min.css" rel="stylesheet"> - <link href="static/hadoop.css" rel="stylesheet"> - <link href="static/nvd3-1.8.5.min.css" rel="stylesheet"> - - <link href="static/ozone.css" rel="stylesheet"> - -</head> - -<body ng-app="scm"> - -<header class="navbar navbar-inverse navbar-fixed-top bs-docs-nav"> - <div class="container-fluid"> - <div class="navbar-header"> - <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" - aria-expanded="false" aria-controls="navbar"> - <span class="sr-only">Toggle navigation</span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <a class="navbar-brand" href="#">HDFS SCM</a> - </div> - - - <navmenu - metrics="{ 'Rpc metrics' : '#!/metrics/rpc'}"></navmenu> - - - </div> -</header> - -<div class="container-fluid" style="margin: 12pt"> - - <ng-view></ng-view> - -</div><!-- /.container --> - -<script src="static/jquery-3.3.1.min.js"></script> -<script src="static/angular-1.6.4.min.js"></script> -<script src="static/angular-route-1.6.4.min.js"></script> -<script src="static/d3-3.5.17.min.js"></script> -<script src="static/nvd3-1.8.5.min.js"></script> -<script src="static/angular-nvd3-1.0.9.min.js"></script> -<script src="static/ozone.js"></script> -<script src="scm.js"></script> -<script src="static/bootstrap-3.3.7/js/bootstrap.min.js"></script> -</body> -</html> http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/server-scm/src/main/webapps/scm/main.html ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/webapps/scm/main.html b/hadoop-hdds/server-scm/src/main/webapps/scm/main.html deleted file mode 100644 index 2666f81..0000000 --- a/hadoop-hdds/server-scm/src/main/webapps/scm/main.html +++ /dev/null @@ -1,20 +0,0 @@ -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> -<overview> - <scm-overview> - </scm-overview> -</overview> http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/server-scm/src/main/webapps/scm/scm-overview.html ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/webapps/scm/scm-overview.html b/hadoop-hdds/server-scm/src/main/webapps/scm/scm-overview.html deleted file mode 100644 index fca23ba..0000000 --- a/hadoop-hdds/server-scm/src/main/webapps/scm/scm-overview.html +++ /dev/null @@ -1,60 +0,0 @@ -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<h2>Node counts</h2> - -<table class="table table-bordered table-striped" class="col-md-6"> - <tbody> - <tr ng-repeat="typestat in $ctrl.nodemanagermetrics.NodeCount | orderBy:'key':false:$ctrl.nodeOrder"> - <td>{{typestat.key}}</td> - <td>{{typestat.value}}</td> - </tr> - </tbody> -</table> - -<h2>Status</h2> -<table class="table table-bordered table-striped" class="col-md-6"> - <tbody> - <tr> - <td>Client Rpc port</td> - <td>{{$ctrl.overview.jmx.ClientRpcPort}}</td> - </tr> - <tr> - <td>Datanode Rpc port</td> - <td>{{$ctrl.overview.jmx.DatanodeRpcPort}}</td> - </tr> - <tr> - <td>Block Manager: Open containers</td> - <td>{{$ctrl.blockmanagermetrics.OpenContainersNo}}</td> - </tr> - <tr> - <td>Node Manager: Minimum chill mode nodes</td> - <td>{{$ctrl.nodemanagermetrics.MinimumChillModeNodes}}</td> - </tr> - <tr> - <td>Node Manager: Out-of-node chill mode</td> - <td>{{$ctrl.nodemanagermetrics.OutOfNodeChillMode}}</td> - </tr> - <tr> - <td>Node Manager: Chill mode status</td> - <td>{{$ctrl.nodemanagermetrics.ChillModeStatus}}</td> - </tr> - <tr> - <td>Node Manager: Manual chill mode</td> - <td>{{$ctrl.nodemanagermetrics.InManualChillMode}}</td> - </tr> - </tbody> -</table> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/server-scm/src/main/webapps/scm/scm.js ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/webapps/scm/scm.js b/hadoop-hdds/server-scm/src/main/webapps/scm/scm.js deleted file mode 100644 index bcfa8b7..0000000 --- a/hadoop-hdds/server-scm/src/main/webapps/scm/scm.js +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -(function () { - "use strict"; - angular.module('scm', ['ozone', 'nvd3']); - - angular.module('scm').component('scmOverview', { - templateUrl: 'scm-overview.html', - require: { - overview: "^overview" - }, - controller: function ($http) { - var ctrl = this; - $http.get("jmx?qry=Hadoop:service=BlockManager,name=*") - .then(function (result) { - ctrl.blockmanagermetrics = result.data.beans[0]; - }); - $http.get("jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo") - .then(function (result) { - ctrl.nodemanagermetrics = result.data.beans[0]; - }); - - var statusSortOrder = { - "HEALTHY": "a", - "STALE": "b", - "DEAD": "c", - "UNKNOWN": "z", - "DECOMMISSIONING": "x", - "DECOMMISSIONED": "y" - }; - ctrl.nodeOrder = function (v1, v2) { - //status with non defined sort order will be "undefined" - return ("" + statusSortOrder[v1.value]).localeCompare("" + statusSortOrder[v2.value]) - } - - } - }); - -})(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/tools/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdds/tools/pom.xml b/hadoop-hdds/tools/pom.xml index 852c81d..37c7d9d 100644 --- a/hadoop-hdds/tools/pom.xml +++ b/hadoop-hdds/tools/pom.xml @@ -29,26 +29,18 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <name>Apache Hadoop HDDS Tools</name> <packaging>jar</packaging> - <properties> - <hadoop.component>hdds</hadoop.component> - <is.hadoop.component>true</is.hadoop.component> - </properties> - <dependencies> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdds-common</artifactId> - <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdds-client</artifactId> - <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> - <scope>provided</scope> </dependency> <dependency> <groupId>commons-cli</groupId> http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/client/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/pom.xml b/hadoop-ozone/client/pom.xml index bbd0759..e471710 100644 --- a/hadoop-ozone/client/pom.xml +++ b/hadoop-ozone/client/pom.xml @@ -28,16 +28,10 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <name>Apache Hadoop Ozone Client</name> <packaging>jar</packaging> - <properties> - <hadoop.component>ozone</hadoop.component> - <is.hadoop.component>true</is.hadoop.component> - </properties> - <dependencies> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-ozone-common</artifactId> - <scope>provided</scope> </dependency> </dependencies> </project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/common/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/pom.xml b/hadoop-ozone/common/pom.xml index 37a2cb9..942576b 100644 --- a/hadoop-ozone/common/pom.xml +++ b/hadoop-ozone/common/pom.xml @@ -28,11 +28,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <name>Apache Hadoop Ozone Common</name> <packaging>jar</packaging> - <properties> - <hadoop.component>ozone</hadoop.component> - <is.hadoop.component>true</is.hadoop.component> - </properties> - <dependencies> </dependencies> http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/common/src/main/bin/ozone ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/bin/ozone b/hadoop-ozone/common/src/main/bin/ozone index 6bf8b01..d173ec6 100755 --- a/hadoop-ozone/common/src/main/bin/ozone +++ b/hadoop-ozone/common/src/main/bin/ozone @@ -68,6 +68,7 @@ function ozonecmd_case datanode) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME=org.apache.hadoop.ozone.HddsDatanodeService + OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-datanode" ;; envvars) echo "JAVA_HOME='${JAVA_HOME}'" @@ -86,42 +87,50 @@ function ozonecmd_case ;; freon) HADOOP_CLASSNAME=org.apache.hadoop.ozone.freon.Freon + OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools" ;; genesis) HADOOP_CLASSNAME=org.apache.hadoop.ozone.genesis.Genesis + OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools" ;; getozoneconf) HADOOP_CLASSNAME=org.apache.hadoop.ozone.freon.OzoneGetConf; + OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools" ;; om) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME=org.apache.hadoop.ozone.om.OzoneManager HDFS_OM_OPTS="${HDFS_OM_OPTS} -Dlog4j.configurationFile=${HADOOP_CONF_DIR}/om-audit-log4j2.properties" HADOOP_OPTS="${HADOOP_OPTS} ${HDFS_OM_OPTS}" + OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-ozone-manager" ;; sh | shell) HADOOP_CLASSNAME=org.apache.hadoop.ozone.web.ozShell.Shell - ;; - noz) - HADOOP_CLASSNAME=org.apache.hadoop.ozone.scm.cli.SQLCLI + OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-ozone-manager" ;; scm) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.hdds.scm.server.StorageContainerManager' hadoop_debug "Appending HDFS_STORAGECONTAINERMANAGER_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HDFS_STORAGECONTAINERMANAGER_OPTS}" + OZONE_RUN_ARTIFACT_NAME="hadoop-hdds-server-scm" + ;; fs) HADOOP_CLASSNAME=org.apache.hadoop.fs.FsShell + OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools" ;; scmcli) HADOOP_CLASSNAME=org.apache.hadoop.hdds.scm.cli.SCMCLI + OZONE_RUN_ARTIFACT_NAME="hadoop-hdds-tools" ;; version) HADOOP_CLASSNAME=org.apache.hadoop.ozone.util.OzoneVersionInfo + OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-common" ;; genconf) HADOOP_CLASSNAME=org.apache.hadoop.ozone.genconf.GenerateOzoneRequiredConfigurations + OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools" ;; *) HADOOP_CLASSNAME="${subcmd}" @@ -161,6 +170,7 @@ fi HADOOP_SUBCMD=$1 shift + if hadoop_need_reexec ozone "${HADOOP_SUBCMD}"; then hadoop_uservar_su ozone "${HADOOP_SUBCMD}" \ "${MYNAME}" \ @@ -180,6 +190,40 @@ else ozonecmd_case "${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}" fi + +# +# Setting up classpath based on the generate classpath descriptors +# +if [ ! "$OZONE_RUN_ARTIFACT_NAME" ]; then + echo "ERROR: Ozone components require to set OZONE_RUN_ARTIFACT_NAME to set the classpath" + exit -1 +fi +export HDDS_LIB_JARS_DIR="${HADOOP_HDFS_HOME}/share/ozone/lib" +CLASSPATH_FILE="${HADOOP_HDFS_HOME}/share/ozone/classpath/${OZONE_RUN_ARTIFACT_NAME}.classpath" +if [ ! "$CLASSPATH_FILE" ]; then + echo "ERROR: Classpath file descriptor $CLASSPATH_FILE is missing" + exit -1 +fi +# shellcheck disable=SC1090,SC2086 +source $CLASSPATH_FILE +OIFS=$IFS +IFS=':' +# shellcheck disable=SC2154 +for jar in $classpath; do + hadoop_add_classpath "$jar" +done +hadoop_add_classpath "${HADOOP_HDFS_HOME}/share/ozone/web" + +#We need to add the artifact manually as it's not part the generated classpath desciptor +ARTIFACT_LIB_DIR="${HADOOP_HDFS_HOME}/share/ozone/lib" +MAIN_ARTIFACT=$(find "$ARTIFACT_LIB_DIR" -name "${OZONE_RUN_ARTIFACT_NAME}-*.jar") +if [ ! "$MAIN_ARTIFACT" ]; then + echo "ERROR: Component jar file $MAIN_ARTIFACT is missing from ${HADOOP_HDFS_HOME}/share/ozone/lib" +fi +hadoop_add_classpath "${MAIN_ARTIFACT}" +IFS=$OIFS + + hadoop_add_client_opts if [[ ${HADOOP_WORKER_MODE} = true ]]; then http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/common/src/main/shellprofile.d/hadoop-ozone.sh ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/shellprofile.d/hadoop-ozone.sh b/hadoop-ozone/common/src/main/shellprofile.d/hadoop-ozone.sh index 99885ab..3fff7f5 100644 --- a/hadoop-ozone/common/src/main/shellprofile.d/hadoop-ozone.sh +++ b/hadoop-ozone/common/src/main/shellprofile.d/hadoop-ozone.sh @@ -19,27 +19,3 @@ if [[ "${HADOOP_SHELL_EXECNAME}" = ozone ]]; then fi -## @description Profile for hdds/ozone components. -## @audience private -## @stability evolving -function _ozone_hadoop_classpath -{ - # - # get all of the ozone jars+config in the path - # - - if [[ -d "${HADOOP_HDFS_HOME}/${HDDS_DIR}/webapps" ]]; then - hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDDS_DIR}" - fi - - if [[ -d "${HADOOP_HDFS_HOME}/${HDDS_DIR}/webapps" ]]; then - hadoop_add_classpath "${HADOOP_HDFS_HOME}/${OZONE_DIR}" - fi - - hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDDS_LIB_JARS_DIR}"'/*' - hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDDS_DIR}"'/*' - hadoop_add_classpath "${HADOOP_HDFS_HOME}/${OZONE_LIB_JARS_DIR}"'/*' - hadoop_add_classpath "${HADOOP_HDFS_HOME}/${OZONE_DIR}"'/*' - hadoop_add_classpath "${HADOOP_HDFS_HOME}/${OZONEFS_DIR}"'/*' - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/datanode/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/datanode/pom.xml b/hadoop-ozone/datanode/pom.xml new file mode 100644 index 0000000..02995f5 --- /dev/null +++ b/hadoop-ozone/datanode/pom.xml @@ -0,0 +1,49 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-ozone</artifactId> + <version>0.3.0-SNAPSHOT</version> + </parent> + <artifactId>hadoop-ozone-datanode</artifactId> + <name>Apache Hadoop Ozone Datanode</name> + <packaging>jar</packaging> + <version>0.3.0-SNAPSHOT</version> + + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdds-container-service</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-ozone-objectstore-service</artifactId> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching new file mode 100755 index 0000000..3f102fa --- /dev/null +++ b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# project.build.directory +BASEDIR=$1 + +#hdds.version +HDDS_VERSION=$2 + +## @audience private +## @stability evolving +function run() +{ + declare res + + echo "\$ ${*}" + "${@}" + res=$? + if [[ ${res} != 0 ]]; then + echo + echo "Failed!" + echo + exit "${res}" + fi +} + +## @audience private +## @stability evolving +function findfileindir() +{ + declare file="$1" + declare dir="${2:-./share}" + declare count + + count=$(find "${dir}" -iname "${file}" | wc -l) + + #shellcheck disable=SC2086 + echo ${count} +} + + +# shellcheck disable=SC2164 +ROOT=$(cd "${BASEDIR}"/../../..;pwd) +echo +echo "Current directory $(pwd)" +echo + +run rm -rf "ozone-${HDDS_VERSION}" +run mkdir "ozone-${HDDS_VERSION}" +run cd "ozone-${HDDS_VERSION}" +run cp -p "${ROOT}/LICENSE.txt" . +run cp -p "${ROOT}/NOTICE.txt" . +run cp -p "${ROOT}/README.txt" . + +run mkdir -p ./share/hadoop/mapreduce +run mkdir -p ./share/hadoop/ozone +run mkdir -p ./share/hadoop/hdds +run mkdir -p ./share/hadoop/yarn +run mkdir -p ./share/hadoop/hdfs +run mkdir -p ./share/hadoop/common +run mkdir -p ./share/ozone/web +run mkdir -p ./bin +run mkdir -p ./sbin +run mkdir -p ./etc +run mkdir -p ./libexec + +run cp -r "${ROOT}/hadoop-common-project/hadoop-common/src/main/conf" "etc/hadoop" + +run cp "${ROOT}/hadoop-common-project/hadoop-common/src/main/bin/hadoop" "bin/" +run cp "${ROOT}/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd" "bin/" +run cp "${ROOT}/hadoop-ozone/common/src/main/bin/ozone" "bin/" + +run cp "${ROOT}/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh" "libexec/" +run cp "${ROOT}/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.cmd" "libexec/" +run cp "${ROOT}/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh" "libexec/" +run cp "${ROOT}/hadoop-ozone/common/src/main/bin/ozone-config.sh" "libexec/" +run cp -r "${ROOT}/hadoop-ozone/common/src/main/shellprofile.d" "libexec/" + + +run cp "${ROOT}/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh" "sbin/" +run cp "${ROOT}/hadoop-common-project/hadoop-common/src/main/bin/workers.sh" "sbin/" +run cp "${ROOT}/hadoop-ozone/common/src/main/bin/start-ozone.sh" "sbin/" +run cp "${ROOT}/hadoop-ozone/common/src/main/bin/stop-ozone.sh" "sbin/" + +#shaded ozonefs +run mkdir -p "./share/hadoop/ozonefs" +run cp "${ROOT}/hadoop-ozone/ozonefs/target/hadoop-ozone-filesystem-${HDDS_VERSION}.jar" "./share/hadoop/ozonefs/hadoop-ozone-filesystem-${HDDS_VERSION}.jar" + +#shaded datanode service +run mkdir -p "./share/hadoop/ozoneplugin" +run cp "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}-plugin.jar" "./share/hadoop/ozoneplugin/hadoop-ozone-datanode-plugin-${HDDS_VERSION}.jar" + + +# Optional documentation, could be missing +cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/ozone/webapps/ozoneManager/ +cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/hdds/webapps/scm/ + +#Copy docker compose files +run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/compose" . +run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/smoketest" . http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/dev-support/bin/dist-tar-stitching ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/dev-support/bin/dist-tar-stitching b/hadoop-ozone/dist/dev-support/bin/dist-tar-stitching new file mode 100755 index 0000000..c94e7d0 --- /dev/null +++ b/hadoop-ozone/dist/dev-support/bin/dist-tar-stitching @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# project.version +VERSION=$1 + +# project.build.directory +BASEDIR=$2 + +## @audience private +## @stability evolving +function run() +{ + declare res + + echo "\$ ${*}" + "${@}" + res=$? + if [[ ${res} != 0 ]]; then + echo + echo "Failed!" + echo + exit "${res}" + fi +} + +run tar -c -f "ozone-${VERSION}.tar" "ozone-${VERSION}" +run gzip -f "ozone-${VERSION}.tar" +echo +echo "Ozone dist tar available at: ${BASEDIR}/ozone-${VERSION}.tar.gz" +echo http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml new file mode 100644 index 0000000..ba9f14f --- /dev/null +++ b/hadoop-ozone/dist/pom.xml @@ -0,0 +1,184 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-ozone</artifactId> + <version>0.3.0-SNAPSHOT</version> + </parent> + <artifactId>hadoop-ozone-dist</artifactId> + <name>Apache Hadoop Ozone Distribution</name> + <packaging>pom</packaging> + <version>0.3.0-SNAPSHOT</version> + <properties> + <file.encoding>UTF-8</file.encoding> + <downloadSources>true</downloadSources> + </properties> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>copy-classpath-files</id> + <phase>package</phase> + <goals> + <goal>copy</goal> + </goals> + <configuration> + <outputDirectory>target/ozone-${ozone.version}/share/ozone/classpath + </outputDirectory> + <artifactItems> + <artifactItem> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdds-server-scm</artifactId> + <version>${hdds.version}</version> + <classifier>classpath</classifier> + <destFileName>hadoop-hdds-server-scm.classpath</destFileName> + </artifactItem> + <artifactItem> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdds-tools</artifactId> + <version>${hdds.version}</version> + <classifier>classpath</classifier> + <destFileName>hadoop-hdds-tools.classpath</destFileName> + </artifactItem> + <artifactItem> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-ozone-ozone-manager</artifactId> + <version>${ozone.version}</version> + <classifier>classpath</classifier> + <destFileName>hadoop-ozone-ozone-manager.classpath + </destFileName> + </artifactItem> + <artifactItem> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-ozone-tools</artifactId> + <version>${ozone.version}</version> + <classifier>classpath</classifier> + <destFileName>hadoop-ozone-tools.classpath</destFileName> + </artifactItem> + <artifactItem> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-ozone-common</artifactId> + <version>${ozone.version}</version> + <classifier>classpath</classifier> + <destFileName>hadoop-ozone-common.classpath</destFileName> + </artifactItem> + <artifactItem> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-ozone-datanode</artifactId> + <version>${ozone.version}</version> + <classifier>classpath</classifier> + <destFileName>hadoop-ozone-datanode.classpath</destFileName> + </artifactItem> + </artifactItems> + </configuration> + </execution> + <execution> + <id>copy-jars</id> + <phase>package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + <configuration> + <outputDirectory>target/ozone-${ozone.version}/share/ozone/lib + </outputDirectory> + <includeScope>runtime</includeScope> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <executions> + <execution> + <id>dist</id> + <phase>prepare-package</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <executable>${shell-executable}</executable> + <workingDirectory>${project.build.directory}</workingDirectory> + <arguments> + <argument> + ${basedir}/dev-support/bin/dist-layout-stitching + </argument> + <argument>${project.build.directory}</argument> + <argument>${hdds.version}</argument> + </arguments> + </configuration> + </execution> + <execution> + <id>tar-ozone</id> + <phase>package</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <executable>${shell-executable}</executable> + <workingDirectory>${project.build.directory} + </workingDirectory> + <arguments> + <argument>${basedir}/dev-support/bin/dist-tar-stitching + </argument> + <argument>${hdds.version}</argument> + <argument>${project.build.directory}</argument> + </arguments> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdds-tools</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdds-server-scm</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdds-container-service</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-ozone-ozone-manager</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-ozone-tools</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-ozone-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-ozone-datanode</artifactId> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/README.md ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/compose/README.md b/hadoop-ozone/dist/src/main/compose/README.md new file mode 100644 index 0000000..8189d2c --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/README.md @@ -0,0 +1,51 @@ +<!--- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +# Docker cluster definitions + +This directory contains multiple docker cluster definitions to start local pseudo cluster with different configuration. + +It helps to start local (multi-node like) pseudo cluster with docker and docker-compose and obviously it's not for production. + +You may find more information in the specific subdirectories but in generic you can use the following commands: + +## Usage + +To start a cluster go to a subdirectory and start the cluster: + +``` +docker-compose up -d +``` + +You can check the logs of all the components with: + +``` +docker-compose logs +``` + +In case of a problem you can destroy the cluster an delete all the local state with: + +``` +docker-compose down +``` + +(Note: a simple docker-compose stop may not delete all the local data). + +You can scale up and down the components: + +``` +docker-compose scale datanode=5 +``` + +Usually the key webui ports are published on the docker host. http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozone-hdfs/.env ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/compose/ozone-hdfs/.env b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/.env new file mode 100644 index 0000000..c437513 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/.env @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HADOOP_VERSION=3.1.0 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-compose.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-compose.yaml new file mode 100644 index 0000000..b89052d --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-compose.yaml @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3" +services: + namenode: + image: flokkr/hadoop:${HADOOP_VERSION} + ports: + - 9870:9870 + env_file: + - ./docker-config + environment: + ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name" + command: ["hdfs", "namenode"] + datanode: + image: flokkr/hadoop:${HADOOP_VERSION} + ports: + - 9864 + volumes: + - ../..:/opt/ozone + command: ["hdfs","datanode"] + environment: + HADOOP_CLASSPATH: /opt/ozone/share/hadoop/ozoneplugin/*.jar + env_file: + - ./docker-config + ozoneManager: + image: apache/hadoop-runner + volumes: + - ../..:/opt/hadoop + ports: + - 9874:9874 + environment: + ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION + env_file: + - ./docker-config + command: ["ozone","om"] + scm: + image: apache/hadoop-runner + volumes: + - ../..:/opt/hadoop + ports: + - 9876:9876 + env_file: + - ./docker-config + environment: + ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + command: ["ozone","scm"] http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config new file mode 100644 index 0000000..3b2819f --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +OZONE-SITE.XML_ozone.om.address=ozoneManager +OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874 +OZONE-SITE.XML_ozone.scm.names=scm +OZONE-SITE.XML_ozone.enabled=true +OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id +OZONE-SITE.XML_ozone.scm.block.client.address=scm +OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata +OZONE-SITE.XML_ozone.scm.client.address=scm +OZONE-SITE.XML_ozone.replication=1 +OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService + +HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService +HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000 +HDFS-SITE.XML_rpc.metrics.quantile.enable=true +HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 + +LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout +LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender +LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n + +#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation. +#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm + +#LOG4J2.PROPERTIES_* are for Ozone Audit Logging +LOG4J2.PROPERTIES_monitorInterval=30 +LOG4J2.PROPERTIES_filter=read,write +LOG4J2.PROPERTIES_filter.read.type=MarkerFilter +LOG4J2.PROPERTIES_filter.read.marker=READ +LOG4J2.PROPERTIES_filter.read.onMatch=DENY +LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL +LOG4J2.PROPERTIES_filter.write.type=MarkerFilter +LOG4J2.PROPERTIES_filter.write.marker=WRITE +LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL +LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL +LOG4J2.PROPERTIES_appenders=console, rolling +LOG4J2.PROPERTIES_appender.console.type=Console +LOG4J2.PROPERTIES_appender.console.name=STDOUT +LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout +LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n +LOG4J2.PROPERTIES_appender.rolling.type=RollingFile +LOG4J2.PROPERTIES_appender.rolling.name=RollingFile +LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log +LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz +LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout +LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n +LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies +LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy +LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400 +LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy +LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB +LOG4J2.PROPERTIES_loggers=audit +LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger +LOG4J2.PROPERTIES_logger.audit.name=OMAudit +LOG4J2.PROPERTIES_logger.audit.level=INFO +LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling +LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile +LOG4J2.PROPERTIES_rootLogger.level=INFO +LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout +LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozone/.env ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/compose/ozone/.env b/hadoop-ozone/dist/src/main/compose/ozone/.env new file mode 100644 index 0000000..67eed25 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone/.env @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HDDS_VERSION=${hdds.version} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml new file mode 100644 index 0000000..0a6a9d8 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3" +services: + datanode: + image: apache/hadoop-runner + volumes: + - ../..:/opt/hadoop + ports: + - 9864 + command: ["/opt/hadoop/bin/ozone","datanode"] + env_file: + - ./docker-config + ozoneManager: + image: apache/hadoop-runner + volumes: + - ../..:/opt/hadoop + ports: + - 9874:9874 + environment: + ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION + env_file: + - ./docker-config + command: ["/opt/hadoop/bin/ozone","om"] + scm: + image: apache/hadoop-runner + volumes: + - ../..:/opt/hadoop + ports: + - 9876:9876 + env_file: + - ./docker-config + environment: + ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + command: ["/opt/hadoop/bin/ozone","scm"] http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozone/docker-config ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-config b/hadoop-ozone/dist/src/main/compose/ozone/docker-config new file mode 100644 index 0000000..f2c8db1 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-config @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +OZONE-SITE.XML_ozone.om.address=ozoneManager +OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874 +OZONE-SITE.XML_ozone.scm.names=scm +OZONE-SITE.XML_ozone.enabled=True +OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id +OZONE-SITE.XML_ozone.scm.block.client.address=scm +OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata +OZONE-SITE.XML_ozone.handler.type=distributed +OZONE-SITE.XML_ozone.scm.client.address=scm +OZONE-SITE.XML_ozone.replication=1 +HDFS-SITE.XML_rpc.metrics.quantile.enable=true +HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 +LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout +LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender +LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n +LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR +LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN +LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR + +#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation. +#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm + +#LOG4J2.PROPERTIES_* are for Ozone Audit Logging +LOG4J2.PROPERTIES_monitorInterval=30 +LOG4J2.PROPERTIES_filter=read,write +LOG4J2.PROPERTIES_filter.read.type=MarkerFilter +LOG4J2.PROPERTIES_filter.read.marker=READ +LOG4J2.PROPERTIES_filter.read.onMatch=DENY +LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL +LOG4J2.PROPERTIES_filter.write.type=MarkerFilter +LOG4J2.PROPERTIES_filter.write.marker=WRITE +LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL +LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL +LOG4J2.PROPERTIES_appenders=console, rolling +LOG4J2.PROPERTIES_appender.console.type=Console +LOG4J2.PROPERTIES_appender.console.name=STDOUT +LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout +LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n +LOG4J2.PROPERTIES_appender.rolling.type=RollingFile +LOG4J2.PROPERTIES_appender.rolling.name=RollingFile +LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log +LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz +LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout +LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n +LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies +LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy +LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400 +LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy +LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB +LOG4J2.PROPERTIES_loggers=audit +LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger +LOG4J2.PROPERTIES_logger.audit.name=OMAudit +LOG4J2.PROPERTIES_logger.audit.level=INFO +LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling +LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile +LOG4J2.PROPERTIES_rootLogger.level=INFO +LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout +LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozonefs/docker-compose.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/compose/ozonefs/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonefs/docker-compose.yaml new file mode 100644 index 0000000..a1e8748 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozonefs/docker-compose.yaml @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3" +services: + datanode: + image: apache/hadoop-runner + volumes: + - ../..:/opt/hadoop + ports: + - 9864 + command: ["/opt/hadoop/bin/ozone","datanode"] + env_file: + - ./docker-config + ozoneManager: + image: apache/hadoop-runner + hostname: ozoneManager + volumes: + - ../..:/opt/hadoop + ports: + - 9874 + environment: + ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION + env_file: + - ./docker-config + command: ["/opt/hadoop/bin/ozone","om"] + scm: + image: apache/hadoop-runner + volumes: + - ../..:/opt/hadoop + ports: + - 9876 + env_file: + - ./docker-config + environment: + ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + command: ["/opt/hadoop/bin/ozone","scm"] + hadooplast: + image: flokkr/hadoop:3.1.0 + volumes: + - ../..:/opt/ozone + env_file: + - ./docker-config + environment: + HADOOP_CLASSPATH: /opt/ozone/share/hadoop/ozonefs/*.jar + command: ["watch","-n","100000","ls"] http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozonefs/docker-config ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/compose/ozonefs/docker-config b/hadoop-ozone/dist/src/main/compose/ozonefs/docker-config new file mode 100644 index 0000000..3171f08 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozonefs/docker-config @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CORE-SITE.XML_fs.o3.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem +OZONE-SITE.XML_ozone.om.address=ozoneManager +OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874 +OZONE-SITE.XML_ozone.scm.names=scm +OZONE-SITE.XML_ozone.enabled=True +OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id +OZONE-SITE.XML_ozone.scm.block.client.address=scm +OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata +OZONE-SITE.XML_ozone.handler.type=distributed +OZONE-SITE.XML_ozone.scm.client.address=scm +OZONE-SITE.XML_ozone.scm.heartbeat.interval=3s +HDFS-SITE.XML_rpc.metrics.quantile.enable=true +HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 +LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout +LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender +LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n +LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR +LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozoneperf/.env ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/.env b/hadoop-ozone/dist/src/main/compose/ozoneperf/.env new file mode 100644 index 0000000..cac418a --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozoneperf/.env @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HDDS_VERSION=${hdds.version} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
