HDDS-223. Create acceptance test for using datanode plugin. Contributed by Sandeep Nemuri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d40912b1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d40912b1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d40912b1 Branch: refs/heads/HDFS-12943 Commit: d40912b1bd8e84592953185e827774d90ed2ce6b Parents: 0a8b1ef Author: Márton Elek <[email protected]> Authored: Thu Nov 15 15:17:18 2018 +0100 Committer: Márton Elek <[email protected]> Committed: Thu Nov 15 15:18:07 2018 +0100 ---------------------------------------------------------------------- hadoop-ozone/dist/src/main/compose/ozone-hdfs/.env | 2 +- .../main/compose/ozone-hdfs/docker-compose.yaml | 13 +++++++++++-- .../dist/src/main/compose/ozone-hdfs/docker-config | 4 ++++ hadoop-ozone/dist/src/main/smoketest/.env | 17 +++++++++++++++++ .../dist/src/main/smoketest/basic/basic.robot | 2 +- hadoop-ozone/dist/src/main/smoketest/test.sh | 4 +++- 6 files changed, 37 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d40912b1/hadoop-ozone/dist/src/main/compose/ozone-hdfs/.env ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/compose/ozone-hdfs/.env b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/.env index c437513..47a25e1 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-hdfs/.env +++ b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/.env @@ -14,4 +14,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -HADOOP_VERSION=3.1.0 \ No newline at end of file +HADOOP_VERSION=3 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/d40912b1/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-compose.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-compose.yaml index b89052d..7b8f5d3 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-compose.yaml @@ -17,7 +17,7 @@ version: "3" services: namenode: - image: flokkr/hadoop:${HADOOP_VERSION} + image: apache/hadoop:${HADOOP_VERSION} ports: - 9870:9870 env_file: @@ -26,7 +26,7 @@ services: ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name" command: ["hdfs", "namenode"] datanode: - image: flokkr/hadoop:${HADOOP_VERSION} + image: apache/hadoop:${HADOOP_VERSION} ports: - 9864 volumes: @@ -58,3 +58,12 @@ services: environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION command: ["ozone","scm"] + s3g: + image: apache/hadoop-runner + volumes: + - ../..:/opt/hadoop + ports: + - 9878:9878 + env_file: + - ./docker-config + command: ["ozone","s3g"] http://git-wip-us.apache.org/repos/asf/hadoop/blob/d40912b1/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config index 9729aef..32931c6 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config @@ -35,6 +35,10 @@ LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n +LOG4J.PROPERTIES_log4j.logger.http.requests.s3gateway=INFO,s3gatewayrequestlog +LOG4J.PROPERTIES_log4j.appender.s3gatewayrequestlog=org.apache.hadoop.http.HttpRequestLogAppender +LOG4J.PROPERTIES_log4j.appender.s3gatewayrequestlog.Filename=/tmp/jetty-s3gateway-yyyy_mm_dd.log +LOG4J.PROPERTIES_log4j.appender.s3gatewayrequestlog.RetainDays=3 #Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation. #BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm http://git-wip-us.apache.org/repos/asf/hadoop/blob/d40912b1/hadoop-ozone/dist/src/main/smoketest/.env ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/smoketest/.env b/hadoop-ozone/dist/src/main/smoketest/.env new file mode 100644 index 0000000..47a25e1 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/.env @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HADOOP_VERSION=3 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/d40912b1/hadoop-ozone/dist/src/main/smoketest/basic/basic.robot ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/smoketest/basic/basic.robot b/hadoop-ozone/dist/src/main/smoketest/basic/basic.robot index a69450d..fb58520 100644 --- a/hadoop-ozone/dist/src/main/smoketest/basic/basic.robot +++ b/hadoop-ozone/dist/src/main/smoketest/basic/basic.robot @@ -20,7 +20,7 @@ Resource ../commonlib.robot *** Variables *** ${COMMON_REST_HEADER} -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE root" -${DATANODE_HOST} localhost +${DATANODE_HOST} datanode *** Test Cases *** http://git-wip-us.apache.org/repos/asf/hadoop/blob/d40912b1/hadoop-ozone/dist/src/main/smoketest/test.sh ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/smoketest/test.sh b/hadoop-ozone/dist/src/main/smoketest/test.sh index 59903c6..5819595 100755 --- a/hadoop-ozone/dist/src/main/smoketest/test.sh +++ b/hadoop-ozone/dist/src/main/smoketest/test.sh @@ -44,7 +44,7 @@ execute_tests(){ TITLE="Ozone $TEST tests with $COMPOSE_DIR cluster" set +e OUTPUT_NAME="$COMPOSE_DIR-${TEST//\//_}" - docker-compose -f "$COMPOSE_FILE" exec datanode python -m robot --log NONE --report NONE "${OZONE_ROBOT_OPTS[@]}" --output "smoketest/$RESULT_DIR/robot-$OUTPUT_NAME.xml" --logtitle "$TITLE" --reporttitle "$TITLE" "smoketest/$TEST" + docker-compose -f "$COMPOSE_FILE" exec ozoneManager python -m robot --log NONE --report NONE "${OZONE_ROBOT_OPTS[@]}" --output "smoketest/$RESULT_DIR/robot-$OUTPUT_NAME.xml" --logtitle "$TITLE" --reporttitle "$TITLE" "smoketest/$TEST" set -e docker-compose -f "$COMPOSE_FILE" logs > "$DIR/$RESULT_DIR/docker-$OUTPUT_NAME.log" done @@ -111,6 +111,8 @@ if [ "$RUN_ALL" = true ]; then execute_tests ozone "${DEFAULT_TESTS[@]}" TESTS=("ozonefs") execute_tests ozonefs "${TESTS[@]}" + TESTS=("ozone-hdfs") + execute_tests ozone-hdfs "${DEFAULT_TESTS[@]}" TESTS=("s3") execute_tests ozones3 "${TESTS[@]}" else --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
