This is an automated email from the ASF dual-hosted git repository. sammichen pushed a commit to branch ozone-0.6.0 in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 6361f4a937d3c4c7b63e94931e940d5af97c43f9 Author: Doroszlai, Attila <[email protected]> AuthorDate: Fri Jul 17 12:45:03 2020 +0200 HDDS-3855. Add upgrade smoketest (#1142) (cherry picked from commit 9b13ab67ca433b44a548bd1ee8c9fa4b01250c50) --- .github/workflows/post-commit.yml | 1 + .../dist/dev-support/bin/dist-layout-stitching | 1 + hadoop-ozone/dist/pom.xml | 2 +- hadoop-ozone/dist/src/main/compose/testlib.sh | 39 +++---- hadoop-ozone/dist/src/main/compose/upgrade/.env | 21 ++++ .../dist/src/main/compose/upgrade/README.md | 29 +++++ .../src/main/compose/upgrade/docker-compose.yaml | 127 +++++++++++++++++++++ .../dist/src/main/compose/upgrade/docker-config | 33 ++++++ hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 70 ++++++++++++ .../src/main/compose/upgrade/versions/README.md | 15 +++ .../main/compose/upgrade/versions/ozone-0.5.0.sh | 18 +++ .../main/compose/upgrade/versions/ozone-0.6.0.sh | 18 +++ hadoop-ozone/dist/src/shell/upgrade/0.6.0.sh | 23 ++++ .../src/shell/upgrade/0.6.0/01-migrate-scm-db.sh | 24 ++++ 14 files changed, 400 insertions(+), 21 deletions(-) diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml index e00018a..bac27e4 100644 --- a/.github/workflows/post-commit.yml +++ b/.github/workflows/post-commit.yml @@ -150,6 +150,7 @@ jobs: env: KEEP_IMAGE: false OZONE_WITH_COVERAGE: true + OZONE_VOLUME_OWNER: 1000 - uses: actions/upload-artifact@master if: always() with: diff --git a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching index e1f5c7e..80455a6 100755 --- a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching +++ b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching @@ -108,6 +108,7 @@ run cp "${ROOT}/hadoop-ozone/dist/src/shell/hdds/hadoop-config.cmd" "libexec/" run cp "${ROOT}/hadoop-ozone/dist/src/shell/hdds/hadoop-functions.sh" "libexec/" run cp "${ROOT}/hadoop-ozone/dist/src/shell/ozone/ozone-config.sh" "libexec/" run cp -r "${ROOT}/hadoop-ozone/dist/src/shell/shellprofile.d" "libexec/" +run cp -r "${ROOT}/hadoop-ozone/dist/src/shell/upgrade" "libexec/" run cp "${ROOT}/hadoop-ozone/dist/src/shell/hdds/hadoop-daemons.sh" "sbin/" diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml index 840f628..a766c0a 100644 --- a/hadoop-ozone/dist/pom.xml +++ b/hadoop-ozone/dist/pom.xml @@ -28,7 +28,7 @@ <properties> <file.encoding>UTF-8</file.encoding> <downloadSources>true</downloadSources> - <docker.ozone-runner.version>20200420-1</docker.ozone-runner.version> + <docker.ozone-runner.version>20200625-1</docker.ozone-runner.version> </properties> <build> diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh index 15d1664..56c35c1 100755 --- a/hadoop-ozone/dist/src/main/compose/testlib.sh +++ b/hadoop-ozone/dist/src/main/compose/testlib.sh @@ -17,7 +17,6 @@ set -e COMPOSE_ENV_NAME=$(basename "$COMPOSE_DIR") -COMPOSE_FILE=$COMPOSE_DIR/docker-compose.yaml RESULT_DIR=${RESULT_DIR:-"$COMPOSE_DIR/result"} RESULT_DIR_INSIDE="/tmp/smoketest/$(basename "$COMPOSE_ENV_NAME")/result" SMOKETEST_DIR_INSIDE="${OZONE_DIR:-/opt/hadoop}/smoketest" @@ -32,7 +31,7 @@ fi ## @description create results directory, purging any prior data create_results_dir() { #delete previous results - rm -rf "$RESULT_DIR" + [[ "${OZONE_KEEP_RESULTS:-}" == "true" ]] || rm -rf "$RESULT_DIR" mkdir -p "$RESULT_DIR" #Should be writeable from the docker containers where user is different. chmod ogu+w "$RESULT_DIR" @@ -40,9 +39,9 @@ create_results_dir() { ## @description wait until safemode exit (or 180 seconds) -## @param the docker-compose file wait_for_safemode_exit(){ - local compose_file=$1 + # version-dependent + : ${OZONE_ADMIN_COMMAND:=admin} #Reset the timer SECONDS=0 @@ -51,11 +50,11 @@ wait_for_safemode_exit(){ while [[ $SECONDS -lt 180 ]]; do #This line checks the safemode status in scm - local command="ozone admin safemode status" + local command="ozone ${OZONE_ADMIN_COMMAND} safemode status" if [[ "${SECURITY_ENABLED}" == 'true' ]]; then - status=$(docker-compose -f "${compose_file}" exec -T scm bash -c "kinit -k HTTP/[email protected] -t /etc/security/keytabs/HTTP.keytab && $command" || true) + status=$(docker-compose exec -T scm bash -c "kinit -k HTTP/[email protected] -t /etc/security/keytabs/HTTP.keytab && $command" || true) else - status=$(docker-compose -f "${compose_file}" exec -T scm bash -c "$command") + status=$(docker-compose exec -T scm bash -c "$command") fi echo $status @@ -80,9 +79,9 @@ start_docker_env(){ create_results_dir export OZONE_SAFEMODE_MIN_DATANODES="${datanode_count}" - docker-compose -f "$COMPOSE_FILE" --no-ansi down - if ! { docker-compose -f "$COMPOSE_FILE" --no-ansi up -d --scale datanode="${datanode_count}" \ - && wait_for_safemode_exit "$COMPOSE_FILE"; }; then + docker-compose --no-ansi down + if ! { docker-compose --no-ansi up -d --scale datanode="${datanode_count}" \ + && wait_for_safemode_exit ; }; then OUTPUT_NAME="$COMPOSE_ENV_NAME" stop_docker_env return 1 @@ -114,11 +113,11 @@ execute_robot_test(){ OUTPUT_PATH="$RESULT_DIR_INSIDE/${OUTPUT_FILE}" # shellcheck disable=SC2068 - docker-compose -f "$COMPOSE_FILE" exec -T "$CONTAINER" mkdir -p "$RESULT_DIR_INSIDE" \ - && docker-compose -f "$COMPOSE_FILE" exec -T "$CONTAINER" robot -v OM_SERVICE_ID:"${OM_SERVICE_ID}" -v SECURITY_ENABLED:"${SECURITY_ENABLED}" -v OM_HA_PARAM:"${OM_HA_PARAM}" ${ARGUMENTS[@]} --log NONE -N "$TEST_NAME" --report NONE "${OZONE_ROBOT_OPTS[@]}" --output "$OUTPUT_PATH" "$SMOKETEST_DIR_INSIDE/$TEST" + docker-compose exec -T "$CONTAINER" mkdir -p "$RESULT_DIR_INSIDE" \ + && docker-compose exec -T "$CONTAINER" robot -v OM_SERVICE_ID:"${OM_SERVICE_ID}" -v SECURITY_ENABLED:"${SECURITY_ENABLED}" -v OM_HA_PARAM:"${OM_HA_PARAM}" ${ARGUMENTS[@]} --log NONE -N "$TEST_NAME" --report NONE "${OZONE_ROBOT_OPTS[@]}" --output "$OUTPUT_PATH" "$SMOKETEST_DIR_INSIDE/$TEST" local -i rc=$? - FULL_CONTAINER_NAME=$(docker-compose -f "$COMPOSE_FILE" ps | grep "_${CONTAINER}_" | head -n 1 | awk '{print $1}') + FULL_CONTAINER_NAME=$(docker-compose ps | grep "_${CONTAINER}_" | head -n 1 | awk '{print $1}') docker cp "$FULL_CONTAINER_NAME:$OUTPUT_PATH" "$RESULT_DIR/" copy_daemon_logs @@ -135,7 +134,7 @@ execute_robot_test(){ ## @description Copy any 'out' files for daemon processes to the result dir copy_daemon_logs() { local c f - for c in $(docker-compose -f "$COMPOSE_FILE" ps | grep "^${COMPOSE_ENV_NAME}_" | awk '{print $1}'); do + for c in $(docker-compose ps | grep "^${COMPOSE_ENV_NAME}_" | awk '{print $1}'); do for f in $(docker exec "${c}" ls -1 /var/log/hadoop | grep -F '.out'); do docker cp "${c}:/var/log/hadoop/${f}" "$RESULT_DIR/" done @@ -149,7 +148,7 @@ copy_daemon_logs() { execute_command_in_container(){ set -e # shellcheck disable=SC2068 - docker-compose -f "$COMPOSE_FILE" exec -T "$@" + docker-compose exec -T "$@" set +e } @@ -157,7 +156,7 @@ execute_command_in_container(){ ## @param List of container names, eg datanode_1 datanode_2 stop_containers() { set -e - docker-compose -f "$COMPOSE_FILE" --no-ansi stop $@ + docker-compose --no-ansi stop $@ set +e } @@ -166,7 +165,7 @@ stop_containers() { ## @param List of container names, eg datanode_1 datanode_2 start_containers() { set -e - docker-compose -f "$COMPOSE_FILE" --no-ansi start $@ + docker-compose --no-ansi start $@ set +e } @@ -185,7 +184,7 @@ wait_for_port(){ while [[ $SECONDS -lt $timeout ]]; do set +e - docker-compose -f "${COMPOSE_FILE}" exec -T scm /bin/bash -c "nc -z $host $port" + docker-compose exec -T scm /bin/bash -c "nc -z $host $port" status=$? set -e if [ $status -eq 0 ] ; then @@ -202,9 +201,9 @@ wait_for_port(){ ## @description Stops a docker-compose based test environment (with saving the logs) stop_docker_env(){ - docker-compose -f "$COMPOSE_FILE" --no-ansi logs > "$RESULT_DIR/docker-$OUTPUT_NAME.log" + docker-compose --no-ansi logs > "$RESULT_DIR/docker-$OUTPUT_NAME.log" if [ "${KEEP_RUNNING:-false}" = false ]; then - docker-compose -f "$COMPOSE_FILE" --no-ansi down + docker-compose --no-ansi down fi } diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/.env b/hadoop-ozone/dist/src/main/compose/upgrade/.env new file mode 100644 index 0000000..6f757c5 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/upgrade/.env @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HDDS_VERSION=${hdds.version} +OZONE_RUNNER_VERSION=${docker.ozone-runner.version} +OZONE_IMAGE=apache/ozone-runner:${docker.ozone-runner.version} +OZONE_DIR=/opt/hadoop +OZONE_VOLUME=. diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/README.md b/hadoop-ozone/dist/src/main/compose/upgrade/README.md new file mode 100644 index 0000000..2a832f4 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/upgrade/README.md @@ -0,0 +1,29 @@ +<!--- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +# Compose file for upgrade + +This directory contains a sample cluster definition and script for +testing upgrade from previous version to the current one. + +Data for each container is persisted in mounted volume (by default it's +`data` under the `compose/upgrade` directory, but can be overridden via +`OZONE_VOLUME` environment variable). + +Prior version is run using an official `apache/ozone` image, while the +current version is run with the `ozone-runner` image using locally built +source code. + +Currently the test script only supports a single version upgrade (eg. +from 0.5.0 to 0.6.0). diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/upgrade/docker-compose.yaml new file mode 100644 index 0000000..ae52759 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/upgrade/docker-compose.yaml @@ -0,0 +1,127 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3.4" + +# reusable fragments (see https://docs.docker.com/compose/compose-file/#extension-fields) +x-common-config: + &common-config + env_file: + - docker-config + image: ${OZONE_IMAGE} + +x-replication: + &replication + OZONE-SITE.XML_ozone.replication: ${OZONE_REPLICATION_FACTOR:-1} + +x-datanode: + &datanode + command: ["ozone","datanode"] + <<: *common-config + environment: + <<: *replication + ports: + - 9864 + - 9882 + +services: + dn1: + <<: *datanode + networks: + net: + ipv4_address: 10.9.0.11 + volumes: + - ${OZONE_VOLUME}/dn1:/data + - ../..:${OZONE_DIR} + dn2: + <<: *datanode + networks: + net: + ipv4_address: 10.9.0.12 + volumes: + - ${OZONE_VOLUME}/dn2:/data + - ../..:${OZONE_DIR} + dn3: + <<: *datanode + networks: + net: + ipv4_address: 10.9.0.13 + volumes: + - ${OZONE_VOLUME}/dn3:/data + - ../..:${OZONE_DIR} + om: + command: ["ozone","om"] + <<: *common-config + environment: + ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION + <<: *replication + networks: + net: + ipv4_address: 10.9.0.14 + ports: + - 9874:9874 + - 9862:9862 + volumes: + - ${OZONE_VOLUME}/om:/data + - ../..:${OZONE_DIR} + recon: + command: ["ozone","recon"] + <<: *common-config + environment: + <<: *replication + networks: + net: + ipv4_address: 10.9.0.15 + ports: + - 9888:9888 + volumes: + - ${OZONE_VOLUME}/recon:/data + - ../..:${OZONE_DIR} + s3g: + command: ["ozone","s3g"] + <<: *common-config + environment: + <<: *replication + networks: + net: + ipv4_address: 10.9.0.16 + ports: + - 9878:9878 + volumes: + - ${OZONE_VOLUME}/s3g:/data + - ../..:${OZONE_DIR} + scm: + command: ["ozone","scm"] + <<: *common-config + environment: + ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_SAFEMODE_MIN_DATANODES:-1} + <<: *replication + networks: + net: + ipv4_address: 10.9.0.17 + ports: + - 9876:9876 + volumes: + - ${OZONE_VOLUME}/scm:/data + - ../..:${OZONE_DIR} + +networks: + net: + driver: bridge + ipam: + config: + - subnet: 10.9.0.0/16 diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/docker-config b/hadoop-ozone/dist/src/main/compose/upgrade/docker-config new file mode 100644 index 0000000..ecc994b --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/upgrade/docker-config @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CORE-SITE.XML_fs.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzoneFileSystem +CORE-SITE.XML_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem +OZONE-SITE.XML_ozone.om.address=om +OZONE-SITE.XML_ozone.om.http-address=om:9874 +OZONE-SITE.XML_ozone.scm.container.size=1GB +OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 +OZONE-SITE.XML_ozone.scm.names=scm +OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data +OZONE-SITE.XML_ozone.scm.block.client.address=scm +OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata +OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon +OZONE-SITE.XML_ozone.scm.client.address=scm +OZONE-SITE.XML_hdds.datanode.dir=/data/hdds +OZONE-SITE.XML_ozone.recon.address=recon:9891 +OZONE-SITE.XML_recon.om.snapshot.task.interval.delay=1m + +no_proxy=om,scm,s3g,kdc,localhost,127.0.0.1 diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh new file mode 100644 index 0000000..0babd17 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +export COMPOSE_DIR + +: "${OZONE_REPLICATION_FACTOR:=3}" +: "${OZONE_UPGRADE_FROM:="0.5.0"}" +: "${OZONE_UPGRADE_TO:="0.6.0"}" +: "${OZONE_VOLUME:="${COMPOSE_DIR}/data"}" + +export OZONE_VOLUME + +mkdir -p "${OZONE_VOLUME}"/{dn1,dn2,dn3,om,recon,s3g,scm} +mkdir -p "${OZONE_VOLUME}/debug" + +if [[ -n "${OZONE_VOLUME_OWNER}" ]]; then + current_user=$(whoami) + if [[ "${OZONE_VOLUME_OWNER}" != "${current_user}" ]]; then + chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}" \ + || sudo chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}" + fi +fi + +# define version-specifics +export OZONE_DIR=/opt/ozone +export OZONE_IMAGE="apache/ozone:${OZONE_UPGRADE_FROM}" +# shellcheck source=/dev/null +source "${COMPOSE_DIR}/versions/ozone-${OZONE_UPGRADE_FROM}.sh" +# shellcheck source=/dev/null +source "${COMPOSE_DIR}/../testlib.sh" + +# prepare pre-upgrade cluster +start_docker_env +execute_robot_test scm topology/loaddata.robot +stop_docker_env + +# run upgrade scripts +SCRIPT_DIR=../../libexec/upgrade +[[ -f "${SCRIPT_DIR}/${OZONE_UPGRADE_TO}.sh" ]] && "${SCRIPT_DIR}/${OZONE_UPGRADE_TO}.sh" + +# update version-specifics +export OZONE_DIR=/opt/hadoop +unset OZONE_IMAGE # use apache/ozone-runner defined in docker-compose.yaml +# shellcheck source=/dev/null +source "${COMPOSE_DIR}/versions/ozone-${OZONE_UPGRADE_TO}.sh" +# shellcheck source=/dev/null +source "${COMPOSE_DIR}/../testlib.sh" + +# re-start cluster with new version and check after upgrade +export OZONE_KEEP_RESULTS=true +start_docker_env +execute_robot_test scm topology/readdata.robot +stop_docker_env + +generate_report diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/versions/README.md b/hadoop-ozone/dist/src/main/compose/upgrade/versions/README.md new file mode 100644 index 0000000..c662c2f --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/upgrade/versions/README.md @@ -0,0 +1,15 @@ +<!--- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +The scripts in this directory define version-specific behavior required for [`testlib.sh`](../../testlib.sh). For example the `ozone admin` command was renamed from `ozone scmcli` in 0.6.0. diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.5.0.sh b/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.5.0.sh new file mode 100644 index 0000000..68f6c75 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.5.0.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +export OZONE_ADMIN_COMMAND=scmcli diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.6.0.sh b/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.6.0.sh new file mode 100644 index 0000000..bab1dba --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.6.0.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +export OZONE_ADMIN_COMMAND=admin diff --git a/hadoop-ozone/dist/src/shell/upgrade/0.6.0.sh b/hadoop-ozone/dist/src/shell/upgrade/0.6.0.sh new file mode 100755 index 0000000..58c78db --- /dev/null +++ b/hadoop-ozone/dist/src/shell/upgrade/0.6.0.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +: "${SCM_DIR:="${OZONE_VOLUME}/scm"}" +: "${OZONE_RUNNER_VERSION:="20200625-1"}" + +docker run --rm -v "${SCM_DIR}":/scm -v "${SCRIPT_DIR}/0.6.0":/upgrade -w /scm/metadata apache/ozone-runner:"${OZONE_RUNNER_VERSION}" /upgrade/01-migrate-scm-db.sh diff --git a/hadoop-ozone/dist/src/shell/upgrade/0.6.0/01-migrate-scm-db.sh b/hadoop-ozone/dist/src/shell/upgrade/0.6.0/01-migrate-scm-db.sh new file mode 100755 index 0000000..dee6768 --- /dev/null +++ b/hadoop-ozone/dist/src/shell/upgrade/0.6.0/01-migrate-scm-db.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo Running upgrade script for HDDS-3499 + +ldb --db=scm.db create_column_family containers +ldb --db=scm.db create_column_family pipelines + +ldb --db=scm-container.db --key_hex --value_hex dump | ldb --db=scm.db --key_hex --value_hex --column_family=containers load +ldb --db=scm-pipeline.db --key_hex --value_hex dump | ldb --db=scm.db --key_hex --value_hex --column_family=pipelines load --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
