This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 3e798e6ce5 HDDS-11698. Use hadoop images from GitHub in CI (#7432)
3e798e6ce5 is described below
commit 3e798e6ce5de825aab13e60d3f4950878639930b
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Sun Nov 17 10:38:23 2024 +0100
HDDS-11698. Use hadoop images from GitHub in CI (#7432)
---
.github/workflows/ci.yml | 1 +
.../src/main/compose/common/hadoop-secure.yaml | 6 ++---
.../dist/src/main/compose/common/hadoop-test.sh | 28 ++++++++++++----------
.../dist/src/main/compose/common/hadoop.yaml | 4 ++--
.../dist/src/main/compose/ozonesecure-ha/.env | 1 +
.../compose/ozonesecure-ha/docker-compose.yaml | 2 +-
.../dist/src/main/compose/ozonesecure-mr/.env | 1 +
.../compose/ozonesecure-mr/docker-compose.yaml | 2 +-
.../dist/src/main/compose/ozonesecure/.env | 1 +
.../main/compose/ozonesecure/docker-compose.yaml | 2 +-
.../dist/src/main/compose/upgrade/compose/ha/.env | 1 +
.../compose/upgrade/compose/ha/docker-compose.yaml | 2 +-
hadoop-ozone/dist/src/main/compose/xcompat/.env | 1 +
.../dist/src/main/compose/xcompat/new-cluster.yaml | 2 +-
.../dist/src/main/compose/xcompat/old-cluster.yaml | 2 +-
15 files changed, 33 insertions(+), 23 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5bd913b53b..bf180b1500 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -31,6 +31,7 @@ env:
# Minimum required Java version for running Ozone is defined in pom.xml
(javac.version).
TEST_JAVA_VERSION: 17 # JDK version used by CI build and tests; should match
the JDK version in apache/ozone-runner image
MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-Dmaven.wagon.http.retryHandler.class=standard
-Dmaven.wagon.http.retryHandler.count=3
+ HADOOP_IMAGE: ghcr.io/apache/hadoop
OZONE_IMAGE: ghcr.io/apache/ozone
OZONE_RUNNER_IMAGE: ghcr.io/apache/ozone-runner
OZONE_WITH_COVERAGE: ${{ github.event_name == 'push' }}
diff --git a/hadoop-ozone/dist/src/main/compose/common/hadoop-secure.yaml
b/hadoop-ozone/dist/src/main/compose/common/hadoop-secure.yaml
index 8600659b78..ee97525fbb 100644
--- a/hadoop-ozone/dist/src/main/compose/common/hadoop-secure.yaml
+++ b/hadoop-ozone/dist/src/main/compose/common/hadoop-secure.yaml
@@ -16,7 +16,7 @@
services:
rm:
- image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
+ image: ${HADOOP_TEST_IMAGE}
hostname: rm
dns_search: .
volumes:
@@ -35,7 +35,7 @@ services:
profiles:
- hadoop
nm:
- image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
+ image: ${HADOOP_TEST_IMAGE}
hostname: nm
dns_search: .
volumes:
@@ -54,7 +54,7 @@ services:
profiles:
- hadoop
jhs:
- image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
+ image: ${HADOOP_TEST_IMAGE}
container_name: jhs
hostname: jhs
dns_search: .
diff --git a/hadoop-ozone/dist/src/main/compose/common/hadoop-test.sh
b/hadoop-ozone/dist/src/main/compose/common/hadoop-test.sh
index 7bf5607443..c16f6fe22f 100755
--- a/hadoop-ozone/dist/src/main/compose/common/hadoop-test.sh
+++ b/hadoop-ozone/dist/src/main/compose/common/hadoop-test.sh
@@ -21,14 +21,18 @@ if [[ ${SECURITY_ENABLED} == "true" ]]; then
fi
export
COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.yaml}":../common/${extra_compose_file}
-# need temp variables because maven filtering replaces only one item per line
-hadoop2_version="${hadoop2.version}"
-hadoop_version="${hadoop.version}"
-
-: ${HADOOP_TEST_VERSIONS:="apache/hadoop:${hadoop2_version}
flokkr/hadoop:3.1.2 apache/hadoop:${hadoop_version}"}
+: ${HADOOP_IMAGE:="apache/hadoop"}
+: ${HADOOP_TEST_IMAGES:=""}
+
+if [[ -z "${HADOOP_TEST_IMAGES}" ]]; then
+ # hadoop2 and flokkr images are only available from Docker Hub
+ HADOOP_TEST_IMAGES="${HADOOP_TEST_IMAGES} apache/hadoop:${hadoop2.version}"
+ HADOOP_TEST_IMAGES="${HADOOP_TEST_IMAGES} flokkr/hadoop:3.1.2"
+ HADOOP_TEST_IMAGES="${HADOOP_TEST_IMAGES} ${HADOOP_IMAGE}:${hadoop.version}"
+fi
export HADOOP_MAJOR_VERSION=3
-export HADOOP_VERSION=unused # will be set for each test version below
+export HADOOP_TEST_IMAGE="${HADOOP_IMAGE}:${hadoop.version}"
export OZONE_REPLICATION_FACTOR=3
# shellcheck source=/dev/null
@@ -48,10 +52,10 @@ export OZONE_DIR=/opt/ozone
# shellcheck source=/dev/null
source "$COMPOSE_DIR/../testlib.sh"
-for test_version in $HADOOP_TEST_VERSIONS; do
- export HADOOP_IMAGE="${test_version%%:*}"
- export HADOOP_VERSION="${test_version##*:}"
- export HADOOP_MAJOR_VERSION=${HADOOP_VERSION%%.*}
+for HADOOP_TEST_IMAGE in $HADOOP_TEST_IMAGES; do
+ export HADOOP_TEST_IMAGE
+ hadoop_version="${HADOOP_TEST_IMAGE##*:}"
+ export HADOOP_MAJOR_VERSION=${hadoop_version%%.*}
docker-compose --ansi never --profile hadoop up -d nm rm
@@ -62,10 +66,10 @@ for test_version in $HADOOP_TEST_VERSIONS; do
fi
for scheme in o3fs ofs; do
- execute_robot_test rm -v "SCHEME:${scheme}" -N
"hadoop-${HADOOP_VERSION}-hadoopfs-${scheme}" ozonefs/hadoopo3fs.robot
+ execute_robot_test rm -v "SCHEME:${scheme}" -N
"hadoop-${hadoop_version}-hadoopfs-${scheme}" ozonefs/hadoopo3fs.robot
# TODO secure MapReduce test is failing with 2.7 due to some token problem
if [[ ${SECURITY_ENABLED} != "true" ]] || [[ ${HADOOP_MAJOR_VERSION} ==
"3" ]]; then
- execute_robot_test rm -v "SCHEME:${scheme}" -N
"hadoop-${HADOOP_VERSION}-mapreduce-${scheme}" mapreduce.robot
+ execute_robot_test rm -v "SCHEME:${scheme}" -N
"hadoop-${hadoop_version}-mapreduce-${scheme}" mapreduce.robot
fi
done
diff --git a/hadoop-ozone/dist/src/main/compose/common/hadoop.yaml
b/hadoop-ozone/dist/src/main/compose/common/hadoop.yaml
index 4fb56e5aa9..c5899eb6b4 100644
--- a/hadoop-ozone/dist/src/main/compose/common/hadoop.yaml
+++ b/hadoop-ozone/dist/src/main/compose/common/hadoop.yaml
@@ -16,7 +16,7 @@
services:
rm:
- image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
+ image: ${HADOOP_TEST_IMAGE}
hostname: rm
volumes:
- ../..:/opt/ozone
@@ -31,7 +31,7 @@ services:
profiles:
- hadoop
nm:
- image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
+ image: ${HADOOP_TEST_IMAGE}
hostname: nm
volumes:
- ../..:/opt/ozone
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
index 5f3e96ed61..c3a2c5329a 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
@@ -15,6 +15,7 @@
# limitations under the License.
HDDS_VERSION=${hdds.version}
+HADOOP_IMAGE=apache/hadoop
HADOOP_VERSION=${hadoop.version}
OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
OZONE_RUNNER_IMAGE=apache/ozone-runner
diff --git
a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml
index d55d5e0e2e..354f718ade 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml
@@ -27,7 +27,7 @@ services:
ozone_net:
ipv4_address: 172.25.0.100
kms:
- image: apache/hadoop:${HADOOP_VERSION}
+ image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
dns_search: .
ports:
- 9600:9600
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env
b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env
index cb5ab1080a..c3a2c5329a 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env
@@ -16,6 +16,7 @@
HDDS_VERSION=${hdds.version}
HADOOP_IMAGE=apache/hadoop
+HADOOP_VERSION=${hadoop.version}
OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
OZONE_RUNNER_IMAGE=apache/ozone-runner
OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image}
diff --git
a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml
b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml
index 625aac7772..f0b094447d 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml
@@ -24,7 +24,7 @@ services:
- ../_keytabs:/etc/security/keytabs
command: ["krb5kdc","-n"]
kms:
- image: apache/hadoop:3
+ image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
dns_search: .
ports:
- 9600:9600
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
index 5f3e96ed61..c3a2c5329a 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
@@ -15,6 +15,7 @@
# limitations under the License.
HDDS_VERSION=${hdds.version}
+HADOOP_IMAGE=apache/hadoop
HADOOP_VERSION=${hadoop.version}
OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
OZONE_RUNNER_IMAGE=apache/ozone-runner
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
index 026dfa1edc..f0b4aee1b3 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
@@ -24,7 +24,7 @@ services:
- ../_keytabs:/etc/security/keytabs
command: ["krb5kdc","-n"]
kms:
- image: apache/hadoop:${HADOOP_VERSION}
+ image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
hostname: kms
dns_search: .
ports:
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/.env
b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/.env
index dee41e9e66..2625c4fbe9 100644
--- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/.env
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/.env
@@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+HADOOP_IMAGE=apache/hadoop
HADOOP_VERSION=${hadoop.version}
HDDS_VERSION=${hdds.version}
OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
diff --git
a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml
b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml
index 0d0af747ee..86ebcc0c2b 100644
--- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml
@@ -83,7 +83,7 @@ services:
- docker-config
environment:
HADOOP_CONF_DIR: /opt/hadoop/etc/hadoop
- image: apache/hadoop:${HADOOP_VERSION}
+ image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
networks:
net:
ipv4_address: 10.9.0.3
diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/.env
b/hadoop-ozone/dist/src/main/compose/xcompat/.env
index 6ace68e893..11979d3432 100644
--- a/hadoop-ozone/dist/src/main/compose/xcompat/.env
+++ b/hadoop-ozone/dist/src/main/compose/xcompat/.env
@@ -17,6 +17,7 @@
HDDS_VERSION=${hdds.version}
OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
OZONE_RUNNER_IMAGE=apache/ozone-runner
+HADOOP_IMAGE=apache/hadoop
HADOOP_VERSION=${hadoop.version}
OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image}
OZONE_IMAGE=${docker.ozone.image}
diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/new-cluster.yaml
b/hadoop-ozone/dist/src/main/compose/xcompat/new-cluster.yaml
index 32059140ce..0f7105b8cc 100644
--- a/hadoop-ozone/dist/src/main/compose/xcompat/new-cluster.yaml
+++ b/hadoop-ozone/dist/src/main/compose/xcompat/new-cluster.yaml
@@ -36,7 +36,7 @@ services:
- ../_keytabs:/etc/security/keytabs
command: [ "krb5kdc","-n" ]
kms:
- image: apache/hadoop:${HADOOP_VERSION}
+ image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
hostname: kms
dns_search: .
ports:
diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/old-cluster.yaml
b/hadoop-ozone/dist/src/main/compose/xcompat/old-cluster.yaml
index 5c57eae514..fceb8e6eb3 100644
--- a/hadoop-ozone/dist/src/main/compose/xcompat/old-cluster.yaml
+++ b/hadoop-ozone/dist/src/main/compose/xcompat/old-cluster.yaml
@@ -36,7 +36,7 @@ services:
- ../_keytabs:/etc/security/keytabs
command: [ "krb5kdc","-n" ]
kms:
- image: apache/hadoop:${HADOOP_VERSION}
+ image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
hostname: kms
dns_search: .
ports:
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
