This is an automated email from the ASF dual-hosted git repository. asf-gitbox-commits pushed a commit to branch HIVE-29591-precommit-image in repository https://gitbox.apache.org/repos/asf/hive.git
commit 19caf33b816254645714a3fead407d215e00fe58 Author: Laszlo Bodor <[email protected]> AuthorDate: Wed Nov 11 17:48:54 2026 +0100 Address review comments - Move the image to dev-support/docker/ci/runner. - Drop the init@<db> stages and precommit-init-metastore.sh (DbInstallBase already covers schematool), and with them Hadoop from the image. - Drop smoke-test.sh. - Version the image with its org.opencontainers.image.version label: publish apache/hive-ci-runner:<version>, never overwrite a published tag, and pin the tag in the Jenkinsfile. - Trim the README to the steps people actually follow. Co-Authored-By: Claude Code <[email protected]> --- .github/workflows/precommit-image.yml | 45 +++-- Jenkinsfile | 26 +-- .../docker/{precommit => ci/runner}/Dockerfile | 30 +--- dev-support/docker/ci/runner/README.md | 82 +++++++++ dev-support/docker/precommit/README.md | 200 --------------------- dev-support/docker/precommit/smoke-test.sh | 75 -------- dev-support/precommit-init-metastore.sh | 120 ------------- 7 files changed, 119 insertions(+), 459 deletions(-) diff --git a/.github/workflows/precommit-image.yml b/.github/workflows/precommit-image.yml index fe91e94335c..b7a37d9d825 100644 --- a/.github/workflows/precommit-image.yml +++ b/.github/workflows/precommit-image.yml @@ -14,27 +14,27 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Builds the precommit image (dev-support/docker/precommit) and smoke tests it. On master it -# publishes the image as apache/hive-ci-runner, the image the Jenkinsfile runs in. +# Builds the precommit image (dev-support/docker/ci/runner). On master it publishes the image as +# apache/hive-ci-runner:<version>, with the org.opencontainers.image.version of the Dockerfile. The +# Jenkinsfile pins that tag. A published tag is never overwritten. name: Precommit image on: pull_request: paths: - - 'dev-support/docker/precommit/**' + - 'dev-support/docker/ci/runner/Dockerfile' - '.github/workflows/precommit-image.yml' push: branches: - master paths: - - 'dev-support/docker/precommit/**' + - 'dev-support/docker/ci/runner/Dockerfile' - '.github/workflows/precommit-image.yml' workflow_dispatch: inputs: tag: - description: 'Tag to publish; use a custom one (e.g. HIVE-12345) to try an image with the IMAGE parameter of the precommit job' + description: 'Tag to publish a candidate under (e.g. HIVE-12345), to try it with the IMAGE parameter of the precommit job; it must not exist yet' required: true - default: 'latest' permissions: contents: read @@ -42,14 +42,29 @@ permissions: jobs: build: runs-on: ubuntu-latest - env: - IMAGE: ${{ github.repository_owner }}/hive-ci-runner:${{ github.event.inputs.tag || 'latest' }} steps: - name: Checkout uses: actions/checkout@v6 with: persist-credentials: false + - name: Check that the tag is not published yet + id: tag + env: + INPUT_TAG: ${{ github.event.inputs.tag }} + run: | + version=$(sed -n 's/^LABEL org.opencontainers.image.version="\(.*\)"$/\1/p' dev-support/docker/ci/runner/Dockerfile) + image=apache/hive-ci-runner:${INPUT_TAG:-$version} + if [[ $image == *: ]]; then + echo "::error::No org.opencontainers.image.version label in the Dockerfile" + exit 1 + fi + if docker manifest inspect "$image" > /dev/null 2>&1; then + echo "::error::$image is already published, and a published tag is never overwritten. Bump org.opencontainers.image.version in the Dockerfile (or pick another tag for a manual run)." + exit 1 + fi + echo "image=$image" >> "$GITHUB_OUTPUT" + - name: Set up QEMU uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 @@ -59,14 +74,12 @@ jobs: - name: Build the image uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: - context: ./dev-support/docker/precommit/ + context: ./dev-support/docker/ci/runner/ load: true - tags: hive-precommit:test + tags: hive-ci-runner:test - - name: Smoke test the image - run: | - docker images hive-precommit:test - dev-support/docker/precommit/smoke-test.sh hive-precommit:test + - name: Show the image size + run: docker images hive-ci-runner:test - name: Login to Docker Hub if: github.event_name != 'pull_request' && github.repository == 'apache/hive' @@ -79,7 +92,7 @@ jobs: if: github.event_name != 'pull_request' && github.repository == 'apache/hive' uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: - context: ./dev-support/docker/precommit/ + context: ./dev-support/docker/ci/runner/ platforms: linux/amd64,linux/arm64 push: true - tags: ${{ env.IMAGE }} + tags: ${{ steps.tag.outputs.image }} diff --git a/Jenkinsfile b/Jenkinsfile index fef0cf9fe5f..eb7df61fa96 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -31,12 +31,13 @@ properties([ parameters([ string(name: 'SPLIT', defaultValue: '22', description: 'Number of buckets to split tests into.'), string(name: 'OPTS', defaultValue: '', description: 'additional maven opts'), - string(name: 'IMAGE', defaultValue: '', description: 'precommit image to use instead of the default one (e.g. a candidate built from dev-support/docker/precommit)'), + string(name: 'IMAGE', defaultValue: '', description: 'precommit image to use instead of the default one (e.g. a candidate built from dev-support/docker/ci/runner)'), ]) ]) -// Built from dev-support/docker/precommit/Dockerfile and pushed by .github/workflows/precommit-image.yml. -this.precommitImage = 'apache/hive-ci-runner' +// Built from dev-support/docker/ci/runner/Dockerfile and published by .github/workflows/precommit-image.yml. +// The tag is the version in the Dockerfile; bump it here once the workflow has published a new one. +this.precommitImage = 'apache/hive-ci-runner:1' this.prHead = null; def checkPrHead() { @@ -301,25 +302,6 @@ fi } def branches = [:] - for (def d in ['derby','postgres',/*'mysql','oracle'*/]) { - def dbType=d - def splitName = "init@$dbType" - branches[splitName] = { - executorNode { - stage('Prepare') { - loadWS(); - } - stage('init-metastore') { - withEnv(["dbType=$dbType"]) { - sh '''#!/bin/bash -e -set -x -dev-support/precommit-init-metastore.sh $dbType -''' - } - } - } - } - } branches['nightly-check'] = { executorNode { stage('Prepare') { diff --git a/dev-support/docker/precommit/Dockerfile b/dev-support/docker/ci/runner/Dockerfile similarity index 72% rename from dev-support/docker/precommit/Dockerfile rename to dev-support/docker/ci/runner/Dockerfile index 9f9e326ef94..68976e39887 100644 --- a/dev-support/docker/precommit/Dockerfile +++ b/dev-support/docker/ci/runner/Dockerfile @@ -36,7 +36,6 @@ RUN rm -rf /opt/java/openjdk/jmods \ FROM debian:${DEBIAN_VERSION}-slim AS downloads ARG MAVEN_VERSION=3.9.16 -ARG HADOOP_VERSION=3.4.3 ARG APACHE_ARCHIVE=https://archive.apache.org/dist RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates curl \ @@ -57,31 +56,11 @@ RUN fetch "${APACHE_ARCHIVE}/maven/maven-3/${MAVEN_VERSION}/binaries/apache-mave && tar -xzf apache-maven-${MAVEN_VERSION}-bin.tar.gz -C /opt/maven --strip-components=1 \ && rm apache-maven-${MAVEN_VERSION}-bin.tar.gz -# Hadoop is only needed by bin/schematool of the Hive binary distribution in the init@<db> -# stages, so keep just the client side of hadoop-common/hdfs/mapreduce/yarn and the launcher scripts. -RUN fetch "${APACHE_ARCHIVE}/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz" \ - && mkdir /opt/hadoop \ - && tar -xzf hadoop-${HADOOP_VERSION}.tar.gz -C /opt/hadoop --strip-components=1 --wildcards \ - "hadoop-${HADOOP_VERSION}/LICENSE*" \ - "hadoop-${HADOOP_VERSION}/NOTICE*" \ - "hadoop-${HADOOP_VERSION}/licenses-binary" \ - "hadoop-${HADOOP_VERSION}/bin/hadoop" \ - "hadoop-${HADOOP_VERSION}/libexec/*.sh" \ - "hadoop-${HADOOP_VERSION}/etc/hadoop/core-site.xml" \ - "hadoop-${HADOOP_VERSION}/etc/hadoop/hadoop-env.sh" \ - "hadoop-${HADOOP_VERSION}/etc/hadoop/log4j.properties" \ - "hadoop-${HADOOP_VERSION}/share/hadoop/common/hadoop-common-${HADOOP_VERSION}.jar" \ - "hadoop-${HADOOP_VERSION}/share/hadoop/common/lib" \ - "hadoop-${HADOOP_VERSION}/share/hadoop/hdfs/hadoop-hdfs-${HADOOP_VERSION}.jar" \ - "hadoop-${HADOOP_VERSION}/share/hadoop/hdfs/hadoop-hdfs-client-${HADOOP_VERSION}.jar" \ - "hadoop-${HADOOP_VERSION}/share/hadoop/mapreduce/hadoop-mapreduce-client-core-${HADOOP_VERSION}.jar" \ - "hadoop-${HADOOP_VERSION}/share/hadoop/yarn/hadoop-yarn-api-${HADOOP_VERSION}.jar" \ - "hadoop-${HADOOP_VERSION}/share/hadoop/yarn/hadoop-yarn-client-${HADOOP_VERSION}.jar" \ - "hadoop-${HADOOP_VERSION}/share/hadoop/yarn/hadoop-yarn-common-${HADOOP_VERSION}.jar" \ - && rm hadoop-${HADOOP_VERSION}.tar.gz - FROM debian:${DEBIAN_VERSION}-slim +# The image is published as apache/hive-ci-runner:<version>, and a published version is never +# overwritten: bump the version with every change of the image (see README.md). +LABEL org.opencontainers.image.version="1" LABEL org.opencontainers.image.title="Apache Hive precommit" \ org.opencontainers.image.description="Build and test environment of the Apache Hive precommit job" \ org.opencontainers.image.source="https://github.com/apache/hive" \ @@ -90,7 +69,7 @@ LABEL org.opencontainers.image.title="Apache Hive precommit" \ # The base image already has the GNU bash, coreutils, diffutils, findutils, grep, sed and tar # that scripts and tests expect (e.g. qtest result diffing, Hadoop's Shell/Stat/DF/DU helpers). # ca-certificates, git: Jenkins checkout. -# docker-cli: tests and the init@postgres stage start databases in the dind sidecar. +# docker-cli: tests start databases in the dind sidecar. # libcom-err2, libk5crypto3, libkeyutils1, libkrb5support0: needed by the Kudu mini cluster # binaries (kudu-binary) that the kudu-handler tests start. # openssl: tests generate self-signed certificates with it (e.g. GcsOAuthTokenMock). @@ -118,7 +97,6 @@ RUN apt-get update \ COPY --from=jdk /opt/java/openjdk /opt/java/openjdk COPY --from=downloads /opt/maven /opt/maven -COPY --from=downloads /opt/hadoop /opt/hadoop ENV JAVA_HOME=/opt/java/openjdk \ MAVEN_HOME=/opt/maven \ diff --git a/dev-support/docker/ci/runner/README.md b/dev-support/docker/ci/runner/README.md new file mode 100644 index 00000000000..aaf413d268f --- /dev/null +++ b/dev-support/docker/ci/runner/README.md @@ -0,0 +1,82 @@ +<!-- +{% comment %} +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to you under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +{% endcomment %} +--> +# Hive precommit image + +The [Jenkinsfile](../../../../Jenkinsfile) runs every precommit stage in this image, with a +`docker:dind` sidecar for the tests that start databases in containers. + +The image is kept small on purpose. It holds: + +* Debian (slim), with the GNU shell tools that scripts and tests expect. It is glibc based on + purpose: tests run prebuilt native code that is linked against glibc, such as the Kudu + mini cluster. +* a JDK (Eclipse Temurin), without jmods and man pages +* Maven +* the docker CLI, git, openssl, python3, rsync, tini and xmlstarlet + +Everything else (Tez, the JDBC drivers, the Hive build dependencies) is fetched by Maven +during the build. Before adding a package, check whether Maven can provide it instead. + +## How the image is published + +The [precommit-image](../../../../.github/workflows/precommit-image.yml) workflow builds the +image on every pull request that changes the Dockerfile. When such a change lands on master, +the workflow publishes the image for `linux/amd64` and `linux/arm64` as +`apache/hive-ci-runner:<version>`, where `<version>` is the `org.opencontainers.image.version` +label of the Dockerfile. + +A published tag is never overwritten: the workflow fails if the tag already exists. The +Jenkinsfile pins the tag in `precommitImage`, so a new image is only used once a commit bumps +that pin. + +## Changing the image + +1. Edit the [Dockerfile](Dockerfile), and bump its `org.opencontainers.image.version`. Keep the + image small: + * Prefer letting Maven fetch a dependency over baking it into the image. + * Pin versions through the existing build arguments (`JAVA_VERSION`, `MAVEN_VERSION`, + `DEBIAN_VERSION`) rather than hard-coding them. + * Install packages with `--no-install-recommends`, and add each new package to the comment + above `apt-get install`, saying which stage or test needs it. + * Do anything heavy in the `jdk` or `downloads` stage and copy only the result into the + final stage. +2. Build the image, and compare its size with the image before your change. Explain any growth + in the pull request. + + ```bash + docker build -t hive-ci-runner:local dev-support/docker/ci/runner + docker images hive-ci-runner:local + ``` +3. Try the image in the precommit job. Push it, for amd64, to a Docker Hub namespace you + control: + + ```bash + docker buildx build --platform linux/amd64 --push \ + -t <your-namespace>/hive-ci-runner:HIVE-12345 dev-support/docker/ci/runner + ``` + + On the precommit job of your pull request in Jenkins, choose *Build with Parameters* and set + `IMAGE` to `<your-namespace>/hive-ci-runner:HIVE-12345`. The parameter is only offered after + the job has run at least once with this Jenkinsfile. Link the run in the pull request. +4. Once the pull request is merged, the workflow publishes `apache/hive-ci-runner:<version>`. + Open a follow-up pull request that bumps `precommitImage` in the Jenkinsfile to the new tag. + Its own precommit run is the first one in the published image. + +To roll back, revert the commit that bumped `precommitImage`. The previous tag is still +published, unchanged. diff --git a/dev-support/docker/precommit/README.md b/dev-support/docker/precommit/README.md deleted file mode 100644 index ea89a314a7c..00000000000 --- a/dev-support/docker/precommit/README.md +++ /dev/null @@ -1,200 +0,0 @@ -<!-- -{% comment %} -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to you under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -{% endcomment %} ---> -# Hive precommit image - -The [Jenkinsfile](../../../Jenkinsfile) runs every precommit stage in this image, with a -`docker:dind` sidecar for the tests that start databases in containers. - -The image is kept small on purpose. It holds: - -* Debian (slim), with the GNU shell tools that scripts and tests expect. It is glibc based on - purpose: tests run prebuilt native code that is linked against glibc, such as the Kudu - mini cluster. -* a JDK (Eclipse Temurin), without jmods and man pages -* Maven -* the client part of Hadoop, used only by `bin/schematool` in the `init@<db>` stages - (see [precommit-init-metastore.sh](../../precommit-init-metastore.sh)) -* the docker CLI, git, openssl, python3, rsync, tini and xmlstarlet - -Everything else (Tez, the JDBC drivers, the Hive build dependencies) is fetched by Maven -during the build. Before adding a package, check whether Maven can provide it instead. - -## How the image is published - -The [precommit-image](../../../.github/workflows/precommit-image.yml) workflow builds the -image and smoke tests it on every pull request that changes this directory. When such a -change lands on master, the workflow publishes the image for `linux/amd64` and `linux/arm64` -as `apache/hive-ci-runner` (the default `latest` tag). That is the `precommitImage` of the -Jenkinsfile, and the precommit job pulls it again on every run. The tag is overwritten on -every publish: once the change is merged, every precommit run, on every branch, uses the new -image. - -## Changing the image, step by step - -All commands run from the root of the Hive source tree. - -### 1. Make the change - -Edit the [Dockerfile](Dockerfile). Keep the image small: - -* Prefer letting Maven fetch a dependency over baking it into the image. -* Pin versions through the existing build arguments (`JAVA_VERSION`, `MAVEN_VERSION`, - `HADOOP_VERSION`, `DEBIAN_VERSION`) rather than hard-coding them. -* Install packages with `--no-install-recommends`, and add each new package to the comment - above `apt-get install`, saying which stage or test needs it. -* Do anything heavy in the `jdk` or `downloads` stage and copy only the result into the - final stage. - -If the Jenkinsfile or a test relies on the new content, add a check for it to -[smoke-test.sh](smoke-test.sh). - -### 2. Build the image - -```bash -docker build -t hive-precommit:local dev-support/docker/precommit -docker images hive-precommit:local -``` - -Compare the size with the image before your change, and explain any growth in the pull -request. On an arm64 machine (e.g. Apple silicon) this builds an arm64 image. The CI runs -amd64: add `--platform linux/amd64` to try that one, which is emulated and slower. - -On Linux, files written into a mounted source tree belong to uid 1000, the `jenkins` user of -the image. If your uid is different, build with `--build-arg UID=$(id -u)`. - -### 3. Run the smoke test - -```bash -dev-support/docker/precommit/smoke-test.sh hive-precommit:local -``` - -Every line must say `ok`. The pull request workflow runs the same script. - -### 4. Start a docker daemon for the tests - -The Jenkins pod runs a `docker:dind` sidecar in the same network namespace. Do the same -locally: - -```bash -docker run -d --name hive-dind --privileged -e DOCKER_TLS_CERTDIR= \ - docker:20.10-dind --host=tcp://0.0.0.0:2375 -``` - -The commands below run the image with -`--network container:hive-dind -e DOCKER_HOST=tcp://localhost:2375`. That way the containers -the tests start are reachable on `localhost`, as in the Jenkins pod. - -### 5. Build Hive in the image - -This is what the `Compile` stage does. Keep the local Maven repository between runs: - -```bash -mkdir -p ~/.m2/hive-precommit-repository -docker run --rm -it \ - -v "$PWD:/home/jenkins/hive" -w /home/jenkins/hive \ - -v ~/.m2/hive-precommit-repository:/home/jenkins/.m2/repository \ - hive-precommit:local \ - mvn -B install -DskipTests -Pitests,dist -``` - -The `dist` profile is needed for the next step. - -### 6. Initialize the metastore - -This is what the `init@derby` and `init@postgres` stages do: - -```bash -docker run --rm \ - -v "$PWD:/home/jenkins/hive" -w /home/jenkins/hive \ - hive-precommit:local \ - dev-support/precommit-init-metastore.sh derby - -docker run --rm \ - --network container:hive-dind -e DOCKER_HOST=tcp://localhost:2375 \ - -v "$PWD:/home/jenkins/hive" -w /home/jenkins/hive \ - -v ~/.m2/hive-precommit-repository:/home/jenkins/.m2/repository \ - hive-precommit:local \ - dev-support/precommit-init-metastore.sh postgres -``` - -Both must end with `Metastore schema initialized on <db>`. The logs, including the log of -the postgres container, are written to `target/init-metastore/<db>`. - -### 7. Run some tests - -Run tests that exercise what you changed. A good baseline is a couple of qtests: -`newline.q` runs python `TRANSFORM` scripts, and `jdbc_project_pushdown.q` starts postgres -in the docker daemon. They belong to different drivers, so they take two runs: - -```bash -for t in TestMiniLlapCliDriver:newline.q TestMiniLlapLocalCliDriver:jdbc_project_pushdown.q; do - docker run --rm \ - --network container:hive-dind -e DOCKER_HOST=tcp://localhost:2375 \ - -v "$PWD:/home/jenkins/hive" -w /home/jenkins/hive \ - -v ~/.m2/hive-precommit-repository:/home/jenkins/.m2/repository \ - hive-precommit:local \ - mvn -B test -Pitests -pl itests/qtest -Dtest=${t%%:*} -Dqfile=${t#*:} || break -done -``` - -When you are done, remove the docker daemon with `docker rm -f -v hive-dind`. - -### 8. Open a pull request - -Open the pull request as usual. The precommit-image workflow builds the image and runs the -smoke test on it. The precommit job of the pull request itself still runs in the published -`apache/hive-ci-runner`, not in your change. - -### 9. Run the precommit job with the candidate image - -For anything beyond a trivial change, run the full precommit job on the new image before -merging. This needs a committer. - -1. Publish the candidate under a tag of its own. Either: - * push the change to a branch of `apache/hive` and run the precommit-image workflow on that - branch from the Actions tab, with `tag` set to e.g. `HIVE-12345`; or - * build it for amd64 and push it to a registry you control: - - ```bash - docker buildx build --platform linux/amd64 --push \ - -t <your-namespace>/hive-precommit:HIVE-12345 dev-support/docker/precommit - ``` - -2. On the precommit job of the pull request in Jenkins, choose *Build with Parameters* and set - `IMAGE` to the candidate, e.g. `apache/hive-ci-runner:HIVE-12345`. The parameter - is only offered after the job has run at least once with this Jenkinsfile. -3. Link the green run in the pull request. - -### 10. Merge and verify - -Once merged, the precommit-image workflow on master publishes -`apache/hive-ci-runner`. Then: - -1. Check that the workflow run succeeded. -2. Check the published image: - - ```bash - docker buildx imagetools inspect apache/hive-ci-runner - ``` - -3. Watch the next precommit runs. - -To roll back, revert the commit. The workflow then publishes the previous Dockerfile as -`latest` again. The rebuilt image picks up the Debian packages current at that time, -so it is equivalent to the old one but not identical. diff --git a/dev-support/docker/precommit/smoke-test.sh b/dev-support/docker/precommit/smoke-test.sh deleted file mode 100755 index e4cba0c7a59..00000000000 --- a/dev-support/docker/precommit/smoke-test.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Checks that a precommit image provides what the Jenkinsfile and the tests rely on. -# -# Usage: dev-support/docker/precommit/smoke-test.sh <image> [<expected java feature version>] - -set -euo pipefail - -image=${1:?usage: $0 <image> [<java version>]} -javaVersion=${2:-21} - -docker run --rm -i -e JAVA_FEATURE="$javaVersion" "$image" bash -s <<'EOF' -set -euo pipefail -check() { - local what=$1 - shift - if out=$("$@" 2>&1); then - echo "ok $what" - else - echo "FAIL $what: $out" >&2 - exit 1 - fi -} - -check "runs as jenkins (uid 1000)" test "$(id -u):$(id -un)" = 1000:jenkins -check "USER is set" test "${USER:-}" = jenkins -check "home is writable" touch "$HOME/.probe" -check "~/.m2 is writable" touch "$HOME/.m2/.probe" -check "java $JAVA_FEATURE" sh -c 'java -version 2>&1 | grep -q "version \"$JAVA_FEATURE"' -check "javac" javac -version -check "default charset is UTF-8" sh -c 'java -XshowSettings:properties -version 2>&1 | grep -q "native.encoding = UTF-8"' -check "maven" mvn -B -v -for tool in bash docker git python python3 rsync tini xmlstarlet ps; do - check "$tool on PATH" command -v $tool -done -for tool in diff find grep sed sort tar; do - check "GNU $tool" sh -c "$tool --version | grep -q GNU" -done -check "openssl at /usr/bin/openssl" sh -c 'cd "$(mktemp -d)" && /usr/bin/openssl req -x509 -newkey rsa:2048 -keyout k.pem -out c.pem -days 1 -nodes -subj /CN=test' -check "glibc" getconf GNU_LIBC_VERSION -for lib in libcom_err.so.2 libk5crypto.so.3 libkeyutils.so.1 libkrb5support.so.0; do - check "$lib for the Kudu mini cluster" sh -c "ldconfig -p | grep -q $lib" -done -check "hadoop launcher" /opt/hadoop/bin/hadoop version -check "python TRANSFORM-style script" sh -c 'test "$(printf "a\tb\n" | python -c "import sys; [print(l.rstrip().upper()) for l in sys.stdin]")" = "$(printf "A\tB")"' - -# The Prechecks stage relies on "xmlstarlet edit" reformatting badly indented poms. -cd "$(mktemp -d)" -printf '<?xml version="1.0" encoding="UTF-8"?>\n<project>\n <a>1</a>\n</project>\n' > good.xml -printf '<?xml version="1.0" encoding="UTF-8"?>\n<project>\n <a>1</a>\n</project>\n' > bad.xml -cp good.xml good.orig -cp bad.xml bad.orig -xmlstarlet edit -L good.xml bad.xml -check "xmlstarlet keeps formatted xml" cmp good.xml good.orig -check "xmlstarlet reformats bad xml" sh -c '! cmp -s bad.xml bad.orig' - -check "git" sh -c 'git init -q repo && git -C repo status --short' -echo "All checks passed" -EOF diff --git a/dev-support/precommit-init-metastore.sh b/dev-support/precommit-init-metastore.sh deleted file mode 100755 index 3c579a5d4b6..00000000000 --- a/dev-support/precommit-init-metastore.sh +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Initializes an empty metastore, and the Hive SYS/INFORMATION_SCHEMA databases on top of it, with -# bin/schematool of the binary distribution built by "mvn install -Pdist". -# Used by the init@<db> stages of the Jenkinsfile, in the precommit image (dev-support/docker/precommit). -# -# Usage: dev-support/precommit-init-metastore.sh <derby|postgres> -# -# Environment: -# HADOOP_HOME Hadoop installation used by bin/schematool (default: /opt/hadoop) -# MAVEN_ARGS extra Maven arguments (settings, local repository) used to fetch the JDBC driver -# DOCKER_NETWORK network of the database container (default: host); its port must be -# reachable on localhost -# POSTGRES_IMAGE image of the postgres container (default: postgres:17-alpine) - -set -euo pipefail - -dbType=${1:?usage: $0 <derby|postgres>} - -cd "$(dirname "$0")/.." - -hiveHomes=(packaging/target/apache-hive-*-bin/apache-hive-*-bin) -if [[ ! -d ${hiveHomes[0]} ]]; then - echo "No Hive binary distribution under packaging/target; build it with -Pdist first." >&2 - exit 1 -fi -export HIVE_HOME=$PWD/${hiveHomes[0]} -export HADOOP_HOME=${HADOOP_HOME:-/opt/hadoop} -export HADOOP_CLIENT_OPTS="${HADOOP_CLIENT_OPTS:-} --add-opens java.base/java.net=ALL-UNNAMED" - -work=$PWD/target/init-metastore/$dbType -rm -rf "$work" -mkdir -p "$work/conf" "$work/lib" -export HIVE_CONF_DIR=$work/conf -export HADOOP_CLIENT_OPTS+=" -Dderby.stream.error.file=$work/derby.log" -export HADOOP_CLASSPATH="$work/lib/*" - -dbContainer= -cleanup() { - if [[ -n $dbContainer ]]; then - docker logs "$dbContainer" > "$work/$dbContainer.log" 2>&1 || true - docker rm -f -v "$dbContainer" > /dev/null 2>&1 || true - fi -} -trap cleanup EXIT - -pomProperty() { - xmlstarlet sel -t -v "/_:project/_:properties/_:$1" pom.xml -} - -case $dbType in - derby) - driver=org.apache.derby.jdbc.EmbeddedDriver - url="jdbc:derby:;databaseName=$work/metastore_db;create=true" - user=APP - password=mine - ;; - postgres) - mvn -B -q dependency:copy -Dartifact="org.postgresql:postgresql:$(pomProperty postgres.version)" \ - -DoutputDirectory="$work/lib" - dbContainer=hive-precommit-postgres - docker rm -f -v "$dbContainer" > /dev/null 2>&1 || true - docker run -d --name "$dbContainer" --network "${DOCKER_NETWORK:-host}" \ - -e POSTGRES_USER=hive -e POSTGRES_PASSWORD=hive -e POSTGRES_DB=metastore \ - "${POSTGRES_IMAGE:-postgres:17-alpine}" > /dev/null - # Probe over TCP: during its first start the entrypoint runs a temporary server on the unix socket only. - for ((i = 0; i < 120; i++)); do - docker exec "$dbContainer" pg_isready -q -h 127.0.0.1 -U hive -d metastore && break - sleep 1 - done - docker exec "$dbContainer" pg_isready -h 127.0.0.1 -U hive -d metastore - driver=org.postgresql.Driver - url=jdbc:postgresql://localhost:5432/metastore - user=hive - password=hive - ;; - *) - echo "Unsupported database type: $dbType" >&2 - exit 1 - ;; -esac - -property() { - printf ' <property><name>%s</name><value>%s</value></property>\n' "$1" "$2" -} -{ - echo '<configuration>' - property javax.jdo.option.ConnectionDriverName "$driver" - property javax.jdo.option.ConnectionURL "$url" - property javax.jdo.option.ConnectionUserName "$user" - property javax.jdo.option.ConnectionPassword "$password" - property hive.metastore.warehouse.dir "file://$work/warehouse" - property hive.metastore.warehouse.external.dir "file://$work/warehouse-external" - property hive.exec.scratchdir "$work/scratch" - property hive.exec.local.scratchdir "$work/local-scratch" - property hive.querylog.location "$work/querylog" - property hive.server2.logging.operation.log.location "$work/operation-logs" - property hive.user.install.directory "file://$work/user" - echo '</configuration>' -} > "$HIVE_CONF_DIR/hive-site.xml" - -"$HIVE_HOME/bin/schematool" -verbose -dbType "$dbType" -initSchema -"$HIVE_HOME/bin/schematool" -verbose -dbType hive -initSchema -metaDbType "$dbType" -echo "Metastore schema initialized on $dbType"
