This is an automated email from the ASF dual-hosted git repository. asf-gitbox-commits pushed a commit to branch HIVE-29591-precommit-image in repository https://gitbox.apache.org/repos/asf/hive.git
commit a1f02e59568ea4b594133ddfe805e437b7dc8159 Author: Laszlo Bodor <[email protected]> AuthorDate: Mon Nov 9 12:01:00 2026 +0100 HIVE-29591: Create Hive precommit image and move Dockerfile into the hive repository --- .github/workflows/precommit-image.yml | 85 +++++++++ Jenkinsfile | 35 ++-- dev-support/docker/precommit/Dockerfile | 143 +++++++++++++++ dev-support/docker/precommit/README.md | 199 +++++++++++++++++++++ dev-support/docker/precommit/smoke-test.sh | 75 ++++++++ dev-support/precommit-init-metastore.sh | 120 +++++++++++++ .../queries/clientnegative/cachingprintstream.q | 3 + ql/src/test/queries/clientnegative/stack_trace.q | 3 + 8 files changed, 641 insertions(+), 22 deletions(-) diff --git a/.github/workflows/precommit-image.yml b/.github/workflows/precommit-image.yml new file mode 100644 index 00000000000..380029acb47 --- /dev/null +++ b/.github/workflows/precommit-image.yml @@ -0,0 +1,85 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Builds the precommit image (dev-support/docker/precommit) and smoke tests it. On master it +# publishes the image as apache/hive:precommit, the image the Jenkinsfile runs in. +name: Precommit image + +on: + pull_request: + paths: + - 'dev-support/docker/precommit/**' + - '.github/workflows/precommit-image.yml' + push: + branches: + - master + paths: + - 'dev-support/docker/precommit/**' + - '.github/workflows/precommit-image.yml' + workflow_dispatch: + inputs: + tag: + description: 'Tag to publish; use a custom one (e.g. precommit-candidate-HIVE-12345) to try an image with the IMAGE parameter of the precommit job' + required: true + default: 'precommit' + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + env: + IMAGE: ${{ github.repository_owner }}/hive:${{ github.event.inputs.tag || 'precommit' }} + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Set up QEMU + uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + - name: Build the image + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: ./dev-support/docker/precommit/ + load: true + tags: hive-precommit:test + + - name: Smoke test the image + run: | + docker images hive-precommit:test + dev-support/docker/precommit/smoke-test.sh hive-precommit:test + + - name: Login to Docker Hub + if: github.event_name != 'pull_request' && github.repository == 'apache/hive' + uses: docker/login-action@8aa5831fb8d48d8f76594c7ddde61ae58019f550 # v4.8.0 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Publish the image + if: github.event_name != 'pull_request' && github.repository == 'apache/hive' + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: ./dev-support/docker/precommit/ + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ env.IMAGE }} diff --git a/Jenkinsfile b/Jenkinsfile index ad8f017e0f1..6d772605753 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -31,9 +31,13 @@ properties([ parameters([ string(name: 'SPLIT', defaultValue: '22', description: 'Number of buckets to split tests into.'), string(name: 'OPTS', defaultValue: '', description: 'additional maven opts'), + string(name: 'IMAGE', defaultValue: '', description: 'precommit image to use instead of the default one (e.g. a candidate built from dev-support/docker/precommit)'), ]) ]) +// Built from dev-support/docker/precommit/Dockerfile and pushed by .github/workflows/precommit-image.yml. +this.precommitImage = 'apache/hive:precommit' + this.prHead = null; def checkPrHead() { if(env.CHANGE_ID) { @@ -72,12 +76,17 @@ def setPrLabel(String prLabel) { setPrLabel("PENDING"); +// Every mvn call uses the settings and the local repository under .m2 of the workspace (set up by buildHive); +// saveWS/loadWS hand them over to the other stages. Maven 3.9+ reads MAVEN_ARGS. def executorNode(run) { hdbPodTemplate { timeout(time: 12, unit: 'HOURS') { node(POD_LABEL) { - container('hdb') { - run() + def m2 = "${env.WORKSPACE}/.m2" + withEnv(["MAVEN_ARGS=-s ${m2}/settings.xml -Dmaven.repo.local=${m2}/repository -B -Dtest.groups= -Pitests,qsplits,dist,errorProne"]) { + container('hdb') { + run() + } } } } @@ -89,17 +98,10 @@ def buildHive(args) { withEnv(["MULTIPLIER=$params.MULTIPLIER","M_OPTS=$params.OPTS"]) { sh '''#!/bin/bash -e set -x -. /etc/profile.d/confs.sh export USER="`whoami`" export MAVEN_OPTS="-Xmx4G" -export -n HIVE_CONF_DIR -sw java 21 && . /etc/profile.d/java.sh mkdir -p .m2/repository cp $SETTINGS .m2/settings.xml -OPTS=" -s $PWD/.m2/settings.xml -B -Dtest.groups= " -OPTS+=" -Pitests,qsplits,dist,errorProne" -OPTS+=" -Dmaven.repo.local=$PWD/.m2/repository" -git config extra.mavenOpts "$OPTS" OPTS=" $M_OPTS -Dmaven.test.failure.ignore " if [ -s inclusions.txt ]; then OPTS+=" -Dsurefire.includesFile=$PWD/inclusions.txt";fi if [ -s exclusions.txt ]; then OPTS+=" -Dsurefire.excludesFile=$PWD/exclusions.txt";fi @@ -123,7 +125,6 @@ def sonarAnalysis(args) { // Sonar scanner runs in a separate JVM so JAVA_OPTS (notably heap size) // must be passed via the appropriate environment variable sh """#!/bin/bash -e - sw java 21 && . /etc/profile.d/java.sh export SONAR_SCANNER_JAVA_OPTS=-Xmx8g """+mvnCmd } @@ -132,7 +133,7 @@ def sonarAnalysis(args) { def hdbPodTemplate(closure) { podTemplate( containers: [ - containerTemplate(name: 'hdb', image: 'ayushtkn/hive-dev-box:executor', ttyEnabled: true, command: 'tini -- cat', + containerTemplate(name: 'hdb', image: params.IMAGE?.trim() ?: precommitImage, ttyEnabled: true, command: 'tini -- cat', alwaysPullImage: true, resourceRequestCpu: '1800m', resourceLimitCpu: '8000m', @@ -200,7 +201,6 @@ def jobWrappers(closure) { def checkstyle() { sh '''#!/bin/bash -e set -x -sw java 21 && . /etc/profile.d/java.sh export MAVEN_OPTS="-Xmx6G" mvn checkstyle:checkstyle -Pitests ''' @@ -312,16 +312,8 @@ fi stage('init-metastore') { withEnv(["dbType=$dbType"]) { sh '''#!/bin/bash -e - sw java 21 && . /etc/profile.d/java.sh set -x -echo 127.0.0.1 dev_$dbType | sudo tee -a /etc/hosts -. /etc/profile.d/confs.sh -sw hive-dev $PWD -export DOCKER_NETWORK=host -export DBNAME=metastore -export HADOOP_CLIENT_OPTS="--add-opens java.base/java.net=ALL-UNNAMED" -reinit_metastore $dbType -time docker rm -f dev_$dbType || true +dev-support/precommit-init-metastore.sh $dbType ''' } } @@ -428,7 +420,6 @@ tar -xzf packaging/target/apache-hive-*-nightly-*-src.tar.gz } stage('Generate javadoc') { sh """#!/bin/bash -e - sw java 21 && . /etc/profile.d/java.sh mvn install javadoc:javadoc javadoc:aggregate -DskipTests -pl '!itests/hive-jmh,!itests/util' """ } diff --git a/dev-support/docker/precommit/Dockerfile b/dev-support/docker/precommit/Dockerfile new file mode 100644 index 00000000000..9f9e326ef94 --- /dev/null +++ b/dev-support/docker/precommit/Dockerfile @@ -0,0 +1,143 @@ +# syntax=docker/dockerfile:1 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The image the Hive precommit (see Jenkinsfile) builds and tests Hive in. +# +# It is deliberately minimal: a JDK, Maven, and the handful of tools the Jenkinsfile +# stages and the tests shell out to. Everything else is fetched by Maven at build time. +# See README.md next to this file before adding anything. + +# glibc based on purpose: tests run prebuilt native code linked against glibc +# (e.g. the Kudu mini cluster binaries, Conscrypt). +ARG DEBIAN_VERSION=trixie +ARG JAVA_VERSION=21 + +# The Temurin JDK is a generic glibc build; only the distribution of its image differs. +FROM eclipse-temurin:${JAVA_VERSION}-jdk AS jdk +# jmods are only used by jlink; classes_nocoops.jsa is only used with heaps > 32GB. +RUN rm -rf /opt/java/openjdk/jmods \ + /opt/java/openjdk/man \ + /opt/java/openjdk/lib/server/classes_nocoops.jsa + +FROM debian:${DEBIAN_VERSION}-slim AS downloads +ARG MAVEN_VERSION=3.9.16 +ARG HADOOP_VERSION=3.4.3 +ARG APACHE_ARCHIVE=https://archive.apache.org/dist +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /tmp + +# Fetches $1 and verifies it against the published .sha512 (plain or BSD-style). +COPY <<'EOF' /usr/local/bin/fetch +#!/bin/sh +set -eu +curl -fsSL --retry 5 -o "${1##*/}" "$1" +echo "$(curl -fsSL --retry 5 "$1.sha512" | awk '{print $NF}') ${1##*/}" | sha512sum -c - +EOF +RUN chmod +x /usr/local/bin/fetch + +RUN fetch "${APACHE_ARCHIVE}/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz" \ + && mkdir /opt/maven \ + && tar -xzf apache-maven-${MAVEN_VERSION}-bin.tar.gz -C /opt/maven --strip-components=1 \ + && rm apache-maven-${MAVEN_VERSION}-bin.tar.gz + +# Hadoop is only needed by bin/schematool of the Hive binary distribution in the init@<db> +# stages, so keep just the client side of hadoop-common/hdfs/mapreduce/yarn and the launcher scripts. +RUN fetch "${APACHE_ARCHIVE}/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz" \ + && mkdir /opt/hadoop \ + && tar -xzf hadoop-${HADOOP_VERSION}.tar.gz -C /opt/hadoop --strip-components=1 --wildcards \ + "hadoop-${HADOOP_VERSION}/LICENSE*" \ + "hadoop-${HADOOP_VERSION}/NOTICE*" \ + "hadoop-${HADOOP_VERSION}/licenses-binary" \ + "hadoop-${HADOOP_VERSION}/bin/hadoop" \ + "hadoop-${HADOOP_VERSION}/libexec/*.sh" \ + "hadoop-${HADOOP_VERSION}/etc/hadoop/core-site.xml" \ + "hadoop-${HADOOP_VERSION}/etc/hadoop/hadoop-env.sh" \ + "hadoop-${HADOOP_VERSION}/etc/hadoop/log4j.properties" \ + "hadoop-${HADOOP_VERSION}/share/hadoop/common/hadoop-common-${HADOOP_VERSION}.jar" \ + "hadoop-${HADOOP_VERSION}/share/hadoop/common/lib" \ + "hadoop-${HADOOP_VERSION}/share/hadoop/hdfs/hadoop-hdfs-${HADOOP_VERSION}.jar" \ + "hadoop-${HADOOP_VERSION}/share/hadoop/hdfs/hadoop-hdfs-client-${HADOOP_VERSION}.jar" \ + "hadoop-${HADOOP_VERSION}/share/hadoop/mapreduce/hadoop-mapreduce-client-core-${HADOOP_VERSION}.jar" \ + "hadoop-${HADOOP_VERSION}/share/hadoop/yarn/hadoop-yarn-api-${HADOOP_VERSION}.jar" \ + "hadoop-${HADOOP_VERSION}/share/hadoop/yarn/hadoop-yarn-client-${HADOOP_VERSION}.jar" \ + "hadoop-${HADOOP_VERSION}/share/hadoop/yarn/hadoop-yarn-common-${HADOOP_VERSION}.jar" \ + && rm hadoop-${HADOOP_VERSION}.tar.gz + +FROM debian:${DEBIAN_VERSION}-slim + +LABEL org.opencontainers.image.title="Apache Hive precommit" \ + org.opencontainers.image.description="Build and test environment of the Apache Hive precommit job" \ + org.opencontainers.image.source="https://github.com/apache/hive" \ + org.opencontainers.image.licenses="Apache-2.0" + +# The base image already has the GNU bash, coreutils, diffutils, findutils, grep, sed and tar +# that scripts and tests expect (e.g. qtest result diffing, Hadoop's Shell/Stat/DF/DU helpers). +# ca-certificates, git: Jenkins checkout. +# docker-cli: tests and the init@postgres stage start databases in the dind sidecar. +# libcom-err2, libk5crypto3, libkeyutils1, libkrb5support0: needed by the Kudu mini cluster +# binaries (kudu-binary) that the kudu-handler tests start. +# openssl: tests generate self-signed certificates with it (e.g. GcsOAuthTokenMock). +# procps: ps, used by tests; python3, python-is-python3: TRANSFORM scripts in qtests; +# rsync: workspace hand-over between stages; tini: reaps the processes tests leave behind; +# xmlstarlet: pom.xml format check. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + docker-cli \ + git \ + libcom-err2 \ + libk5crypto3 \ + libkeyutils1 \ + libkrb5support0 \ + openssl \ + procps \ + python-is-python3 \ + python3 \ + rsync \ + tini \ + xmlstarlet \ + && rm -rf /var/lib/apt/lists/* \ + && git config --system --add safe.directory '*' + +COPY --from=jdk /opt/java/openjdk /opt/java/openjdk +COPY --from=downloads /opt/maven /opt/maven +COPY --from=downloads /opt/hadoop /opt/hadoop + +ENV JAVA_HOME=/opt/java/openjdk \ + MAVEN_HOME=/opt/maven \ + PATH=/opt/java/openjdk/bin:/opt/maven/bin:$PATH \ + LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 \ + TZ=UTC + +# The Jenkins agent runs as uid 1000 and the pod uses fsGroup 1000; tests must not run as root. +# ~/.m2 exists up front so that a local repository mounted below it leaves it writable. +ARG UID=1000 +RUN groupadd -g ${UID} jenkins \ + && useradd -m -u ${UID} -g jenkins -d /home/jenkins -s /bin/bash jenkins \ + && install -d -o jenkins -g jenkins /home/jenkins/.m2 + +# docker run sets no USER, but LLAP reads it to submit work (LlapTaskCommunicator). +ENV USER=jenkins +USER jenkins +WORKDIR /home/jenkins + +ENTRYPOINT ["/usr/bin/tini", "--"] +CMD ["bash"] diff --git a/dev-support/docker/precommit/README.md b/dev-support/docker/precommit/README.md new file mode 100644 index 00000000000..6bcffbf0458 --- /dev/null +++ b/dev-support/docker/precommit/README.md @@ -0,0 +1,199 @@ +<!-- +{% comment %} +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to you under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +{% endcomment %} +--> +# Hive precommit image + +The [Jenkinsfile](../../../Jenkinsfile) runs every precommit stage in this image, with a +`docker:dind` sidecar for the tests that start databases in containers. + +The image is kept small on purpose. It holds: + +* Debian (slim), with the GNU shell tools that scripts and tests expect. It is glibc based on + purpose: tests run prebuilt native code that is linked against glibc, such as the Kudu + mini cluster. +* a JDK (Eclipse Temurin), without jmods and man pages +* Maven +* the client part of Hadoop, used only by `bin/schematool` in the `init@<db>` stages + (see [precommit-init-metastore.sh](../../precommit-init-metastore.sh)) +* the docker CLI, git, openssl, python3, rsync, tini and xmlstarlet + +Everything else (Tez, the JDBC drivers, the Hive build dependencies) is fetched by Maven +during the build. Before adding a package, check whether Maven can provide it instead. + +## How the image is published + +The [precommit-image](../../../.github/workflows/precommit-image.yml) workflow builds the +image and smoke tests it on every pull request that changes this directory. When such a +change lands on master, the workflow publishes the image for `linux/amd64` and `linux/arm64` +as `apache/hive:precommit`. That tag is the `precommitImage` of the Jenkinsfile, and +the precommit job pulls it again on every run. The tag is overwritten on every publish: +once the change is merged, every precommit run, on every branch, uses the new image. + +## Changing the image, step by step + +All commands run from the root of the Hive source tree. + +### 1. Make the change + +Edit the [Dockerfile](Dockerfile). Keep the image small: + +* Prefer letting Maven fetch a dependency over baking it into the image. +* Pin versions through the existing build arguments (`JAVA_VERSION`, `MAVEN_VERSION`, + `HADOOP_VERSION`, `DEBIAN_VERSION`) rather than hard-coding them. +* Install packages with `--no-install-recommends`, and add each new package to the comment + above `apt-get install`, saying which stage or test needs it. +* Do anything heavy in the `jdk` or `downloads` stage and copy only the result into the + final stage. + +If the Jenkinsfile or a test relies on the new content, add a check for it to +[smoke-test.sh](smoke-test.sh). + +### 2. Build the image + +```bash +docker build -t hive-precommit:local dev-support/docker/precommit +docker images hive-precommit:local +``` + +Compare the size with the image before your change, and explain any growth in the pull +request. On an arm64 machine (e.g. Apple silicon) this builds an arm64 image. The CI runs +amd64: add `--platform linux/amd64` to try that one, which is emulated and slower. + +On Linux, files written into a mounted source tree belong to uid 1000, the `jenkins` user of +the image. If your uid is different, build with `--build-arg UID=$(id -u)`. + +### 3. Run the smoke test + +```bash +dev-support/docker/precommit/smoke-test.sh hive-precommit:local +``` + +Every line must say `ok`. The pull request workflow runs the same script. + +### 4. Start a docker daemon for the tests + +The Jenkins pod runs a `docker:dind` sidecar in the same network namespace. Do the same +locally: + +```bash +docker run -d --name hive-dind --privileged -e DOCKER_TLS_CERTDIR= \ + docker:20.10-dind --host=tcp://0.0.0.0:2375 +``` + +The commands below run the image with +`--network container:hive-dind -e DOCKER_HOST=tcp://localhost:2375`. That way the containers +the tests start are reachable on `localhost`, as in the Jenkins pod. + +### 5. Build Hive in the image + +This is what the `Compile` stage does. Keep the local Maven repository between runs: + +```bash +mkdir -p ~/.m2/hive-precommit-repository +docker run --rm -it \ + -v "$PWD:/home/jenkins/hive" -w /home/jenkins/hive \ + -v ~/.m2/hive-precommit-repository:/home/jenkins/.m2/repository \ + hive-precommit:local \ + mvn -B install -DskipTests -Pitests,dist +``` + +The `dist` profile is needed for the next step. + +### 6. Initialize the metastore + +This is what the `init@derby` and `init@postgres` stages do: + +```bash +docker run --rm \ + -v "$PWD:/home/jenkins/hive" -w /home/jenkins/hive \ + hive-precommit:local \ + dev-support/precommit-init-metastore.sh derby + +docker run --rm \ + --network container:hive-dind -e DOCKER_HOST=tcp://localhost:2375 \ + -v "$PWD:/home/jenkins/hive" -w /home/jenkins/hive \ + -v ~/.m2/hive-precommit-repository:/home/jenkins/.m2/repository \ + hive-precommit:local \ + dev-support/precommit-init-metastore.sh postgres +``` + +Both must end with `Metastore schema initialized on <db>`. The logs, including the log of +the postgres container, are written to `target/init-metastore/<db>`. + +### 7. Run some tests + +Run tests that exercise what you changed. A good baseline is a couple of qtests: +`newline.q` runs python `TRANSFORM` scripts, and `jdbc_project_pushdown.q` starts postgres +in the docker daemon. They belong to different drivers, so they take two runs: + +```bash +for t in TestMiniLlapCliDriver:newline.q TestMiniLlapLocalCliDriver:jdbc_project_pushdown.q; do + docker run --rm \ + --network container:hive-dind -e DOCKER_HOST=tcp://localhost:2375 \ + -v "$PWD:/home/jenkins/hive" -w /home/jenkins/hive \ + -v ~/.m2/hive-precommit-repository:/home/jenkins/.m2/repository \ + hive-precommit:local \ + mvn -B test -Pitests -pl itests/qtest -Dtest=${t%%:*} -Dqfile=${t#*:} || break +done +``` + +When you are done, remove the docker daemon with `docker rm -f -v hive-dind`. + +### 8. Open a pull request + +Open the pull request as usual. The precommit-image workflow builds the image and runs the +smoke test on it. The precommit job of the pull request itself still runs in the published +`apache/hive:precommit`, not in your change. + +### 9. Run the precommit job with the candidate image + +For anything beyond a trivial change, run the full precommit job on the new image before +merging. This needs a committer. + +1. Publish the candidate under a tag of its own. Either: + * push the change to a branch of `apache/hive` and run the precommit-image workflow on that + branch from the Actions tab, with `tag` set to e.g. `precommit-candidate-HIVE-12345`; or + * build it for amd64 and push it to a registry you control: + + ```bash + docker buildx build --platform linux/amd64 --push \ + -t <your-namespace>/hive-precommit:HIVE-12345 dev-support/docker/precommit + ``` + +2. On the precommit job of the pull request in Jenkins, choose *Build with Parameters* and set + `IMAGE` to the candidate, e.g. `apache/hive:precommit-candidate-HIVE-12345`. The parameter + is only offered after the job has run at least once with this Jenkinsfile. +3. Link the green run in the pull request. + +### 10. Merge and verify + +Once merged, the precommit-image workflow on master publishes +`apache/hive:precommit`. Then: + +1. Check that the workflow run succeeded. +2. Check the published image: + + ```bash + docker buildx imagetools inspect apache/hive:precommit + ``` + +3. Watch the next precommit runs. + +To roll back, revert the commit. The workflow then publishes the previous Dockerfile as +`precommit` again. The rebuilt image picks up the Debian packages current at that time, +so it is equivalent to the old one but not identical. diff --git a/dev-support/docker/precommit/smoke-test.sh b/dev-support/docker/precommit/smoke-test.sh new file mode 100755 index 00000000000..e4cba0c7a59 --- /dev/null +++ b/dev-support/docker/precommit/smoke-test.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Checks that a precommit image provides what the Jenkinsfile and the tests rely on. +# +# Usage: dev-support/docker/precommit/smoke-test.sh <image> [<expected java feature version>] + +set -euo pipefail + +image=${1:?usage: $0 <image> [<java version>]} +javaVersion=${2:-21} + +docker run --rm -i -e JAVA_FEATURE="$javaVersion" "$image" bash -s <<'EOF' +set -euo pipefail +check() { + local what=$1 + shift + if out=$("$@" 2>&1); then + echo "ok $what" + else + echo "FAIL $what: $out" >&2 + exit 1 + fi +} + +check "runs as jenkins (uid 1000)" test "$(id -u):$(id -un)" = 1000:jenkins +check "USER is set" test "${USER:-}" = jenkins +check "home is writable" touch "$HOME/.probe" +check "~/.m2 is writable" touch "$HOME/.m2/.probe" +check "java $JAVA_FEATURE" sh -c 'java -version 2>&1 | grep -q "version \"$JAVA_FEATURE"' +check "javac" javac -version +check "default charset is UTF-8" sh -c 'java -XshowSettings:properties -version 2>&1 | grep -q "native.encoding = UTF-8"' +check "maven" mvn -B -v +for tool in bash docker git python python3 rsync tini xmlstarlet ps; do + check "$tool on PATH" command -v $tool +done +for tool in diff find grep sed sort tar; do + check "GNU $tool" sh -c "$tool --version | grep -q GNU" +done +check "openssl at /usr/bin/openssl" sh -c 'cd "$(mktemp -d)" && /usr/bin/openssl req -x509 -newkey rsa:2048 -keyout k.pem -out c.pem -days 1 -nodes -subj /CN=test' +check "glibc" getconf GNU_LIBC_VERSION +for lib in libcom_err.so.2 libk5crypto.so.3 libkeyutils.so.1 libkrb5support.so.0; do + check "$lib for the Kudu mini cluster" sh -c "ldconfig -p | grep -q $lib" +done +check "hadoop launcher" /opt/hadoop/bin/hadoop version +check "python TRANSFORM-style script" sh -c 'test "$(printf "a\tb\n" | python -c "import sys; [print(l.rstrip().upper()) for l in sys.stdin]")" = "$(printf "A\tB")"' + +# The Prechecks stage relies on "xmlstarlet edit" reformatting badly indented poms. +cd "$(mktemp -d)" +printf '<?xml version="1.0" encoding="UTF-8"?>\n<project>\n <a>1</a>\n</project>\n' > good.xml +printf '<?xml version="1.0" encoding="UTF-8"?>\n<project>\n <a>1</a>\n</project>\n' > bad.xml +cp good.xml good.orig +cp bad.xml bad.orig +xmlstarlet edit -L good.xml bad.xml +check "xmlstarlet keeps formatted xml" cmp good.xml good.orig +check "xmlstarlet reformats bad xml" sh -c '! cmp -s bad.xml bad.orig' + +check "git" sh -c 'git init -q repo && git -C repo status --short' +echo "All checks passed" +EOF diff --git a/dev-support/precommit-init-metastore.sh b/dev-support/precommit-init-metastore.sh new file mode 100755 index 00000000000..3c579a5d4b6 --- /dev/null +++ b/dev-support/precommit-init-metastore.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Initializes an empty metastore, and the Hive SYS/INFORMATION_SCHEMA databases on top of it, with +# bin/schematool of the binary distribution built by "mvn install -Pdist". +# Used by the init@<db> stages of the Jenkinsfile, in the precommit image (dev-support/docker/precommit). +# +# Usage: dev-support/precommit-init-metastore.sh <derby|postgres> +# +# Environment: +# HADOOP_HOME Hadoop installation used by bin/schematool (default: /opt/hadoop) +# MAVEN_ARGS extra Maven arguments (settings, local repository) used to fetch the JDBC driver +# DOCKER_NETWORK network of the database container (default: host); its port must be +# reachable on localhost +# POSTGRES_IMAGE image of the postgres container (default: postgres:17-alpine) + +set -euo pipefail + +dbType=${1:?usage: $0 <derby|postgres>} + +cd "$(dirname "$0")/.." + +hiveHomes=(packaging/target/apache-hive-*-bin/apache-hive-*-bin) +if [[ ! -d ${hiveHomes[0]} ]]; then + echo "No Hive binary distribution under packaging/target; build it with -Pdist first." >&2 + exit 1 +fi +export HIVE_HOME=$PWD/${hiveHomes[0]} +export HADOOP_HOME=${HADOOP_HOME:-/opt/hadoop} +export HADOOP_CLIENT_OPTS="${HADOOP_CLIENT_OPTS:-} --add-opens java.base/java.net=ALL-UNNAMED" + +work=$PWD/target/init-metastore/$dbType +rm -rf "$work" +mkdir -p "$work/conf" "$work/lib" +export HIVE_CONF_DIR=$work/conf +export HADOOP_CLIENT_OPTS+=" -Dderby.stream.error.file=$work/derby.log" +export HADOOP_CLASSPATH="$work/lib/*" + +dbContainer= +cleanup() { + if [[ -n $dbContainer ]]; then + docker logs "$dbContainer" > "$work/$dbContainer.log" 2>&1 || true + docker rm -f -v "$dbContainer" > /dev/null 2>&1 || true + fi +} +trap cleanup EXIT + +pomProperty() { + xmlstarlet sel -t -v "/_:project/_:properties/_:$1" pom.xml +} + +case $dbType in + derby) + driver=org.apache.derby.jdbc.EmbeddedDriver + url="jdbc:derby:;databaseName=$work/metastore_db;create=true" + user=APP + password=mine + ;; + postgres) + mvn -B -q dependency:copy -Dartifact="org.postgresql:postgresql:$(pomProperty postgres.version)" \ + -DoutputDirectory="$work/lib" + dbContainer=hive-precommit-postgres + docker rm -f -v "$dbContainer" > /dev/null 2>&1 || true + docker run -d --name "$dbContainer" --network "${DOCKER_NETWORK:-host}" \ + -e POSTGRES_USER=hive -e POSTGRES_PASSWORD=hive -e POSTGRES_DB=metastore \ + "${POSTGRES_IMAGE:-postgres:17-alpine}" > /dev/null + # Probe over TCP: during its first start the entrypoint runs a temporary server on the unix socket only. + for ((i = 0; i < 120; i++)); do + docker exec "$dbContainer" pg_isready -q -h 127.0.0.1 -U hive -d metastore && break + sleep 1 + done + docker exec "$dbContainer" pg_isready -h 127.0.0.1 -U hive -d metastore + driver=org.postgresql.Driver + url=jdbc:postgresql://localhost:5432/metastore + user=hive + password=hive + ;; + *) + echo "Unsupported database type: $dbType" >&2 + exit 1 + ;; +esac + +property() { + printf ' <property><name>%s</name><value>%s</value></property>\n' "$1" "$2" +} +{ + echo '<configuration>' + property javax.jdo.option.ConnectionDriverName "$driver" + property javax.jdo.option.ConnectionURL "$url" + property javax.jdo.option.ConnectionUserName "$user" + property javax.jdo.option.ConnectionPassword "$password" + property hive.metastore.warehouse.dir "file://$work/warehouse" + property hive.metastore.warehouse.external.dir "file://$work/warehouse-external" + property hive.exec.scratchdir "$work/scratch" + property hive.exec.local.scratchdir "$work/local-scratch" + property hive.querylog.location "$work/querylog" + property hive.server2.logging.operation.log.location "$work/operation-logs" + property hive.user.install.directory "file://$work/user" + echo '</configuration>' +} > "$HIVE_CONF_DIR/hive-site.xml" + +"$HIVE_HOME/bin/schematool" -verbose -dbType "$dbType" -initSchema +"$HIVE_HOME/bin/schematool" -verbose -dbType hive -initSchema -metaDbType "$dbType" +echo "Metastore schema initialized on $dbType" diff --git a/ql/src/test/queries/clientnegative/cachingprintstream.q b/ql/src/test/queries/clientnegative/cachingprintstream.q index a1939b6ccb0..85d7fa512a1 100644 --- a/ql/src/test/queries/clientnegative/cachingprintstream.q +++ b/ql/src/test/queries/clientnegative/cachingprintstream.q @@ -1,4 +1,7 @@ --! qt:dataset:src +-- Recent JDKs report a failed exec as "Exec failed, error: 2 (No such file or directory) ", +-- older ones as "error=2, No such file or directory"; the expected output uses the older form. +--! qt:replace:/Exec failed, error: (\d+) \(([^)]*)\) ?/error=$1, $2/ set hive.llap.execution.mode=auto; set hive.exec.failure.hooks=org.apache.hadoop.hive.ql.hooks.VerifyCachingPrintStreamHook; set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.VerifyCachingPrintStreamHook; diff --git a/ql/src/test/queries/clientnegative/stack_trace.q b/ql/src/test/queries/clientnegative/stack_trace.q index 953a7e44165..0d6e307df62 100644 --- a/ql/src/test/queries/clientnegative/stack_trace.q +++ b/ql/src/test/queries/clientnegative/stack_trace.q @@ -1,4 +1,7 @@ --! qt:dataset:src +-- Recent JDKs report a failed exec as "Exec failed, error: 2 (No such file or directory) ", +-- older ones as "error=2, No such file or directory"; the expected output uses the older form. +--! qt:replace:/Exec failed, error: (\d+) \(([^)]*)\) ?/error=$1, $2/ set hive.vectorized.execution.enabled=false; set hive.exec.mode.local.auto=false;
