This is an automated email from the ASF dual-hosted git repository.

asf-gitbox-commits pushed a commit to branch HIVE-29591-precommit-image
in repository https://gitbox.apache.org/repos/asf/hive.git

commit a1f02e59568ea4b594133ddfe805e437b7dc8159
Author: Laszlo Bodor <[email protected]>
AuthorDate: Mon Nov 9 12:01:00 2026 +0100

    HIVE-29591: Create Hive precommit image and move Dockerfile into the hive 
repository
---
 .github/workflows/precommit-image.yml              |  85 +++++++++
 Jenkinsfile                                        |  35 ++--
 dev-support/docker/precommit/Dockerfile            | 143 +++++++++++++++
 dev-support/docker/precommit/README.md             | 199 +++++++++++++++++++++
 dev-support/docker/precommit/smoke-test.sh         |  75 ++++++++
 dev-support/precommit-init-metastore.sh            | 120 +++++++++++++
 .../queries/clientnegative/cachingprintstream.q    |   3 +
 ql/src/test/queries/clientnegative/stack_trace.q   |   3 +
 8 files changed, 641 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/precommit-image.yml 
b/.github/workflows/precommit-image.yml
new file mode 100644
index 00000000000..380029acb47
--- /dev/null
+++ b/.github/workflows/precommit-image.yml
@@ -0,0 +1,85 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Builds the precommit image (dev-support/docker/precommit) and smoke tests 
it. On master it
+# publishes the image as apache/hive:precommit, the image the Jenkinsfile runs 
in.
+name: Precommit image
+
+on:
+  pull_request:
+    paths:
+      - 'dev-support/docker/precommit/**'
+      - '.github/workflows/precommit-image.yml'
+  push:
+    branches:
+      - master
+    paths:
+      - 'dev-support/docker/precommit/**'
+      - '.github/workflows/precommit-image.yml'
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Tag to publish; use a custom one (e.g. 
precommit-candidate-HIVE-12345) to try an image with the IMAGE parameter of the 
precommit job'
+        required: true
+        default: 'precommit'
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      IMAGE: ${{ github.repository_owner }}/hive:${{ github.event.inputs.tag 
|| 'precommit' }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Set up QEMU
+        uses: 
docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0
+
+      - name: Set up Docker Buildx
+        uses: 
docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
+
+      - name: Build the image
+        uses: 
docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
+        with:
+          context: ./dev-support/docker/precommit/
+          load: true
+          tags: hive-precommit:test
+
+      - name: Smoke test the image
+        run: |
+          docker images hive-precommit:test
+          dev-support/docker/precommit/smoke-test.sh hive-precommit:test
+
+      - name: Login to Docker Hub
+        if: github.event_name != 'pull_request' && github.repository == 
'apache/hive'
+        uses: docker/login-action@8aa5831fb8d48d8f76594c7ddde61ae58019f550 # 
v4.8.0
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Publish the image
+        if: github.event_name != 'pull_request' && github.repository == 
'apache/hive'
+        uses: 
docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
+        with:
+          context: ./dev-support/docker/precommit/
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ env.IMAGE }}
diff --git a/Jenkinsfile b/Jenkinsfile
index ad8f017e0f1..6d772605753 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -31,9 +31,13 @@ properties([
     parameters([
         string(name: 'SPLIT', defaultValue: '22', description: 'Number of 
buckets to split tests into.'),
         string(name: 'OPTS', defaultValue: '', description: 'additional maven 
opts'),
+        string(name: 'IMAGE', defaultValue: '', description: 'precommit image 
to use instead of the default one (e.g. a candidate built from 
dev-support/docker/precommit)'),
     ])
 ])
 
+// Built from dev-support/docker/precommit/Dockerfile and pushed by 
.github/workflows/precommit-image.yml.
+this.precommitImage = 'apache/hive:precommit'
+
 this.prHead = null;
 def checkPrHead() {
   if(env.CHANGE_ID) {
@@ -72,12 +76,17 @@ def setPrLabel(String prLabel) {
 
 setPrLabel("PENDING");
 
+// Every mvn call uses the settings and the local repository under .m2 of the 
workspace (set up by buildHive);
+// saveWS/loadWS hand them over to the other stages. Maven 3.9+ reads 
MAVEN_ARGS.
 def executorNode(run) {
   hdbPodTemplate {
     timeout(time: 12, unit: 'HOURS') {
       node(POD_LABEL) {
-        container('hdb') {
-          run()
+        def m2 = "${env.WORKSPACE}/.m2"
+        withEnv(["MAVEN_ARGS=-s ${m2}/settings.xml 
-Dmaven.repo.local=${m2}/repository -B -Dtest.groups= 
-Pitests,qsplits,dist,errorProne"]) {
+          container('hdb') {
+            run()
+          }
         }
       }
     }
@@ -89,17 +98,10 @@ def buildHive(args) {
     withEnv(["MULTIPLIER=$params.MULTIPLIER","M_OPTS=$params.OPTS"]) {
       sh '''#!/bin/bash -e
 set -x
-. /etc/profile.d/confs.sh
 export USER="`whoami`"
 export MAVEN_OPTS="-Xmx4G"
-export -n HIVE_CONF_DIR
-sw java 21 && . /etc/profile.d/java.sh
 mkdir -p .m2/repository
 cp $SETTINGS .m2/settings.xml
-OPTS=" -s $PWD/.m2/settings.xml -B -Dtest.groups= "
-OPTS+=" -Pitests,qsplits,dist,errorProne"
-OPTS+=" -Dmaven.repo.local=$PWD/.m2/repository"
-git config extra.mavenOpts "$OPTS"
 OPTS=" $M_OPTS -Dmaven.test.failure.ignore "
 if [ -s inclusions.txt ]; then OPTS+=" 
-Dsurefire.includesFile=$PWD/inclusions.txt";fi
 if [ -s exclusions.txt ]; then OPTS+=" 
-Dsurefire.excludesFile=$PWD/exclusions.txt";fi
@@ -123,7 +125,6 @@ def sonarAnalysis(args) {
       // Sonar scanner runs in a separate JVM so JAVA_OPTS (notably heap size)
       // must be passed via the appropriate environment variable
       sh """#!/bin/bash -e
-      sw java 21 && . /etc/profile.d/java.sh
       export SONAR_SCANNER_JAVA_OPTS=-Xmx8g
       """+mvnCmd
   }
@@ -132,7 +133,7 @@ def sonarAnalysis(args) {
 def hdbPodTemplate(closure) {
   podTemplate(
   containers: [
-    containerTemplate(name: 'hdb', image: 'ayushtkn/hive-dev-box:executor', 
ttyEnabled: true, command: 'tini -- cat',
+    containerTemplate(name: 'hdb', image: params.IMAGE?.trim() ?: 
precommitImage, ttyEnabled: true, command: 'tini -- cat',
         alwaysPullImage: true,
         resourceRequestCpu: '1800m',
         resourceLimitCpu: '8000m',
@@ -200,7 +201,6 @@ def jobWrappers(closure) {
 def checkstyle() {
   sh '''#!/bin/bash -e
 set -x
-sw java 21 && . /etc/profile.d/java.sh
 export MAVEN_OPTS="-Xmx6G"
 mvn checkstyle:checkstyle -Pitests
 '''
@@ -312,16 +312,8 @@ fi
         stage('init-metastore') {
            withEnv(["dbType=$dbType"]) {
              sh '''#!/bin/bash -e
-             sw java 21 && . /etc/profile.d/java.sh
 set -x
-echo 127.0.0.1 dev_$dbType | sudo tee -a /etc/hosts
-. /etc/profile.d/confs.sh
-sw hive-dev $PWD
-export DOCKER_NETWORK=host
-export DBNAME=metastore
-export HADOOP_CLIENT_OPTS="--add-opens java.base/java.net=ALL-UNNAMED"
-reinit_metastore $dbType
-time docker rm -f dev_$dbType || true
+dev-support/precommit-init-metastore.sh $dbType
 '''
           }
         }
@@ -428,7 +420,6 @@ tar -xzf packaging/target/apache-hive-*-nightly-*-src.tar.gz
       }
       stage('Generate javadoc') {
           sh """#!/bin/bash -e
-          sw java 21 && . /etc/profile.d/java.sh
 mvn install javadoc:javadoc javadoc:aggregate -DskipTests -pl 
'!itests/hive-jmh,!itests/util'
 """
       }
diff --git a/dev-support/docker/precommit/Dockerfile 
b/dev-support/docker/precommit/Dockerfile
new file mode 100644
index 00000000000..9f9e326ef94
--- /dev/null
+++ b/dev-support/docker/precommit/Dockerfile
@@ -0,0 +1,143 @@
+# syntax=docker/dockerfile:1
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# The image the Hive precommit (see Jenkinsfile) builds and tests Hive in.
+#
+# It is deliberately minimal: a JDK, Maven, and the handful of tools the 
Jenkinsfile
+# stages and the tests shell out to. Everything else is fetched by Maven at 
build time.
+# See README.md next to this file before adding anything.
+
+# glibc based on purpose: tests run prebuilt native code linked against glibc
+# (e.g. the Kudu mini cluster binaries, Conscrypt).
+ARG DEBIAN_VERSION=trixie
+ARG JAVA_VERSION=21
+
+# The Temurin JDK is a generic glibc build; only the distribution of its image 
differs.
+FROM eclipse-temurin:${JAVA_VERSION}-jdk AS jdk
+# jmods are only used by jlink; classes_nocoops.jsa is only used with heaps > 
32GB.
+RUN rm -rf /opt/java/openjdk/jmods \
+           /opt/java/openjdk/man \
+           /opt/java/openjdk/lib/server/classes_nocoops.jsa
+
+FROM debian:${DEBIAN_VERSION}-slim AS downloads
+ARG MAVEN_VERSION=3.9.16
+ARG HADOOP_VERSION=3.4.3
+ARG APACHE_ARCHIVE=https://archive.apache.org/dist
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends ca-certificates curl \
+    && rm -rf /var/lib/apt/lists/*
+WORKDIR /tmp
+
+# Fetches $1 and verifies it against the published .sha512 (plain or 
BSD-style).
+COPY <<'EOF' /usr/local/bin/fetch
+#!/bin/sh
+set -eu
+curl -fsSL --retry 5 -o "${1##*/}" "$1"
+echo "$(curl -fsSL --retry 5 "$1.sha512" | awk '{print $NF}')  ${1##*/}" | 
sha512sum -c -
+EOF
+RUN chmod +x /usr/local/bin/fetch
+
+RUN fetch 
"${APACHE_ARCHIVE}/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz"
 \
+    && mkdir /opt/maven \
+    && tar -xzf apache-maven-${MAVEN_VERSION}-bin.tar.gz -C /opt/maven 
--strip-components=1 \
+    && rm apache-maven-${MAVEN_VERSION}-bin.tar.gz
+
+# Hadoop is only needed by bin/schematool of the Hive binary distribution in 
the init@<db>
+# stages, so keep just the client side of hadoop-common/hdfs/mapreduce/yarn 
and the launcher scripts.
+RUN fetch 
"${APACHE_ARCHIVE}/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz"
 \
+    && mkdir /opt/hadoop \
+    && tar -xzf hadoop-${HADOOP_VERSION}.tar.gz -C /opt/hadoop 
--strip-components=1 --wildcards \
+         "hadoop-${HADOOP_VERSION}/LICENSE*" \
+         "hadoop-${HADOOP_VERSION}/NOTICE*" \
+         "hadoop-${HADOOP_VERSION}/licenses-binary" \
+         "hadoop-${HADOOP_VERSION}/bin/hadoop" \
+         "hadoop-${HADOOP_VERSION}/libexec/*.sh" \
+         "hadoop-${HADOOP_VERSION}/etc/hadoop/core-site.xml" \
+         "hadoop-${HADOOP_VERSION}/etc/hadoop/hadoop-env.sh" \
+         "hadoop-${HADOOP_VERSION}/etc/hadoop/log4j.properties" \
+         
"hadoop-${HADOOP_VERSION}/share/hadoop/common/hadoop-common-${HADOOP_VERSION}.jar"
 \
+         "hadoop-${HADOOP_VERSION}/share/hadoop/common/lib" \
+         
"hadoop-${HADOOP_VERSION}/share/hadoop/hdfs/hadoop-hdfs-${HADOOP_VERSION}.jar" \
+         
"hadoop-${HADOOP_VERSION}/share/hadoop/hdfs/hadoop-hdfs-client-${HADOOP_VERSION}.jar"
 \
+         
"hadoop-${HADOOP_VERSION}/share/hadoop/mapreduce/hadoop-mapreduce-client-core-${HADOOP_VERSION}.jar"
 \
+         
"hadoop-${HADOOP_VERSION}/share/hadoop/yarn/hadoop-yarn-api-${HADOOP_VERSION}.jar"
 \
+         
"hadoop-${HADOOP_VERSION}/share/hadoop/yarn/hadoop-yarn-client-${HADOOP_VERSION}.jar"
 \
+         
"hadoop-${HADOOP_VERSION}/share/hadoop/yarn/hadoop-yarn-common-${HADOOP_VERSION}.jar"
 \
+    && rm hadoop-${HADOOP_VERSION}.tar.gz
+
+FROM debian:${DEBIAN_VERSION}-slim
+
+LABEL org.opencontainers.image.title="Apache Hive precommit" \
+      org.opencontainers.image.description="Build and test environment of the 
Apache Hive precommit job" \
+      org.opencontainers.image.source="https://github.com/apache/hive"; \
+      org.opencontainers.image.licenses="Apache-2.0"
+
+# The base image already has the GNU bash, coreutils, diffutils, findutils, 
grep, sed and tar
+# that scripts and tests expect (e.g. qtest result diffing, Hadoop's 
Shell/Stat/DF/DU helpers).
+# ca-certificates, git: Jenkins checkout.
+# docker-cli: tests and the init@postgres stage start databases in the dind 
sidecar.
+# libcom-err2, libk5crypto3, libkeyutils1, libkrb5support0: needed by the Kudu 
mini cluster
+#   binaries (kudu-binary) that the kudu-handler tests start.
+# openssl: tests generate self-signed certificates with it (e.g. 
GcsOAuthTokenMock).
+# procps: ps, used by tests; python3, python-is-python3: TRANSFORM scripts in 
qtests;
+# rsync: workspace hand-over between stages; tini: reaps the processes tests 
leave behind;
+# xmlstarlet: pom.xml format check.
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        ca-certificates \
+        docker-cli \
+        git \
+        libcom-err2 \
+        libk5crypto3 \
+        libkeyutils1 \
+        libkrb5support0 \
+        openssl \
+        procps \
+        python-is-python3 \
+        python3 \
+        rsync \
+        tini \
+        xmlstarlet \
+    && rm -rf /var/lib/apt/lists/* \
+    && git config --system --add safe.directory '*'
+
+COPY --from=jdk /opt/java/openjdk /opt/java/openjdk
+COPY --from=downloads /opt/maven /opt/maven
+COPY --from=downloads /opt/hadoop /opt/hadoop
+
+ENV JAVA_HOME=/opt/java/openjdk \
+    MAVEN_HOME=/opt/maven \
+    PATH=/opt/java/openjdk/bin:/opt/maven/bin:$PATH \
+    LANG=C.UTF-8 \
+    LC_ALL=C.UTF-8 \
+    TZ=UTC
+
+# The Jenkins agent runs as uid 1000 and the pod uses fsGroup 1000; tests must 
not run as root.
+# ~/.m2 exists up front so that a local repository mounted below it leaves it 
writable.
+ARG UID=1000
+RUN groupadd -g ${UID} jenkins \
+    && useradd -m -u ${UID} -g jenkins -d /home/jenkins -s /bin/bash jenkins \
+    && install -d -o jenkins -g jenkins /home/jenkins/.m2
+
+# docker run sets no USER, but LLAP reads it to submit work 
(LlapTaskCommunicator).
+ENV USER=jenkins
+USER jenkins
+WORKDIR /home/jenkins
+
+ENTRYPOINT ["/usr/bin/tini", "--"]
+CMD ["bash"]
diff --git a/dev-support/docker/precommit/README.md 
b/dev-support/docker/precommit/README.md
new file mode 100644
index 00000000000..6bcffbf0458
--- /dev/null
+++ b/dev-support/docker/precommit/README.md
@@ -0,0 +1,199 @@
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+# Hive precommit image
+
+The [Jenkinsfile](../../../Jenkinsfile) runs every precommit stage in this 
image, with a
+`docker:dind` sidecar for the tests that start databases in containers.
+
+The image is kept small on purpose. It holds:
+
+* Debian (slim), with the GNU shell tools that scripts and tests expect. It is 
glibc based on
+  purpose: tests run prebuilt native code that is linked against glibc, such 
as the Kudu
+  mini cluster.
+* a JDK (Eclipse Temurin), without jmods and man pages
+* Maven
+* the client part of Hadoop, used only by `bin/schematool` in the `init@<db>` 
stages
+  (see [precommit-init-metastore.sh](../../precommit-init-metastore.sh))
+* the docker CLI, git, openssl, python3, rsync, tini and xmlstarlet
+
+Everything else (Tez, the JDBC drivers, the Hive build dependencies) is 
fetched by Maven
+during the build. Before adding a package, check whether Maven can provide it 
instead.
+
+## How the image is published
+
+The [precommit-image](../../../.github/workflows/precommit-image.yml) workflow 
builds the
+image and smoke tests it on every pull request that changes this directory. 
When such a
+change lands on master, the workflow publishes the image for `linux/amd64` and 
`linux/arm64`
+as `apache/hive:precommit`. That tag is the `precommitImage` of the 
Jenkinsfile, and
+the precommit job pulls it again on every run. The tag is overwritten on every 
publish:
+once the change is merged, every precommit run, on every branch, uses the new 
image.
+
+## Changing the image, step by step
+
+All commands run from the root of the Hive source tree.
+
+### 1. Make the change
+
+Edit the [Dockerfile](Dockerfile). Keep the image small:
+
+* Prefer letting Maven fetch a dependency over baking it into the image.
+* Pin versions through the existing build arguments (`JAVA_VERSION`, 
`MAVEN_VERSION`,
+  `HADOOP_VERSION`, `DEBIAN_VERSION`) rather than hard-coding them.
+* Install packages with `--no-install-recommends`, and add each new package to 
the comment
+  above `apt-get install`, saying which stage or test needs it.
+* Do anything heavy in the `jdk` or `downloads` stage and copy only the result 
into the
+  final stage.
+
+If the Jenkinsfile or a test relies on the new content, add a check for it to
+[smoke-test.sh](smoke-test.sh).
+
+### 2. Build the image
+
+```bash
+docker build -t hive-precommit:local dev-support/docker/precommit
+docker images hive-precommit:local
+```
+
+Compare the size with the image before your change, and explain any growth in 
the pull
+request. On an arm64 machine (e.g. Apple silicon) this builds an arm64 image. 
The CI runs
+amd64: add `--platform linux/amd64` to try that one, which is emulated and 
slower.
+
+On Linux, files written into a mounted source tree belong to uid 1000, the 
`jenkins` user of
+the image. If your uid is different, build with `--build-arg UID=$(id -u)`.
+
+### 3. Run the smoke test
+
+```bash
+dev-support/docker/precommit/smoke-test.sh hive-precommit:local
+```
+
+Every line must say `ok`. The pull request workflow runs the same script.
+
+### 4. Start a docker daemon for the tests
+
+The Jenkins pod runs a `docker:dind` sidecar in the same network namespace. Do 
the same
+locally:
+
+```bash
+docker run -d --name hive-dind --privileged -e DOCKER_TLS_CERTDIR= \
+  docker:20.10-dind --host=tcp://0.0.0.0:2375
+```
+
+The commands below run the image with
+`--network container:hive-dind -e DOCKER_HOST=tcp://localhost:2375`. That way 
the containers
+the tests start are reachable on `localhost`, as in the Jenkins pod.
+
+### 5. Build Hive in the image
+
+This is what the `Compile` stage does. Keep the local Maven repository between 
runs:
+
+```bash
+mkdir -p ~/.m2/hive-precommit-repository
+docker run --rm -it \
+  -v "$PWD:/home/jenkins/hive" -w /home/jenkins/hive \
+  -v ~/.m2/hive-precommit-repository:/home/jenkins/.m2/repository \
+  hive-precommit:local \
+  mvn -B install -DskipTests -Pitests,dist
+```
+
+The `dist` profile is needed for the next step.
+
+### 6. Initialize the metastore
+
+This is what the `init@derby` and `init@postgres` stages do:
+
+```bash
+docker run --rm \
+  -v "$PWD:/home/jenkins/hive" -w /home/jenkins/hive \
+  hive-precommit:local \
+  dev-support/precommit-init-metastore.sh derby
+
+docker run --rm \
+  --network container:hive-dind -e DOCKER_HOST=tcp://localhost:2375 \
+  -v "$PWD:/home/jenkins/hive" -w /home/jenkins/hive \
+  -v ~/.m2/hive-precommit-repository:/home/jenkins/.m2/repository \
+  hive-precommit:local \
+  dev-support/precommit-init-metastore.sh postgres
+```
+
+Both must end with `Metastore schema initialized on <db>`. The logs, including 
the log of
+the postgres container, are written to `target/init-metastore/<db>`.
+
+### 7. Run some tests
+
+Run tests that exercise what you changed. A good baseline is a couple of 
qtests:
+`newline.q` runs python `TRANSFORM` scripts, and `jdbc_project_pushdown.q` 
starts postgres
+in the docker daemon. They belong to different drivers, so they take two runs:
+
+```bash
+for t in TestMiniLlapCliDriver:newline.q 
TestMiniLlapLocalCliDriver:jdbc_project_pushdown.q; do
+  docker run --rm \
+    --network container:hive-dind -e DOCKER_HOST=tcp://localhost:2375 \
+    -v "$PWD:/home/jenkins/hive" -w /home/jenkins/hive \
+    -v ~/.m2/hive-precommit-repository:/home/jenkins/.m2/repository \
+    hive-precommit:local \
+    mvn -B test -Pitests -pl itests/qtest -Dtest=${t%%:*} -Dqfile=${t#*:} || 
break
+done
+```
+
+When you are done, remove the docker daemon with `docker rm -f -v hive-dind`.
+
+### 8. Open a pull request
+
+Open the pull request as usual. The precommit-image workflow builds the image 
and runs the
+smoke test on it. The precommit job of the pull request itself still runs in 
the published
+`apache/hive:precommit`, not in your change.
+
+### 9. Run the precommit job with the candidate image
+
+For anything beyond a trivial change, run the full precommit job on the new 
image before
+merging. This needs a committer.
+
+1. Publish the candidate under a tag of its own. Either:
+   * push the change to a branch of `apache/hive` and run the precommit-image 
workflow on that
+     branch from the Actions tab, with `tag` set to e.g. 
`precommit-candidate-HIVE-12345`; or
+   * build it for amd64 and push it to a registry you control:
+
+     ```bash
+     docker buildx build --platform linux/amd64 --push \
+       -t <your-namespace>/hive-precommit:HIVE-12345 
dev-support/docker/precommit
+     ```
+
+2. On the precommit job of the pull request in Jenkins, choose *Build with 
Parameters* and set
+   `IMAGE` to the candidate, e.g. 
`apache/hive:precommit-candidate-HIVE-12345`. The parameter
+   is only offered after the job has run at least once with this Jenkinsfile.
+3. Link the green run in the pull request.
+
+### 10. Merge and verify
+
+Once merged, the precommit-image workflow on master publishes
+`apache/hive:precommit`. Then:
+
+1. Check that the workflow run succeeded.
+2. Check the published image:
+
+   ```bash
+   docker buildx imagetools inspect apache/hive:precommit
+   ```
+
+3. Watch the next precommit runs.
+
+To roll back, revert the commit. The workflow then publishes the previous 
Dockerfile as
+`precommit` again. The rebuilt image picks up the Debian packages current at 
that time,
+so it is equivalent to the old one but not identical.
diff --git a/dev-support/docker/precommit/smoke-test.sh 
b/dev-support/docker/precommit/smoke-test.sh
new file mode 100755
index 00000000000..e4cba0c7a59
--- /dev/null
+++ b/dev-support/docker/precommit/smoke-test.sh
@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Checks that a precommit image provides what the Jenkinsfile and the tests 
rely on.
+#
+# Usage: dev-support/docker/precommit/smoke-test.sh <image> [<expected java 
feature version>]
+
+set -euo pipefail
+
+image=${1:?usage: $0 <image> [<java version>]}
+javaVersion=${2:-21}
+
+docker run --rm -i -e JAVA_FEATURE="$javaVersion" "$image" bash -s <<'EOF'
+set -euo pipefail
+check() {
+  local what=$1
+  shift
+  if out=$("$@" 2>&1); then
+    echo "ok   $what"
+  else
+    echo "FAIL $what: $out" >&2
+    exit 1
+  fi
+}
+
+check "runs as jenkins (uid 1000)" test "$(id -u):$(id -un)" = 1000:jenkins
+check "USER is set" test "${USER:-}" = jenkins
+check "home is writable" touch "$HOME/.probe"
+check "~/.m2 is writable" touch "$HOME/.m2/.probe"
+check "java $JAVA_FEATURE" sh -c 'java -version 2>&1 | grep -q "version 
\"$JAVA_FEATURE"'
+check "javac" javac -version
+check "default charset is UTF-8" sh -c 'java -XshowSettings:properties 
-version 2>&1 | grep -q "native.encoding = UTF-8"'
+check "maven" mvn -B -v
+for tool in bash docker git python python3 rsync tini xmlstarlet ps; do
+  check "$tool on PATH" command -v $tool
+done
+for tool in diff find grep sed sort tar; do
+  check "GNU $tool" sh -c "$tool --version | grep -q GNU"
+done
+check "openssl at /usr/bin/openssl" sh -c 'cd "$(mktemp -d)" && 
/usr/bin/openssl req -x509 -newkey rsa:2048 -keyout k.pem -out c.pem -days 1 
-nodes -subj /CN=test'
+check "glibc" getconf GNU_LIBC_VERSION
+for lib in libcom_err.so.2 libk5crypto.so.3 libkeyutils.so.1 
libkrb5support.so.0; do
+  check "$lib for the Kudu mini cluster" sh -c "ldconfig -p | grep -q $lib"
+done
+check "hadoop launcher" /opt/hadoop/bin/hadoop version
+check "python TRANSFORM-style script" sh -c 'test "$(printf "a\tb\n" | python 
-c "import sys; [print(l.rstrip().upper()) for l in sys.stdin]")" = "$(printf 
"A\tB")"'
+
+# The Prechecks stage relies on "xmlstarlet edit" reformatting badly indented 
poms.
+cd "$(mktemp -d)"
+printf '<?xml version="1.0" encoding="UTF-8"?>\n<project>\n  
<a>1</a>\n</project>\n' > good.xml
+printf '<?xml version="1.0" encoding="UTF-8"?>\n<project>\n 
<a>1</a>\n</project>\n' > bad.xml
+cp good.xml good.orig
+cp bad.xml bad.orig
+xmlstarlet edit -L good.xml bad.xml
+check "xmlstarlet keeps formatted xml" cmp good.xml good.orig
+check "xmlstarlet reformats bad xml" sh -c '! cmp -s bad.xml bad.orig'
+
+check "git" sh -c 'git init -q repo && git -C repo status --short'
+echo "All checks passed"
+EOF
diff --git a/dev-support/precommit-init-metastore.sh 
b/dev-support/precommit-init-metastore.sh
new file mode 100755
index 00000000000..3c579a5d4b6
--- /dev/null
+++ b/dev-support/precommit-init-metastore.sh
@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Initializes an empty metastore, and the Hive SYS/INFORMATION_SCHEMA 
databases on top of it, with
+# bin/schematool of the binary distribution built by "mvn install -Pdist".
+# Used by the init@<db> stages of the Jenkinsfile, in the precommit image 
(dev-support/docker/precommit).
+#
+# Usage: dev-support/precommit-init-metastore.sh <derby|postgres>
+#
+# Environment:
+#   HADOOP_HOME     Hadoop installation used by bin/schematool (default: 
/opt/hadoop)
+#   MAVEN_ARGS      extra Maven arguments (settings, local repository) used to 
fetch the JDBC driver
+#   DOCKER_NETWORK  network of the database container (default: host); its 
port must be
+#                   reachable on localhost
+#   POSTGRES_IMAGE  image of the postgres container (default: 
postgres:17-alpine)
+
+set -euo pipefail
+
+dbType=${1:?usage: $0 <derby|postgres>}
+
+cd "$(dirname "$0")/.."
+
+hiveHomes=(packaging/target/apache-hive-*-bin/apache-hive-*-bin)
+if [[ ! -d ${hiveHomes[0]} ]]; then
+  echo "No Hive binary distribution under packaging/target; build it with 
-Pdist first." >&2
+  exit 1
+fi
+export HIVE_HOME=$PWD/${hiveHomes[0]}
+export HADOOP_HOME=${HADOOP_HOME:-/opt/hadoop}
+export HADOOP_CLIENT_OPTS="${HADOOP_CLIENT_OPTS:-} --add-opens 
java.base/java.net=ALL-UNNAMED"
+
+work=$PWD/target/init-metastore/$dbType
+rm -rf "$work"
+mkdir -p "$work/conf" "$work/lib"
+export HIVE_CONF_DIR=$work/conf
+export HADOOP_CLIENT_OPTS+=" -Dderby.stream.error.file=$work/derby.log"
+export HADOOP_CLASSPATH="$work/lib/*"
+
+dbContainer=
+cleanup() {
+  if [[ -n $dbContainer ]]; then
+    docker logs "$dbContainer" > "$work/$dbContainer.log" 2>&1 || true
+    docker rm -f -v "$dbContainer" > /dev/null 2>&1 || true
+  fi
+}
+trap cleanup EXIT
+
+pomProperty() {
+  xmlstarlet sel -t -v "/_:project/_:properties/_:$1" pom.xml
+}
+
+case $dbType in
+  derby)
+    driver=org.apache.derby.jdbc.EmbeddedDriver
+    url="jdbc:derby:;databaseName=$work/metastore_db;create=true"
+    user=APP
+    password=mine
+    ;;
+  postgres)
+    mvn -B -q dependency:copy 
-Dartifact="org.postgresql:postgresql:$(pomProperty postgres.version)" \
+      -DoutputDirectory="$work/lib"
+    dbContainer=hive-precommit-postgres
+    docker rm -f -v "$dbContainer" > /dev/null 2>&1 || true
+    docker run -d --name "$dbContainer" --network "${DOCKER_NETWORK:-host}" \
+      -e POSTGRES_USER=hive -e POSTGRES_PASSWORD=hive -e POSTGRES_DB=metastore 
\
+      "${POSTGRES_IMAGE:-postgres:17-alpine}" > /dev/null
+    # Probe over TCP: during its first start the entrypoint runs a temporary 
server on the unix socket only.
+    for ((i = 0; i < 120; i++)); do
+      docker exec "$dbContainer" pg_isready -q -h 127.0.0.1 -U hive -d 
metastore && break
+      sleep 1
+    done
+    docker exec "$dbContainer" pg_isready -h 127.0.0.1 -U hive -d metastore
+    driver=org.postgresql.Driver
+    url=jdbc:postgresql://localhost:5432/metastore
+    user=hive
+    password=hive
+    ;;
+  *)
+    echo "Unsupported database type: $dbType" >&2
+    exit 1
+    ;;
+esac
+
+property() {
+  printf '  <property><name>%s</name><value>%s</value></property>\n' "$1" "$2"
+}
+{
+  echo '<configuration>'
+  property javax.jdo.option.ConnectionDriverName "$driver"
+  property javax.jdo.option.ConnectionURL "$url"
+  property javax.jdo.option.ConnectionUserName "$user"
+  property javax.jdo.option.ConnectionPassword "$password"
+  property hive.metastore.warehouse.dir "file://$work/warehouse"
+  property hive.metastore.warehouse.external.dir 
"file://$work/warehouse-external"
+  property hive.exec.scratchdir "$work/scratch"
+  property hive.exec.local.scratchdir "$work/local-scratch"
+  property hive.querylog.location "$work/querylog"
+  property hive.server2.logging.operation.log.location "$work/operation-logs"
+  property hive.user.install.directory "file://$work/user"
+  echo '</configuration>'
+} > "$HIVE_CONF_DIR/hive-site.xml"
+
+"$HIVE_HOME/bin/schematool" -verbose -dbType "$dbType" -initSchema
+"$HIVE_HOME/bin/schematool" -verbose -dbType hive -initSchema -metaDbType 
"$dbType"
+echo "Metastore schema initialized on $dbType"
diff --git a/ql/src/test/queries/clientnegative/cachingprintstream.q 
b/ql/src/test/queries/clientnegative/cachingprintstream.q
index a1939b6ccb0..85d7fa512a1 100644
--- a/ql/src/test/queries/clientnegative/cachingprintstream.q
+++ b/ql/src/test/queries/clientnegative/cachingprintstream.q
@@ -1,4 +1,7 @@
 --! qt:dataset:src
+-- Recent JDKs report a failed exec as "Exec failed, error: 2 (No such file or 
directory) ",
+-- older ones as "error=2, No such file or directory"; the expected output 
uses the older form.
+--! qt:replace:/Exec failed, error: (\d+) \(([^)]*)\) ?/error=$1, $2/
 set hive.llap.execution.mode=auto;
 set 
hive.exec.failure.hooks=org.apache.hadoop.hive.ql.hooks.VerifyCachingPrintStreamHook;
 set 
hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.VerifyCachingPrintStreamHook;
diff --git a/ql/src/test/queries/clientnegative/stack_trace.q 
b/ql/src/test/queries/clientnegative/stack_trace.q
index 953a7e44165..0d6e307df62 100644
--- a/ql/src/test/queries/clientnegative/stack_trace.q
+++ b/ql/src/test/queries/clientnegative/stack_trace.q
@@ -1,4 +1,7 @@
 --! qt:dataset:src
+-- Recent JDKs report a failed exec as "Exec failed, error: 2 (No such file or 
directory) ",
+-- older ones as "error=2, No such file or directory"; the expected output 
uses the older form.
+--! qt:replace:/Exec failed, error: (\d+) \(([^)]*)\) ?/error=$1, $2/
 
 set hive.vectorized.execution.enabled=false;
 set hive.exec.mode.local.auto=false;

Reply via email to