This is an automated email from the ASF dual-hosted git repository.
yashmayya pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 6d5cdfb93ab Verify the Java 11 pinned client artifacts actually run on
a Java 11 JVM in CI (#19102)
6d5cdfb93ab is described below
commit 6d5cdfb93ab6513bb02c31e1172d2604899698ef
Author: Yash Mayya <[email protected]>
AuthorDate: Tue Jul 28 17:06:37 2026 -0400
Verify the Java 11 pinned client artifacts actually run on a Java 11 JVM in
CI (#19102)
---
.../pinot_java11_client_compatibility.yml | 114 +++
.../scripts/.pinot_java11_client_compat.sh | 97 +++
pinot-java11-client-verifier/pom.xml | 140 ++++
.../pinot/java11/CannedResponseTransport.java | 73 ++
.../pinot/java11/ClasspathClosureScanner.java | 334 +++++++++
.../pinot/java11/Java11CompatibilityVerifier.java | 803 +++++++++++++++++++++
.../src/main/resources/sample-broker-response.json | 53 ++
.../pinot/java11/ClasspathClosureScannerTest.java | 367 ++++++++++
pom.xml | 1 +
9 files changed, 1982 insertions(+)
diff --git a/.github/workflows/pinot_java11_client_compatibility.yml
b/.github/workflows/pinot_java11_client_compatibility.yml
new file mode 100644
index 00000000000..44598d5d982
--- /dev/null
+++ b/.github/workflows/pinot_java11_client_compatibility.yml
@@ -0,0 +1,114 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Pinot Java 11 Client Compatibility
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - "contrib/**"
+ - "docs/**"
+ - "docker/**"
+ - "kubernetes/**"
+ - "licenses/**"
+ - "licenses-binary/**"
+ - "**.md"
+ pull_request:
+ branches:
+ - master
+ paths-ignore:
+ - "contrib/**"
+ - "docs/**"
+ - "docker/**"
+ - "kubernetes/**"
+ - "licenses/**"
+ - "licenses-binary/**"
+ - "**.md"
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number ||
github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ # pinot-spi, pinot-segment-spi, pinot-timeseries-spi, pinot-common,
pinot-java-client and
+ # pinot-jdbc-client hard-code <release>11</release> so that third-party
plugins and applications
+ # embedding the Java/JDBC client are not forced onto the JDK that Pinot's
services require.
+ #
+ # --release 11 keeps Pinot's own code in those modules Java 11 clean, but
nothing checks their
+ # transitive dependency closure. A routine dependency bump can drop a Java
17+ jar into the client
+ # classpath, and today the first sign of that would be a user reporting
+ # UnsupportedClassVersionError. This job loads and exercises the clients on
a real Java 11 JVM.
+ java11-client-compatibility:
+ if: github.repository == 'apache/pinot'
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ # build-java is the JDK the artifacts are compiled with; the verifier
always runs on the
+ # Java version the clients are pinned to.
+ build-java: [ 25 ]
+ distribution: [ "temurin" ]
+ name: Pinot Java 11 Client Compatibility (built on JDK ${{
matrix.build-java }}-${{ matrix.distribution }})
+ steps:
+ - uses: actions/checkout@v7
+ # Installed first so that the build JDK below ends up as JAVA_HOME.
setup-java also exports
+ # JAVA_HOME_11_<arch>, which the script falls back on if this step's
output is unavailable.
+ - name: Set up Java 11 (verification runtime)
+ id: java11
+ uses: actions/setup-java@v5
+ with:
+ java-version: 11
+ distribution: ${{ matrix.distribution }}
+ - name: Set up JDK ${{ matrix.build-java }} (build)
+ uses: actions/setup-java@v5
+ with:
+ java-version: ${{ matrix.build-java }}
+ distribution: ${{ matrix.distribution }}
+ cache: 'maven'
+ - uses: actions/cache@v6
+ env:
+ SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
+ with:
+ path: ~/.m2/repository
+ key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-maven-
+ - name: Verify the Java 11 clients on a Java 11 JVM
+ timeout-minutes: 40
+ env:
+ JAVA11_HOME: ${{ steps.java11.outputs.path }}
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
+ # Same download hardening the other Maven jobs use, so a transient
Central hiccup retries
+ # here too instead of failing the job.
+ #
+ # Only JVM arguments and -D system properties belong here:
MAVEN_OPTS is handed to the JVM
+ # that runs Maven, not to Maven itself. The peer jobs also list -B
and -ntp, which the JVM
+ # rejects outright ("Unrecognized option: -B") and which only
survive there because those
+ # jobs additionally pass -XX:+IgnoreUnrecognizedVMOptions. The
script passes -B -ntp on the
+ # mvn command line where they belong, so they are left out rather
than masked.
+ #
+ # Deliberately no -DskipShade either: the script passes
-Dshade.phase.prop=none, which
+ # disables shading for both client modules (-DskipShade only covers
pinot-jdbc-client).
+ MAVEN_OPTS: >
+ -Xmx2G -DfailIfNoTests=false
-Dmaven.wagon.httpconnectionManager.ttlSeconds=25
+ -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false
-Dmaven.wagon.http.pool=false
+ run: |
+ .github/workflows/scripts/.pinot_java11_client_compat.sh
diff --git a/.github/workflows/scripts/.pinot_java11_client_compat.sh
b/.github/workflows/scripts/.pinot_java11_client_compat.sh
new file mode 100755
index 00000000000..8f8399d62a2
--- /dev/null
+++ b/.github/workflows/scripts/.pinot_java11_client_compat.sh
@@ -0,0 +1,97 @@
+#!/bin/bash -x
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Verifies that Pinot's Java-11-pinned client and SPI artifacts work on a Java
11 JVM.
+#
+# The build cannot run on Java 11 -- the root pom enforces requireJavaVersion
[25,) and Pinot's
+# services have a genuine Java 25 floor. So this is necessarily a two-JDK job:
build the artifacts
+# with the build JDK, then run the verifier under Java 11.
+#
+# Environment:
+# JAVA11_HOME Optional. Home of the JVM to verify against. Falls back to the
+# JAVA_HOME_11_<arch> variables that GitHub-hosted runners
export.
+# MVN Optional. Maven command, defaults to "mvn".
+
+TARGET_JAVA_VERSION=11
+VERIFIER_MODULE="pinot-java11-client-verifier"
+VERIFIER_MAIN_CLASS="org.apache.pinot.java11.Java11CompatibilityVerifier"
+MVN="${MVN:-mvn}"
+
+# Build JDK, for the record.
+java -version
+
+# GitHub-hosted runners export JAVA_HOME_<version>_<arch> for every JDK that
setup-java installed.
+# The arch suffix differs between x64 and arm64 runners, so try both rather
than assuming.
+if [ -z "${JAVA11_HOME}" ]; then
+ JAVA11_HOME="${JAVA_HOME_11_X64:-${JAVA_HOME_11_ARM64:-}}"
+fi
+if [ -z "${JAVA11_HOME}" ]; then
+ echo "No Java ${TARGET_JAVA_VERSION} JVM found. Set JAVA11_HOME, or install
one with actions/setup-java" \
+ "and pass its path through."
+ exit 1
+fi
+
+JAVA11_BIN="${JAVA11_HOME}/bin/java"
+if [ ! -x "${JAVA11_BIN}" ]; then
+ echo "Not an executable JVM launcher: ${JAVA11_BIN}"
+ exit 1
+fi
+"${JAVA11_BIN}" -version || exit 1
+
+# Build the verifier and everything it depends on, which is exactly the six
Java-11-pinned modules
+# and the third-party closure underneath them. Linting is covered by the
linter job, so skip it here.
+#
+# -Dshade.phase.prop=none matters: pinot-java-client and pinot-jdbc-client
each produce a ~150 MB
+# shaded jar, and shadedArtifactAttached=true means those jars never even
appear on the runtime
+# classpath this job verifies. Without the flag the job spends minutes
building them and then pushes
+# 300 MB into ~/.m2, which actions/cache uploads under a key the other Maven
jobs share. Note that
+# -DskipShade=true is not enough: it only deactivates the pinot-jdbc-client
profile, while
+# pinot-java-client sets shade.phase.prop=package unconditionally.
+${MVN} clean install -B -ntp -T1C -pl "${VERIFIER_MODULE}" -am \
+ -DskipTests \
+ -Dshade.phase.prop=none \
+ -Dmaven.javadoc.skip=true \
+ -Dlicense.skip=true \
+ -Dcheckstyle.skip=true \
+ -Dspotless.check.skip=true || exit 1
+
+CLASSPATH_FILE="${VERIFIER_MODULE}/target/runtime-classpath.txt"
+if [ ! -s "${CLASSPATH_FILE}" ]; then
+ echo "Expected the build to write the resolved runtime closure to
${CLASSPATH_FILE}"
+ exit 1
+fi
+
+VERIFIER_CLASSPATH="${VERIFIER_MODULE}/target/classes:$(cat
"${CLASSPATH_FILE}")"
+
+# Tracing off for the run itself: echoing a few hundred jar paths buries the
verifier's own output.
+set +x
+echo "Running ${VERIFIER_MAIN_CLASS} on Java ${TARGET_JAVA_VERSION} against a
closure of" \
+ "$(tr ':' '\n' <<< "${VERIFIER_CLASSPATH}" | wc -l | tr -d ' ') classpath
entries"
+
+# No --add-opens or -Dio.netty.tryReflectionSetAccessible here on purpose.
Those flags exist in
+# Pinot's own launch scripts for JDK 17+, and adding them would paper over
exactly the kind of
+# runtime breakage this job is meant to catch. The verifier asserts it is
really running on Java
+# ${TARGET_JAVA_VERSION}, so a mis-wired JDK fails the job instead of passing
it vacuously.
+if ! "${JAVA11_BIN}" -cp "${VERIFIER_CLASSPATH}" "${VERIFIER_MAIN_CLASS}"
"${TARGET_JAVA_VERSION}"; then
+ # Hosted runners discard the workspace, so dump the closure that was
verified while we still can.
+ echo
+ echo "Verification failed. The runtime closure that was verified:"
+ tr ':' '\n' <<< "${VERIFIER_CLASSPATH}"
+ exit 1
+fi
diff --git a/pinot-java11-client-verifier/pom.xml
b/pinot-java11-client-verifier/pom.xml
new file mode 100644
index 00000000000..410cd519ba3
--- /dev/null
+++ b/pinot-java11-client-verifier/pom.xml
@@ -0,0 +1,140 @@
+<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>pinot</artifactId>
+ <groupId>org.apache.pinot</groupId>
+ <version>1.6.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>pinot-java11-client-verifier</artifactId>
+ <name>Pinot Java 11 Client Verifier</name>
+ <url>https://pinot.apache.org/</url>
+ <packaging>jar</packaging>
+
+ <properties>
+ <pinot.root>${basedir}/..</pinot.root>
+ <!--
+ A CI harness, not something a consumer should depend on. Keep it out of
the published
+ artifacts; nothing in the reactor depends on it either.
+ -->
+ <maven.deploy.skip>true</maven.deploy.skip>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <!-- Load-bearing rather than copied boilerplate:
.pinot_java11_client_compat.sh launches
+ this module's own classes on a Java 11 JVM, so its bytecode has
to be Java 11 as well.
+ Hard-coded literal so -Djdk.version=21 on the CLI cannot
silently bump the level. -->
+ <release>11</release>
+ <source>11</source>
+ <target>11</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <!--
+ Writes the resolved runtime closure of this module (the union of the
pinot-java-client and
+ pinot-jdbc-client closures) to a file, so the CI driver script can
hand exactly that
+ classpath to a Java 11 JVM.
+ -->
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>write-runtime-classpath</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>build-classpath</goal>
+ </goals>
+ <configuration>
+ <includeScope>runtime</includeScope>
+
<outputFile>${project.build.directory}/runtime-classpath.txt</outputFile>
+ <regenerateFile>true</regenerateFile>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <!-- The two consumer-facing clients. Depending on both makes this
module's runtime closure the
+ union of theirs, which is what gets scanned for bytecode a Java 11
JVM cannot load. -->
+ <dependency>
+ <groupId>org.apache.pinot</groupId>
+ <artifactId>pinot-java-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.pinot</groupId>
+ <artifactId>pinot-jdbc-client</artifactId>
+ </dependency>
+
+ <!-- The remaining Java-11-pinned modules. All four already arrive
transitively, so declaring
+ them changes nothing about the closure; it is done because the
verifier imports them
+ directly, and so that dropping one breaks the build instead of
silently shrinking coverage.
+ Java11CompatibilityVerifier also asserts at runtime that all six are
on the classpath. -->
+ <dependency>
+ <groupId>org.apache.pinot</groupId>
+ <artifactId>pinot-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.pinot</groupId>
+ <artifactId>pinot-spi</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.pinot</groupId>
+ <artifactId>pinot-segment-spi</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.pinot</groupId>
+ <artifactId>pinot-timeseries-spi</artifactId>
+ </dependency>
+
+ <!-- Third-party types the verifier constructs directly. -->
+ <dependency>
+ <groupId>org.apache.helix</groupId>
+ <artifactId>helix-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git
a/pinot-java11-client-verifier/src/main/java/org/apache/pinot/java11/CannedResponseTransport.java
b/pinot-java11-client-verifier/src/main/java/org/apache/pinot/java11/CannedResponseTransport.java
new file mode 100644
index 00000000000..65619cdd68b
--- /dev/null
+++
b/pinot-java11-client-verifier/src/main/java/org/apache/pinot/java11/CannedResponseTransport.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.java11;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.util.concurrent.CompletableFuture;
+import javax.annotation.Nullable;
+import org.apache.pinot.client.BrokerResponse;
+import org.apache.pinot.client.PinotClientException;
+import org.apache.pinot.client.PinotClientTransport;
+
+
+/// A {@link PinotClientTransport} that replays a fixed broker response
instead of talking to a broker, so the verifier
+/// can drive the real {@code Connection} / {@code ResultSetGroup} / JDBC code
paths without standing up a cluster.
+///
+/// Records the last query it was handed so callers can assert that query
construction
+/// (for example {@code PreparedStatement} parameter binding) happened on the
way in.
+///
+/// Not thread-safe: the verifier drives it from a single thread.
+final class CannedResponseTransport implements PinotClientTransport<Void> {
+ private final JsonNode _cannedResponse;
+ private String _lastQuery;
+ private boolean _closed;
+
+ CannedResponseTransport(JsonNode cannedResponse) {
+ _cannedResponse = cannedResponse;
+ }
+
+ @Override
+ public BrokerResponse executeQuery(String brokerAddress, String query)
+ throws PinotClientException {
+ _lastQuery = query;
+ return BrokerResponse.fromJson(_cannedResponse);
+ }
+
+ @Override
+ public CompletableFuture<BrokerResponse> executeQueryAsync(String
brokerAddress, String query)
+ throws PinotClientException {
+ return CompletableFuture.completedFuture(executeQuery(brokerAddress,
query));
+ }
+
+ @Override
+ public void close()
+ throws PinotClientException {
+ _closed = true;
+ }
+
+ /// The last query handed to this transport, or null if it has not been
asked to execute one yet.
+ @Nullable
+ String getLastQuery() {
+ return _lastQuery;
+ }
+
+ boolean isClosed() {
+ return _closed;
+ }
+}
diff --git
a/pinot-java11-client-verifier/src/main/java/org/apache/pinot/java11/ClasspathClosureScanner.java
b/pinot-java11-client-verifier/src/main/java/org/apache/pinot/java11/ClasspathClosureScanner.java
new file mode 100644
index 00000000000..c691c70edbd
--- /dev/null
+++
b/pinot-java11-client-verifier/src/main/java/org/apache/pinot/java11/ClasspathClosureScanner.java
@@ -0,0 +1,334 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.java11;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.DataInputStream;
+import java.io.EOFException;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.TreeMap;
+import java.util.jar.Attributes;
+import java.util.jar.Manifest;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipException;
+import java.util.zip.ZipFile;
+
+
+/// Scans every entry of a classpath and reports any class file whose bytecode
version is newer than a given Java
+/// feature release can load.
+///
+/// This complements the hard-coded compiler {@code release} of 11 on Pinot's
client and SPI modules. That pin
+/// guarantees Pinot's _own_ bytecode is Java 11 clean, but says nothing about
the transitive third-party closure those
+/// modules drag in. A dependency bump that pulls in a Java 17+ jar would
otherwise only be discovered by a user hitting
+/// {@link UnsupportedClassVersionError} at runtime.
+///
+/// Two categories of entry are deliberately _not_ violations, because a Java
11 JVM never loads them:
+/// - {@code module-info.class} descriptors, which are compiled at the
version of the JDK that built the jar and are
+/// ignored entirely on the classpath.
+/// - Entries under {@code META-INF/versions/<n>/} that this JVM would not
select. A JVM only performs multi-release
+/// lookup at all when the jar manifest carries {@code Multi-Release:
true}, so in a jar without that attribute
+/// every versioned entry is an inert resource and is skipped outright. In
a genuine multi-release jar, only the
+/// entries at or below the target release are selected, so entries above
it are skipped and the rest are checked.
+/// Directory classpath entries never get multi-release treatment, so
their versioned entries are skipped too.
+///
+/// This class is not thread-safe; a {@link Result} is produced by a single
{@link #scan} call.
+final class ClasspathClosureScanner {
+ /// Class file major version for Java 11 is 55; each later feature release
adds one.
+ private static final int CLASS_FILE_MAJOR_VERSION_OFFSET = 44;
+ private static final int CLASS_FILE_MAGIC = 0xCAFEBABE;
+ private static final String MULTI_RELEASE_PREFIX = "META-INF/versions/";
+ private static final String MODULE_INFO_ENTRY = "module-info.class";
+ private static final String MANIFEST_ENTRY = "META-INF/MANIFEST.MF";
+ /// Cap the reported violations so a wholesale mistake does not produce
megabytes of CI log.
+ @VisibleForTesting
+ static final int MAX_REPORTED_VIOLATIONS = 40;
+
+ private ClasspathClosureScanner() {
+ }
+
+ /// A single class file that a JVM at the target feature release could not
load.
+ static final class Violation {
+ private final String _classpathEntry;
+ private final String _classFile;
+ private final int _majorVersion;
+
+ Violation(String classpathEntry, String classFile, int majorVersion) {
+ _classpathEntry = classpathEntry;
+ _classFile = classFile;
+ _majorVersion = majorVersion;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("%s!/%s (class file major version %d, needs Java
%d)", _classpathEntry, _classFile,
+ _majorVersion, _majorVersion - CLASS_FILE_MAJOR_VERSION_OFFSET);
+ }
+ }
+
+ /// Outcome of a scan, including enough counters to tell a real pass from a
vacuous one.
+ static final class Result {
+ private final List<Violation> _reportedViolations = new ArrayList<>();
+ private final List<String> _archiveNames = new ArrayList<>();
+ private final List<String> _skippedEntries = new ArrayList<>();
+ private final Map<Integer, Integer> _majorVersionHistogram = new
TreeMap<>();
+ private int _totalViolationCount;
+ private int _archivesScanned;
+ private int _directoriesScanned;
+ private int _classFilesInArchives;
+ private int _classFilesInDirectories;
+
+ /// The first {@value #MAX_REPORTED_VIOLATIONS} violations, so a wholesale
mistake does not produce megabytes of CI
+ /// log. {@link #getTotalViolationCount()} is the authoritative count.
+ List<Violation> getReportedViolations() {
+ return _reportedViolations;
+ }
+
+ int getTotalViolationCount() {
+ return _totalViolationCount;
+ }
+
+ /// File names of every archive that was opened, for asserting which
artifacts were covered.
+ List<String> getArchiveNames() {
+ return _archiveNames;
+ }
+
+ /// Classpath entries that carried no bytecode to check, each with a
reason. Callers should treat a non-empty list
+ /// as suspicious rather than routine -- see the allow-list in the
verifier.
+ List<String> getSkippedEntries() {
+ return _skippedEntries;
+ }
+
+ int getArchivesScanned() {
+ return _archivesScanned;
+ }
+
+ int getDirectoriesScanned() {
+ return _directoriesScanned;
+ }
+
+ /// Class files inspected inside jars, counted separately from {@link
#getClassFilesInDirectories()} so a vacuity
+ /// guard can assert that third-party bytecode was actually looked at.
Counting both together would let the
+ /// verifier's own {@code target/classes} satisfy the guard on its own.
+ int getClassFilesInArchives() {
+ return _classFilesInArchives;
+ }
+
+ int getClassFilesInDirectories() {
+ return _classFilesInDirectories;
+ }
+
+ /// Major version to class file count, so the CI log shows what the scan
actually looked at. A pass whose histogram
+ /// is empty, or whose highest bucket is implausibly low, is a broken scan
rather than a clean closure.
+ Map<Integer, Integer> getMajorVersionHistogram() {
+ return _majorVersionHistogram;
+ }
+
+ private void record(String classpathEntry, String classFile, int
majorVersion, int maxAllowedMajorVersion,
+ boolean insideArchive) {
+ if (insideArchive) {
+ _classFilesInArchives++;
+ } else {
+ _classFilesInDirectories++;
+ }
+ _majorVersionHistogram.merge(majorVersion, 1, Integer::sum);
+ if (majorVersion > maxAllowedMajorVersion) {
+ _totalViolationCount++;
+ if (_reportedViolations.size() < MAX_REPORTED_VIOLATIONS) {
+ _reportedViolations.add(new Violation(classpathEntry, classFile,
majorVersion));
+ }
+ }
+ }
+ }
+
+ /// Scans {@code classpath} (a {@link File#pathSeparator}-delimited list)
for class files that a JVM at {@code
+ /// targetJavaFeatureVersion} could not load.
+ ///
+ /// @throws IOException if a {@code .jar}/{@code .zip} entry on the
classpath cannot be opened or read. A corrupt
+ /// archive is a hard error, never a silent skip.
+ static Result scan(String classpath, int targetJavaFeatureVersion)
+ throws IOException {
+ int maxAllowedMajorVersion = targetJavaFeatureVersion +
CLASS_FILE_MAJOR_VERSION_OFFSET;
+ Result result = new Result();
+ for (String entry : classpath.split(File.pathSeparator)) {
+ if (entry.isEmpty()) {
+ continue;
+ }
+ File file = new File(entry);
+ if (!file.exists()) {
+ result._skippedEntries.add(entry + " (does not exist)");
+ continue;
+ }
+ if (file.isDirectory()) {
+ result._directoriesScanned++;
+ scanDirectory(file, file, result, targetJavaFeatureVersion,
maxAllowedMajorVersion);
+ continue;
+ }
+ String lowerCaseName = file.getName().toLowerCase(Locale.ROOT);
+ if (!lowerCaseName.endsWith(".jar") && !lowerCaseName.endsWith(".zip")) {
+ // Maven puts `pom`-type dependencies (e.g. groovy-all) on the
classpath as .pom files. Those
+ // carry no bytecode, so there is nothing to check -- but report them
so the skip is visible
+ // and the caller can decide whether the reason is one it tolerates.
+ result._skippedEntries.add(entry + " (not an archive)");
+ continue;
+ }
+ result._archivesScanned++;
+ result._archiveNames.add(file.getName());
+ scanArchive(file, result, targetJavaFeatureVersion,
maxAllowedMajorVersion);
+ }
+ return result;
+ }
+
+ private static void scanArchive(File archive, Result result, int
targetJavaFeatureVersion,
+ int maxAllowedMajorVersion)
+ throws IOException {
+ try (ZipFile zipFile = new ZipFile(archive)) {
+ Enumeration<? extends ZipEntry> entries = zipFile.entries();
+ boolean multiRelease = isMultiRelease(zipFile);
+ while (entries.hasMoreElements()) {
+ ZipEntry zipEntry = entries.nextElement();
+ String name = zipEntry.getName();
+ if (zipEntry.isDirectory() || !isLoadableClassEntry(name,
targetJavaFeatureVersion, multiRelease)) {
+ continue;
+ }
+ try (InputStream inputStream = zipFile.getInputStream(zipEntry)) {
+ readMajorVersion(inputStream).ifPresent(
+ majorVersion -> result.record(archive.getPath(), name,
majorVersion, maxAllowedMajorVersion, true));
+ }
+ }
+ } catch (ZipException e) {
+ throw new IOException("Failed to open archive on the classpath: " +
archive, e);
+ }
+ }
+
+ private static void scanDirectory(File root, File current, Result result,
int targetJavaFeatureVersion,
+ int maxAllowedMajorVersion)
+ throws IOException {
+ File[] children = current.listFiles();
+ if (children == null) {
+ result._skippedEntries.add(current.getPath() + " (unreadable
directory)");
+ return;
+ }
+ for (File child : children) {
+ if (child.isDirectory()) {
+ scanDirectory(root, child, result, targetJavaFeatureVersion,
maxAllowedMajorVersion);
+ continue;
+ }
+ String relativePath =
root.toPath().relativize(child.toPath()).toString().replace(File.separatorChar,
'/');
+ // A directory classpath entry never gets multi-release treatment,
whatever a MANIFEST.MF inside
+ // it might claim, so any versioned entry under it is inert.
+ if (!isLoadableClassEntry(relativePath, targetJavaFeatureVersion,
false)) {
+ continue;
+ }
+ try (InputStream inputStream = Files.newInputStream(child.toPath())) {
+ readMajorVersion(inputStream).ifPresent(
+ majorVersion -> result.record(root.getPath(), relativePath,
majorVersion, maxAllowedMajorVersion, false));
+ }
+ }
+ }
+
+ /// Returns whether the archive declares {@code Multi-Release: true} in its
manifest. Only such an archive gets
+ /// versioned-entry lookup from a JVM; in any other jar the {@code
META-INF/versions/} tree is inert data.
+ private static boolean isMultiRelease(ZipFile zipFile)
+ throws IOException {
+ ZipEntry manifestEntry = zipFile.getEntry(MANIFEST_ENTRY);
+ if (manifestEntry == null) {
+ // Some archives spell the manifest with different casing; the JAR spec
treats it case-insensitively.
+ Enumeration<? extends ZipEntry> entries = zipFile.entries();
+ while (entries.hasMoreElements()) {
+ ZipEntry candidate = entries.nextElement();
+ if (candidate.getName().equalsIgnoreCase(MANIFEST_ENTRY)) {
+ manifestEntry = candidate;
+ break;
+ }
+ }
+ }
+ if (manifestEntry == null) {
+ return false;
+ }
+ try (InputStream inputStream = zipFile.getInputStream(manifestEntry)) {
+ String value = new
Manifest(inputStream).getMainAttributes().getValue(Attributes.Name.MULTI_RELEASE);
+ // The attribute value is defined to be case-insensitive; anything other
than "true" means not multi-release.
+ return value != null && Boolean.parseBoolean(value.trim());
+ } catch (IOException e) {
+ // A jar we cannot read the manifest of is a real problem, not something
to silently treat as single-release.
+ throw new IOException("Failed to read the manifest of archive on the
classpath: " + zipFile.getName(), e);
+ }
+ }
+
+ /// Returns true if {@code name} is a class file that a JVM at {@code
targetJavaFeatureVersion} would actually load
+ /// from the classpath. {@code multiRelease} says whether the enclosing
archive declared
+ /// {@code Multi-Release: true}; when it did not, versioned entries are
never selected and so are never checked.
+ private static boolean isLoadableClassEntry(String name, int
targetJavaFeatureVersion, boolean multiRelease) {
+ if (!name.endsWith(".class")) {
+ return false;
+ }
+ String effectiveName = name;
+ if (name.startsWith(MULTI_RELEASE_PREFIX)) {
+ if (!multiRelease) {
+ return false;
+ }
+ int versionEnd = name.indexOf('/', MULTI_RELEASE_PREFIX.length());
+ if (versionEnd < 0) {
+ return false;
+ }
+ int version;
+ try {
+ version =
Integer.parseInt(name.substring(MULTI_RELEASE_PREFIX.length(), versionEnd));
+ } catch (NumberFormatException e) {
+ // Not a well-formed multi-release directory; treat it as an inert
resource.
+ return false;
+ }
+ if (version > targetJavaFeatureVersion) {
+ return false;
+ }
+ effectiveName = name.substring(versionEnd + 1);
+ }
+ // module-info descriptors are ignored on the classpath, whatever version
they were compiled at.
+ return !effectiveName.equals(MODULE_INFO_ENTRY) &&
!effectiveName.endsWith('/' + MODULE_INFO_ENTRY);
+ }
+
+ /// Reads the class file major version from the first 8 bytes. Returns empty
for anything that is not a class file
+ /// despite the {@code .class} name, which does occur in the wild as
packaged test data.
+ private static Optional<Integer> readMajorVersion(InputStream inputStream)
+ throws IOException {
+ DataInputStream dataInputStream = new DataInputStream(inputStream);
+ int magic;
+ int major;
+ try {
+ magic = dataInputStream.readInt();
+ dataInputStream.readUnsignedShort(); // minor version
+ major = dataInputStream.readUnsignedShort();
+ } catch (EOFException e) {
+ return Optional.empty();
+ }
+ if (magic != CLASS_FILE_MAGIC) {
+ return Optional.empty();
+ }
+ return Optional.of(major);
+ }
+}
diff --git
a/pinot-java11-client-verifier/src/main/java/org/apache/pinot/java11/Java11CompatibilityVerifier.java
b/pinot-java11-client-verifier/src/main/java/org/apache/pinot/java11/Java11CompatibilityVerifier.java
new file mode 100644
index 00000000000..c93b6665e6e
--- /dev/null
+++
b/pinot-java11-client-verifier/src/main/java/org/apache/pinot/java11/Java11CompatibilityVerifier.java
@@ -0,0 +1,803 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.java11;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.protobuf.ByteString;
+import io.grpc.ManagedChannel;
+import java.io.IOException;
+import java.io.InputStream;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.charset.StandardCharsets;
+import java.sql.Driver;
+import java.sql.DriverManager;
+import java.sql.DriverPropertyInfo;
+import java.sql.ResultSetMetaData;
+import java.sql.Types;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+import org.apache.helix.model.ExternalView;
+import org.apache.helix.zookeeper.datamodel.ZNRecord;
+import org.apache.helix.zookeeper.datamodel.serializer.ZNRecordSerializer;
+import org.apache.pinot.client.Connection;
+import org.apache.pinot.client.ConnectionFactory;
+import org.apache.pinot.client.ExecutionStats;
+import org.apache.pinot.client.JsonAsyncHttpPinotClientTransportFactory;
+import org.apache.pinot.client.PinotClientTransport;
+import org.apache.pinot.client.PinotResultSet;
+import org.apache.pinot.client.PreparedStatement;
+import org.apache.pinot.client.ResultSet;
+import org.apache.pinot.client.ResultSetGroup;
+import org.apache.pinot.client.grpc.GrpcUtils;
+import org.apache.pinot.common.compression.CompressionFactory;
+import org.apache.pinot.common.compression.Compressor;
+import org.apache.pinot.common.config.GrpcConfig;
+import org.apache.pinot.common.metadata.segment.SegmentZKMetadata;
+import org.apache.pinot.common.proto.Broker;
+import org.apache.pinot.common.request.PinotQuery;
+import org.apache.pinot.common.response.broker.BrokerResponseNative;
+import org.apache.pinot.common.response.broker.ResultTable;
+import org.apache.pinot.common.response.encoder.ResponseEncoder;
+import org.apache.pinot.common.response.encoder.ResponseEncoderFactory;
+import org.apache.pinot.common.utils.DataSchema;
+import org.apache.pinot.common.utils.DataSchema.ColumnDataType;
+import org.apache.pinot.common.utils.grpc.BrokerGrpcQueryClient;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.CommonConstants;
+import org.apache.pinot.spi.utils.JsonUtils;
+import org.apache.pinot.sql.parsers.CalciteSqlParser;
+import org.apache.pinot.sql.parsers.SqlNodeAndOptions;
+import org.apache.pinot.tsdb.spi.AggInfo;
+import org.apache.pinot.tsdb.spi.TimeBuckets;
+import org.apache.pinot.tsdb.spi.plan.BaseTimeSeriesPlanNode;
+import org.apache.pinot.tsdb.spi.plan.LeafTimeSeriesPlanNode;
+import org.apache.pinot.tsdb.spi.plan.serde.TimeSeriesPlanSerde;
+
+
+/// Verifies that Pinot's consumer-facing client and SPI artifacts actually
work on an older JVM, by being run _by_ that
+/// JVM.
+///
+/// Six modules (pinot-spi, pinot-segment-spi, pinot-timeseries-spi,
pinot-common, pinot-java-client and
+/// pinot-jdbc-client) hard-code a compiler {@code release} of 11 so that
third-party plugins and embedding applications
+/// are not forced onto the JDK that Pinot's services require. {@code
--release} makes Pinot's own code in those modules
+/// Java 11 clean, but it says nothing about their transitive dependency
closure: a routine dependency bump can drop a
+/// Java 17+ jar into the client's classpath, and nothing in the build would
notice. This verifier closes that gap.
+///
+/// It must be launched by a JVM at the target feature release (11 by default)
-- running it on the build JDK proves
+/// nothing, so {@link #checkJvmIsAtTargetFeatureVersion} fails loudly in that
case. The build itself still requires JDK
+/// 25 ({@code requireJavaVersion} in the root pom), so CI builds the
artifacts with JDK 25 and then runs this class
+/// under Java 11.
+///
+/// Usage: {@code java -cp <runtime closure>
org.apache.pinot.java11.Java11CompatibilityVerifier
+/// [targetJavaFeatureVersion]}. Exits 0 if every check passes, 1 otherwise.
+///
+/// The assertions go through a local {@link #require} helper rather than
TestNG on purpose. A test framework on the
+/// classpath would be scanned by {@link #checkClasspathClosureIsLoadable}
alongside the real dependencies, so a TestNG
+/// or byte-buddy release that dropped Java 11 support would fail this job for
a reason no Pinot consumer would ever
+/// hit.
+///
+/// Not thread-safe, and deliberately single-threaded: checks run sequentially
on the calling thread so that a failure
+/// maps to exactly one named check.
+public final class Java11CompatibilityVerifier {
+ private static final int DEFAULT_TARGET_JAVA_FEATURE_VERSION = 11;
+ private static final String SAMPLE_BROKER_RESPONSE_RESOURCE =
"sample-broker-response.json";
+ private static final String JDBC_SERVICE_DESCRIPTOR =
"META-INF/services/java.sql.Driver";
+ private static final String SAMPLE_TABLE_NAME = "baseballStats";
+ private static final String JVM_VERSION_CHECK =
"jvm-is-at-target-feature-version";
+
+ /// Every module that hard-codes a Java 11 compiler release. All of them
have to be on the verified closure, otherwise
+ /// this job quietly checks less than it advertises. Add to this list when a
seventh module gets pinned.
+ private static final List<String> JAVA11_PINNED_MODULES = List.of(
+ "pinot-spi", "pinot-segment-spi", "pinot-timeseries-spi",
"pinot-common", "pinot-java-client",
+ "pinot-jdbc-client");
+
+ /// Encoders and compression codecs that must stay covered. Iterating
whatever the factories happen to return would
+ /// let a refactor drop ARROW or ZSTD -- and with them arrow-memory-netty
and the JNI codecs, the most likely sources
+ /// of a Java version floor bump -- while the job stayed green.
+ private static final List<String> REQUIRED_RESPONSE_ENCODERS =
List.of("JSON", "ARROW");
+ private static final List<String> REQUIRED_COMPRESSION_CODECS =
List.of("ZSTD", "SNAPPY", "LZ4", "GZIP", "DEFLATE");
+
+ /// Floors for the closure scan's vacuity guards, set well below the real
numbers (~220 jars, ~69k class files at the
+ /// time of writing) so ordinary dependency churn does not trip them, but
far enough above zero to catch a closure
+ /// that failed to resolve.
+ private static final int MIN_EXPECTED_ARCHIVES = 100;
+ private static final int MIN_EXPECTED_CLASS_FILES_IN_ARCHIVES = 20_000;
+
+ private final int _targetJavaFeatureVersion;
+ private final String _sampleBrokerResponseJson;
+ private final JsonNode _sampleBrokerResponse;
+
+ private Java11CompatibilityVerifier(int targetJavaFeatureVersion)
+ throws IOException {
+ _targetJavaFeatureVersion = targetJavaFeatureVersion;
+ _sampleBrokerResponseJson = readResource(SAMPLE_BROKER_RESPONSE_RESOURCE);
+ _sampleBrokerResponse =
JsonUtils.stringToJsonNode(_sampleBrokerResponseJson);
+ }
+
+ /// A single named verification. Throwing anything -- including {@link
AssertionError} -- fails it.
+ @FunctionalInterface
+ private interface Check {
+ void run()
+ throws Exception;
+ }
+
+ public static void main(String[] args)
+ throws IOException {
+ int targetJavaFeatureVersion = DEFAULT_TARGET_JAVA_FEATURE_VERSION;
+ if (args.length > 1) {
+ throw new IllegalArgumentException(
+ "Usage: Java11CompatibilityVerifier [targetJavaFeatureVersion], got:
" + Arrays.toString(args));
+ }
+ if (args.length == 1) {
+ targetJavaFeatureVersion = Integer.parseInt(args[0].trim());
+ }
+ System.exit(new
Java11CompatibilityVerifier(targetJavaFeatureVersion).run());
+ }
+
+ private int run() {
+ Map<String, Check> checks = new LinkedHashMap<>();
+ // Guards against the whole job passing vacuously on the build JDK. Must
stay first; failing it
+ // aborts the run.
+ checks.put(JVM_VERSION_CHECK, this::checkJvmIsAtTargetFeatureVersion);
+ checks.put("classpath-closure-is-loadable",
this::checkClasspathClosureIsLoadable);
+ checks.put("spi-schema-deserialization",
this::checkSpiSchemaDeserialization);
+ checks.put("spi-table-config-deserialization",
this::checkSpiTableConfigDeserialization);
+ checks.put("segment-spi-data-buffer", this::checkSegmentSpiDataBuffer);
+ checks.put("timeseries-spi-plan-serde", this::checkTimeSeriesSpiPlanSerde);
+ checks.put("common-data-schema-round-trip",
this::checkCommonDataSchemaRoundTrip);
+ checks.put("common-broker-response-deserialization",
this::checkCommonBrokerResponseDeserialization);
+ checks.put("common-response-encoders", this::checkCommonResponseEncoders);
+ checks.put("common-calcite-sql-parsing",
this::checkCommonCalciteSqlParsing);
+ checks.put("common-helix-segment-metadata",
this::checkCommonHelixSegmentMetadata);
+ checks.put("common-grpc-response-decoding",
this::checkCommonGrpcResponseDecoding);
+ checks.put("common-grpc-channel-construction",
this::checkCommonGrpcChannelConstruction);
+ checks.put("java-client-http-transport",
this::checkJavaClientHttpTransport);
+ checks.put("java-client-query-execution",
this::checkJavaClientQueryExecution);
+ checks.put("java-client-prepared-statement",
this::checkJavaClientPreparedStatement);
+ checks.put("jdbc-driver-registration", this::checkJdbcDriverRegistration);
+ checks.put("jdbc-result-set", this::checkJdbcResultSet);
+
+ System.out.printf("Verifying Pinot clients on %s %s (%s), targeting Java
%d%n", System.getProperty("java.vm.name"),
+ System.getProperty("java.version"), System.getProperty("java.vendor"),
_targetJavaFeatureVersion);
+ System.out.println();
+
+ List<String> failed = new ArrayList<>();
+ for (Map.Entry<String, Check> entry : checks.entrySet()) {
+ String name = entry.getKey();
+ try {
+ entry.getValue().run();
+ System.out.printf(" PASS %s%n", name);
+ } catch (Throwable t) {
+ failed.add(name);
+ System.out.printf(" FAIL %s%n", name);
+ System.out.printf(" %s: %s%n", t.getClass().getName(),
t.getMessage());
+ t.printStackTrace(System.out);
+ if (name.equals(JVM_VERSION_CHECK)) {
+ // Running on the wrong JVM makes everything below meaningless. Stop
rather than emit a wall
+ // of passes that a reader could mistake for real coverage.
+ System.out.printf("%nAborting: the remaining %d checks would not
tell us anything on this JVM.%n",
+ checks.size() - 1);
+ break;
+ }
+ }
+ }
+
+ System.out.println();
+ if (failed.isEmpty()) {
+ System.out.printf("All %d checks passed on Java %d.%n", checks.size(),
_targetJavaFeatureVersion);
+ return 0;
+ }
+ System.out.printf("%d of %d checks FAILED on Java %d: %s%n",
failed.size(), checks.size(),
+ _targetJavaFeatureVersion, String.join(", ", failed));
+ return 1;
+ }
+
+ //
---------------------------------------------------------------------------------------------
+ // Checks
+ //
---------------------------------------------------------------------------------------------
+
+ /// The point of this harness is that an _old_ JVM runs it. If CI ever hands
it the build JDK, every other check would
+ /// pass for the wrong reason, so refuse to run.
+ private void checkJvmIsAtTargetFeatureVersion() {
+ int actual = Runtime.version().feature();
+ require(actual == _targetJavaFeatureVersion,
+ "expected to be running on a Java %d JVM but this is Java %d (%s);
verifying the clients on the build JDK "
+ + "would make every other check vacuous",
_targetJavaFeatureVersion, actual,
+ System.getProperty("java.home"));
+ }
+
+ /// Walks the whole runtime closure looking for bytecode this JVM could not
load. {@code --release 11} covers Pinot's
+ /// own class files; this is the part that covers everybody else's.
+ private void checkClasspathClosureIsLoadable()
+ throws IOException {
+ String classpath = System.getProperty("java.class.path");
+ require(classpath != null && !classpath.isEmpty(), "java.class.path is
empty");
+ ClasspathClosureScanner.Result result =
ClasspathClosureScanner.scan(classpath, _targetJavaFeatureVersion);
+
+ System.out.printf(" scanned %d archives (%d class files) and %d
directories (%d class files); "
+ + "major versions: %s%n", result.getArchivesScanned(),
result.getClassFilesInArchives(),
+ result.getDirectoriesScanned(), result.getClassFilesInDirectories(),
result.getMajorVersionHistogram());
+
+ // Vacuity guards. Every one of these has to be able to fail, or a green
run means nothing:
+ // - the closure must actually have been resolved, not collapsed to a
handful of entries;
+ // - third-party bytecode specifically must have been read. Counting the
verifier's own
+ // target/classes towards that would make the guard unfalsifiable,
which is why the scanner
+ // keeps the archive and directory counts apart;
+ // - every Java-11-pinned module must be present, so that making one of
them optional or
+ // provided shrinks coverage loudly instead of silently.
+ require(result.getArchivesScanned() >= MIN_EXPECTED_ARCHIVES,
+ "only %d jars on the classpath, expected at least %d -- the runtime
closure was not resolved properly",
+ result.getArchivesScanned(), MIN_EXPECTED_ARCHIVES);
+ require(result.getClassFilesInArchives() >=
MIN_EXPECTED_CLASS_FILES_IN_ARCHIVES,
+ "only %d class files inspected inside jars, expected at least %d --
the scan is broken, not the closure",
+ result.getClassFilesInArchives(),
MIN_EXPECTED_CLASS_FILES_IN_ARCHIVES);
+ assertPinnedModulesArePresent(result.getArchiveNames());
+
+ for (String skipped : result.getSkippedEntries()) {
+ // A `pom`-type dependency (groovy-all) legitimately lands on the
classpath as a .pom. Anything
+ // else that carried no bytecode means the closure is not what we think
it is.
+ require(skipped.endsWith("(not an archive)") && skipped.contains(".pom
"),
+ "unexpected classpath entry that could not be scanned: %s", skipped);
+ System.out.printf(" skipped classpath entry: %s%n", skipped);
+ }
+
+ if (result.getTotalViolationCount() > 0) {
+ StringBuilder message = new StringBuilder(
+ String.format("%d class file(s) on the client runtime closure cannot
be loaded by Java %d. A dependency was "
+ + "bumped to a release that no longer supports Java %d; pin
it back or drop it from the client "
+ + "closure. Offenders:", result.getTotalViolationCount(),
_targetJavaFeatureVersion,
+ _targetJavaFeatureVersion));
+ for (ClasspathClosureScanner.Violation violation :
result.getReportedViolations()) {
+ message.append("\n ").append(violation);
+ }
+ if (result.getTotalViolationCount() >
result.getReportedViolations().size()) {
+ message.append("\n ... and ")
+ .append(result.getTotalViolationCount() -
result.getReportedViolations().size())
+ .append(" more");
+ }
+ throw new AssertionError(message.toString());
+ }
+ }
+
+ /// Fails unless a jar is on the closure for every module that pins its
bytecode to Java 11. Without this, dropping
+ /// one of them from the closure leaves every check passing while silently
verifying less than the job claims to.
+ private static void assertPinnedModulesArePresent(List<String> archiveNames)
{
+ List<String> missing = new ArrayList<>();
+ for (String artifactId : JAVA11_PINNED_MODULES) {
+ boolean found = false;
+ for (String archiveName : archiveNames) {
+ // Jar names are <artifactId>-<version>.jar; require the '-' so
pinot-spi does not match
+ // pinot-spi-something-else.
+ if (archiveName.startsWith(artifactId + "-")) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ missing.add(artifactId);
+ }
+ }
+ require(missing.isEmpty(),
+ "these Java 11 pinned modules are not on the verified closure, so this
job is no longer checking them: %s. "
+ + "Either restore the dependency or update
JAVA11_PINNED_MODULES.", missing);
+ }
+
+ /// pinot-spi schema deserialization, which is Jackson plus the SPI's own
field spec model.
+ private void checkSpiSchemaDeserialization()
+ throws IOException {
+ String schemaJson = "{"
+ + "\"schemaName\":\"" + SAMPLE_TABLE_NAME + "\","
+ + "\"dimensionFieldSpecs\":["
+ + "{\"name\":\"playerName\",\"dataType\":\"STRING\"},"
+ +
"{\"name\":\"teams\",\"dataType\":\"STRING\",\"singleValueField\":false}],"
+ +
"\"metricFieldSpecs\":[{\"name\":\"numGames\",\"dataType\":\"LONG\"}],"
+ +
"\"dateTimeFieldSpecs\":[{\"name\":\"gameDate\",\"dataType\":\"LONG\","
+ + "\"format\":\"1:MILLISECONDS:EPOCH\",\"granularity\":\"1:DAYS\"}]}";
+
+ Schema schema = Schema.fromString(schemaJson);
+ require(SAMPLE_TABLE_NAME.equals(schema.getSchemaName()), "unexpected
schema name: %s", schema.getSchemaName());
+ require(schema.getColumnNames().size() == 4, "expected 4 columns, got %s",
schema.getColumnNames());
+ require(schema.getFieldSpecFor("playerName").getDataType() ==
DataType.STRING, "playerName should be STRING");
+ require(!schema.getFieldSpecFor("teams").isSingleValueField(), "teams
should be multi-valued");
+ require(schema.getDateTimeSpec("gameDate") != null, "gameDate date-time
spec missing");
+
+ Schema reparsed = Schema.fromString(schema.toSingleLineJsonString());
+ require(schema.equals(reparsed), "schema did not survive a JSON round
trip");
+ }
+
+ /// pinot-spi table config deserialization, the other half of what a
third-party plugin reads.
+ private void checkSpiTableConfigDeserialization()
+ throws IOException {
+ String tableConfigJson = "{"
+ + "\"tableName\":\"" + SAMPLE_TABLE_NAME + "\","
+ + "\"tableType\":\"OFFLINE\","
+ +
"\"segmentsConfig\":{\"replication\":\"2\",\"timeColumnName\":\"gameDate\"},"
+ +
"\"tenants\":{\"broker\":\"DefaultTenant\",\"server\":\"DefaultTenant\"},"
+ +
"\"tableIndexConfig\":{\"invertedIndexColumns\":[\"playerName\"],\"loadMode\":\"MMAP\"},"
+ + "\"metadata\":{}}";
+
+ TableConfig tableConfig = JsonUtils.stringToObject(tableConfigJson,
TableConfig.class);
+ require(tableConfig.getTableType() == TableType.OFFLINE, "unexpected table
type: %s", tableConfig.getTableType());
+ require((SAMPLE_TABLE_NAME +
"_OFFLINE").equals(tableConfig.getTableName()), "unexpected table name: %s",
+ tableConfig.getTableName());
+ require("2".equals(tableConfig.getValidationConfig().getReplication()),
"unexpected replication: %s",
+ tableConfig.getValidationConfig().getReplication());
+
require(tableConfig.getIndexingConfig().getInvertedIndexColumns().contains("playerName"),
+ "inverted index column lost: %s",
tableConfig.getIndexingConfig().getInvertedIndexColumns());
+
+ TableConfig reserialized =
JsonUtils.stringToObject(JsonUtils.objectToString(tableConfig),
TableConfig.class);
+ require(tableConfig.getTableName().equals(reserialized.getTableName()),
+ "table config did not survive a JSON round trip");
+ }
+
+ /// pinot-segment-spi's off-heap buffer, which is what a third-party index
or reader plugin built against the SPI
+ /// actually uses. Worth exercising rather than merely scanning: the
allocation path goes through {@code Unsafe} and
+ /// {@code sun.misc.Cleaner} reflection whose availability is exactly what
shifts between JDK releases, and the
+ /// class-file scan cannot see that.
+ private void checkSegmentSpiDataBuffer()
+ throws IOException {
+ int size = 64;
+ try (PinotDataBuffer buffer = PinotDataBuffer.allocateDirect(size,
ByteOrder.LITTLE_ENDIAN,
+ "java11-verifier")) {
+ require(buffer.size() == size, "unexpected buffer size: %d",
buffer.size());
+ buffer.putInt(0, 0xCAFEBABE);
+ buffer.putLong(8, 1750000000000L);
+ buffer.putDouble(16, 0.305d);
+ require(buffer.getInt(0) == 0xCAFEBABE, "int lost in the off-heap
buffer: %d", buffer.getInt(0));
+ require(buffer.getLong(8) == 1750000000000L, "long lost in the off-heap
buffer: %d", buffer.getLong(8));
+ require(buffer.getDouble(16) == 0.305d, "double lost in the off-heap
buffer: %s", buffer.getDouble(16));
+
+ byte[] bytes = "baseballStats".getBytes(StandardCharsets.UTF_8);
+ buffer.readFrom(24, bytes, 0, bytes.length);
+ byte[] readBack = new byte[bytes.length];
+ buffer.copyTo(24, readBack, 0, readBack.length);
+ require(Arrays.equals(bytes, readBack), "bytes lost in the off-heap
buffer: %s",
+ new String(readBack, StandardCharsets.UTF_8));
+ }
+ }
+
+ /// pinot-timeseries-spi's plan model, which a time series language plugin
serializes and ships.
+ private void checkTimeSeriesSpiPlanSerde() {
+ LeafTimeSeriesPlanNode leaf = new LeafTimeSeriesPlanNode("sfp#0",
List.of(), SAMPLE_TABLE_NAME, "gameDate",
+ TimeUnit.MILLISECONDS, 0L, "battingAverage > 0.3", "numGames",
+ new AggInfo("SUM", false, Map.of("window", "60")),
List.of("playerName"), 100, Map.of("timeoutMs", "20000"));
+
+ String serialized = TimeSeriesPlanSerde.serialize(leaf);
+ require(serialized.contains(SAMPLE_TABLE_NAME), "table name lost during
plan serialization: %s", serialized);
+
+ BaseTimeSeriesPlanNode deserialized =
TimeSeriesPlanSerde.deserialize(serialized);
+ require(deserialized instanceof LeafTimeSeriesPlanNode, "unexpected plan
node type: %s",
+ deserialized.getClass().getName());
+ LeafTimeSeriesPlanNode roundTripped = (LeafTimeSeriesPlanNode)
deserialized;
+ require(SAMPLE_TABLE_NAME.equals(roundTripped.getTableName()), "unexpected
table name: %s",
+ roundTripped.getTableName());
+ require("numGames".equals(roundTripped.getValueExpression()), "unexpected
value expression: %s",
+ roundTripped.getValueExpression());
+ require(roundTripped.getAggInfo() != null &&
"SUM".equals(roundTripped.getAggInfo().getAggFunction()),
+ "aggregation lost during the plan round trip: %s",
roundTripped.getAggInfo());
+
require(List.of("playerName").equals(roundTripped.getGroupByExpressions()),
"group by lost: %s",
+ roundTripped.getGroupByExpressions());
+ require(roundTripped.getLimit() == 100, "unexpected limit: %d",
roundTripped.getLimit());
+
+ TimeBuckets timeBuckets = TimeBuckets.ofSeconds(1750000000L,
Duration.ofSeconds(60), 10);
+ require(timeBuckets.getNumBuckets() == 10, "unexpected bucket count: %d",
timeBuckets.getNumBuckets());
+ require(timeBuckets.getTimeBuckets().length == 10, "unexpected bucket
array length: %d",
+ timeBuckets.getTimeBuckets().length);
+ }
+
+ /// pinot-common's wire-format schema, both its binary encoding and its JSON
encoding.
+ private void checkCommonDataSchemaRoundTrip()
+ throws IOException {
+ DataSchema dataSchema = sampleDataSchema();
+
+ DataSchema fromBytes =
DataSchema.fromBytes(ByteBuffer.wrap(dataSchema.toBytes()));
+ require(dataSchema.equals(fromBytes), "DataSchema did not survive a binary
round trip: %s", fromBytes);
+
+ DataSchema fromJson =
JsonUtils.stringToObject(JsonUtils.objectToString(dataSchema),
DataSchema.class);
+ require(dataSchema.equals(fromJson), "DataSchema did not survive a JSON
round trip: %s", fromJson);
+ require(dataSchema.getColumnDataType(1) == ColumnDataType.INT, "unexpected
column type: %s",
+ dataSchema.getColumnDataType(1));
+ }
+
+ /// A realistic broker response parsed by pinot-common's own response model.
+ private void checkCommonBrokerResponseDeserialization()
+ throws IOException {
+ BrokerResponseNative response =
BrokerResponseNative.fromJsonString(_sampleBrokerResponseJson);
+ require(response.getExceptions().isEmpty(), "unexpected exceptions: %s",
response.getExceptions());
+ require(response.getNumDocsScanned() == 4231, "unexpected numDocsScanned:
%d", response.getNumDocsScanned());
+ require(response.getTotalDocs() == 97889, "unexpected totalDocs: %d",
response.getTotalDocs());
+ require(response.getTimeUsedMs() == 37, "unexpected timeUsedMs: %d",
response.getTimeUsedMs());
+ require("Broker_192.168.1.10_8000".equals(response.getBrokerId()),
"unexpected brokerId: %s",
+ response.getBrokerId());
+
+ ResultTable resultTable = response.getResultTable();
+ require(resultTable != null, "resultTable was dropped during
deserialization");
+ require(resultTable.getRows().size() == 3, "expected 3 rows, got %d",
resultTable.getRows().size());
+ require("Hank Aaron".equals(resultTable.getRows().get(0)[0]), "unexpected
first cell: %s",
+ resultTable.getRows().get(0)[0]);
+ require(Arrays.equals(new String[]{
+ "playerName", "playerId", "numGames", "battingAverage", "isActive",
"teams"
+ }, resultTable.getDataSchema().getColumnNames()), "unexpected columns: %s",
+ Arrays.toString(resultTable.getDataSchema().getColumnNames()));
+
+ String reserialized = JsonUtils.objectToString(response);
+
require(BrokerResponseNative.fromJsonString(reserialized).getNumDocsScanned()
== 4231,
+ "broker response did not survive a JSON round trip");
+ }
+
+ /// Round-trips a result table through every encoder the gRPC client can be
asked to decode. The Arrow encoder is the
+ /// interesting one: arrow-vector and arrow-memory-netty are the client's
most likely source of a Java version floor
+ /// bump, and they only fail when actually exercised.
+ private void checkCommonResponseEncoders()
+ throws IOException {
+ ResultTable resultTable = sampleResultTable();
+ int numRows = resultTable.getRows().size();
+ List<String> encoderTypes =
Arrays.asList(ResponseEncoderFactory.getResponseEncoderTypes());
+ require(encoderTypes.containsAll(REQUIRED_RESPONSE_ENCODERS),
+ "response encoders %s no longer cover %s, so this check would stop
exercising them",
+ encoderTypes, REQUIRED_RESPONSE_ENCODERS);
+
+ for (String encoderType : encoderTypes) {
+ ResponseEncoder encoder =
ResponseEncoderFactory.getResponseEncoder(encoderType);
+ byte[] encoded = encoder.encodeResultTable(resultTable, 0, numRows);
+ require(encoded.length > 0, "%s encoder produced no bytes", encoderType);
+
+ ResultTable decoded = encoder.decodeResultTable(encoded, numRows,
resultTable.getDataSchema());
+ require(decoded.getRows().size() == numRows, "%s encoder lost rows: %d
of %d", encoderType,
+ decoded.getRows().size(), numRows);
+ assertSampleRowsMatch(encoderType, decoded);
+ }
+ }
+
+ /// Calcite SQL parsing, which third-party consumers of pinot-common reach
through the SQL compiler.
+ private void checkCommonCalciteSqlParsing() {
+ String sql = "SELECT playerName, SUM(numGames) AS totalGames FROM " +
SAMPLE_TABLE_NAME
+ + " WHERE battingAverage > 0.3 AND playerName IN ('Hank Aaron',
'Willie Mays') "
+ + "GROUP BY playerName ORDER BY totalGames DESC LIMIT 5";
+
+ PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery(sql);
+
require(SAMPLE_TABLE_NAME.equals(pinotQuery.getDataSource().getTableName()),
"unexpected table: %s",
+ pinotQuery.getDataSource().getTableName());
+ require(pinotQuery.getSelectListSize() == 2, "unexpected select list size:
%d", pinotQuery.getSelectListSize());
+ require(pinotQuery.getGroupByListSize() == 1, "unexpected group by size:
%d", pinotQuery.getGroupByListSize());
+ require(pinotQuery.getOrderByListSize() == 1, "unexpected order by size:
%d", pinotQuery.getOrderByListSize());
+ require(pinotQuery.getLimit() == 5, "unexpected limit: %d",
pinotQuery.getLimit());
+ require(pinotQuery.getFilterExpression() != null, "filter was dropped");
+
+ SqlNodeAndOptions sqlNodeAndOptions =
CalciteSqlParser.compileToSqlNodeAndOptions("SET timeoutMs = 20000; " + sql);
+ require("20000".equals(sqlNodeAndOptions.getOptions().get("timeoutMs")),
"query option lost: %s",
+ sqlNodeAndOptions.getOptions());
+ }
+
+ /// pinot-common's ZooKeeper metadata model, which is Helix's {@code
ZNRecord} plus Helix serialization.
+ private void checkCommonHelixSegmentMetadata() {
+ SegmentZKMetadata metadata = new SegmentZKMetadata(SAMPLE_TABLE_NAME +
"_0");
+ metadata.setCrc(3141592653L);
+ metadata.setTotalDocs(97889);
+ metadata.setIndexVersion("v3");
+ metadata.setCreationTime(1750000000000L);
+ metadata.setStartTime(1749000000000L);
+ metadata.setEndTime(1750000000000L);
+ metadata.setTimeUnit(TimeUnit.MILLISECONDS);
+
+ ZNRecordSerializer serializer = new ZNRecordSerializer();
+ byte[] serialized = serializer.serialize(metadata.toZNRecord());
+ require(serialized.length > 0, "ZNRecordSerializer produced no bytes");
+ ZNRecord deserialized = (ZNRecord) serializer.deserialize(serialized);
+ require(deserialized != null, "ZNRecordSerializer returned null");
+
+ SegmentZKMetadata roundTripped = new SegmentZKMetadata(deserialized);
+ require(roundTripped.getCrc() == 3141592653L, "unexpected crc: %d",
roundTripped.getCrc());
+ require(roundTripped.getTotalDocs() == 97889, "unexpected totalDocs: %d",
roundTripped.getTotalDocs());
+ require("v3".equals(roundTripped.getIndexVersion()), "unexpected
indexVersion: %s",
+ roundTripped.getIndexVersion());
+ require(roundTripped.getStartTimeMs() == 1749000000000L, "unexpected
startTimeMs: %d",
+ roundTripped.getStartTimeMs());
+ require(roundTripped.getEndTimeMs() == 1750000000000L, "unexpected
endTimeMs: %d", roundTripped.getEndTimeMs());
+ require(roundTripped.getCreationTime() == 1750000000000L, "unexpected
creationTime: %d",
+ roundTripped.getCreationTime());
+
+ // The broker list the java client discovers from ZooKeeper is a Helix
external view.
+ ExternalView externalView = new
ExternalView(CommonConstants.Helix.BROKER_RESOURCE_INSTANCE);
+ externalView.setState(SAMPLE_TABLE_NAME + "_OFFLINE",
"Broker_192.168.1.10_8000", "ONLINE");
+ require("ONLINE".equals(externalView.getStateMap(SAMPLE_TABLE_NAME +
"_OFFLINE").get("Broker_192.168.1.10_8000")),
+ "external view state lost: %s",
externalView.getStateMap(SAMPLE_TABLE_NAME + "_OFFLINE"));
+ }
+
+ /// The gRPC client's decode path end to end: a protobuf {@code
BrokerResponse} carrying a compressed, encoded
+ /// payload, unpacked exactly the way {@code GrpcConnection} unpacks a
server response. Covers protobuf, the
+ /// compression codecs and the response encoders in one go.
+ private void checkCommonGrpcResponseDecoding()
+ throws IOException {
+ DataSchema dataSchema = sampleResultTable().getDataSchema();
+ Broker.BrokerResponse schemaResponse =
+
Broker.BrokerResponse.newBuilder().setPayload(ByteString.copyFrom(dataSchema.toBytes())).build();
+ require(dataSchema.equals(GrpcUtils.extractSchema(schemaResponse)),
"schema lost through the gRPC payload");
+
+ String metadataJson = "{\"requestId\":\"42\",\"numDocsScanned\":4231}";
+ Broker.BrokerResponse metadataResponse = Broker.BrokerResponse.newBuilder()
+
.setPayload(ByteString.copyFrom(metadataJson.getBytes(StandardCharsets.UTF_8)))
+ .putMetadata("rowSize", "3")
+ .build();
+
require("42".equals(GrpcUtils.extractMetadataJson(metadataResponse).get("requestId").asText()),
+ "requestId lost through the gRPC metadata payload");
+ ExecutionStats executionStats =
+
GrpcUtils.extractExecutionStats(JsonUtils.stringToJsonNode(_sampleBrokerResponseJson));
+ require(executionStats.getNumDocsScanned() == 4231, "unexpected
numDocsScanned: %d",
+ executionStats.getNumDocsScanned());
+
+ ResultTable resultTable = sampleResultTable();
+ int numRows = resultTable.getRows().size();
+ List<String> compressionTypes =
Arrays.asList(CompressionFactory.getCompressionTypes());
+ require(compressionTypes.containsAll(REQUIRED_COMPRESSION_CODECS),
+ "compression codecs %s no longer cover %s, so the JNI codecs would
stop being loaded here",
+ compressionTypes, REQUIRED_COMPRESSION_CODECS);
+
+ for (String encoderType :
ResponseEncoderFactory.getResponseEncoderTypes()) {
+ // Loop-invariant: encoding once per encoder avoids repeating the Arrow
allocator setup for
+ // every codec.
+ byte[] encoded = ResponseEncoderFactory.getResponseEncoder(encoderType)
+ .encodeResultTable(resultTable, 0, numRows);
+ for (String compressionType : compressionTypes) {
+ Compressor compressor =
CompressionFactory.getCompressor(compressionType);
+ byte[] compressed;
+ try {
+ compressed = compressor.compress(encoded);
+ } catch (Exception e) {
+ throw new AssertionError("compressor " + compressionType + " failed
on Java "
+ + _targetJavaFeatureVersion, e);
+ }
+ Broker.BrokerResponse dataResponse = Broker.BrokerResponse.newBuilder()
+ .setPayload(ByteString.copyFrom(compressed))
+ .putMetadata("rowSize", String.valueOf(numRows))
+ .putMetadata(CommonConstants.Broker.Grpc.COMPRESSION,
compressionType)
+ .putMetadata(CommonConstants.Broker.Grpc.ENCODING, encoderType)
+ .build();
+ ResultTable decoded = GrpcUtils.extractResultTable(dataResponse,
dataSchema);
+ require(decoded.getRows().size() == numRows, "%s/%s lost rows: %d of
%d", encoderType, compressionType,
+ decoded.getRows().size(), numRows);
+ assertSampleRowsMatch(encoderType + "/" + compressionType, decoded);
+ }
+ }
+ }
+
+ /// Builds the Netty-backed gRPC channel the client uses, without
connecting. Exercises grpc-netty's provider lookup
+ /// and the pooled direct-buffer allocator, both of which are version
sensitive.
+ private void checkCommonGrpcChannelConstruction()
+ throws IOException {
+ // Construction is deliberately outside the try: it builds the Netty
channel and the pooled
+ // direct-buffer allocator, and either step throwing on an older JVM is
the failure we are looking
+ // for, so it should propagate as-is -- and there is nothing to close if
it never returned.
+ BrokerGrpcQueryClient client = new BrokerGrpcQueryClient("localhost",
8010, new GrpcConfig(Map.of()));
+ ManagedChannel channel = client.getChannel();
+ try {
+ require(!channel.isShutdown(), "the gRPC channel was already shut down
on creation");
+ } finally {
+ client.close();
+ }
+ // close() swallows its own exceptions, so assert the channel state rather
than trusting it to throw.
+ require(channel.isTerminated(), "the gRPC channel did not terminate on
close");
+ }
+
+ /// Builds the async-http-client transport the java client queries brokers
with, without connecting.
+ private void checkJavaClientHttpTransport()
+ throws Exception {
+ JsonAsyncHttpPinotClientTransportFactory factory = new
JsonAsyncHttpPinotClientTransportFactory();
+ Properties properties = new Properties();
+ properties.setProperty("brokerReadTimeoutMs", "12000");
+ properties.setProperty("brokerConnectTimeoutMs", "3000");
+ PinotClientTransport<?> transport =
factory.withConnectionProperties(properties).buildTransport();
+ require(transport != null, "transport was not built");
+ transport.close();
+ }
+
+ /// The java client's query path, driven against a canned broker response
instead of a live cluster.
+ private void checkJavaClientQueryExecution() {
+ CannedResponseTransport transport = new
CannedResponseTransport(_sampleBrokerResponse);
+ Connection connection =
+ ConnectionFactory.fromHostList(new Properties(),
List.of("localhost:8000"), transport);
+ try {
+ ResultSetGroup resultSetGroup = connection.execute(
+ "SELECT playerName, playerId, numGames, battingAverage, isActive,
teams FROM " + SAMPLE_TABLE_NAME
+ + " LIMIT 3");
+ require(transport.getLastQuery() != null &&
transport.getLastQuery().contains(SAMPLE_TABLE_NAME),
+ "the transport never saw the query: %s", transport.getLastQuery());
+ require(!resultSetGroup.getBrokerResponse().hasExceptions(), "unexpected
exceptions: %s",
+ resultSetGroup.getExceptions());
+ require(resultSetGroup.getResultSetCount() == 1, "expected 1 result set,
got %d",
+ resultSetGroup.getResultSetCount());
+
+ ResultSet resultSet = resultSetGroup.getResultSet(0);
+ require(resultSet.getRowCount() == 3, "expected 3 rows, got %d",
resultSet.getRowCount());
+ require(resultSet.getColumnCount() == 6, "expected 6 columns, got %d",
resultSet.getColumnCount());
+ require("playerName".equals(resultSet.getColumnName(0)), "unexpected
column name: %s",
+ resultSet.getColumnName(0));
+ require("Hank Aaron".equals(resultSet.getString(0, 0)), "unexpected
string value: %s",
+ resultSet.getString(0, 0));
+ require(resultSet.getInt(0, 1) == 1001, "unexpected int value: %d",
resultSet.getInt(0, 1));
+ require(resultSet.getLong(0, 2) == 3298L, "unexpected long value: %d",
resultSet.getLong(0, 2));
+ require(Math.abs(resultSet.getDouble(0, 3) - 0.305) < 1e-9, "unexpected
double value: %s",
+ resultSet.getDouble(0, 3));
+ require("Shohei Ohtani".equals(resultSet.getString(2, 0)), "unexpected
last row: %s",
+ resultSet.getString(2, 0));
+
+ ExecutionStats stats = resultSetGroup.getExecutionStats();
+ require(stats.getNumDocsScanned() == 4231, "unexpected numDocsScanned:
%d", stats.getNumDocsScanned());
+ require(stats.getTotalDocs() == 97889, "unexpected totalDocs: %d",
stats.getTotalDocs());
+ require(stats.getNumServersQueried() == 2, "unexpected
numServersQueried: %d", stats.getNumServersQueried());
+ require(stats.getTimeUsedMs() == 37, "unexpected timeUsedMs: %d",
stats.getTimeUsedMs());
+ } finally {
+ connection.close();
+ }
+ require(transport.isClosed(), "closing the connection did not close the
transport");
+ }
+
+ /// Parameter binding in the java client's prepared statement.
+ private void checkJavaClientPreparedStatement() {
+ CannedResponseTransport transport = new
CannedResponseTransport(_sampleBrokerResponse);
+ Connection connection =
+ ConnectionFactory.fromHostList(new Properties(),
List.of("localhost:8000"), transport);
+ try {
+ PreparedStatement statement = connection.prepareStatement(
+ "SELECT playerName FROM " + SAMPLE_TABLE_NAME + " WHERE playerName =
? AND playerId > ?");
+ statement.setString(0, "Hank Aaron");
+ statement.setInt(1, 1000);
+ ResultSetGroup resultSetGroup = statement.execute();
+
+ String executedQuery = transport.getLastQuery();
+ require(executedQuery != null && executedQuery.contains("'Hank Aaron'"),
+ "string parameter was not bound: %s", executedQuery);
+ require(executedQuery.contains("1000"), "int parameter was not bound:
%s", executedQuery);
+ require(!executedQuery.contains("?"), "query still has unbound
parameters: %s", executedQuery);
+ require(resultSetGroup.getResultSetCount() == 1, "expected 1 result set,
got %d",
+ resultSetGroup.getResultSetCount());
+ } finally {
+ connection.close();
+ }
+ }
+
+ /// The JDBC driver has to be discoverable through {@link DriverManager}
without an explicit {@code Class.forName},
+ /// which means the {@code META-INF/services} descriptor and Java's {@code
ServiceLoader} have to work together on
+ /// this JVM.
+ private void checkJdbcDriverRegistration()
+ throws Exception {
+ require(getClass().getClassLoader().getResource(JDBC_SERVICE_DESCRIPTOR)
!= null,
+ "%s is missing from the classpath, so DriverManager cannot
auto-register the driver",
+ JDBC_SERVICE_DESCRIPTOR);
+
+ Driver driver = DriverManager.getDriver("jdbc:pinot://localhost:8000");
+
require("org.apache.pinot.client.PinotDriver".equals(driver.getClass().getName()),
+ "DriverManager resolved an unexpected driver: %s",
driver.getClass().getName());
+ require(driver.acceptsURL("jdbc:pinot://localhost:8000"), "the driver
rejected a pinot URL");
+ require(driver.acceptsURL("jdbc:pinotgrpc://localhost:8010"), "the driver
rejected a pinotgrpc URL");
+ require(!driver.acceptsURL("jdbc:mysql://localhost:3306/db"), "the driver
accepted a non-pinot URL");
+
+ DriverPropertyInfo[] propertyInfo =
driver.getPropertyInfo("jdbc:pinot://localhost:8000", new Properties());
+ List<String> propertyNames = new ArrayList<>();
+ for (DriverPropertyInfo info : propertyInfo) {
+ propertyNames.add(info.name);
+ }
+ // Membership, not position: a future driver property added ahead of
"tenant" is not a Java 11
+ // problem and must not turn this job red.
+ require(propertyNames.contains("tenant"), "the driver did not report a
tenant property: %s", propertyNames);
+ require(driver.getMajorVersion() > 0, "unexpected driver major version:
%d", driver.getMajorVersion());
+ }
+
+ /// The JDBC result set over a real broker response, including its metadata
and type mapping.
+ private void checkJdbcResultSet()
+ throws Exception {
+ // PinotResultSet.fromJson swallows failures and returns an empty result
set, so asserting on the
+ // row contents is what makes this check meaningful.
+ try (PinotResultSet resultSet =
PinotResultSet.fromJson(_sampleBrokerResponseJson)) {
+ ResultSetMetaData metaData = resultSet.getMetaData();
+ require(metaData.getColumnCount() == 6, "expected 6 columns, got %d",
metaData.getColumnCount());
+ require("playerName".equals(metaData.getColumnName(1)), "unexpected
column name: %s",
+ metaData.getColumnName(1));
+ require(metaData.getColumnType(1) == Types.VARCHAR, "expected VARCHAR
for a STRING column, got %d",
+ metaData.getColumnType(1));
+ require(metaData.getColumnType(4) == Types.DOUBLE, "expected DOUBLE for
a DOUBLE column, got %d",
+ metaData.getColumnType(4));
+
+ require(resultSet.next(), "the JDBC result set was empty --
PinotResultSet.fromJson swallowed a failure");
+ require("Hank Aaron".equals(resultSet.getString(1)), "unexpected string
value: %s", resultSet.getString(1));
+ require(resultSet.getInt(2) == 1001, "unexpected int value: %d",
resultSet.getInt(2));
+ require(resultSet.getLong(3) == 3298L, "unexpected long value: %d",
resultSet.getLong(3));
+ require(Math.abs(resultSet.getDouble(4) - 0.305) < 1e-9, "unexpected
double value: %s",
+ resultSet.getDouble(4));
+ require(new BigDecimal("0.305").compareTo(resultSet.getBigDecimal(4)) ==
0, "unexpected big decimal value: %s",
+ resultSet.getBigDecimal(4));
+ require("Hank Aaron".equals(resultSet.getString("playerName")), "column
lookup by name failed: %s",
+ resultSet.getString("playerName"));
+
+ int rowCount = 1;
+ while (resultSet.next()) {
+ rowCount++;
+ }
+ require(rowCount == 3, "expected 3 rows, got %d", rowCount);
+ }
+ }
+
+ //
---------------------------------------------------------------------------------------------
+ // Fixtures and helpers
+ //
---------------------------------------------------------------------------------------------
+
+ private static DataSchema sampleDataSchema() {
+ return new DataSchema(new String[]{
+ "playerName", "playerId", "numGames", "battingAverage", "isActive"
+ }, new ColumnDataType[]{
+ ColumnDataType.STRING, ColumnDataType.INT, ColumnDataType.LONG,
ColumnDataType.DOUBLE, ColumnDataType.BOOLEAN
+ });
+ }
+
+ /// A result table matching {@link #sampleDataSchema()}. Values are chosen
so that the assertions in {@link
+ /// #assertSampleRowsMatch} are exact rather than approximate.
+ private static ResultTable sampleResultTable() {
+ return new ResultTable(sampleDataSchema(), List.of(
+ new Object[]{"Hank Aaron", 1001, 3298L, 0.305d, false},
+ new Object[]{"Willie Mays", 1002, 2992L, 0.301d, false},
+ new Object[]{"Shohei Ohtani", 1003, 716L, 0.277d, true}));
+ }
+
+ /// Asserts that a decoded result table still carries the values from {@link
#sampleResultTable()}. Compares rendered
+ /// values because the encoders are free to pick their own boxed
representation for a given {@link ColumnDataType}.
+ private static void assertSampleRowsMatch(String label, ResultTable decoded)
{
+ List<Object[]> expectedRows = sampleResultTable().getRows();
+ for (int rowId = 0; rowId < expectedRows.size(); rowId++) {
+ Object[] expected = expectedRows.get(rowId);
+ Object[] actual = decoded.getRows().get(rowId);
+ require(actual.length == expected.length, "%s: row %d has %d columns,
expected %d", label, rowId, actual.length,
+ expected.length);
+ for (int colId = 0; colId < expected.length; colId++) {
+ String expectedValue = String.valueOf(expected[colId]);
+ String actualValue = String.valueOf(actual[colId]);
+ require(expectedValue.equals(actualValue), "%s: row %d column %d is
%s, expected %s", label, rowId, colId,
+ actualValue, expectedValue);
+ }
+ }
+ }
+
+ private static String readResource(String resource)
+ throws IOException {
+ try (InputStream inputStream =
Java11CompatibilityVerifier.class.getClassLoader()
+ .getResourceAsStream(resource)) {
+ if (inputStream == null) {
+ throw new IOException("Resource not found on the classpath: " +
resource);
+ }
+ return new String(inputStream.readAllBytes(), StandardCharsets.UTF_8);
+ }
+ }
+
+ private static void require(boolean condition, String message, Object...
args) {
+ if (!condition) {
+ throw new AssertionError(args.length == 0 ? message :
String.format(message, args));
+ }
+ }
+}
diff --git
a/pinot-java11-client-verifier/src/main/resources/sample-broker-response.json
b/pinot-java11-client-verifier/src/main/resources/sample-broker-response.json
new file mode 100644
index 00000000000..710fc5a567b
--- /dev/null
+++
b/pinot-java11-client-verifier/src/main/resources/sample-broker-response.json
@@ -0,0 +1,53 @@
+{
+ "requestId": "8465172461064265729",
+ "brokerId": "Broker_192.168.1.10_8000",
+ "resultTable": {
+ "dataSchema": {
+ "columnNames": [
+ "playerName",
+ "playerId",
+ "numGames",
+ "battingAverage",
+ "isActive",
+ "teams"
+ ],
+ "columnDataTypes": [
+ "STRING",
+ "INT",
+ "LONG",
+ "DOUBLE",
+ "BOOLEAN",
+ "STRING_ARRAY"
+ ]
+ },
+ "rows": [
+ ["Hank Aaron", 1001, 3298, 0.305, false, ["MLN", "ATL", "MIL"]],
+ ["Willie Mays", 1002, 2992, 0.301, false, ["NY1", "SFN", "NYN"]],
+ ["Shohei Ohtani", 1003, 716, 0.277, true, ["LAA", "LAD"]]
+ ]
+ },
+ "exceptions": [],
+ "numRowsResultSet": 3,
+ "numServersQueried": 2,
+ "numServersResponded": 2,
+ "numSegmentsQueried": 12,
+ "numSegmentsProcessed": 8,
+ "numSegmentsMatched": 5,
+ "numConsumingSegmentsQueried": 1,
+ "numSegmentsPrunedByBroker": 2,
+ "numSegmentsPrunedByServer": 2,
+ "numDocsScanned": 4231,
+ "numEntriesScannedInFilter": 97889,
+ "numEntriesScannedPostFilter": 12693,
+ "numGroupsLimitReached": false,
+ "groupsTrimmed": false,
+ "partialResult": false,
+ "totalDocs": 97889,
+ "timeUsedMs": 37,
+ "brokerReduceTimeMs": 4,
+ "minConsumingFreshnessTimeMs": 1750000000000,
+ "offlineThreadCpuTimeNs": 1250000,
+ "realtimeThreadCpuTimeNs": 310000,
+ "tablesQueried": ["baseballStats"],
+ "traceInfo": {}
+}
diff --git
a/pinot-java11-client-verifier/src/test/java/org/apache/pinot/java11/ClasspathClosureScannerTest.java
b/pinot-java11-client-verifier/src/test/java/org/apache/pinot/java11/ClasspathClosureScannerTest.java
new file mode 100644
index 00000000000..10893fdf816
--- /dev/null
+++
b/pinot-java11-client-verifier/src/test/java/org/apache/pinot/java11/ClasspathClosureScannerTest.java
@@ -0,0 +1,367 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.java11;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertThrows;
+import static org.testng.Assert.assertTrue;
+
+
+/// Guards the fail path of {@link ClasspathClosureScanner}.
+///
+/// The scanner is the only part of the Java 11 verifier that can go red on a
dependency bump, and both of its filters
+/// ({@code isLoadableClassEntry} and {@code readMajorVersion}) fail _open_ --
an input they do not recognise is treated
+/// as "not a violation". A regression in either therefore produces a
permanently green CI job rather than a red one,
+/// which is worse than having no job at all. These tests pin the behaviour
that keeps it honest, and need no Java 11
+/// JVM, so they run in the normal unit test job.
+public class ClasspathClosureScannerTest {
+ private static final int JAVA_11_FEATURE_VERSION = 11;
+ private static final int JAVA_11_MAJOR_VERSION = 55;
+ private static final int JAVA_17_MAJOR_VERSION = 61;
+ private static final int JAVA_21_MAJOR_VERSION = 65;
+ private static final String MANIFEST = "META-INF/MANIFEST.MF";
+
+ private Path _tempDir;
+
+ @BeforeClass
+ public void setUp()
+ throws IOException {
+ _tempDir = Files.createTempDirectory("closure-scanner-test");
+ }
+
+ @AfterClass
+ public void tearDown()
+ throws IOException {
+ if (_tempDir == null) {
+ return;
+ }
+ try (Stream<Path> paths = Files.walk(_tempDir)) {
+ paths.sorted(Comparator.reverseOrder()).forEach(path -> {
+ try {
+ Files.delete(path);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+ }
+
+ @Test
+ public void testPostTargetBytecodeAtJarRootIsAViolation()
+ throws IOException {
+ Path jar = writeJar("too-new.jar", Map.of("com/example/Foo.class",
classFile(JAVA_21_MAJOR_VERSION)));
+
+ ClasspathClosureScanner.Result result =
ClasspathClosureScanner.scan(jar.toString(), JAVA_11_FEATURE_VERSION);
+
+ assertEquals(result.getTotalViolationCount(), 1);
+ assertEquals(result.getClassFilesInArchives(), 1);
+
assertTrue(result.getReportedViolations().get(0).toString().contains("com/example/Foo.class"),
+ "the violation should name the offending entry: " +
result.getReportedViolations().get(0));
+ assertTrue(result.getReportedViolations().get(0).toString().contains("Java
21"),
+ "the violation should translate the major version: " +
result.getReportedViolations().get(0));
+ }
+
+ @Test
+ public void testTargetBytecodeIsNotAViolation()
+ throws IOException {
+ Path jar = writeJar("just-right.jar", Map.of("com/example/Foo.class",
classFile(JAVA_11_MAJOR_VERSION)));
+
+ ClasspathClosureScanner.Result result =
ClasspathClosureScanner.scan(jar.toString(), JAVA_11_FEATURE_VERSION);
+
+ assertEquals(result.getTotalViolationCount(), 0);
+ assertEquals(result.getClassFilesInArchives(), 1);
+ assertEquals(result.getMajorVersionHistogram(),
Map.of(JAVA_11_MAJOR_VERSION, 1));
+ }
+
+ /// A module descriptor is never loaded from the classpath, whatever version
built it.
+ @Test
+ public void testModuleInfoIsIgnoredAtAnyVersion()
+ throws IOException {
+ Map<String, byte[]> entries = new LinkedHashMap<>();
+ entries.put("module-info.class", classFile(JAVA_21_MAJOR_VERSION));
+ entries.put("META-INF/versions/9/module-info.class",
classFile(JAVA_21_MAJOR_VERSION));
+ Path jar = writeJar("with-module-info.jar", entries);
+
+ ClasspathClosureScanner.Result result =
ClasspathClosureScanner.scan(jar.toString(), JAVA_11_FEATURE_VERSION);
+
+ assertEquals(result.getTotalViolationCount(), 0);
+ assertEquals(result.getClassFilesInArchives(), 0);
+ }
+
+ /// Real multi-release jars on the client closure (jackson-core,
jersey-common) ship Java 17 and 21 bytecode under
+ /// META-INF/versions. Flagging those would fail the job for classes a Java
11 JVM never loads, so this is the
+ /// false-positive guard.
+ @Test
+ public void testMultiReleaseEntriesAboveTargetAreIgnored()
+ throws IOException {
+ Map<String, byte[]> entries = new LinkedHashMap<>();
+ entries.put(MANIFEST, multiReleaseManifest());
+ entries.put("com/example/Foo.class", classFile(JAVA_11_MAJOR_VERSION));
+ entries.put("META-INF/versions/17/com/example/Foo.class",
classFile(JAVA_17_MAJOR_VERSION));
+ entries.put("META-INF/versions/21/com/example/Foo.class",
classFile(JAVA_21_MAJOR_VERSION));
+ Path jar = writeJar("multi-release.jar", entries);
+
+ ClasspathClosureScanner.Result result =
ClasspathClosureScanner.scan(jar.toString(), JAVA_11_FEATURE_VERSION);
+
+ assertEquals(result.getTotalViolationCount(), 0);
+ assertEquals(result.getClassFilesInArchives(), 1, "only the baseline entry
should have been inspected");
+ }
+
+ /// The other half of the multi-release rule: in a real multi-release jar a
versioned directory at or below the target
+ /// _is_ selected, so bytecode too new for it is a real violation. Without
this the scanner could skip every
+ /// META-INF/versions entry and still look correct.
+ @Test
+ public void testMultiReleaseEntriesAtOrBelowTargetAreChecked()
+ throws IOException {
+ Map<String, byte[]> entries = new LinkedHashMap<>();
+ entries.put(MANIFEST, multiReleaseManifest());
+ entries.put("META-INF/versions/9/com/example/Foo.class",
classFile(JAVA_21_MAJOR_VERSION));
+ Path jar = writeJar("multi-release-9.jar", entries);
+
+ ClasspathClosureScanner.Result result =
ClasspathClosureScanner.scan(jar.toString(), JAVA_11_FEATURE_VERSION);
+
+ assertEquals(result.getTotalViolationCount(), 1);
+ }
+
+ /// A JVM only performs versioned lookup when the manifest says {@code
Multi-Release: true}. Without it the whole
+ /// META-INF/versions tree is inert data, so nothing under it may be
reported -- including entries at or below the
+ /// target, which would otherwise be a false positive on a shaded jar that
merged versioned entries but lost the
+ /// attribute.
+ @Test
+ public void
testVersionedEntriesAreIgnoredWhenTheManifestDoesNotDeclareMultiRelease()
+ throws IOException {
+ Map<String, byte[]> entries = new LinkedHashMap<>();
+ entries.put(MANIFEST, manifest("Manifest-Version: 1.0\n"));
+ entries.put("META-INF/versions/9/com/example/Below.class",
classFile(JAVA_21_MAJOR_VERSION));
+ entries.put("META-INF/versions/21/com/example/Above.class",
classFile(JAVA_21_MAJOR_VERSION));
+ Path jar = writeJar("not-multi-release.jar", entries);
+
+ ClasspathClosureScanner.Result result =
ClasspathClosureScanner.scan(jar.toString(), JAVA_11_FEATURE_VERSION);
+
+ assertEquals(result.getTotalViolationCount(), 0);
+ assertEquals(result.getClassFilesInArchives(), 0);
+ }
+
+ /// Same rule when there is no manifest at all.
+ @Test
+ public void testVersionedEntriesAreIgnoredWhenThereIsNoManifest()
+ throws IOException {
+ Path jar = writeJar("no-manifest.jar",
+ Map.of("META-INF/versions/9/com/example/Foo.class",
classFile(JAVA_21_MAJOR_VERSION)));
+
+ ClasspathClosureScanner.Result result =
ClasspathClosureScanner.scan(jar.toString(), JAVA_11_FEATURE_VERSION);
+
+ assertEquals(result.getTotalViolationCount(), 0);
+ assertEquals(result.getClassFilesInArchives(), 0);
+ }
+
+ /// The attribute value is case-insensitive per the JAR spec.
+ @Test
+ public void testMultiReleaseAttributeValueIsCaseInsensitive()
+ throws IOException {
+ Map<String, byte[]> entries = new LinkedHashMap<>();
+ entries.put(MANIFEST, manifest("Manifest-Version: 1.0\nMulti-Release:
TRUE\n"));
+ entries.put("META-INF/versions/9/com/example/Foo.class",
classFile(JAVA_21_MAJOR_VERSION));
+ Path jar = writeJar("multi-release-caps.jar", entries);
+
+ ClasspathClosureScanner.Result result =
ClasspathClosureScanner.scan(jar.toString(), JAVA_11_FEATURE_VERSION);
+
+ assertEquals(result.getTotalViolationCount(), 1);
+ }
+
+ @Test
+ public void testMalformedMultiReleaseDirectoryIsIgnored()
+ throws IOException {
+ Map<String, byte[]> entries = new LinkedHashMap<>();
+ entries.put(MANIFEST, multiReleaseManifest());
+ entries.put("META-INF/versions/notanumber/com/example/Foo.class",
classFile(JAVA_21_MAJOR_VERSION));
+ entries.put("META-INF/versions/Foo.class",
classFile(JAVA_21_MAJOR_VERSION));
+ Path jar = writeJar("malformed-multi-release.jar", entries);
+
+ ClasspathClosureScanner.Result result =
ClasspathClosureScanner.scan(jar.toString(), JAVA_11_FEATURE_VERSION);
+
+ assertEquals(result.getTotalViolationCount(), 0);
+ assertEquals(result.getClassFilesInArchives(), 0);
+ }
+
+ /// Jars do package non-bytecode files under a .class name; those must not
be misread as violations.
+ @Test
+ public void testEntryWithoutClassFileMagicIsNotCounted()
+ throws IOException {
+ Map<String, byte[]> entries = new LinkedHashMap<>();
+ entries.put("com/example/NotReallyAClass.class", "this is not
bytecode".getBytes(StandardCharsets.UTF_8));
+ entries.put("com/example/Truncated.class", new byte[]{(byte) 0xCA, (byte)
0xFE});
+ entries.put("com/example/Real.class", classFile(JAVA_11_MAJOR_VERSION));
+ Path jar = writeJar("not-bytecode.jar", entries);
+
+ ClasspathClosureScanner.Result result =
ClasspathClosureScanner.scan(jar.toString(), JAVA_11_FEATURE_VERSION);
+
+ assertEquals(result.getTotalViolationCount(), 0);
+ assertEquals(result.getClassFilesInArchives(), 1, "only the real class
file should have been counted");
+ }
+
+ @Test
+ public void testNonClassEntriesAreIgnored()
+ throws IOException {
+ Map<String, byte[]> entries = new LinkedHashMap<>();
+ entries.put("META-INF/MANIFEST.MF", "Manifest-Version:
1.0\n".getBytes(StandardCharsets.UTF_8));
+ entries.put("com/example/resource.json",
"{}".getBytes(StandardCharsets.UTF_8));
+ Path jar = writeJar("resources-only.jar", entries);
+
+ ClasspathClosureScanner.Result result =
ClasspathClosureScanner.scan(jar.toString(), JAVA_11_FEATURE_VERSION);
+
+ assertEquals(result.getTotalViolationCount(), 0);
+ assertEquals(result.getClassFilesInArchives(), 0);
+ assertEquals(result.getArchivesScanned(), 1);
+ }
+
+ /// Directory entries are scanned too, but counted apart so they cannot
satisfy the vacuity guard.
+ @Test
+ public void testDirectoryEntriesAreScannedAndCountedSeparately()
+ throws IOException {
+ Path classesDir =
Files.createDirectories(_tempDir.resolve("classes/com/example"));
+ Files.write(classesDir.resolve("Foo.class"),
classFile(JAVA_21_MAJOR_VERSION));
+ Files.write(classesDir.resolve("module-info.class"),
classFile(JAVA_21_MAJOR_VERSION));
+ Files.write(classesDir.resolve("notes.txt"),
"hello".getBytes(StandardCharsets.UTF_8));
+
+ ClasspathClosureScanner.Result result =
+ ClasspathClosureScanner.scan(_tempDir.resolve("classes").toString(),
JAVA_11_FEATURE_VERSION);
+
+ assertEquals(result.getTotalViolationCount(), 1);
+ assertEquals(result.getClassFilesInDirectories(), 1);
+ assertEquals(result.getClassFilesInArchives(), 0);
+ assertEquals(result.getDirectoriesScanned(), 1);
+ }
+
+ @Test
+ public void testCorruptArchiveIsAHardFailure()
+ throws IOException {
+ Path notAJar = _tempDir.resolve("corrupt.jar");
+ Files.write(notAJar, "definitely not a
zip".getBytes(StandardCharsets.UTF_8));
+
+ assertThrows(IOException.class, () ->
ClasspathClosureScanner.scan(notAJar.toString(), JAVA_11_FEATURE_VERSION));
+ }
+
+ @Test
+ public void testNonArchiveAndMissingEntriesAreReportedAsSkipped()
+ throws IOException {
+ Path pom = _tempDir.resolve("groovy-all-3.0.25.pom");
+ Files.write(pom, "<project/>".getBytes(StandardCharsets.UTF_8));
+ String missing = _tempDir.resolve("does-not-exist.jar").toString();
+ String classpath = pom + File.pathSeparator + missing;
+
+ ClasspathClosureScanner.Result result =
ClasspathClosureScanner.scan(classpath, JAVA_11_FEATURE_VERSION);
+
+ assertEquals(result.getSkippedEntries().size(), 2, "both entries should be
reported: "
+ + result.getSkippedEntries());
+ assertTrue(result.getSkippedEntries().get(0).endsWith("(not an archive)"),
result.getSkippedEntries().get(0));
+ assertTrue(result.getSkippedEntries().get(1).endsWith("(does not exist)"),
result.getSkippedEntries().get(1));
+ assertEquals(result.getArchivesScanned(), 0);
+ }
+
+ @Test
+ public void testViolationReportingIsCappedButCountIsNot()
+ throws IOException {
+ Map<String, byte[]> entries = new LinkedHashMap<>();
+ for (int i = 0; i < 100; i++) {
+ entries.put("com/example/Foo" + i + ".class",
classFile(JAVA_21_MAJOR_VERSION));
+ }
+ Path jar = writeJar("many-violations.jar", entries);
+
+ ClasspathClosureScanner.Result result =
ClasspathClosureScanner.scan(jar.toString(), JAVA_11_FEATURE_VERSION);
+
+ assertEquals(result.getTotalViolationCount(), 100);
+ assertEquals(result.getReportedViolations().size(),
ClasspathClosureScanner.MAX_REPORTED_VIOLATIONS,
+ "reporting should be capped for log sanity while the total count stays
exact");
+ }
+
+ /// The target version is a parameter, so the same scanner has to work if
the floor ever moves.
+ @Test
+ public void testTargetVersionIsHonoured()
+ throws IOException {
+ Path jar = writeJar("java17.jar", Map.of("com/example/Foo.class",
classFile(JAVA_17_MAJOR_VERSION)));
+
+ assertEquals(ClasspathClosureScanner.scan(jar.toString(),
11).getTotalViolationCount(), 1);
+ assertEquals(ClasspathClosureScanner.scan(jar.toString(),
17).getTotalViolationCount(), 0);
+ assertEquals(ClasspathClosureScanner.scan(jar.toString(),
21).getTotalViolationCount(), 0);
+ }
+
+ @Test
+ public void testArchiveNamesAreRecordedForCoverageAssertions()
+ throws IOException {
+ Path first = writeJar("pinot-spi-1.6.0-SNAPSHOT.jar",
+ Map.of("com/example/Foo.class", classFile(JAVA_11_MAJOR_VERSION)));
+ Path second = writeJar("pinot-common-1.6.0-SNAPSHOT.jar",
+ Map.of("com/example/Bar.class", classFile(JAVA_11_MAJOR_VERSION)));
+
+ ClasspathClosureScanner.Result result =
+ ClasspathClosureScanner.scan(first + File.pathSeparator + second,
JAVA_11_FEATURE_VERSION);
+
+ assertEquals(result.getArchiveNames(),
List.of("pinot-spi-1.6.0-SNAPSHOT.jar", "pinot-common-1.6.0-SNAPSHOT.jar"));
+ }
+
+ private Path writeJar(String name, Map<String, byte[]> entries)
+ throws IOException {
+ Path jar = _tempDir.resolve(name);
+ try (OutputStream fileOut = Files.newOutputStream(jar); ZipOutputStream
zipOut = new ZipOutputStream(fileOut)) {
+ for (Map.Entry<String, byte[]> entry : entries.entrySet()) {
+ zipOut.putNextEntry(new ZipEntry(entry.getKey()));
+ zipOut.write(entry.getValue());
+ zipOut.closeEntry();
+ }
+ }
+ return jar;
+ }
+
+ private static byte[] multiReleaseManifest() {
+ return manifest("Manifest-Version: 1.0\nMulti-Release: true\n");
+ }
+
+ private static byte[] manifest(String content) {
+ return content.getBytes(StandardCharsets.UTF_8);
+ }
+
+ /// The first 8 bytes of a class file are all the scanner reads: magic,
minor version, major version.
+ private static byte[] classFile(int majorVersion) {
+ return ByteBuffer.allocate(10)
+ .putInt(0xCAFEBABE)
+ .putShort((short) 0)
+ .putShort((short) majorVersion)
+ .putShort((short) 0)
+ .array();
+ }
+}
diff --git a/pom.xml b/pom.xml
index f251d146816..dd7ff7df93d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -59,6 +59,7 @@
<module>pinot-connectors</module>
<module>pinot-segment-local</module>
<module>pinot-compatibility-verifier</module>
+ <module>pinot-java11-client-verifier</module>
<module>pinot-query-planner-spi</module>
<module>pinot-query-planner</module>
<module>pinot-query-runtime</module>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
