This is an automated email from the ASF dual-hosted git repository.
ijuma pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/trunk by this push:
new 1f4cbc5d532 MINOR: Add JDK 20 CI build and remove some branch builds
(#12948)
1f4cbc5d532 is described below
commit 1f4cbc5d53259031123b6e9e6bb9a5bbe1e084e8
Author: Ismael Juma <[email protected]>
AuthorDate: Fri Jun 30 01:12:00 2023 -0700
MINOR: Add JDK 20 CI build and remove some branch builds (#12948)
It's good for us to add support for Java 20 in preparation for Java 21 -
the next LTS.
Given that Scala 2.12 support has been deprecated, a Scala 2.12 variant is
not included.
Also remove some branch builds that add load to the CI, but have
low value: JDK 8 & Scala 2.13 (JDK 8 support has been deprecated),
JDK 11 & Scala 2.12 (Scala 2.12 support has been deprecated) and
JDK 17 & Scala 2.12 (Scala 2.12 support has been deprecated).
A newer version of Mockito (4.9.0 -> 4.11.0) is required for Java 20
support, but we
only use it with Scala 2.13+ since it causes compilation errors with Scala
2.12. Similarly,
we upgrade easymock when the Java version is 16 or newer as it's
incompatible
with powermock (which doesn't support Java 16 or newer).
Filed KAFKA-15117 for a test that fails with Java 20
(SslTransportLayerTest.testValidEndpointIdentificationCN).
Finally, fixed some lossy conversions that were added after #13582 was
submitted.
Reviewers: Ismael Juma <[email protected]>
---
Jenkinsfile | 66 ++--------------------
build.gradle | 3 +
.../kafka/common/utils/ChunkedBytesStream.java | 14 +++--
.../common/network/SslTransportLayerTest.java | 3 +
.../kafka/connect/mirror/OffsetSyncStoreTest.java | 2 +-
.../kafka/server/KafkaRequestHandlerTest.scala | 2 +-
gradle/dependencies.gradle | 28 +++++++--
.../CachingInMemoryKeyValueStoreTest.java | 2 +-
.../CachingPersistentWindowStoreTest.java | 2 +-
...imeOrderedCachingPersistentWindowStoreTest.java | 2 +-
.../internals/TimeOrderedWindowStoreTest.java | 2 +-
11 files changed, 47 insertions(+), 79 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index fbd3fce4b5f..70f956bb4fc 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -155,79 +155,23 @@ pipeline {
echo 'Skipping Kafka Streams archetype test for Java 17'
}
}
-
- // To avoid excessive Jenkins resource usage, we only run the stages
- // above at the PR stage. The ones below are executed after changes
- // are pushed to trunk and/or release branches. We achieve this via
- // the `when` clause.
-
- stage('JDK 8 and Scala 2.13') {
- when {
- not { changeRequest() }
- beforeAgent true
- }
- agent { label 'ubuntu' }
- tools {
- jdk 'jdk_1.8_latest'
- maven 'maven_3_latest'
- }
- options {
- timeout(time: 8, unit: 'HOURS')
- timestamps()
- }
- environment {
- SCALA_VERSION=2.13
- }
- steps {
- doValidation()
- doTest(env)
- tryStreamsArchetype()
- }
- }
-
- stage('JDK 11 and Scala 2.12') {
- when {
- not { changeRequest() }
- beforeAgent true
- }
- agent { label 'ubuntu' }
- tools {
- jdk 'jdk_11_latest'
- }
- options {
- timeout(time: 8, unit: 'HOURS')
- timestamps()
- }
- environment {
- SCALA_VERSION=2.12
- }
- steps {
- doValidation()
- doTest(env)
- echo 'Skipping Kafka Streams archetype test for Java 11'
- }
- }
- stage('JDK 17 and Scala 2.12') {
- when {
- not { changeRequest() }
- beforeAgent true
- }
+ stage('JDK 20 and Scala 2.13') {
agent { label 'ubuntu' }
tools {
- jdk 'jdk_17_latest'
+ jdk 'jdk_20_latest'
}
options {
- timeout(time: 8, unit: 'HOURS')
+ timeout(time: 8, unit: 'HOURS')
timestamps()
}
environment {
- SCALA_VERSION=2.12
+ SCALA_VERSION=2.13
}
steps {
doValidation()
doTest(env)
- echo 'Skipping Kafka Streams archetype test for Java 17'
+ echo 'Skipping Kafka Streams archetype test for Java 20'
}
}
}
diff --git a/build.gradle b/build.gradle
index 62c1e701991..023477a7994 100644
--- a/build.gradle
+++ b/build.gradle
@@ -276,6 +276,9 @@ subprojects {
// set --source and --target via `sourceCompatibility` and
`targetCompatibility` a couple of lines below
if (JavaVersion.current().isJava9Compatible())
options.release = minJavaVersion
+ // --source/--target 8 is deprecated in Java 20, suppress warning until
Java 8 support is dropped in Kafka 4.0
+ if (JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_20))
+ options.compilerArgs << "-Xlint:-options"
}
// We should only set this if Java version is < 9 (--release is recommended
for >= 9), but the Scala plugin for IntelliJ sets
diff --git
a/clients/src/main/java/org/apache/kafka/common/utils/ChunkedBytesStream.java
b/clients/src/main/java/org/apache/kafka/common/utils/ChunkedBytesStream.java
index c4dca031e7d..6be225ae045 100644
---
a/clients/src/main/java/org/apache/kafka/common/utils/ChunkedBytesStream.java
+++
b/clients/src/main/java/org/apache/kafka/common/utils/ChunkedBytesStream.java
@@ -290,7 +290,7 @@ public class ChunkedBytesStream extends FilterInputStream {
// Skip bytes stored in intermediate buffer first
int avail = count - pos;
- long bytesSkipped = (avail < remaining) ? avail : remaining;
+ int bytesSkipped = (int) Math.min(avail, remaining);
pos += bytesSkipped;
remaining -= bytesSkipped;
@@ -298,17 +298,19 @@ public class ChunkedBytesStream extends FilterInputStream
{
if (delegateSkipToSourceStream) {
// Use sourceStream's skip() to skip the rest.
// conversion to int is acceptable because toSkip and
remaining are int.
- bytesSkipped = getInIfOpen().skip(remaining);
- if (bytesSkipped == 0) {
+ long delegateBytesSkipped = getInIfOpen().skip(remaining);
+ if (delegateBytesSkipped == 0) {
// read one byte to check for EOS
if (read() == -1) {
break;
}
// one byte read so decrement number to skip
remaining--;
- } else if (bytesSkipped > remaining || bytesSkipped < 0) { //
skipped negative or too many bytes
+ } else if (delegateBytesSkipped > remaining ||
delegateBytesSkipped < 0) { // skipped negative or too many bytes
throw new IOException("Unable to skip exactly");
+
}
+ remaining -= delegateBytesSkipped;
} else {
// skip from intermediate buffer, filling it first (if
required)
if (pos >= count) {
@@ -318,10 +320,10 @@ public class ChunkedBytesStream extends FilterInputStream
{
break;
}
avail = count - pos;
- bytesSkipped = (avail < remaining) ? avail : remaining;
+ bytesSkipped = (int) Math.min(avail, remaining);
pos += bytesSkipped;
+ remaining -= bytesSkipped;
}
- remaining -= bytesSkipped;
}
return toSkip - remaining;
}
diff --git
a/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java
b/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java
index 26987e30da8..eb104a2dc04 100644
---
a/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java
+++
b/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java
@@ -36,6 +36,8 @@ import org.apache.kafka.common.utils.Utils;
import org.apache.kafka.test.TestSslUtils;
import org.apache.kafka.test.TestUtils;
import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.condition.DisabledOnJre;
+import org.junit.jupiter.api.condition.JRE;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
@@ -193,6 +195,7 @@ public class SslTransportLayerTest {
*/
@ParameterizedTest
@ArgumentsSource(SslTransportLayerArgumentsProvider.class)
+ @DisabledOnJre(value = JRE.JAVA_20, disabledReason = "KAFKA-15117")
public void testValidEndpointIdentificationCN(Args args) throws Exception {
args.serverCertStores = certBuilder(true, "localhost",
args.useInlinePem).build();
args.clientCertStores = certBuilder(false, "localhost",
args.useInlinePem).build();
diff --git
a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/OffsetSyncStoreTest.java
b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/OffsetSyncStoreTest.java
index b2fa9fa4da3..66c699f13b7 100644
---
a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/OffsetSyncStoreTest.java
+++
b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/OffsetSyncStoreTest.java
@@ -110,7 +110,7 @@ public class OffsetSyncStoreTest {
@Test
public void testPastOffsetTranslation() {
try (FakeOffsetSyncStore store = new FakeOffsetSyncStore()) {
- long maxOffsetLag = 10;
+ int maxOffsetLag = 10;
int offset = 0;
for (; offset <= 1000; offset += maxOffsetLag) {
store.sync(tp, offset, offset);
diff --git a/core/src/test/scala/kafka/server/KafkaRequestHandlerTest.scala
b/core/src/test/scala/kafka/server/KafkaRequestHandlerTest.scala
index d29267e2a90..5b0c0746054 100644
--- a/core/src/test/scala/kafka/server/KafkaRequestHandlerTest.scala
+++ b/core/src/test/scala/kafka/server/KafkaRequestHandlerTest.scala
@@ -53,7 +53,7 @@ class KafkaRequestHandlerTest {
val context = new RequestContext(requestHeader, "0",
mock(classOf[InetAddress]), new KafkaPrincipal("", ""),
new ListenerName(""), SecurityProtocol.PLAINTEXT,
mock(classOf[ClientInformation]), false)
val request = new RequestChannel.Request(0, context, time.nanoseconds(),
- mock(classOf[MemoryPool]), mock(classOf[ByteBuffer]), metrics)
+ mock(classOf[MemoryPool]), ByteBuffer.allocate(0), metrics)
val handler = new KafkaRequestHandler(0, 0, mock(classOf[Meter]), new
AtomicInteger(1), requestChannel, apiHandler, time)
diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle
index ca30e14e11e..cc74819c897 100644
--- a/gradle/dependencies.gradle
+++ b/gradle/dependencies.gradle
@@ -53,6 +53,23 @@ if ( !versions.scala.contains('-') ) {
versions["baseScala"] = versions.scala
}
+// We use Mockito 4.11 with Scala 2.13+ for Java 20 support and Mockito 4.9
with Scala 2.12
+// to workaround ambiguous reference to `Mockito.spy` compiler errors. Since
Scala 2.12 support
+// is going away soon, this is simpler than adjusting the code.
+String mockitoVersion
+if (scalaVersion == "2.12")
+ mockitoVersion = "4.9.0"
+else
+ mockitoVersion = "4.11.0"
+
+// easymock 5.1 is required for Java 20 support, but it breaks tests using
powermock
+// powermock doesn't work with Java 16 or newer and hence it's safe to use
easymock 5.1 in this case only
+String easymockVersion
+if (JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_16))
+ easymockVersion = "5.1.0"
+else
+ easymockVersion = "4.3"
+
// When adding, removing or updating dependencies, please also update the
LICENSE-binary file accordingly.
// See https://issues.apache.org/jira/browse/KAFKA-12622 for steps to verify
the LICENSE-binary file is correct.
versions += [
@@ -69,7 +86,6 @@ versions += [
gradle: "8.1.1",
grgit: "4.1.1",
httpclient: "4.5.14",
- easymock: "4.3",
jackson: "2.13.5",
jacksonDatabind: "2.13.5",
jacoco: "0.8.10",
@@ -110,7 +126,6 @@ versions += [
lz4: "1.8.0",
mavenArtifact: "3.8.4",
metrics: "2.2.0",
- mockito: "4.9.0",
netty: "4.1.92.Final",
pcollections: "4.0.1",
powermock: "2.0.9",
@@ -133,6 +148,7 @@ versions += [
zookeeper: "3.6.4",
zstd: "1.5.5-1"
]
+
libs += [
activation: "javax.activation:activation:$versions.activation",
apacheda: "org.apache.directory.api:api-all:$versions.apacheda",
@@ -149,7 +165,7 @@ libs += [
caffeine: "com.github.ben-manes.caffeine:caffeine:$versions.caffeine",
commonsCli: "commons-cli:commons-cli:$versions.commonsCli",
commonsValidator:
"commons-validator:commons-validator:$versions.commonsValidator",
- easymock: "org.easymock:easymock:$versions.easymock",
+ easymock: "org.easymock:easymock:$easymockVersion",
jacksonAnnotations:
"com.fasterxml.jackson.core:jackson-annotations:$versions.jackson",
jacksonDatabind:
"com.fasterxml.jackson.core:jackson-databind:$versions.jacksonDatabind",
jacksonDataformatCsv:
"com.fasterxml.jackson.dataformat:jackson-dataformat-csv:$versions.jackson",
@@ -200,9 +216,9 @@ libs += [
lz4: "org.lz4:lz4-java:$versions.lz4",
metrics: "com.yammer.metrics:metrics-core:$versions.metrics",
dropwizardMetrics:
"io.dropwizard.metrics:metrics-core:$versions.dropwizardMetrics",
- mockitoCore: "org.mockito:mockito-core:$versions.mockito",
- mockitoInline: "org.mockito:mockito-inline:$versions.mockito",
- mockitoJunitJupiter: "org.mockito:mockito-junit-jupiter:$versions.mockito",
+ mockitoCore: "org.mockito:mockito-core:$mockitoVersion",
+ mockitoInline: "org.mockito:mockito-inline:$mockitoVersion",
+ mockitoJunitJupiter: "org.mockito:mockito-junit-jupiter:$mockitoVersion",
nettyHandler: "io.netty:netty-handler:$versions.netty",
nettyTransportNativeEpoll:
"io.netty:netty-transport-native-epoll:$versions.netty",
pcollections: "org.pcollections:pcollections:$versions.pcollections",
diff --git
a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingInMemoryKeyValueStoreTest.java
b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingInMemoryKeyValueStoreTest.java
index 9be8b3c58bb..d790863cdfd 100644
---
a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingInMemoryKeyValueStoreTest.java
+++
b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingInMemoryKeyValueStoreTest.java
@@ -616,7 +616,7 @@ public class CachingInMemoryKeyValueStoreTest extends
AbstractKeyValueStoreTest
}
private int addItemsToCache() {
- int cachedSize = 0;
+ long cachedSize = 0;
int i = 0;
while (cachedSize < maxCacheSizeBytes) {
final String kv = String.valueOf(i++);
diff --git
a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingPersistentWindowStoreTest.java
b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingPersistentWindowStoreTest.java
index e077d34daeb..a4ae8d99ec1 100644
---
a/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingPersistentWindowStoreTest.java
+++
b/streams/src/test/java/org/apache/kafka/streams/state/internals/CachingPersistentWindowStoreTest.java
@@ -1091,7 +1091,7 @@ public class CachingPersistentWindowStoreTest {
}
private int addItemsToCache() {
- int cachedSize = 0;
+ long cachedSize = 0;
int i = 0;
while (cachedSize < MAX_CACHE_SIZE_BYTES) {
final String kv = String.valueOf(i++);
diff --git
a/streams/src/test/java/org/apache/kafka/streams/state/internals/TimeOrderedCachingPersistentWindowStoreTest.java
b/streams/src/test/java/org/apache/kafka/streams/state/internals/TimeOrderedCachingPersistentWindowStoreTest.java
index 70cd401fb5e..a078e8aae83 100644
---
a/streams/src/test/java/org/apache/kafka/streams/state/internals/TimeOrderedCachingPersistentWindowStoreTest.java
+++
b/streams/src/test/java/org/apache/kafka/streams/state/internals/TimeOrderedCachingPersistentWindowStoreTest.java
@@ -1192,7 +1192,7 @@ public class TimeOrderedCachingPersistentWindowStoreTest {
}
private int addItemsToCache() {
- int cachedSize = 0;
+ long cachedSize = 0;
int i = 0;
while (cachedSize < MAX_CACHE_SIZE_BYTES) {
final String kv = String.valueOf(i++);
diff --git
a/streams/src/test/java/org/apache/kafka/streams/state/internals/TimeOrderedWindowStoreTest.java
b/streams/src/test/java/org/apache/kafka/streams/state/internals/TimeOrderedWindowStoreTest.java
index 53b54c7226c..4c30a79567a 100644
---
a/streams/src/test/java/org/apache/kafka/streams/state/internals/TimeOrderedWindowStoreTest.java
+++
b/streams/src/test/java/org/apache/kafka/streams/state/internals/TimeOrderedWindowStoreTest.java
@@ -1206,7 +1206,7 @@ public class TimeOrderedWindowStoreTest {
}
private int addItemsToCache() {
- int cachedSize = 0;
+ long cachedSize = 0;
int i = 0;
while (cachedSize < MAX_CACHE_SIZE_BYTES) {
final String kv = String.valueOf(i++);
