This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/storm.git
The following commit(s) were added to refs/heads/master by this push:
new 653a0f878 Fix the integration test cluster and window test
expectations (#9152)
653a0f878 is described below
commit 653a0f878bd58aed5bd4bfd9d6cfb4301d49edb6
Author: Richard Zowalla <[email protected]>
AuthorDate: Thu Oct 1 12:43:51 2026 +0200
Fix the integration test cluster and window test expectations (#9152)
* Fix the integration test cluster and window test expectations
- worker.childopts in the integration storm.yaml still used Java 8 GC
logging flags, so no worker JVM could start on Java 9+. Use the
unified -Xlog syntax and drop storm-java9.yaml, which was only
appended for JDK 11.
- The window tests asserted IllegalArgumentException for invalid sizes
but then went on to build the topology again, failing with the same
exception. Return after the assertion.
* Enable the logviewer HTTP binding on the integration test cluster
TopoWrap fetches worker logs from http://<host>:8000, but the logviewer
has disabled its HTTP binding by default since 2018, so every window
test failed with Connection refused.
* Ship the UI static files in the integration test distribution
The binary assembly builds public/ from storm-webapp/target/classes/WEB-INF.
The Integration-Test job only downloads the built jars, so that directory
was missing, the distribution had no public/, and the UI and logviewer
failed to start. Unpack WEB-INF from the storm-webapp jar first.
* Create the extraction directory before unpacking WEB-INF
* Parse worker log timestamps as UTC in the integration tests
The log date pattern carries no zone, so ZonedDateTime.parse failed for
every decorated log line.
---
integration-test/config/storm-java9.yaml | 20 --------------------
integration-test/config/storm.yaml | 5 ++++-
integration-test/run-it.sh | 11 +++++++----
.../storm/st/tests/window/SlidingWindowTest.java | 2 ++
.../storm/st/tests/window/TumblingWindowTest.java | 1 +
.../apache/storm/st/wrapper/DecoratedLogLine.java | 4 +++-
6 files changed, 17 insertions(+), 26 deletions(-)
diff --git a/integration-test/config/storm-java9.yaml
b/integration-test/config/storm-java9.yaml
deleted file mode 100644
index 245fb0ed6..000000000
--- a/integration-test/config/storm-java9.yaml
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Java 9 replaced a bunch of GC logging parameters for the JDK. This uses the
Java 9+ syntax, whereas defaults.yaml currently uses the Java 8 syntax.
-
-worker.childopts: "-Xmx%HEAP-MEM%m
-Xlog:gc*:artifacts/gc.log:time,uptime:filecount=10,filesize=1M
-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=artifacts/heapdump"
\ No newline at end of file
diff --git a/integration-test/config/storm.yaml
b/integration-test/config/storm.yaml
index 774497fca..da01f9a1f 100644
--- a/integration-test/config/storm.yaml
+++ b/integration-test/config/storm.yaml
@@ -34,10 +34,13 @@ executor.metrics.frequency.secs: 3
drpc.servers:
- "node1"
+# The tests fetch worker logs from the logviewer over plain HTTP on
logviewer.port
+logviewer.disable.http.binding: false
+
supervisor.slots.ports: [6700, 6701, 6702, 6703, 6704, 6705, 6706, 6707, 6708,
6709]
# Enable assertions
nimbus.childopts: "-Xmx1024m -ea"
supervisor.childopts: "-Xmx256m -ea"
-worker.childopts: "-Xmx%HEAP-MEM%m -XX:+PrintGCDetails
-Xloggc:artifacts/gc.log -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps
-XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=1M
-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=artifacts/heapdump -ea"
+worker.childopts: "-Xmx%HEAP-MEM%m
-Xlog:gc*:artifacts/gc.log:time,uptime:filecount=10,filesize=1M
-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=artifacts/heapdump -ea"
ui.childopts: "-Xmx768m -ea"
\ No newline at end of file
diff --git a/integration-test/run-it.sh b/integration-test/run-it.sh
index 4043ba289..6823b5255 100755
--- a/integration-test/run-it.sh
+++ b/integration-test/run-it.sh
@@ -60,6 +60,13 @@ if [[ "${USER}" == "vagrant" ]]; then # install oracle jdk8
or openjdk11
export MAVEN_OPTS="-Xmx3000m"
else
if [[ "${USER}" == "github" ]]; then
+ # The distribution takes public/ from
storm-webapp/target/classes/WEB-INF, which this job
+ # does not build; it only has the storm-webapp jar from the build job.
+ webapp_jar=$(find ~/.m2/repository/org/apache/storm/storm-webapp -name
'storm-webapp-*.jar' \
+ -not -name '*-tests.jar' -not -name '*-sources.jar' -not -name
'*-javadoc.jar' | head -1)
+ [[ -n "${webapp_jar}" ]] || die "storm-webapp jar not found in the
local Maven repository"
+ mkdir -p "${STORM_SRC_DIR}/storm-webapp/target/classes"
+ unzip -q -o "${webapp_jar}" 'WEB-INF/*' -d
"${STORM_SRC_DIR}/storm-webapp/target/classes"
( cd "${STORM_SRC_DIR}/storm-dist/binary" && mvn clean package
-Dgpg.skip=true )
fi
# The binary build also produces the lean "storm-lite" distribution; the
integration
@@ -78,10 +85,6 @@ sudo bash "${SCRIPT_DIR}/config/common.sh"
sudo bash "${SCRIPT_DIR}/config/install-storm.sh" "$storm_binary_zip"
# storm.yaml points ZooKeeper, Nimbus and DRPC at node1; make it resolve when
the host does not already
grep -qw node1 /etc/hosts || echo "127.0.0.1 node1" | sudo tee -a /etc/hosts
-if [[ "$TRAVIS_JDK_VERSION" == "openjdk11" ]] || [[ "${JDK_VERSION}" == "11" ]]
-then
- cat "${SCRIPT_DIR}/config/storm-java9.yaml" | sudo tee -a
/usr/share/storm/conf/storm.yaml
-fi
export JAVA_HOME="${JAVA_HOME}"
env
function start_storm_process() {
diff --git
a/integration-test/src/test/java/org/apache/storm/st/tests/window/SlidingWindowTest.java
b/integration-test/src/test/java/org/apache/storm/st/tests/window/SlidingWindowTest.java
index 11ca2ae3c..b20615cfd 100644
---
a/integration-test/src/test/java/org/apache/storm/st/tests/window/SlidingWindowTest.java
+++
b/integration-test/src/test/java/org/apache/storm/st/tests/window/SlidingWindowTest.java
@@ -55,6 +55,7 @@ public final class SlidingWindowTest extends AbstractTest {
final String topologyName = this.getClass().getSimpleName() +
"-size-window" + windowSize + "-slide" + slideSize;
if (windowSize <= 0 || slideSize <= 0) {
assertThrows(IllegalArgumentException.class, () ->
testable.newTopology());
+ return;
}
topo = new TopoWrap(cluster, topologyName, testable.newTopology());
windowVerifier.runAndVerifyCount(windowSize, slideSize, testable,
topo);
@@ -83,6 +84,7 @@ public final class SlidingWindowTest extends AbstractTest {
final String topologyName = this.getClass().getSimpleName() +
"-sec-window" + windowSec + "-slide" + slideSec;
if (windowSec <= 0 || slideSec <= 0) {
assertThrows(IllegalArgumentException.class, () ->
testable.newTopology());
+ return;
}
topo = new TopoWrap(cluster, topologyName, testable.newTopology());
windowVerifier.runAndVerifyTime(windowSec, slideSec, testable, topo);
diff --git
a/integration-test/src/test/java/org/apache/storm/st/tests/window/TumblingWindowTest.java
b/integration-test/src/test/java/org/apache/storm/st/tests/window/TumblingWindowTest.java
index 99c8d313c..e8bb507ff 100644
---
a/integration-test/src/test/java/org/apache/storm/st/tests/window/TumblingWindowTest.java
+++
b/integration-test/src/test/java/org/apache/storm/st/tests/window/TumblingWindowTest.java
@@ -80,6 +80,7 @@ public final class TumblingWindowTest extends AbstractTest {
final String topologyName = this.getClass().getSimpleName() + "-sec" +
tumbleSec;
if (tumbleSec <= 0) {
assertThrows(IllegalArgumentException.class, () ->
testable.newTopology());
+ return;
}
topo = new TopoWrap(cluster, topologyName, testable.newTopology());
windowVerifier.runAndVerifyTime(tumbleSec, tumbleSec, testable, topo);
diff --git
a/integration-test/src/test/java/org/apache/storm/st/wrapper/DecoratedLogLine.java
b/integration-test/src/test/java/org/apache/storm/st/wrapper/DecoratedLogLine.java
index c3090c1a9..334f7d203 100644
---
a/integration-test/src/test/java/org/apache/storm/st/wrapper/DecoratedLogLine.java
+++
b/integration-test/src/test/java/org/apache/storm/st/wrapper/DecoratedLogLine.java
@@ -21,6 +21,7 @@ import org.apache.storm.st.utils.AssertUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.storm.st.utils.StringDecorator;
+import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Arrays;
@@ -34,7 +35,8 @@ public class DecoratedLogLine implements
Comparable<DecoratedLogLine> {
private final ZonedDateTime logDate;
private final String data;
private static final int DATE_LEN = "2016-05-04 23:38:10.702".length();
//format of date in worker logs
- private static final DateTimeFormatter DATE_FORMAT =
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS");
+ private static final DateTimeFormatter DATE_FORMAT =
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS")
+ .withZone(ZoneOffset.UTC);
public DecoratedLogLine(String logLine) {
final List<String> splitOnDecorator =
Arrays.asList(StringDecorator.split2(StringUtils.strip(logLine)));
