This is an automated email from the ASF dual-hosted git repository.
tbonelee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new 0bd5b0329a [ZEPPELIN-6322] Filter download messages in ProcessData
error stream
0bd5b0329a is described below
commit 0bd5b0329ad4dd50ed006a2fac904e67e072dd04
Author: YeonKyung Ryu <[email protected]>
AuthorDate: Tue Jul 21 00:18:21 2026 +0900
[ZEPPELIN-6322] Filter download messages in ProcessData error stream
### What is this PR for?
This PR improves error stream output filtering in the `ProcessData` class
to exclude download-related messages. Currently, Maven/npm download progress
information clutters the error stream during integration tests, making it
difficult to identify actual errors.
This change filters out download messages (e.g., "Downloading:", "Progress:
45%", "1024/2048 KB") while preserving real error messages.
### What type of PR is it?
Improvement
### Todos
* [x] - Add DOWNLOAD_PATTERNS array with 6 regex patterns
* [x] - Implement isDownloadMessage() method
* [x] - Apply filtering logic in buildOutputAndErrorStreamData()
### What is the Jira issue?
[ZEPPELIN-6322](https://issues.apache.org/jira/browse/ZEPPELIN-6322)
### How should this be tested?
### Screenshots (if appropriate)
### Questions:
* Does the license files need to update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Closes #5104 from celinayk/ZEPPELIN-6322.
Signed-off-by: ChanHo Lee <[email protected]>
---
.../test/java/org/apache/zeppelin/ProcessData.java | 91 ++++++++++++++++++++--
.../java/org/apache/zeppelin/ProcessDataTest.java | 63 +++++++++++++++
2 files changed, 147 insertions(+), 7 deletions(-)
diff --git
a/zeppelin-integration/src/test/java/org/apache/zeppelin/ProcessData.java
b/zeppelin-integration/src/test/java/org/apache/zeppelin/ProcessData.java
index f4a578a32b..a7ad3fc3d9 100644
--- a/zeppelin-integration/src/test/java/org/apache/zeppelin/ProcessData.java
+++ b/zeppelin-integration/src/test/java/org/apache/zeppelin/ProcessData.java
@@ -25,6 +25,7 @@ import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,6 +40,19 @@ public class ProcessData {
private static final Logger LOGGER =
LoggerFactory.getLogger(ProcessData.class);
+ // Patterns to identify download-related messages in the error stream.
+ // Anchored to the start of a (single, already-split) line to avoid matching
+ // error messages that merely mention a size or percentage figure.
+ private static final Pattern[] DOWNLOAD_PATTERNS = {
+ Pattern.compile("^\\[INFO\\]\\s+(Downloading|Downloaded):.*"), // Maven
download messages
+ Pattern.compile("^Downloading\\s+.*"), // Generic
downloading messages
+ Pattern.compile("^Downloaded\\s+.*"), // Generic
downloaded messages
+ Pattern.compile("^Progress\\s*(\\(\\d+\\))?:?\\s*\\d+%.*",
+ Pattern.CASE_INSENSITIVE), //
Progress indicators, e.g. "Progress (1): 45%"
+ Pattern.compile("^[\\d.]+/[\\d.]+\\s*(KB|MB|GB|bytes|kB)\\b.*",
+ Pattern.CASE_INSENSITIVE) // Size
progress (e.g., "1024/2048 KB")
+ };
+
private Process checked_process;
private boolean printToConsole = false;
@@ -149,6 +163,51 @@ public class ProcessData {
return this.errorStream;
}
+ /**
+ * Checks whether a single (already trimmed) line looks like a
download-related
+ * message (Maven downloads, progress indicators, size information, etc.).
+ *
+ * <p>This is only used to decide the console log level for a line - a line
that
+ * matches is still logged (at TRACE), never dropped, so a misclassified
line is
+ * merely quieter rather than lost. Callers must check for "error"/"failed"
content
+ * before calling this, since that check always takes precedence.
+ *
+ * @param line The single line to check
+ * @return true if the line looks like a download message, false otherwise
+ */
+ boolean isDownloadMessage(String line) {
+ if (line == null || line.isEmpty()) {
+ return false;
+ }
+
+ for (Pattern pattern : DOWNLOAD_PATTERNS) {
+ if (pattern.matcher(line).matches()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Classifies and logs a single, complete line of error-stream output to the
console.
+ * A line is never dropped: real error/failed lines always win at WARN, and
only lines
+ * that look like download noise are downgraded to TRACE instead of being
discarded.
+ */
+ private void logErrorLine(String line) {
+ String trimmedLine = line.trim();
+ if (trimmedLine.isEmpty()) {
+ return;
+ }
+ String lowerLine = trimmedLine.toLowerCase();
+ if (lowerLine.contains("error") || lowerLine.contains("failed")) {
+ LOGGER.warn(trimmedLine);
+ } else if (isDownloadMessage(trimmedLine)) {
+ LOGGER.trace(trimmedLine);
+ } else {
+ LOGGER.debug(trimmedLine);
+ }
+ }
+
@Override
public String toString() {
StringBuilder result = new StringBuilder();
@@ -161,6 +220,10 @@ public class ProcessData {
private void buildOutputAndErrorStreamData() throws IOException {
StringBuilder sbInStream = new StringBuilder();
StringBuilder sbErrorStream = new StringBuilder();
+ // Carries an incomplete trailing line across chunk reads, since a single
line of
+ // output can be split across two BUFFER_LEN-sized reads (or even two
outer-loop
+ // iterations). Only complete lines are ever classified/logged from this
buffer.
+ StringBuilder pendingErrorLine = new StringBuilder();
try {
InputStream in = this.checked_process.getInputStream();
@@ -203,19 +266,25 @@ public class ProcessData {
break;
}
tempSB.append(charBuffer, 0, readCount);
+ // The full, unfiltered chunk is always kept here, so
getErrorStream() always
+ // returns the complete error output. Download-message filtering
below only
+ // affects the verbosity of the console log, not the returned stream
content.
sbErrorStream.append(tempSB);
if (tempSB.length() > 0) {
outputProduced = true;
String temp = new String(tempSB);
temp = temp.replaceAll("Pseudo-terminal will not be allocated
because stdin is not a terminal.", "");
- //TODO : error stream output need to be improved, because it
outputs downloading information.
if (printToConsole) {
- if (!temp.trim().equals("")) {
- if (temp.toLowerCase().contains("error") ||
temp.toLowerCase().contains("failed")) {
- LOGGER.warn(temp.trim());
- } else {
- LOGGER.debug(temp.trim());
- }
+ // Buffer chunks can hold several lines, and a line can itself
be split
+ // across chunks/iterations, so accumulate into pendingErrorLine
and only
+ // classify/log complete lines. The trailing remainder (no
newline yet)
+ // stays buffered until more data (or stream close) completes it.
+ pendingErrorLine.append(temp);
+ int newlineIndex;
+ while ((newlineIndex = pendingErrorLine.indexOf("\n")) >= 0) {
+ String line = pendingErrorLine.substring(0, newlineIndex);
+ pendingErrorLine.delete(0, newlineIndex + 1);
+ logErrorLine(line);
}
}
}
@@ -246,6 +315,14 @@ public class ProcessData {
}
}
+ // Stream ended (or we gave up waiting) - the process will send no more
data, so
+ // whatever is left in pendingErrorLine is a final, unterminated line.
Flush it
+ // rather than silently dropping it.
+ if (printToConsole && pendingErrorLine.length() > 0) {
+ logErrorLine(pendingErrorLine.toString());
+ pendingErrorLine.setLength(0);
+ }
+
in.close();
inErrors.close();
} finally {
diff --git
a/zeppelin-integration/src/test/java/org/apache/zeppelin/ProcessDataTest.java
b/zeppelin-integration/src/test/java/org/apache/zeppelin/ProcessDataTest.java
new file mode 100644
index 0000000000..a7df1dca30
--- /dev/null
+++
b/zeppelin-integration/src/test/java/org/apache/zeppelin/ProcessDataTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.junit.jupiter.api.Test;
+
+public class ProcessDataTest {
+
+ private final ProcessData processData = new ProcessData(null, false);
+
+ @Test
+ public void detectsMavenDownloadMessages() {
+ assertTrue(processData.isDownloadMessage(
+ "[INFO] Downloading: https://repo1.maven.org/maven2/foo/foo.jar";));
+ assertTrue(processData.isDownloadMessage(
+ "[INFO] Downloaded: https://repo1.maven.org/maven2/foo/foo.jar (12 kB
at 45 kB/s)"));
+ assertTrue(processData.isDownloadMessage("Downloading from central:
https://example.com/foo.jar";));
+ assertTrue(processData.isDownloadMessage("Downloaded from central:
https://example.com/foo.jar";));
+ }
+
+ @Test
+ public void detectsProgressAndSizeMessages() {
+ assertTrue(processData.isDownloadMessage("Progress (1): 45%"));
+ assertTrue(processData.isDownloadMessage("progress: 45%"));
+ assertTrue(processData.isDownloadMessage("1024/2048 KB"));
+ assertTrue(processData.isDownloadMessage("1.5/12 kB"));
+ }
+
+ @Test
+ public void doesNotFilterOutRealErrorLines() {
+ // These lines contain download-shaped figures but are genuine failures
and must
+ // never be classified as a download message, otherwise they would only
ever be
+ // reachable via the download-noise path instead of being surfaced as
warnings.
+ assertFalse(processData.isDownloadMessage("Task failed after writing
1024/2048 MB"));
+ assertFalse(processData.isDownloadMessage("Build failed at progress:
50%"));
+ assertFalse(processData.isDownloadMessage("ERROR: could not resolve
dependency foo:bar:1.0"));
+ assertFalse(processData.isDownloadMessage("Compilation error in
Main.java"));
+ }
+
+ @Test
+ public void ignoresBlankOrNullInput() {
+ assertFalse(processData.isDownloadMessage(""));
+ assertFalse(processData.isDownloadMessage(null));
+ }
+}
\ No newline at end of file
