This is an automated email from the ASF dual-hosted git repository.
paulk-asert pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/groovy.git
The following commit(s) were added to refs/heads/master by this push:
new c98c2573e2 GROOVY-12155: decode process output streams using the
native encoding
c98c2573e2 is described below
commit c98c2573e2a1b402e5900b9350139f8d26e41004
Author: Paul King <[email protected]>
AuthorDate: Mon Jul 13 15:11:01 2026 +1000
GROOVY-12155: decode process output streams using the native encoding
with thanks to https://github.com/netliomax25-code for identifying the
problem
---
.../groovy/runtime/FlushingStreamWriter.java | 13 ++
.../groovy/runtime/ProcessGroovyMethods.java | 226 +++++++++++++++++++--
src/test/groovy/groovy/ProcessTest.groovy | 93 ++++++++-
3 files changed, 303 insertions(+), 29 deletions(-)
diff --git
a/src/main/java/org/codehaus/groovy/runtime/FlushingStreamWriter.java
b/src/main/java/org/codehaus/groovy/runtime/FlushingStreamWriter.java
index a1fd825244..bc028c44cc 100644
--- a/src/main/java/org/codehaus/groovy/runtime/FlushingStreamWriter.java
+++ b/src/main/java/org/codehaus/groovy/runtime/FlushingStreamWriter.java
@@ -21,6 +21,7 @@ package org.codehaus.groovy.runtime;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
+import java.nio.charset.Charset;
/**
* Stream writer which flushes after each write operation.
@@ -37,6 +38,18 @@ public class FlushingStreamWriter extends OutputStreamWriter
{
super(out);
}
+ /**
+ * Constructs a FlushingStreamWriter that flushes on every write.
+ *
+ * @param out the underlying output stream to write to
+ * @param charset the charset to encode with
+ * @throws NullPointerException if out or charset is null
+ * @since 6.0.0
+ */
+ public FlushingStreamWriter(OutputStream out, Charset charset) {
+ super(out, charset);
+ }
+
@Override
public void write(char[] cbuf, int off, int len) throws IOException {
super.write(cbuf, off, len);
diff --git
a/src/main/java/org/codehaus/groovy/runtime/ProcessGroovyMethods.java
b/src/main/java/org/codehaus/groovy/runtime/ProcessGroovyMethods.java
index fd20400fc2..cca3ba7b77 100644
--- a/src/main/java/org/codehaus/groovy/runtime/ProcessGroovyMethods.java
+++ b/src/main/java/org/codehaus/groovy/runtime/ProcessGroovyMethods.java
@@ -33,6 +33,7 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.Writer;
+import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -57,6 +58,49 @@ import java.util.concurrent.TimeUnit;
*/
public class ProcessGroovyMethods extends DefaultGroovyMethodsSupport {
+ /**
+ * The system property naming the charset used for process streams when no
explicit
+ * charset is supplied. It overrides the native encoding and is useful for
a child
+ * known to emit (or expect) something other than the platform's native
encoding,
+ * e.g. a UTF-8 child on a Windows host.
+ *
+ * @since 6.0.0
+ */
+ public static final String PROCESS_ENCODING_PROPERTY =
"groovy.process.encoding";
+
+ /**
+ * The operating system's native encoding, as named by the {@code
native.encoding}
+ * system property (JDK 17+), falling back to {@link
Charset#defaultCharset()} when
+ * that property is absent or names an unsupported charset. This cannot
change during
+ * the life of the JVM, so it is resolved once.
+ */
+ private static final Charset NATIVE_ENCODING =
+ charsetOrDefault(System.getProperty("native.encoding"),
Charset.defaultCharset());
+
+ /**
+ * The charset used for a process's streams when no explicit charset is
supplied.
+ * A child process reads and writes text in the operating system's native
encoding,
+ * whereas {@link Charset#defaultCharset()} has been UTF-8 on every
platform since
+ * JEP 400, so on a non-UTF-8 host the default charset no longer describes
what the
+ * child emits or expects. The native encoding is used instead, matching
the charset
+ * selection of {@link Process#inputReader()} and {@link
Process#outputWriter()}.
+ * Setting {@value #PROCESS_ENCODING_PROPERTY} overrides it.
+ */
+ private static Charset processEncoding() {
+ return charsetOrDefault(System.getProperty(PROCESS_ENCODING_PROPERTY),
NATIVE_ENCODING);
+ }
+
+ private static Charset charsetOrDefault(String name, Charset fallback) {
+ if (name != null && !name.isEmpty()) {
+ try {
+ return Charset.forName(name);
+ } catch (IllegalArgumentException ignore) {
+ // unknown or unsupported charset name
+ }
+ }
+ return fallback;
+ }
+
/**
* An alias method so that a process appears similar to System.out,
System.in, System.err;
* you can use process.in, process.out, process.err in a similar fashion.
@@ -70,7 +114,8 @@ public class ProcessGroovyMethods extends
DefaultGroovyMethodsSupport {
}
/**
- * Read the text of the output stream of the Process.
+ * Read the text of the output stream of the Process, decoding using the
operating
+ * system's native encoding (or {@value #PROCESS_ENCODING_PROPERTY} when
set).
* Closes all the streams associated with the process after retrieving the
text.
*
* @param self a Process instance
@@ -79,9 +124,26 @@ public class ProcessGroovyMethods extends
DefaultGroovyMethodsSupport {
* @since 1.0
*/
public static String getText(Process self) throws IOException {
- String text = IOGroovyMethods.getText(new BufferedReader(new
InputStreamReader(self.getInputStream())));
- closeStreams(self);
- return text;
+ return getText(self, processEncoding());
+ }
+
+ /**
+ * Read the text of the output stream of the Process using the given
charset.
+ * Closes all the streams associated with the process after retrieving the
text.
+ *
+ * @param self a Process instance
+ * @param charset the charset the process writes its output in, or null to
use the process encoding
+ * @return the text of the output
+ * @throws java.io.IOException if an IOException occurs.
+ * @since 6.0.0
+ */
+ public static String getText(Process self, Charset charset) throws
IOException {
+ if (charset == null) charset = processEncoding();
+ try {
+ return IOGroovyMethods.getText(new BufferedReader(new
InputStreamReader(self.getInputStream(), charset)));
+ } finally {
+ closeStreams(self);
+ }
}
/**
@@ -110,7 +172,9 @@ public class ProcessGroovyMethods extends
DefaultGroovyMethodsSupport {
/**
* Overloads the left shift operator (<<) to provide an append
mechanism
- * to pipe data to a Process.
+ * to pipe data to a Process. The value is encoded using the operating
system's
+ * native encoding (or {@value #PROCESS_ENCODING_PROPERTY} when set),
which is what
+ * the child process expects to read.
*
* @param self a Process instance
* @param value a value to append
@@ -119,7 +183,7 @@ public class ProcessGroovyMethods extends
DefaultGroovyMethodsSupport {
* @since 1.0
*/
public static Writer leftShift(Process self, Object value) throws
IOException {
- return IOGroovyMethods.leftShift(self.getOutputStream(), value);
+ return IOGroovyMethods.leftShift(new
FlushingStreamWriter(self.getOutputStream(), processEncoding()), value);
}
/**
@@ -192,8 +256,27 @@ public class ProcessGroovyMethods extends
DefaultGroovyMethodsSupport {
* @since 1.7.5
*/
public static void consumeProcessOutput(Process self, Appendable output,
Appendable error) {
- consumeProcessOutputStream(self, output);
- consumeProcessErrorStream(self, error);
+ consumeProcessOutput(self, output, error, processEncoding());
+ }
+
+ /**
+ * Gets the output and error streams from a process and reads them
+ * to keep the process from blocking due to a full output buffer.
+ * The processed stream data is decoded using the given charset and
appended
+ * to the supplied Appendable.
+ * For this, two Threads are started, so this method will return
immediately.
+ * The threads will not be join()ed, even if waitFor() is called. To wait
+ * for the output to be fully consumed call waitForProcessOutput().
+ *
+ * @param self a Process
+ * @param output an Appendable to capture the process stdout
+ * @param error an Appendable to capture the process stderr
+ * @param charset the charset the process writes its output in
+ * @since 6.0.0
+ */
+ public static void consumeProcessOutput(Process self, Appendable output,
Appendable error, Charset charset) {
+ consumeProcessOutputStream(self, output, charset);
+ consumeProcessErrorStream(self, error, charset);
}
/**
@@ -246,8 +329,27 @@ public class ProcessGroovyMethods extends
DefaultGroovyMethodsSupport {
* @since 1.7.5
*/
public static void waitForProcessOutput(Process self, Appendable output,
Appendable error) {
- Thread tout = consumeProcessOutputStream(self, output);
- Thread terr = consumeProcessErrorStream(self, error);
+ waitForProcessOutput(self, output, error, processEncoding());
+ }
+
+ /**
+ * Gets the output and error streams from a process and reads them
+ * to keep the process from blocking due to a full output buffer.
+ * The processed stream data is decoded using the given charset and
appended
+ * to the supplied Appendable.
+ * For this, two Threads are started, but join()ed, so we wait.
+ * As implied by the waitFor... name, we also wait until we finish
+ * as well. Finally, the input, output and error streams are closed.
+ *
+ * @param self a Process
+ * @param output an Appendable to capture the process stdout
+ * @param error an Appendable to capture the process stderr
+ * @param charset the charset the process writes its output in
+ * @since 6.0.0
+ */
+ public static void waitForProcessOutput(Process self, Appendable output,
Appendable error, Charset charset) {
+ Thread tout = consumeProcessOutputStream(self, output, charset);
+ Thread terr = consumeProcessErrorStream(self, error, charset);
boolean interrupted = false;
try {
try { tout.join(); } catch (InterruptedException ignore) {
interrupted = true; }
@@ -303,10 +405,25 @@ public class ProcessGroovyMethods extends
DefaultGroovyMethodsSupport {
* @since 6.0.0
*/
public static ProcessResult waitForResult(Process self) throws
InterruptedException {
+ return waitForResult(self, processEncoding());
+ }
+
+ /**
+ * Executes the process and waits for it to complete, capturing
+ * the standard output and standard error decoded using the given charset,
+ * plus the exit code, into a {@link ProcessResult}.
+ *
+ * @param self a Process
+ * @param charset the charset the process writes its output in
+ * @return a ProcessResult containing stdout, stderr, and exit code
+ * @throws InterruptedException if the current thread is interrupted.
+ * @since 6.0.0
+ */
+ public static ProcessResult waitForResult(Process self, Charset charset)
throws InterruptedException {
StringBuilder sout = new StringBuilder();
StringBuilder serr = new StringBuilder();
- Thread tout = consumeProcessOutputStream(self, sout);
- Thread terr = consumeProcessErrorStream(self, serr);
+ Thread tout = consumeProcessOutputStream(self, sout, charset);
+ Thread terr = consumeProcessErrorStream(self, serr, charset);
boolean interrupted = false;
try {
try { tout.join(); } catch (InterruptedException ignore) {
interrupted = true; }
@@ -337,10 +454,28 @@ public class ProcessGroovyMethods extends
DefaultGroovyMethodsSupport {
* @since 6.0.0
*/
public static ProcessResult waitForResult(Process self, long timeout,
TimeUnit unit) throws InterruptedException {
+ return waitForResult(self, timeout, unit, processEncoding());
+ }
+
+ /**
+ * Executes the process and waits for it to complete within the given
timeout,
+ * capturing the standard output and standard error decoded using the
given charset,
+ * plus the exit code, into a {@link ProcessResult}. If the process does
not complete
+ * within the timeout, it is forcibly destroyed.
+ *
+ * @param self a Process
+ * @param timeout the maximum time to wait
+ * @param unit the time unit of the timeout argument
+ * @param charset the charset the process writes its output in
+ * @return a ProcessResult containing stdout, stderr, and exit code
+ * @throws InterruptedException if the current thread is interrupted.
+ * @since 6.0.0
+ */
+ public static ProcessResult waitForResult(Process self, long timeout,
TimeUnit unit, Charset charset) throws InterruptedException {
StringBuilder sout = new StringBuilder();
StringBuilder serr = new StringBuilder();
- Thread tout = consumeProcessOutputStream(self, sout);
- Thread terr = consumeProcessErrorStream(self, serr);
+ Thread tout = consumeProcessOutputStream(self, sout, charset);
+ Thread terr = consumeProcessErrorStream(self, serr, charset);
boolean interrupted = false;
try {
try {
@@ -412,7 +547,24 @@ public class ProcessGroovyMethods extends
DefaultGroovyMethodsSupport {
* @since 1.7.5
*/
public static Thread consumeProcessErrorStream(Process self, Appendable
error) {
- Thread thread = new Thread(new TextDumper(self.getErrorStream(),
error));
+ return consumeProcessErrorStream(self, error, processEncoding());
+ }
+
+ /**
+ * Gets the error stream from a process and reads it
+ * to keep the process from blocking due to a full buffer.
+ * The stream data is decoded using the given charset and appended
+ * to the supplied Appendable.
+ * A new Thread is started, so this method will return immediately.
+ *
+ * @param self a Process
+ * @param error an Appendable to capture the process stderr
+ * @param charset the charset the process writes its error output in
+ * @return the Thread
+ * @since 6.0.0
+ */
+ public static Thread consumeProcessErrorStream(Process self, Appendable
error, Charset charset) {
+ Thread thread = new Thread(new TextDumper(self.getErrorStream(),
error, charset));
thread.start();
return thread;
}
@@ -429,7 +581,24 @@ public class ProcessGroovyMethods extends
DefaultGroovyMethodsSupport {
* @since 1.7.5
*/
public static Thread consumeProcessOutputStream(Process self, Appendable
output) {
- Thread thread = new Thread(new TextDumper(self.getInputStream(),
output));
+ return consumeProcessOutputStream(self, output, processEncoding());
+ }
+
+ /**
+ * Gets the output stream from a process and reads it
+ * to keep the process from blocking due to a full output buffer.
+ * The stream data is decoded using the given charset and appended
+ * to the supplied Appendable.
+ * A new Thread is started, so this method will return immediately.
+ *
+ * @param self a Process
+ * @param output an Appendable to capture the process stdout
+ * @param charset the charset the process writes its output in
+ * @return the Thread
+ * @since 6.0.0
+ */
+ public static Thread consumeProcessOutputStream(Process self, Appendable
output, Charset charset) {
+ Thread thread = new Thread(new TextDumper(self.getInputStream(),
output, charset));
thread.start();
return thread;
}
@@ -455,6 +624,8 @@ public class ProcessGroovyMethods extends
DefaultGroovyMethodsSupport {
* Creates a new BufferedWriter as stdin for this process,
* passes it to the closure, and ensures the stream is flushed
* and closed after the closure returns.
+ * Text is encoded using the operating system's native encoding
+ * (or {@value #PROCESS_ENCODING_PROPERTY} when set).
* A new Thread is started, so this method will return immediately.
*
* @param self a Process
@@ -462,9 +633,24 @@ public class ProcessGroovyMethods extends
DefaultGroovyMethodsSupport {
* @since 1.5.2
*/
public static void withWriter(final Process self, final Closure closure) {
+ withWriter(self, processEncoding(), closure);
+ }
+
+ /**
+ * Creates a new BufferedWriter as stdin for this process using the given
charset,
+ * passes it to the closure, and ensures the stream is flushed
+ * and closed after the closure returns.
+ * A new Thread is started, so this method will return immediately.
+ *
+ * @param self a Process
+ * @param charset the charset the process expects to read its input in
+ * @param closure a closure
+ * @since 6.0.0
+ */
+ public static void withWriter(final Process self, final Charset charset,
final Closure closure) {
new Thread(() -> {
try {
- IOGroovyMethods.withWriter(new
BufferedOutputStream(getOut(self)), closure);
+ IOGroovyMethods.withWriter(new
BufferedOutputStream(getOut(self)), charset.name(), closure);
} catch (IOException e) {
throw new GroovyRuntimeException("exception while reading
process stream", e);
}
@@ -621,15 +807,17 @@ public class ProcessGroovyMethods extends
DefaultGroovyMethodsSupport {
private static class TextDumper implements Runnable {
final InputStream in;
final Appendable app;
+ final Charset charset;
- public TextDumper(InputStream in, Appendable app) {
+ public TextDumper(InputStream in, Appendable app, Charset charset) {
this.in = in;
this.app = app;
+ this.charset = charset;
}
@Override
public void run() {
- InputStreamReader isr = new InputStreamReader(in);
+ InputStreamReader isr = new InputStreamReader(in, charset);
BufferedReader br = new BufferedReader(isr);
String next;
try {
diff --git a/src/test/groovy/groovy/ProcessTest.groovy
b/src/test/groovy/groovy/ProcessTest.groovy
index e8c1270c18..6fbd0bf489 100644
--- a/src/test/groovy/groovy/ProcessTest.groovy
+++ b/src/test/groovy/groovy/ProcessTest.groovy
@@ -22,14 +22,23 @@ import org.junit.jupiter.api.AfterEach
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
+import java.nio.charset.Charset
+
+import static java.nio.charset.StandardCharsets.ISO_8859_1
+import static java.nio.charset.StandardCharsets.UTF_8
+
/**
* check that groovy Process methods do their job.
*/
class ProcessTest {
+ private static final String PROCESS_ENCODING = 'groovy.process.encoding'
+
def myProcess
+ private String previousProcessEncoding
@BeforeEach
void setUp() {
+ previousProcessEncoding = System.getProperty(PROCESS_ENCODING)
myProcess = new MockProcess()
}
@@ -73,6 +82,69 @@ class ProcessTest {
assert "" == myProcess.text
}
+ @Test
+ void testProcessTextUsesSuppliedCharset() {
+ // a single 0xE9 byte is 'é' in ISO-8859-1 but is not valid UTF-8
+ def latin1 = new MockProcess([0xE9] as byte[])
+ assert "é" == latin1.getText(ISO_8859_1)
+
+ def utf8 = new MockProcess("é".getBytes(UTF_8))
+ assert "é" == utf8.getText(UTF_8)
+ }
+
+ @Test
+ void testProcessTextDefaultsToTheNativeEncoding() {
+ def bytes = [0xE9] as byte[]
+ def expected = new String(bytes,
Charset.forName(System.getProperty("native.encoding")))
+ assert expected == new MockProcess(bytes).text
+ }
+
+ @Test
+ void testProcessEncodingPropertyOverridesTheNativeEncoding() {
+ System.setProperty(PROCESS_ENCODING, "ISO-8859-1")
+ assert "é" == new MockProcess([0xE9] as byte[]).text
+
+ System.setProperty(PROCESS_ENCODING, "UTF-8")
+ assert "é" == new MockProcess("é".getBytes(UTF_8)).text
+ }
+
+ @Test
+ void testProcessEncodingPropertyIgnoredWhenNotAValidCharset() {
+ System.setProperty(PROCESS_ENCODING, "no-such-charset")
+ def bytes = [0xE9] as byte[]
+ def expected = new String(bytes,
Charset.forName(System.getProperty("native.encoding")))
+ assert expected == new MockProcess(bytes).text
+ }
+
+ @Test
+ void testConsumeProcessOutputUsesSuppliedCharset() {
+ def out = new StringBuilder()
+ def err = new StringBuilder()
+ def proc = new MockProcess([0xE9] as byte[], [0xE8] as byte[])
+
+ proc.waitForProcessOutput(out, err, ISO_8859_1)
+
+ assert "é\n" == out.toString()
+ assert "è\n" == err.toString()
+ }
+
+ @Test
+ void testWaitForResultUsesSuppliedCharset() {
+ def result = new MockProcess([0xE9] as
byte[]).waitForResult(ISO_8859_1)
+
+ assert "é\n" == result.out
+ assert result.ok
+ }
+
+ @Test
+ void testProcessAppendEncodesUsingTheProcessEncoding() {
+ System.setProperty(PROCESS_ENCODING, "ISO-8859-1")
+
+ myProcess << "é"
+
+ assert [0xE9] as byte[] == myProcess.outputStream.toByteArray()
+ }
+
@Test
void testProcessErrorStream() {
assert myProcess.err instanceof InputStream
@@ -89,6 +161,11 @@ class ProcessTest {
@AfterEach
void tearDown() {
+ if (previousProcessEncoding == null) {
+ System.clearProperty(PROCESS_ENCODING)
+ } else {
+ System.setProperty(PROCESS_ENCODING, previousProcessEncoding)
+ }
myProcess.destroy()
}
}
@@ -102,8 +179,12 @@ class MockProcess extends Process {
private def o
MockProcess() {
- e = new AnotherMockInputStream()
- i = new AnotherMockInputStream()
+ this(new byte[0], new byte[0])
+ }
+
+ MockProcess(byte[] stdout, byte[] stderr = new byte[0]) {
+ i = new ByteArrayInputStream(stdout)
+ e = new ByteArrayInputStream(stderr)
o = new ByteArrayOutputStream()
}
@@ -119,11 +200,3 @@ class MockProcess extends Process {
int waitFor() { return 0 }
}
-
-/**
- * only needed for workaround in groovy,
- * new ByteArrayInputStream(myByteArray) doesn't work at mo...
(28-Sep-2004)
- */
-class AnotherMockInputStream extends InputStream {
- int read() { return -1 }
-}
