This is an automated email from the ASF dual-hosted git repository.
SvenO3 pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/streampipes.git
The following commit(s) were added to refs/heads/dev by this push:
new b6820fde4c fix(#4564): Ensure correct escaping and quoting of string
values in csv export (#4616)
b6820fde4c is described below
commit b6820fde4cc94a50a164e08087d406d9b77fd825
Author: Sven Oehler <[email protected]>
AuthorDate: Tue Jun 23 14:50:19 2026 +0200
fix(#4564): Ensure correct escaping and quoting of string values in csv
export (#4616)
---
.../export/ConfiguredCsvOutputWriter.java | 4 +-
.../dataexplorer/export/item/CsvItemGenerator.java | 25 +++++++++++-
.../export/TestConfiguredCsvOutputWriter.java | 24 +++++++++++
.../dataexplorer/export/TestCsvItemGenerator.java | 34 ++++++++++++++++
.../dataexplorer/influx/PropertyHandler.java | 8 +++-
.../influx/TimeSeriesStorageInfluxTest.java | 46 ++++++++++++++++++++++
6 files changed, 138 insertions(+), 3 deletions(-)
diff --git
a/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredCsvOutputWriter.java
b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredCsvOutputWriter.java
index cbc7eaba95..fd0012d475 100644
---
a/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredCsvOutputWriter.java
+++
b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredCsvOutputWriter.java
@@ -78,7 +78,9 @@ public class ConfiguredCsvOutputWriter extends
ConfiguredOutputWriter {
private String makeHeaderLine(List<String> columns) {
StringJoiner joiner = new StringJoiner(this.delimiter);
- columns.forEach(c -> joiner.add(getHeaderName(schema, c,
headerColumnNameStrategy)));
+ columns.forEach(c -> joiner.add(
+ itemGenerator.encodeCsvValue(getHeaderName(schema, c,
headerColumnNameStrategy))
+ ));
return joiner + LINE_SEPARATOR;
}
}
diff --git
a/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/item/CsvItemGenerator.java
b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/item/CsvItemGenerator.java
index 824c05f97c..0ec0c5004d 100644
---
a/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/item/CsvItemGenerator.java
+++
b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/item/CsvItemGenerator.java
@@ -23,17 +23,40 @@ import
org.apache.streampipes.dataexplorer.export.ExportUtils;
public class CsvItemGenerator extends ItemGenerator {
+ private static final String QUOTE = "\"";
+ private static final String ESCAPED_QUOTE = "\"\"";
+ private static final String CARRIAGE_RETURN = "\r";
+ private static final String LINE_FEED = "\n";
+
+ private final String delimiter;
+
public CsvItemGenerator(String delimiter) {
super(delimiter);
+ this.delimiter = delimiter;
}
@Override
protected String makeItemString(String key, Object value) {
- return value != null ? ExportUtils.formatValue(value) : "";
+ return value != null ? encodeCsvValue(ExportUtils.formatValue(value)) : "";
}
@Override
protected String finalizeItem(String item) {
return item;
}
+
+ public String encodeCsvValue(String value) {
+ if (requiresQuoting(value)) {
+ return QUOTE + value.replace(QUOTE, ESCAPED_QUOTE) + QUOTE;
+ } else {
+ return value;
+ }
+ }
+
+ private boolean requiresQuoting(String value) {
+ return value.contains(delimiter)
+ || value.contains(QUOTE)
+ || value.contains(CARRIAGE_RETURN)
+ || value.contains(LINE_FEED);
+ }
}
diff --git
a/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredCsvOutputWriter.java
b/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredCsvOutputWriter.java
index 15a6b66e28..60c3b67388 100644
---
a/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredCsvOutputWriter.java
+++
b/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredCsvOutputWriter.java
@@ -25,8 +25,10 @@ import org.junit.jupiter.api.Test;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
import java.util.HashMap;
+import static
org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_CSV_DELIMITER;
import static org.junit.jupiter.api.Assertions.assertEquals;
public class TestConfiguredCsvOutputWriter extends TestConfiguredOutputWriter {
@@ -49,4 +51,26 @@ public class TestConfiguredCsvOutputWriter extends
TestConfiguredOutputWriter {
assertEquals(Expected, outputStream.toString(StandardCharsets.UTF_8));
}
}
+
+ @Test
+ public void testCsvOutputWriterEscapesConfiguredDelimiter() throws
IOException {
+ var writer = new ConfiguredCsvOutputWriter();
+ var params = new HashMap<String, String>();
+ params.put(QP_CSV_DELIMITER, "semicolon");
+ writer.configure(null, new ProvidedRestQueryParams(null, params), true);
+ var columns = Arrays.asList("time", "string;header");
+ var row = Arrays.<Object>asList(1781515964488L, "abc=;,");
+
+ try (var outputStream = new ByteArrayOutputStream()) {
+ writer.beforeFirstItem(outputStream);
+
+ writer.writeItem(outputStream, row, columns, true);
+
+ writer.afterLastItem(outputStream);
+ assertEquals(
+ "time;\"string;header\"\n1781515964488;\"abc=;,\"\n",
+ outputStream.toString(StandardCharsets.UTF_8)
+ );
+ }
+ }
}
diff --git
a/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestCsvItemGenerator.java
b/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestCsvItemGenerator.java
index d1b088bcd3..2efa579c41 100644
---
a/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestCsvItemGenerator.java
+++
b/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestCsvItemGenerator.java
@@ -22,6 +22,8 @@ import
org.apache.streampipes.dataexplorer.export.item.CsvItemGenerator;
import org.junit.jupiter.api.Test;
+import java.util.Arrays;
+
import static org.junit.jupiter.api.Assertions.assertEquals;
public class TestCsvItemGenerator extends TestItemGenerator {
@@ -46,4 +48,36 @@ public class TestCsvItemGenerator extends TestItemGenerator {
assertEquals(ExpectedSemicolon, result);
}
+
+ @Test
+ public void testCsvItemWriterEscapesSpecialCharactersCommaSeparated() {
+ var writer = new CsvItemGenerator(",");
+ var row = Arrays.<Object>asList(
+ 1668578077051L,
+ "abc=;,",
+ "quoted \"value\"",
+ "line\nbreak"
+ );
+ var columns = Arrays.asList("time", "string", "quoted", "multiline");
+
+ String result = writer.createItem(row, columns);
+
+ assertEquals("1668578077051,\"abc=;,\",\"quoted
\"\"value\"\"\",\"line\nbreak\"", result);
+ }
+
+ @Test
+ public void testCsvItemWriterEscapesSpecialCharactersSemicolonSeparated() {
+ var writer = new CsvItemGenerator(";");
+ var row = Arrays.<Object>asList(
+ 1668578077051L,
+ "abc=;,",
+ "quoted \"value\"",
+ "line\nbreak"
+ );
+ var columns = Arrays.asList("time", "string", "quoted", "multiline");
+
+ String result = writer.createItem(row, columns);
+
+ assertEquals("1668578077051;\"abc=;,\";\"quoted
\"\"value\"\"\";\"line\nbreak\"", result);
+ }
}
diff --git
a/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/PropertyHandler.java
b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/PropertyHandler.java
index 388f9cedee..d64a106797 100644
---
a/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/PropertyHandler.java
+++
b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/PropertyHandler.java
@@ -100,7 +100,13 @@ public class PropertyHandler {
PrimitiveField primitiveField,
String sanitizedRuntimeName
) {
- point.tag(sanitizedRuntimeName, primitiveField.getAsString());
+ point.tag(sanitizedRuntimeName,
escapeSpecialCharacters(primitiveField.getAsString()));
+ }
+
+ private String escapeSpecialCharacters(String value) {
+ return value
+ .replace("\r", "\\r")
+ .replace("\n", "\\n");
}
/**
diff --git
a/streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx/TimeSeriesStorageInfluxTest.java
b/streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx/TimeSeriesStorageInfluxTest.java
index 9874f081c8..cd35280034 100644
---
a/streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx/TimeSeriesStorageInfluxTest.java
+++
b/streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx/TimeSeriesStorageInfluxTest.java
@@ -52,7 +52,9 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
public class TimeSeriesStorageInfluxTest {
@@ -188,6 +190,21 @@ public class TimeSeriesStorageInfluxTest {
assertEquals(expected, actualPoint);
}
+ @Test
+ public void onEventWithStringSpecialCharacters() {
+ var value = "abc=;,\nquoted \"value\"\r\n";
+ var expected = getPointBuilderWithTimestamp()
+ .addField(FIELD_NAME, value)
+ .build();
+
+ var actualPoint = testEventWithOneField(
+ XSD.STRING,
+ value
+ );
+
+ assertEquals(expected, actualPoint);
+ }
+
@Test
public void onEventReservedKeyWord() {
// name is a reserved keyword
@@ -249,6 +266,35 @@ public class TimeSeriesStorageInfluxTest {
assertEquals(expected, actualPoint);
}
+ @Test
+ public void onEventWithTagSpecialCharacters() {
+ var eventSchema = getEventSchemaBuilderWithTimestamp()
+ .withEventProperty(
+ EventPropertyPrimitiveTestBuilder
+ .create()
+ .withRuntimeName(FIELD_NAME)
+ .withRuntimeType(XSD.STRING)
+ .build())
+ .withEventProperty(
+ EventPropertyPrimitiveTestBuilder
+ .create()
+ .withRuntimeName("id")
+ .withRuntimeType(XSD.STRING)
+ .withPropertyScope(PropertyScope.DIMENSION_PROPERTY)
+ .build())
+ .build();
+
+ var event = getEvent(eventSchema, Map.of(FIELD_NAME, "value", "id",
"line\nbreak"));
+
+ var influxStore = getInfluxStore(eventSchema);
+
+ var actualPoint = executeOnEvent(influxStore, event);
+ var lineProtocol = actualPoint.lineProtocol(TimeUnit.MILLISECONDS);
+
+ assertFalse(lineProtocol.contains("\n"));
+ assertTrue(lineProtocol.contains(",id=line\\nbreak "));
+ }
+
@Test
public void onEventWithNestedProperty() {
