This is an automated email from the ASF dual-hosted git repository.
karan pushed a commit to branch 25.0.0
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/25.0.0 by this push:
new 8bf4b68b1a Convert errors based on implicit type conversion in multi
value arrays to parse exception in MSQ (#13366) (#13454)
8bf4b68b1a is described below
commit 8bf4b68b1a2f84cb6737f6ee50a7fab99c0673aa
Author: Laksh Singla <[email protected]>
AuthorDate: Wed Nov 30 15:38:35 2022 +0530
Convert errors based on implicit type conversion in multi value arrays to
parse exception in MSQ (#13366) (#13454)
* initial commit
* fix test
* push the json changes
* reduce the area of the try..catch
* Trigger Build
* review
---
.../org/apache/druid/msq/exec/MSQSelectTest.java | 58 ++++++++++++++++++++++
.../resources/unparseable-mv-string-array.json | 3 ++
.../druid/frame/write/RowBasedFrameWriter.java | 10 +++-
.../frame/write/columnar/ColumnarFrameWriter.java | 12 +++--
4 files changed, 78 insertions(+), 5 deletions(-)
diff --git
a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java
b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java
index c83481757d..26d566b31e 100644
---
a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java
+++
b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java
@@ -31,6 +31,7 @@ import org.apache.druid.msq.indexing.ColumnMapping;
import org.apache.druid.msq.indexing.ColumnMappings;
import org.apache.druid.msq.indexing.MSQSpec;
import org.apache.druid.msq.indexing.MSQTuningConfig;
+import org.apache.druid.msq.indexing.error.CannotParseExternalDataFault;
import org.apache.druid.msq.shuffle.DurableStorageUtils;
import org.apache.druid.msq.test.MSQTestBase;
import org.apache.druid.query.InlineDataSource;
@@ -1253,6 +1254,63 @@ public class MSQSelectTest extends MSQTestBase
.write(ArgumentMatchers.endsWith("__success"));
}
+ @Test
+ public void testMultiValueStringWithIncorrectType() throws IOException
+ {
+ final File toRead =
getResourceAsTemporaryFile("/unparseable-mv-string-array.json");
+ final String toReadAsJson =
queryFramework().queryJsonMapper().writeValueAsString(toRead.getAbsolutePath());
+
+ RowSignature rowSignature = RowSignature.builder()
+ .add("__time", ColumnType.LONG)
+ .add("language",
ColumnType.STRING_ARRAY)
+ .build();
+
+ final GroupByQuery expectedQuery =
+ GroupByQuery.builder()
+ .setDataSource(CalciteTests.DATASOURCE1)
+ .setInterval(querySegmentSpec(Filtration.eternity()))
+ .setGranularity(Granularities.ALL)
+ .setDimensions(dimensions(new
DefaultDimensionSpec("__time", "d0", ColumnType.LONG)))
+ .build();
+
+
+ testSelectQuery()
+ .setSql("WITH\n"
+ + "kttm_data AS (\n"
+ + "SELECT * FROM TABLE(\n"
+ + " EXTERN(\n"
+ + " '{ \"files\": [" + toReadAsJson +
"],\"type\":\"local\"}',\n"
+ + " '{\"type\":\"json\"}',\n"
+ + "
'[{\"name\":\"timestamp\",\"type\":\"string\"},{\"name\":\"agent_category\",\"type\":\"string\"},{\"name\":\"agent_type\",\"type\":\"string\"},{\"name\":\"browser\",\"type\":\"string\"},{\"name\":\"browser_version\",\"type\":\"string\"},{\"name\":\"city\",\"type\":\"string\"},{\"name\":\"continent\",\"type\":\"string\"},{\"name\":\"country\",\"type\":\"string\"},{\"name\":\"version\",\"type\":\"string\"},{\"name\":\"event_type\",\"type\":\"string\"},{\"name\":\"eve
[...]
+ + " )\n"
+ + "))\n"
+ + "\n"
+ + "SELECT\n"
+ + " FLOOR(TIME_PARSE(\"timestamp\") TO MINUTE) AS __time,\n"
+ + " MV_TO_ARRAY(\"language\") AS \"language\"\n"
+ + "FROM kttm_data")
+ .setExpectedRowSignature(rowSignature)
+ .setExpectedResultRows(ImmutableList.of(
+ new Object[]{1566691200000L, ImmutableList.of("en")},
+ new Object[]{1566691200000L, ImmutableList.of("en", "es",
"es-419", "es-MX")},
+ new Object[]{1566691200000L, ImmutableList.of("en", "es",
"es-419", "es-US")}
+ ))
+ .setExpectedMSQSpec(
+ MSQSpec
+ .builder()
+ .query(expectedQuery)
+ .columnMappings(new ColumnMappings(
+ ImmutableList.of(
+ new ColumnMapping("d0", "__time"),
+ new ColumnMapping("a0", "cnt")
+ )
+ ))
+ .tuningConfig(MSQTuningConfig.defaultConfig())
+ .build())
+ .setExpectedMSQFault(new CannotParseExternalDataFault(
+ "Unable to add the row to the frame. Type conversion might be
required."))
+ .verifyResults();
+ }
@Nonnull
private List<Object[]> expectedMultiValueFooRowsGroup()
diff --git
a/extensions-core/multi-stage-query/src/test/resources/unparseable-mv-string-array.json
b/extensions-core/multi-stage-query/src/test/resources/unparseable-mv-string-array.json
new file mode 100644
index 0000000000..57b9a7709b
--- /dev/null
+++
b/extensions-core/multi-stage-query/src/test/resources/unparseable-mv-string-array.json
@@ -0,0 +1,3 @@
+{"timestamp":"2019-08-25T00:00:00.031Z","agent_category":"Personal
computer","agent_type":"Browser","browser":"Chrome","browser_version":"76.0.3809.100","city":"Rosario","continent":"South
America","country":"Argentina","version":"1.9.6","event_type":"PercentClear","event_subtype":"55","loaded_image":"http://www.koalastothemax.com/img/koalas2.jpg","adblock_list":"NoAdblock","forwarded_for":"181.13.41.82","language":[{},{}],"number":"16","os":"Windows
7","path":"http://www.koalastothemax. [...]
+{"timestamp":"2019-08-25T00:00:00.059Z","agent_category":"Smartphone","agent_type":"Mobile
Browser","browser":"Chrome
Mobile","browser_version":"50.0.2661.89","city":"Nuevo Casas
Grandes","continent":"North
America","country":"Mexico","version":"1.9.6","event_type":"PercentClear","event_subtype":"85","loaded_image":"https://koalastothemax.com/img/koalas1.jpg","adblock_list":"NoAdblock","forwarded_for":"177.242.100.0","language":["en","es","es-419","es-MX"],"number":"24","os":"Android","p
[...]
+{"timestamp":"2019-08-25T00:00:00.178Z","agent_category":"Personal
computer","agent_type":"Browser","browser":"Chrome","browser_version":"76.0.3809.100","city":"Luis
Guillon","continent":"South
America","country":"Argentina","version":"1.9.6","event_type":"PercentClear","event_subtype":"90","loaded_image":"http://www.koalastothemax.com/img/koalas.jpg","adblock_list":"NoAdblock","forwarded_for":"181.46.136.44","language":["en","es","es-419","es-US"],"number":"24","os":"Windows
7","path":" [...]
diff --git
a/processing/src/main/java/org/apache/druid/frame/write/RowBasedFrameWriter.java
b/processing/src/main/java/org/apache/druid/frame/write/RowBasedFrameWriter.java
index 2bf4b34308..2a8af4b9e8 100644
---
a/processing/src/main/java/org/apache/druid/frame/write/RowBasedFrameWriter.java
+++
b/processing/src/main/java/org/apache/druid/frame/write/RowBasedFrameWriter.java
@@ -32,6 +32,7 @@ import org.apache.druid.frame.read.FrameReader;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.io.Closer;
+import org.apache.druid.java.util.common.parsers.ParseException;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.utils.CloseableUtils;
@@ -118,8 +119,13 @@ public class RowBasedFrameWriter implements FrameWriter
return false;
}
- if (!writeData()) {
- return false;
+ try {
+ if (!writeData()) {
+ return false;
+ }
+ }
+ catch (Exception e) {
+ throw new ParseException("", e, "Unable to add the row to the frame.
Type conversion might be required.");
}
final MemoryRange<WritableMemory> rowOffsetCursor =
rowOffsetMemory.cursor();
diff --git
a/processing/src/main/java/org/apache/druid/frame/write/columnar/ColumnarFrameWriter.java
b/processing/src/main/java/org/apache/druid/frame/write/columnar/ColumnarFrameWriter.java
index 76f5a17d27..eff5213ae1 100644
---
a/processing/src/main/java/org/apache/druid/frame/write/columnar/ColumnarFrameWriter.java
+++
b/processing/src/main/java/org/apache/druid/frame/write/columnar/ColumnarFrameWriter.java
@@ -30,6 +30,7 @@ import org.apache.druid.frame.write.FrameSort;
import org.apache.druid.frame.write.FrameWriter;
import org.apache.druid.frame.write.FrameWriterUtils;
import org.apache.druid.java.util.common.ISE;
+import org.apache.druid.java.util.common.parsers.ParseException;
import org.apache.druid.segment.column.RowSignature;
import javax.annotation.Nullable;
@@ -74,11 +75,16 @@ public class ColumnarFrameWriter implements FrameWriter
}
int i = 0;
- for (; i < columnWriters.size(); i++) {
- if (!columnWriters.get(i).addSelection()) {
- break;
+ try {
+ for (; i < columnWriters.size(); i++) {
+ if (!columnWriters.get(i).addSelection()) {
+ break;
+ }
}
}
+ catch (Exception e) {
+ throw new ParseException("", e, "Unable to add the row to the frame.
Type conversion might be required.");
+ }
if (i < columnWriters.size()) {
// Add failed, clean up.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
![http://www.koalastothemax.com/img/koalas2.jpg","adblock_list":"NoAdblock","forwarded_for":"181.13.41.82","language":[{},{}],"number":"16","os":"Windows](http://www.koalastothemax.com/img/koalas2.jpg","adblock_list":"NoAdblock","forwarded_for":"181.13.41.82","language":[{},{}],"number":"16","os":"Windows){kind=link}
![https://koalastothemax.com/img/koalas1.jpg","adblock_list":"NoAdblock","forwarded_for":"177.242.100.0","language":["en","es","es-419","es-MX"],"number":"24","os":"Android","p](https://koalastothemax.com/img/koalas1.jpg","adblock_list":"NoAdblock","forwarded_for":"177.242.100.0","language":["en","es","es-419","es-MX"],"number":"24","os":"Android","p){kind=link}
![http://www.koalastothemax.com/img/koalas.jpg","adblock_list":"NoAdblock","forwarded_for":"181.46.136.44","language":["en","es","es-419","es-US"],"number":"24","os":"Windows](http://www.koalastothemax.com/img/koalas.jpg","adblock_list":"NoAdblock","forwarded_for":"181.46.136.44","language":["en","es","es-419","es-US"],"number":"24","os":"Windows){kind=link}