[
https://issues.apache.org/jira/browse/DRILL-4653?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15378336#comment-15378336
]
ASF GitHub Bot commented on DRILL-4653:
---------------------------------------
Github user chunhui-shi commented on a diff in the pull request:
https://github.com/apache/drill/pull/518#discussion_r70879554
--- Diff:
exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java
---
@@ -159,64 +164,91 @@ public void drill_3353() throws Exception {
test("create table dfs_test.tmp.drill_3353 as select a from
dfs.`${WORKING_PATH}/src/test/resources/jsoninput/drill_3353` where e = true");
String query = "select t.a.d cnt from dfs_test.tmp.drill_3353 t
where t.a.d is not null";
test(query);
- testBuilder()
- .sqlQuery(query)
- .unOrdered()
- .baselineColumns("cnt")
- .baselineValues("1")
- .go();
+ testBuilder().sqlQuery(query).unOrdered().baselineColumns("cnt")
+ .baselineValues("1").go();
} finally {
testNoResult("alter session set `store.json.all_text_mode` = false");
}
}
- @Test // See DRILL-3476
+ @Test
+ // See DRILL-3476
public void testNestedFilter() throws Exception {
String query = "select a from cp.`jsoninput/nestedFilter.json` t where
t.a.b = 1";
String baselineQuery = "select * from cp.`jsoninput/nestedFilter.json`
t where t.a.b = 1";
- testBuilder()
- .sqlQuery(query)
- .unOrdered()
- .sqlBaselineQuery(baselineQuery)
+
testBuilder().sqlQuery(query).unOrdered().sqlBaselineQuery(baselineQuery)
.go();
}
- @Test // See DRILL-4653
-public void testSkippingInvalidJSONRecords() throws Exception {
- try
- {
- String set = "alter session set `" +
ExecConstants.JSON_READER_SKIP_INVALID_RECORDS_FLAG+ "` = true";
- String query = "select count(*) from cp.`jsoninput/DRILL-4653.json`";
+ @Test
+ // See DRILL-4653
+ /* Test for CountingJSONReader */
+ public void testCountingQuerySkippingInvalidJSONRecords() throws
Exception {
+ try {
+ String set = "alter session set `"
+ + ExecConstants.JSON_READER_SKIP_INVALID_RECORDS_FLAG + "` =
true";
+ String set1 = "alter session set `"
+ + ExecConstants.JSON_READER_PRINT_INVALID_RECORDS_LINE_NOS_FLAG
+ + "` = true";
+ String query = "select count(*) from
cp.`jsoninput/drill4653/file.json`";
+ testNoResult(set);
+ testNoResult(set1);
+
testBuilder().unOrdered().sqlQuery(query).sqlBaselineQuery(query).build()
+ .run();
+ } finally {
+ String set = "alter session set `"
+ + ExecConstants.JSON_READER_SKIP_INVALID_RECORDS_FLAG + "` =
false";
testNoResult(set);
- testBuilder()
- .unOrdered()
- .sqlQuery(query)
- .sqlBaselineQuery(query)
- .build().run();
}
- finally
- {
- String set = "alter session set `" +
ExecConstants.JSON_READER_SKIP_INVALID_RECORDS_FLAG+ "` = false";
+ }
+
+ @Test
+ // See DRILL-4653
+ /* Test for CountingJSONReader */
+ public void testCountingQueryNotSkippingInvalidJSONRecords() throws
Exception {
+ try {
+ String query = "select count(*) from
cp.`jsoninput/drill4653/file.json`";
+
testBuilder().unOrdered().sqlQuery(query).sqlBaselineQuery(query).build()
--- End diff --
Should we compare with expected count here?
> Malformed JSON should not stop the entire query from progressing
> ----------------------------------------------------------------
>
> Key: DRILL-4653
> URL: https://issues.apache.org/jira/browse/DRILL-4653
> Project: Apache Drill
> Issue Type: Improvement
> Components: Storage - JSON
> Affects Versions: 1.6.0
> Reporter: subbu srinivasan
> Fix For: Future
>
>
> Currently Drill query terminates upon first encounter of a invalid JSON line.
> Drill has to continue progressing after ignoring the bad records. Something
> similar to a setting of (ignore.malformed.json) would help.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
