This is an automated email from the ASF dual-hosted git repository. volodymyr pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git
commit 7bac742aa0ba8289325fd7c71bae3236087a6e48 Author: Volodymyr Vysotskyi <[email protected]> AuthorDate: Fri May 15 14:30:47 2020 +0300 DRILL-7741: Columns are missing when using convert_from function closes #2081 --- .../physical/impl/project/ProjectRecordBatch.java | 10 ++++++++++ .../exec/vector/complex/writer/TestJsonReader.java | 22 ++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java index 62bd0c6..948f283 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java @@ -332,6 +332,16 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> { return IterOutcome.OK_NEW_SCHEMA; } + @Override + protected IterOutcome getFinalOutcome(boolean hasMoreRecordInBoundary) { + // In a case of complex writers vectors are added at runtime, so the schema + // may change (e.g. when a batch contains new column(s) not present in previous batches) + if (complexWriters != null) { + return IterOutcome.OK_NEW_SCHEMA; + } + return super.getFinalOutcome(hasMoreRecordInBoundary); + } + private void setupNewSchema(RecordBatch incomingBatch, int configuredBatchSize) { memoryManager = new ProjectMemoryManager(configuredBatchSize); memoryManager.init(incomingBatch, ProjectRecordBatch.this); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java index 04bc67d..3fae12b 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java @@ -782,4 +782,26 @@ public class TestJsonReader extends BaseTestQuery { resetSessionOption(ExecConstants.ENABLE_UNION_TYPE_KEY); } } + + @Test + public void testConvertFromJson() throws Exception { + String fileName = "table.tsv"; + try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dirTestWatcher.getRootDir(), fileName)))) { + for (int i = 0; i < JSONRecordReader.DEFAULT_ROWS_PER_BATCH; i++) { + writer.write("{\"id\":\"1\"}\n"); + } + writer.write("{\"id\":\"2\",\"v\":[\"abc\"]}"); + } + + String sql = "SELECT t.m.id AS id, t.m.v[0] v FROM \n" + + "(SELECT convert_from(columns[0], 'json') AS m FROM dfs.`%s`) t\n" + + "where t.m.id='2'"; + + testBuilder() + .sqlQuery(sql, fileName) + .unOrdered() + .baselineColumns("id", "v") + .baselineValues("2", "abc") + .go(); + } }
