This is an automated email from the ASF dual-hosted git repository. volodymyr pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git
commit 807a22f6920e550649be9c1a7f836324eacbe9e9 Author: Arina Ielchiieva <[email protected]> AuthorDate: Wed Dec 11 17:26:48 2019 +0200 DRILL-7472: Fix ser / de for sys and information_schema schemas queries closes #1925 --- .../apache/drill/exec/physical/base/GroupScan.java | 2 +- .../exec/store/ischema/InfoSchemaGroupScan.java | 20 ++++++++-------- .../InfoSchemaPushFilterIntoRecordGenerator.java | 5 ++-- .../drill/exec/store/sys/SystemTableScan.java | 11 +++++---- .../test/java/org/apache/drill/PlanTestBase.java | 10 ++++++++ .../org/apache/drill/exec/sql/TestInfoSchema.java | 28 +++++++++++++++------- .../ischema/TestInfoSchemaFilterPushDown.java | 20 ++++++++-------- .../drill/exec/store/sys/TestSystemTable.java | 10 ++++++++ .../java/org/apache/drill/test/TestBuilder.java | 5 ++++ 9 files changed, 75 insertions(+), 36 deletions(-) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java index 72c26e9..40ce22f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java @@ -37,7 +37,7 @@ import org.apache.hadoop.fs.Path; /** * A GroupScan operator represents all data which will be scanned by a given physical - * plan. It is the superset of all SubScans for the plan. + * plan. It is the superset of all SubScans for the plan. */ public interface GroupScan extends Scan, HasAffinity { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaGroupScan.java index 7b44a4d..8ae95cf 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaGroupScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaGroupScan.java @@ -39,8 +39,6 @@ public class InfoSchemaGroupScan extends AbstractGroupScan { private final InfoSchemaTableType table; private final InfoSchemaFilter filter; - private boolean isFilterPushedDown = false; - public InfoSchemaGroupScan(InfoSchemaTableType table) { this(table, null); } @@ -57,7 +55,6 @@ public class InfoSchemaGroupScan extends AbstractGroupScan { super(that); this.table = that.table; this.filter = that.filter; - this.isFilterPushedDown = that.isFilterPushedDown; } @JsonProperty("table") @@ -70,6 +67,12 @@ public class InfoSchemaGroupScan extends AbstractGroupScan { return filter; } + @JsonIgnore + @Override + public List<SchemaPath> getColumns() { + return super.getColumns(); + } + @Override public void applyAssignments(List<DrillbitEndpoint> endpoints) { Preconditions.checkArgument(endpoints.size() == 1); @@ -81,7 +84,8 @@ public class InfoSchemaGroupScan extends AbstractGroupScan { return new InfoSchemaSubScan(table, filter); } - public ScanStats getScanStats(){ + @Override + public ScanStats getScanStats() { if (filter == null) { return ScanStats.TRIVIAL_TABLE; } else { @@ -103,7 +107,7 @@ public class InfoSchemaGroupScan extends AbstractGroupScan { @Override public String getDigest() { - return this.table.toString() + ", filter=" + filter; + return table.toString() + ", filter=" + filter; } @Override @@ -111,12 +115,8 @@ public class InfoSchemaGroupScan extends AbstractGroupScan { return new InfoSchemaGroupScan(this); } - public void setFilterPushedDown(boolean status) { - this.isFilterPushedDown = status; - } - @JsonIgnore public boolean isFilterPushedDown() { - return isFilterPushedDown; + return filter != null; } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaPushFilterIntoRecordGenerator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaPushFilterIntoRecordGenerator.java index 2ce0cc1..e53f7fb 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaPushFilterIntoRecordGenerator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaPushFilterIntoRecordGenerator.java @@ -80,7 +80,7 @@ public abstract class InfoSchemaPushFilterIntoRecordGenerator extends StoragePlu } protected void doMatch(RelOptRuleCall call, ScanPrel scan, ProjectPrel project, FilterPrel filter) { - final RexNode condition = filter.getCondition(); + RexNode condition = filter.getCondition(); InfoSchemaGroupScan groupScan = (InfoSchemaGroupScan) scan.getGroupScan(); if (groupScan.isFilterPushedDown()) { @@ -95,8 +95,7 @@ public abstract class InfoSchemaPushFilterIntoRecordGenerator extends StoragePlu return; //no filter push down ==> No transformation. } - final InfoSchemaGroupScan newGroupsScan = new InfoSchemaGroupScan(groupScan.getTable(), infoSchemaFilter); - newGroupsScan.setFilterPushedDown(true); + InfoSchemaGroupScan newGroupsScan = new InfoSchemaGroupScan(groupScan.getTable(), infoSchemaFilter); RelNode input = new ScanPrel(scan.getCluster(), filter.getTraitSet(), newGroupsScan, scan.getRowType(), scan.getTable()); if (project != null) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTableScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTableScan.java index 787e243..50d65a2 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTableScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTableScan.java @@ -42,7 +42,6 @@ import org.apache.drill.exec.store.StoragePluginRegistry; @JsonTypeName("sys") public class SystemTableScan extends AbstractGroupScan implements SubScan { - // private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SystemTableScan.class); private final SystemTable table; private final SystemTablePlugin plugin; @@ -51,7 +50,6 @@ public class SystemTableScan extends AbstractGroupScan implements SubScan { @JsonCreator public SystemTableScan(@JsonProperty("table") SystemTable table, @JsonProperty("maxRecordsToRead") int maxRecordsToRead, - @JsonProperty("columns") List<SchemaPath> columns, @JacksonInject StoragePluginRegistry engineRegistry) throws ExecutionSetupException { this(table, maxRecordsToRead, (SystemTablePlugin) engineRegistry.getPlugin(SystemTablePluginConfig.INSTANCE)); } @@ -85,6 +83,12 @@ public class SystemTableScan extends AbstractGroupScan implements SubScan { public void applyAssignments(List<DrillbitEndpoint> endpoints) { } + @JsonIgnore + @Override + public List<SchemaPath> getColumns() { + return super.getColumns(); + } + @Override public SubScan getSpecificScan(int minorFragmentId) { return this; @@ -178,7 +182,7 @@ public class SystemTableScan extends AbstractGroupScan implements SubScan { @Override public GroupScan applyLimit(int maxRecords) { - if (this.maxRecordsToRead == maxRecords) { + if (maxRecordsToRead == maxRecords) { return null; } return clone(this, maxRecords); @@ -196,5 +200,4 @@ public class SystemTableScan extends AbstractGroupScan implements SubScan { public SystemTablePlugin getPlugin() { return plugin; } - } diff --git a/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java b/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java index cba86ed..a2764c2 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java +++ b/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java @@ -72,6 +72,16 @@ public class PlanTestBase extends BaseTestQuery { } /** + * Converts given sql query into physical JSON plan representation. + * + * @param sql sql query + * @return physical plan in JSON format + */ + public static String getPhysicalJsonPlan(String sql) throws Exception { + return getPlanInString("EXPLAIN PLAN for " + sql, JSON_FORMAT); + } + + /** * Runs an explain plan query and check for expected regex patterns (in optiq * text format), also ensure excluded patterns are not found. Either list can * be empty or null to skip that part of the check. diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java index f55ee1a..509809c 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java @@ -18,6 +18,7 @@ package org.apache.drill.exec.sql; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.drill.PlanTestBase; import org.apache.drill.categories.SqlTest; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.exec.record.RecordBatchLoader; @@ -435,25 +436,25 @@ public class TestInfoSchema extends BaseTestQuery { @Test public void describeSchemaOutput() throws Exception { - final List<QueryDataBatch> result = testSqlWithResults("describe schema dfs.tmp"); + List<QueryDataBatch> result = testSqlWithResults("describe schema dfs.tmp"); assertEquals(1, result.size()); - final QueryDataBatch batch = result.get(0); - final RecordBatchLoader loader = new RecordBatchLoader(getDrillbitContext().getAllocator()); + QueryDataBatch batch = result.get(0); + RecordBatchLoader loader = new RecordBatchLoader(getDrillbitContext().getAllocator()); loader.load(batch.getHeader().getDef(), batch.getData()); // check schema column value - final VectorWrapper schemaValueVector = loader.getValueAccessorById( + VectorWrapper<?> schemaValueVector = loader.getValueAccessorById( NullableVarCharVector.class, loader.getValueVectorId(SchemaPath.getCompoundPath("schema")).getFieldIds()); String schema = schemaValueVector.getValueVector().getAccessor().getObject(0).toString(); assertEquals("dfs.tmp", schema); // check properties column value - final VectorWrapper propertiesValueVector = loader.getValueAccessorById( + VectorWrapper<?> propertiesValueVector = loader.getValueAccessorById( NullableVarCharVector.class, loader.getValueVectorId(SchemaPath.getCompoundPath("properties")).getFieldIds()); String properties = propertiesValueVector.getValueVector().getAccessor().getObject(0).toString(); - final Map configMap = mapper.readValue(properties, Map.class); + Map<?, ?> configMap = mapper.readValue(properties, Map.class); // check some stable properties existence assertTrue(configMap.containsKey("connection")); @@ -464,8 +465,8 @@ public class TestInfoSchema extends BaseTestQuery { // check some stable properties values assertEquals("file", configMap.get("type")); - final FileSystemConfig testConfig = (FileSystemConfig) bits[0].getContext().getStorage().getPlugin("dfs").getConfig(); - final String tmpSchemaLocation = testConfig.getWorkspaces().get("tmp").getLocation(); + FileSystemConfig testConfig = (FileSystemConfig) bits[0].getContext().getStorage().getPlugin("dfs").getConfig(); + String tmpSchemaLocation = testConfig.getWorkspaces().get("tmp").getLocation(); assertEquals(tmpSchemaLocation, configMap.get("location")); batch.release(); @@ -483,4 +484,15 @@ public class TestInfoSchema extends BaseTestQuery { test("desc information_schema.`catalogs`"); test("desc table information_schema.`catalogs`"); } + + @Test + public void testSerDe() throws Exception { + String sql = "select * from information_schema.`tables` where table_name = 'schemata' order by 1"; + + testBuilder() + .sqlQuery(sql) + .unOrdered() + .physicalPlanBaseline(PlanTestBase.getPhysicalJsonPlan(sql)) + .go(); + } } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/TestInfoSchemaFilterPushDown.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/TestInfoSchemaFilterPushDown.java index a25bd1f..04a7635 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/TestInfoSchemaFilterPushDown.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/TestInfoSchemaFilterPushDown.java @@ -25,7 +25,7 @@ public class TestInfoSchemaFilterPushDown extends PlanTestBase { @Test public void testFilterPushdown_Equal() throws Exception { final String query = "SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_SCHEMA='INFORMATION_SCHEMA'"; - final String scan = "Scan.*groupscan=\\[TABLES, filter=equal\\(Field=TABLE_SCHEMA,Literal=INFORMATION_SCHEMA\\)\\]"; + final String scan = "Scan.*groupscan=\\[TABLES, filter=equal\\(Field=TABLE_SCHEMA,Literal=INFORMATION_SCHEMA\\)"; testHelper(query, scan, false); } @@ -33,7 +33,7 @@ public class TestInfoSchemaFilterPushDown extends PlanTestBase { @Test public void testFilterPushdown_NonEqual() throws Exception { final String query = "SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_SCHEMA <> 'INFORMATION_SCHEMA'"; - final String scan = "Scan.*groupscan=\\[TABLES, filter=not_equal\\(Field=TABLE_SCHEMA,Literal=INFORMATION_SCHEMA\\)\\]"; + final String scan = "Scan.*groupscan=\\[TABLES, filter=not_equal\\(Field=TABLE_SCHEMA,Literal=INFORMATION_SCHEMA\\)"; testHelper(query, scan, false); } @@ -41,7 +41,7 @@ public class TestInfoSchemaFilterPushDown extends PlanTestBase { @Test public void testFilterPushdown_Like() throws Exception { final String query = "SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_SCHEMA LIKE '%SCH%'"; - final String scan = "Scan.*groupscan=\\[TABLES, filter=like\\(Field=TABLE_SCHEMA,Literal=%SCH%\\)\\]"; + final String scan = "Scan.*groupscan=\\[TABLES, filter=like\\(Field=TABLE_SCHEMA,Literal=%SCH%\\)"; testHelper(query, scan, false); } @@ -49,7 +49,7 @@ public class TestInfoSchemaFilterPushDown extends PlanTestBase { @Test public void testFilterPushdown_LikeWithEscape() throws Exception { final String query = "SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_SCHEMA LIKE '%\\\\SCH%' ESCAPE '\\'"; - final String scan = "Scan.*groupscan=\\[TABLES, filter=like\\(Field=TABLE_SCHEMA,Literal=%\\\\\\\\SCH%,Literal=\\\\\\)\\]"; + final String scan = "Scan.*groupscan=\\[TABLES, filter=like\\(Field=TABLE_SCHEMA,Literal=%\\\\\\\\SCH%,Literal=\\\\\\)"; testHelper(query, scan, false); } @@ -60,7 +60,7 @@ public class TestInfoSchemaFilterPushDown extends PlanTestBase { "TABLE_SCHEMA = 'sys' AND " + "TABLE_NAME <> 'version'"; final String scan = "Scan.*groupscan=\\[COLUMNS, filter=booleanand\\(equal\\(Field=TABLE_SCHEMA,Literal=sys\\)," + - "not_equal\\(Field=TABLE_NAME,Literal=version\\)\\)\\]"; + "not_equal\\(Field=TABLE_NAME,Literal=version\\)\\)"; testHelper(query, scan, false); } @@ -72,7 +72,7 @@ public class TestInfoSchemaFilterPushDown extends PlanTestBase { "TABLE_NAME <> 'version' OR " + "TABLE_SCHEMA like '%sdfgjk%'"; final String scan = "Scan.*groupscan=\\[COLUMNS, filter=booleanor\\(equal\\(Field=TABLE_SCHEMA,Literal=sys\\)," + - "not_equal\\(Field=TABLE_NAME,Literal=version\\),like\\(Field=TABLE_SCHEMA,Literal=%sdfgjk%\\)\\)\\]"; + "not_equal\\(Field=TABLE_NAME,Literal=version\\),like\\(Field=TABLE_SCHEMA,Literal=%sdfgjk%\\)\\)"; testHelper(query, scan, false); } @@ -80,21 +80,21 @@ public class TestInfoSchemaFilterPushDown extends PlanTestBase { @Test public void testFilterPushDownWithProject_Equal() throws Exception { final String query = "SELECT COLUMN_NAME from INFORMATION_SCHEMA.`COLUMNS` WHERE TABLE_SCHEMA = 'INFORMATION_SCHEMA'"; - final String scan = "Scan.*groupscan=\\[COLUMNS, filter=equal\\(Field=TABLE_SCHEMA,Literal=INFORMATION_SCHEMA\\)\\]"; + final String scan = "Scan.*groupscan=\\[COLUMNS, filter=equal\\(Field=TABLE_SCHEMA,Literal=INFORMATION_SCHEMA\\)"; testHelper(query, scan, false); } @Test public void testFilterPushDownWithProject_NotEqual() throws Exception { final String query = "SELECT COLUMN_NAME from INFORMATION_SCHEMA.`COLUMNS` WHERE TABLE_NAME <> 'TABLES'"; - final String scan = "Scan.*groupscan=\\[COLUMNS, filter=not_equal\\(Field=TABLE_NAME,Literal=TABLES\\)\\]"; + final String scan = "Scan.*groupscan=\\[COLUMNS, filter=not_equal\\(Field=TABLE_NAME,Literal=TABLES\\)"; testHelper(query, scan, false); } @Test public void testFilterPushDownWithProject_Like() throws Exception { final String query = "SELECT COLUMN_NAME from INFORMATION_SCHEMA.`COLUMNS` WHERE TABLE_NAME LIKE '%BL%'"; - final String scan = "Scan.*groupscan=\\[COLUMNS, filter=like\\(Field=TABLE_NAME,Literal=%BL%\\)\\]"; + final String scan = "Scan.*groupscan=\\[COLUMNS, filter=like\\(Field=TABLE_NAME,Literal=%BL%\\)"; testHelper(query, scan, false); } @@ -107,7 +107,7 @@ public class TestInfoSchemaFilterPushDown extends PlanTestBase { "IS_NULLABLE = 'YES'"; // this is not expected to pushdown into scan final String scan = "Scan.*groupscan=\\[COLUMNS, " + "filter=booleanand\\(equal\\(Field=TABLE_SCHEMA,Literal=sys\\),equal\\(Field=TABLE_NAME,Literal=version\\)," + - "like\\(Field=COLUMN_NAME,Literal=commit%s\\)\\)\\]"; + "like\\(Field=COLUMN_NAME,Literal=commit%s\\)\\)"; testHelper(query, scan, true); } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java index 2ed5f6e..0fbd242 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java @@ -62,6 +62,16 @@ public class TestSystemTable extends PlanTestBase { } @Test + public void testSerDe() throws Exception { + String sql = "select * from sys.functions where name = 'avg' limit 100"; + testBuilder() + .sqlQuery(sql) + .unOrdered() + .physicalPlanBaseline(PlanTestBase.getPhysicalJsonPlan(sql)) + .go(); + } + + @Test public void threadsTable() throws Exception { test("select * from sys.threads"); } diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/TestBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/TestBuilder.java index 0b88670..029d259 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/TestBuilder.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/TestBuilder.java @@ -441,6 +441,11 @@ public class TestBuilder { baselineTypeMap, baselineOptionSettingQueries, testOptionSettingQueries, highPerformanceComparison, expectedNumBatches); } + public BaselineQueryTestBuilder physicalPlanBaseline(String physicalPlan) { + return new BaselineQueryTestBuilder(physicalPlan, UserBitShared.QueryType.PHYSICAL, services, query, queryType, ordered, approximateEquality, + baselineTypeMap, baselineOptionSettingQueries, testOptionSettingQueries, highPerformanceComparison, expectedNumBatches); + } + private String getDecimalPrecisionScaleInfo(TypeProtos.MajorType type) { String precision = ""; switch(type.getMinorType()) {
