This is an automated email from the ASF dual-hosted git repository. dzamo pushed a commit to branch 1.20 in repository https://gitbox.apache.org/repos/asf/drill.git
commit 94917849f93465e19726a29e0fbbc76a3483fcee Author: James Turton <[email protected]> AuthorDate: Fri Jul 8 08:03:26 2022 +0200 DRILL-8182: File scan nodes not differentiated by format config (#2583) --- .../drill/exec/store/excel/TestExcelFormat.java | 30 +++++++++++++ .../resources/excel/test_cross_sheet_join.xlsx | Bin 0 -> 6426 bytes .../cassandra/schema/CassandraDynamicTable.java | 3 +- .../drill/exec/store/druid/DruidScanSpec.java | 8 +++- .../schema/ElasticsearchDynamicTable.java | 3 +- .../exec/store/hbase/AbstractHBaseDrillTable.java | 3 +- .../drill/exec/store/hbase/HBaseScanSpec.java | 20 ++++++--- .../drill/hbase/TestHBaseFilterPushDown.java | 44 +++++++++--------- .../drill/exec/store/hive/HiveReadEntry.java | 17 ++++++- .../apache/drill/exec/store/http/HttpScanSpec.java | 8 +++- .../drill/exec/store/kafka/KafkaScanSpec.java | 13 +++++- .../apache/drill/exec/store/kudu/KuduScanSpec.java | 16 ++++++- .../drill/exec/store/mongo/MongoScanSpec.java | 8 +++- .../exec/store/openTSDB/OpenTSDBScanSpec.java | 15 +++++-- .../drill/exec/store/splunk/SplunkScanSpec.java | 8 +++- .../planner/FileSystemPartitionDescriptor.java | 10 +++-- .../drill/exec/planner/logical/DrillTable.java | 17 ++++--- .../exec/planner/logical/DrillTableSelection.java | 36 +++++++++++++++ .../exec/planner/logical/DynamicDrillTable.java | 6 +-- ...TableScan.java => SelectionBasedTableScan.java} | 16 +++---- .../planner/logical/partition/PruneScanRule.java | 5 +++ .../apache/drill/exec/store/dfs/FileSelection.java | 8 +++- .../drill/exec/store/dfs/FormatSelection.java | 13 +++++- .../drill/exec/store/dfs/easy/EasyGroupScan.java | 8 ++++ .../exec/store/ischema/InfoSchemaTableType.java | 8 +++- .../drill/exec/store/mock/MockStorageEngine.java | 21 ++++----- .../apache/drill/exec/store/mock/MockTableDef.java | 30 +++++++++++-- .../exec/store/plan/rel/PluginDrillTable.java | 3 +- .../drill/exec/store/sys/StaticDrillTable.java | 3 +- .../apache/drill/exec/store/sys/SystemTable.java | 8 +++- .../exec/store/enumerable/plan/EnumMockTable.java | 3 +- .../conv/conversionTestWithLogicalPlan.json | 50 +++++++++++---------- .../src/test/resources/scan_screen_logical.json | 18 ++++---- pom.xml | 2 +- 34 files changed, 342 insertions(+), 119 deletions(-) diff --git a/contrib/format-excel/src/test/java/org/apache/drill/exec/store/excel/TestExcelFormat.java b/contrib/format-excel/src/test/java/org/apache/drill/exec/store/excel/TestExcelFormat.java index 9faf64e95e..f4fc80a80f 100644 --- a/contrib/format-excel/src/test/java/org/apache/drill/exec/store/excel/TestExcelFormat.java +++ b/contrib/format-excel/src/test/java/org/apache/drill/exec/store/excel/TestExcelFormat.java @@ -780,4 +780,34 @@ public class TestExcelFormat extends ClusterTest { new RowSetComparison(expected).verifyAndClearAll(results); } + + // DRILL-8182 + @Test + public void testTableFuncsThatDifferOnlyByFormatConfig() throws Exception { + String sql = "WITH prod AS (" + + " SELECT id, name FROM table(cp.`excel/test_cross_sheet_join.xlsx` (type=> 'excel', sheetName => 'products'))" + + "), cust AS (" + + " SELECT id, name FROM table(cp.`excel/test_cross_sheet_join.xlsx` (type=> 'excel', sheetName => 'customers'))" + + ")" + + "SELECT prod.*, cust.* from prod JOIN cust ON prod.id = cust.id"; + + RowSet results = client.queryBuilder().sql(sql).rowSet(); + + TupleMetadata expectedSchema = new SchemaBuilder() + .addNullable("id", MinorType.FLOAT8) + .addNullable("name", MinorType.VARCHAR) + .addNullable("id0", MinorType.FLOAT8) + .addNullable("name0", MinorType.VARCHAR) + .buildSchema(); + + RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema) + .addRow(1.0, "Doughnut", 1.0, "Alice") + .addRow(2.0, "Coffee", 2.0, "Bob") + .addRow(3.0, "Coke", 3.0, "Carol") + .addRow(4.0, "Cheesecake", 4.0, "Dave") + .addRow(5.0, "Popsicle", 5.0, "Eve") + .build(); + + new RowSetComparison(expected).verifyAndClearAll(results); + } } diff --git a/contrib/format-excel/src/test/resources/excel/test_cross_sheet_join.xlsx b/contrib/format-excel/src/test/resources/excel/test_cross_sheet_join.xlsx new file mode 100644 index 0000000000..b057361229 Binary files /dev/null and b/contrib/format-excel/src/test/resources/excel/test_cross_sheet_join.xlsx differ diff --git a/contrib/storage-cassandra/src/main/java/org/apache/drill/exec/store/cassandra/schema/CassandraDynamicTable.java b/contrib/storage-cassandra/src/main/java/org/apache/drill/exec/store/cassandra/schema/CassandraDynamicTable.java index 8b1d58d899..7d5c4022ba 100644 --- a/contrib/storage-cassandra/src/main/java/org/apache/drill/exec/store/cassandra/schema/CassandraDynamicTable.java +++ b/contrib/storage-cassandra/src/main/java/org/apache/drill/exec/store/cassandra/schema/CassandraDynamicTable.java @@ -34,6 +34,7 @@ import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.schema.TranslatableTable; import org.apache.calcite.schema.Wrapper; import org.apache.drill.exec.planner.logical.DrillTable; +import org.apache.drill.exec.planner.logical.DrillTableSelection; import org.apache.drill.exec.store.StoragePlugin; import java.lang.reflect.Type; @@ -44,7 +45,7 @@ public class CassandraDynamicTable extends DrillTable implements TranslatableTab private final CassandraTable table; - public CassandraDynamicTable(StoragePlugin plugin, String storageEngineName, Object selection, CassandraTable table) { + public CassandraDynamicTable(StoragePlugin plugin, String storageEngineName, DrillTableSelection selection, CassandraTable table) { super(storageEngineName, plugin, selection); this.table = table; } diff --git a/contrib/storage-druid/src/main/java/org/apache/drill/exec/store/druid/DruidScanSpec.java b/contrib/storage-druid/src/main/java/org/apache/drill/exec/store/druid/DruidScanSpec.java index dcd74315ef..f4e6114664 100755 --- a/contrib/storage-druid/src/main/java/org/apache/drill/exec/store/druid/DruidScanSpec.java +++ b/contrib/storage-druid/src/main/java/org/apache/drill/exec/store/druid/DruidScanSpec.java @@ -21,9 +21,10 @@ package org.apache.drill.exec.store.druid; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.drill.common.PlanStringBuilder; +import org.apache.drill.exec.planner.logical.DrillTableSelection; import org.apache.drill.exec.store.druid.common.DruidFilter; -public class DruidScanSpec { +public class DruidScanSpec implements DrillTableSelection { private final String dataSourceName; private final long dataSourceSize; @@ -84,4 +85,9 @@ public class DruidScanSpec { .field("filter", filter) .toString(); } + + @Override + public String digest() { + return toString(); + } } diff --git a/contrib/storage-elasticsearch/src/main/java/org/apache/drill/exec/store/elasticsearch/schema/ElasticsearchDynamicTable.java b/contrib/storage-elasticsearch/src/main/java/org/apache/drill/exec/store/elasticsearch/schema/ElasticsearchDynamicTable.java index a1919f5929..1e18fd3bc1 100644 --- a/contrib/storage-elasticsearch/src/main/java/org/apache/drill/exec/store/elasticsearch/schema/ElasticsearchDynamicTable.java +++ b/contrib/storage-elasticsearch/src/main/java/org/apache/drill/exec/store/elasticsearch/schema/ElasticsearchDynamicTable.java @@ -31,6 +31,7 @@ import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.schema.Table; import org.apache.calcite.schema.TranslatableTable; import org.apache.calcite.schema.Wrapper; +import org.apache.drill.exec.planner.logical.DrillTableSelection; import org.apache.drill.exec.planner.logical.DynamicDrillTable; import org.apache.drill.exec.store.StoragePlugin; @@ -40,7 +41,7 @@ public class ElasticsearchDynamicTable extends DynamicDrillTable implements Tran private final ElasticsearchTable table; - public ElasticsearchDynamicTable(StoragePlugin plugin, String storageEngineName, Object selection, Table table) { + public ElasticsearchDynamicTable(StoragePlugin plugin, String storageEngineName, DrillTableSelection selection, Table table) { super(plugin, storageEngineName, selection); this.table = (ElasticsearchTable) table; } diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/AbstractHBaseDrillTable.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/AbstractHBaseDrillTable.java index 93d87392e9..21df6b591c 100644 --- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/AbstractHBaseDrillTable.java +++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/AbstractHBaseDrillTable.java @@ -22,6 +22,7 @@ import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.exec.planner.logical.DrillTable; +import org.apache.drill.exec.planner.logical.DrillTableSelection; import org.apache.drill.exec.store.StoragePlugin; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; @@ -41,7 +42,7 @@ public abstract class AbstractHBaseDrillTable extends DrillTable { protected HTableDescriptor tableDesc; - public AbstractHBaseDrillTable(String storageEngineName, StoragePlugin plugin, Object selection) { + public AbstractHBaseDrillTable(String storageEngineName, StoragePlugin plugin, DrillTableSelection selection) { super(storageEngineName, plugin, selection); } diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseScanSpec.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseScanSpec.java index 793d924f43..cfcf7402e4 100644 --- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseScanSpec.java +++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseScanSpec.java @@ -18,6 +18,8 @@ package org.apache.drill.exec.store.hbase; +import org.apache.drill.common.PlanStringBuilder; +import org.apache.drill.exec.planner.logical.DrillTableSelection; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.util.Bytes; @@ -26,7 +28,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; -public class HBaseScanSpec { +public class HBaseScanSpec implements DrillTableSelection { protected String tableName; protected byte[] startRow; @@ -87,10 +89,16 @@ public class HBaseScanSpec { @Override public String toString() { - return "HBaseScanSpec [tableName=" + tableName - + ", startRow=" + (startRow == null ? null : Bytes.toStringBinary(startRow)) - + ", stopRow=" + (stopRow == null ? null : Bytes.toStringBinary(stopRow)) - + ", filter=" + (filter == null ? null : filter.toString()) - + "]"; + return new PlanStringBuilder(this) + .field("tableName", tableName) + .field("startRow", startRow == null ? null : Bytes.toStringBinary(startRow)) + .field("stopRow", stopRow == null ? null : Bytes.toStringBinary(stopRow)) + .field("filter", filter == null ? null : filter.toString()) + .toString(); + } + + @Override + public String digest() { + return toString(); } } diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java index cccaeb1c0c..edba290fc3 100644 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java @@ -38,7 +38,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { runHBaseSQLVerifyCount(sql, 1); - final String[] expectedPlan = {".*startRow=b4, stopRow=b4\\\\x00, filter=null.*"}; + final String[] expectedPlan = {".*startRow=\"b4\", stopRow=\"b4\\\\x00\".*"}; final String[] excludedPlan ={}; final String sqlHBase = canonizeHBaseSQL(sql); PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan); @@ -56,7 +56,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { runHBaseSQLVerifyCount(sql, 7); - final String[] expectedPlan = {".*startRow=, stopRow=, filter=RowFilter \\(NOT_EQUAL, b4\\).*"}; + final String[] expectedPlan = {".*startRow=\"\", stopRow=\"\", filter=\"RowFilter \\(NOT_EQUAL, b4\\)\".*"}; final String[] excludedPlan ={}; final String sqlHBase = canonizeHBaseSQL(sql); PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan); @@ -74,7 +74,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { runHBaseSQLVerifyCount(sql, 1); - final String[] expectedPlan = {".*startRow=b4, stopRow=b4\\\\x00, filter=null.*"}; + final String[] expectedPlan = {".*startRow=\"b4\", stopRow=\"b4\\\\x00\".*"}; final String[] excludedPlan ={".*startRow=null, stopRow=null.*"}; final String sqlHBase = canonizeHBaseSQL(sql); PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan); @@ -489,7 +489,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { runHBaseSQLVerifyCount(sql, 21); - final String[] expectedPlan = {".*filter=FilterList OR.*EQUAL.*EQUAL.*"}; + final String[] expectedPlan = {".*filter=\"FilterList OR.*EQUAL.*EQUAL.*\""}; final String[] excludedPlan ={}; final String sqlHBase = canonizeHBaseSQL(sql); PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan); @@ -507,7 +507,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { runHBaseSQLVerifyCount(sql, 2); - final String[] expectedPlan = {".*startRow=\\%_AS_PREFIX_, stopRow=\\%_AS_PREFIX`, filter=RowFilter.*EQUAL.*"}; + final String[] expectedPlan = {".*startRow=\"\\%_AS_PREFIX_\", stopRow=\"\\%_AS_PREFIX`\", filter=\"RowFilter.*EQUAL.*\""}; final String[] excludedPlan ={}; final String sqlHBase = canonizeHBaseSQL(sql); PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan); @@ -525,7 +525,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { runHBaseSQLVerifyCount(sql, 22); - final String[] expectedPlan = {".*startRow=07, stopRow=09, filter=FilterList AND.*RowFilter \\(GREATER_OR_EQUAL, 07\\), RowFilter \\(LESS, 09\\), SingleColumnValueFilter \\(f, c, EQUAL.*"}; + final String[] expectedPlan = {".*startRow=\"07\", stopRow=\"09\", filter=\"FilterList AND.*RowFilter \\(GREATER_OR_EQUAL, 07\\), RowFilter \\(LESS, 09\\), SingleColumnValueFilter \\(f, c, EQUAL.*\""}; final String[] excludedPlan ={}; final String sqlHBase = canonizeHBaseSQL(sql); PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan); @@ -543,7 +543,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { runHBaseSQLVerifyCount(sql, 4); - final String[] expectedPlan = {".*startRow=b4\\\\x00.*stopRow=,.*"}; + final String[] expectedPlan = {".*startRow=\"b4\\\\x00\", stopRow=\"\".*"}; final String[] excludedPlan ={}; final String sqlHBase = canonizeHBaseSQL(sql); PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan); @@ -561,7 +561,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { runHBaseSQLVerifyCount(sql, 2); - final String[] expectedPlan = {".*startRow=b4\\\\x00.*stopRow=, filter=null.*"}; + final String[] expectedPlan = {".*startRow=\"b4\\\\x00\".*stopRow=.*"}; final String[] excludedPlan ={}; final String sqlHBase = canonizeHBaseSQL(sql); PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan); @@ -579,7 +579,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { runHBaseSQLVerifyCount(sql, 3); - final String[] expectedPlan = {".*startRow=a2, stopRow=b4\\\\x00, filter=FilterList AND.*GREATER_OR_EQUAL, a2.*LESS_OR_EQUAL, b4.*"}; + final String[] expectedPlan = {".*startRow=\"a2\", stopRow=\"b4\\\\x00\", filter=\"FilterList AND.*GREATER_OR_EQUAL, a2.*LESS_OR_EQUAL, b4.*\""}; final String[] excludedPlan ={}; final String sqlHBase = canonizeHBaseSQL(sql); PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan); @@ -597,7 +597,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { runHBaseSQLVerifyCount(sql, 3); - final String[] expectedPlan = {".*startRow=a2, stopRow=b4\\\\x00, filter=FilterList AND.*GREATER_OR_EQUAL, a2.*LESS_OR_EQUAL, b4.*"}; + final String[] expectedPlan = {".*startRow=\"a2\", stopRow=\"b4\\\\x00\", filter=\"FilterList AND.*GREATER_OR_EQUAL, a2.*LESS_OR_EQUAL, b4.*\""}; final String[] excludedPlan ={}; final String sqlHBase = canonizeHBaseSQL(sql); PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan); @@ -615,7 +615,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { runHBaseSQLVerifyCount(sql, 5); - final String[] expectedPlan = {".*startRow=, stopRow=, filter=FilterList OR.*GREATER_OR_EQUAL, b5.*LESS_OR_EQUAL, a2.*"}; + final String[] expectedPlan = {".*startRow=\"\", stopRow=\"\", filter=\"FilterList OR.*GREATER_OR_EQUAL, b5.*LESS_OR_EQUAL, a2.*\""}; final String[] excludedPlan ={}; final String sqlHBase = canonizeHBaseSQL(sql); PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan); @@ -631,7 +631,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { + "WHERE\n" + " (row_key >= 'b5' OR row_key <= 'a2') AND (t.f.c1 >= '1' OR t.f.c1 is null)"; - final String[] expectedPlan = {".*startRow=, stopRow=, filter=FilterList OR.*GREATER_OR_EQUAL, b5.*LESS_OR_EQUAL, a2.*"}; + final String[] expectedPlan = {".*startRow=\"\", stopRow=\"\", filter=\"FilterList OR.*GREATER_OR_EQUAL, b5.*LESS_OR_EQUAL, a2.*\""}; final String[] excludedPlan ={}; final String sqlHBase = canonizeHBaseSQL(sql); PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan); @@ -649,7 +649,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { runHBaseSQLVerifyCount(sql, 4); - final String[] expectedPlan = {".*startRow=b4\\\\x00, stopRow=,.*"}; + final String[] expectedPlan = {".*startRow=\"b4\\\\x00\", stopRow=\"\".*"}; final String[] excludedPlan ={}; final String sqlHBase = canonizeHBaseSQL(sql); PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan); @@ -667,7 +667,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { runHBaseSQLVerifyCount(sql, 2); - final String[] expectedPlan = {".*startRow=b4\\\\x00, stopRow=,.*"}; + final String[] expectedPlan = {".*startRow=\"b4\\\\x00\", stopRow=\"\".*"}; final String[] excludedPlan ={}; final String sqlHBase = canonizeHBaseSQL(sql); PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan); @@ -698,7 +698,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { runHBaseSQLVerifyCount(sql, 4); - final String[] expectedPlan = {".*startRow=, stopRow=b4\\\\x00, filter=null.*"}; + final String[] expectedPlan = {".*startRow=\"\", stopRow=\"b4\\\\x00\".*"}; final String[] excludedPlan ={}; final String sqlHBase = canonizeHBaseSQL(sql); PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan); @@ -716,7 +716,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { runHBaseSQLVerifyCount(sql, 4); - final String[] expectedPlan = {".*startRow=, stopRow=b4\\\\x00, filter=null.*"}; + final String[] expectedPlan = {".*startRow=\"\", stopRow=\"b4\\\\x00\".*"}; final String[] excludedPlan ={}; final String sqlHBase = canonizeHBaseSQL(sql); PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan); @@ -734,7 +734,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { runHBaseSQLVerifyCount(sql, 2); - final String[] expectedPlan = {".*startRow=a2, stopRow=b4\\\\x00, filter=FilterList OR \\(2/2\\): \\[RowFilter \\(EQUAL, b4\\), RowFilter \\(EQUAL, a2\\).*"}; + final String[] expectedPlan = {".*startRow=\"a2\", stopRow=\"b4\\\\x00\", filter=\"FilterList OR \\(2/2\\): \\[RowFilter \\(EQUAL, b4\\), RowFilter \\(EQUAL, a2\\).*\""}; final String[] excludedPlan ={}; final String sqlHBase = canonizeHBaseSQL(sql); PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan); @@ -753,7 +753,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { runHBaseSQLVerifyCount(sql, 2); - final String[] expectedPlan = {".*startRow=a2, stopRow=b4\\\\x00, filter=FilterList OR \\(2/2\\): \\[RowFilter \\(EQUAL, b4\\), RowFilter \\(EQUAL, a2\\).*"}; + final String[] expectedPlan = {".*startRow=\"a2\", stopRow=\"b4\\\\x00\", filter=\"FilterList OR \\(2/2\\): \\[RowFilter \\(EQUAL, b4\\), RowFilter \\(EQUAL, a2\\).*\""}; final String[] excludedPlan ={}; final String sqlHBase = canonizeHBaseSQL(sql); PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan); @@ -772,7 +772,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { runHBaseSQLVerifyCount(sql, 3); - final String[] expectedPlan = {".*startRow=a2, stopRow=b6\\\\x00, filter=FilterList OR \\(2/2\\): \\[RowFilter \\(EQUAL, a2\\), FilterList AND \\(2/2\\): \\[RowFilter \\(GREATER_OR_EQUAL, b5\\), RowFilter \\(LESS_OR_EQUAL, b6.*"}; + final String[] expectedPlan = {".*startRow=\"a2\", stopRow=\"b6\\\\x00\", filter=\"FilterList OR \\(2/2\\): \\[RowFilter \\(EQUAL, a2\\), FilterList AND \\(2/2\\): \\[RowFilter \\(GREATER_OR_EQUAL, b5\\), RowFilter \\(LESS_OR_EQUAL, b6.*\""}; final String[] excludedPlan ={}; final String sqlHBase = canonizeHBaseSQL(sql); PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan); @@ -808,9 +808,9 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { String query = "select d from dfs.tmp.pd_view where d > date '2015-06-13' and d < DATE '2015-06-18'"; String[] expectedPlan = { - "startRow=\\\\x00\\\\x00\\\\x01M\\\\xEF\\]\\\\xA0\\\\x00, " + - "stopRow=\\\\x00\\\\x00\\\\x01N\\\\x03\\\\xF7\\\\x10\\\\x00, " + - "filter=null"}; + "startRow=\"\\\\x00\\\\x00\\\\x01M\\\\xEF\\]\\\\xA0\\\\x00\", " + + "stopRow=\"\\\\x00\\\\x00\\\\x01N\\\\x03\\\\xF7\\\\x10\\\\x00\"" + }; String[] excludedPlan ={"Filter\\("}; PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java index d8bf75072b..7710a2b42a 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java @@ -21,6 +21,8 @@ import java.util.List; import org.apache.calcite.schema.Schema.TableType; +import org.apache.drill.common.PlanStringBuilder; +import org.apache.drill.exec.planner.logical.DrillTableSelection; import org.apache.drill.exec.store.hive.HiveTableWrapper.HivePartitionWrapper; import com.fasterxml.jackson.annotation.JsonCreator; @@ -28,7 +30,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.drill.shaded.guava.com.google.common.collect.Lists; -public class HiveReadEntry { +public class HiveReadEntry implements DrillTableSelection { @JsonProperty("table") public HiveTableWrapper table; @@ -93,5 +95,18 @@ public class HiveReadEntry { return partitionPath; } + + @Override + public String toString() { + return new PlanStringBuilder(this) + .field("tableName", table) + .field("partitions", partitions) + .toString(); + } + + @Override + public String digest() { + return toString(); + } } diff --git a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpScanSpec.java b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpScanSpec.java index e43490c8f5..22c056e7f3 100644 --- a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpScanSpec.java +++ b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpScanSpec.java @@ -24,10 +24,11 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; import org.apache.drill.common.PlanStringBuilder; import org.apache.drill.exec.oauth.PersistentTokenTable; +import org.apache.drill.exec.planner.logical.DrillTableSelection; import org.apache.drill.exec.store.StoragePluginRegistry; @JsonTypeName("http-scan-spec") -public class HttpScanSpec { +public class HttpScanSpec implements DrillTableSelection { private final String pluginName; private final String connectionName; @@ -100,4 +101,9 @@ public class HttpScanSpec { .field("config", config) .toString(); } + + @Override + public String digest() { + return toString(); + } } diff --git a/contrib/storage-kafka/src/main/java/org/apache/drill/exec/store/kafka/KafkaScanSpec.java b/contrib/storage-kafka/src/main/java/org/apache/drill/exec/store/kafka/KafkaScanSpec.java index d0590991cf..6bd88f84b9 100644 --- a/contrib/storage-kafka/src/main/java/org/apache/drill/exec/store/kafka/KafkaScanSpec.java +++ b/contrib/storage-kafka/src/main/java/org/apache/drill/exec/store/kafka/KafkaScanSpec.java @@ -19,8 +19,10 @@ package org.apache.drill.exec.store.kafka; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.drill.common.PlanStringBuilder; +import org.apache.drill.exec.planner.logical.DrillTableSelection; -public class KafkaScanSpec { +public class KafkaScanSpec implements DrillTableSelection { private final String topicName; @JsonCreator @@ -34,6 +36,13 @@ public class KafkaScanSpec { @Override public String toString() { - return "KafkaScanSpec [topicName=" + topicName + "]"; + return new PlanStringBuilder(this) + .field("topicName", topicName) + .toString(); + } + + @Override + public String digest() { + return toString(); } } diff --git a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduScanSpec.java b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduScanSpec.java index 371cf2bb2a..78abdbefbd 100644 --- a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduScanSpec.java +++ b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduScanSpec.java @@ -20,8 +20,10 @@ package org.apache.drill.exec.store.kudu; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.drill.common.PlanStringBuilder; +import org.apache.drill.exec.planner.logical.DrillTableSelection; -public class KuduScanSpec { +public class KuduScanSpec implements DrillTableSelection { private final String tableName; @@ -33,4 +35,16 @@ public class KuduScanSpec { public String getTableName() { return tableName; } + + @Override + public String toString() { + return new PlanStringBuilder(this) + .field("tableName", tableName) + .toString(); + } + + @Override + public String digest() { + return toString(); + } } diff --git a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoScanSpec.java b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoScanSpec.java index 2c97785457..41426332c5 100644 --- a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoScanSpec.java +++ b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoScanSpec.java @@ -20,11 +20,12 @@ package org.apache.drill.exec.store.mongo; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.drill.common.PlanStringBuilder; +import org.apache.drill.exec.planner.logical.DrillTableSelection; import java.util.ArrayList; import java.util.List; -public class MongoScanSpec { +public class MongoScanSpec implements DrillTableSelection { private final String dbName; private final String collectionName; @@ -71,4 +72,9 @@ public class MongoScanSpec { .field("operations", operations) .toString(); } + + @Override + public String digest() { + return toString(); + } } diff --git a/contrib/storage-opentsdb/src/main/java/org/apache/drill/exec/store/openTSDB/OpenTSDBScanSpec.java b/contrib/storage-opentsdb/src/main/java/org/apache/drill/exec/store/openTSDB/OpenTSDBScanSpec.java index f93758de9f..50931d1eb2 100644 --- a/contrib/storage-opentsdb/src/main/java/org/apache/drill/exec/store/openTSDB/OpenTSDBScanSpec.java +++ b/contrib/storage-opentsdb/src/main/java/org/apache/drill/exec/store/openTSDB/OpenTSDBScanSpec.java @@ -19,8 +19,10 @@ package org.apache.drill.exec.store.openTSDB; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.drill.common.PlanStringBuilder; +import org.apache.drill.exec.planner.logical.DrillTableSelection; -public class OpenTSDBScanSpec { +public class OpenTSDBScanSpec implements DrillTableSelection { private final String tableName; @@ -35,8 +37,13 @@ public class OpenTSDBScanSpec { @Override public String toString() { - return "OpenTSDBScanSpec{" + - "tableName='" + tableName + '\'' + - '}'; + return new PlanStringBuilder(this) + .field("tableName", tableName) + .toString(); + } + + @Override + public String digest() { + return toString(); } } diff --git a/contrib/storage-splunk/src/main/java/org/apache/drill/exec/store/splunk/SplunkScanSpec.java b/contrib/storage-splunk/src/main/java/org/apache/drill/exec/store/splunk/SplunkScanSpec.java index 513db63707..2d736bbd4b 100644 --- a/contrib/storage-splunk/src/main/java/org/apache/drill/exec/store/splunk/SplunkScanSpec.java +++ b/contrib/storage-splunk/src/main/java/org/apache/drill/exec/store/splunk/SplunkScanSpec.java @@ -22,9 +22,10 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; import org.apache.drill.common.PlanStringBuilder; +import org.apache.drill.exec.planner.logical.DrillTableSelection; @JsonTypeName("splunk-scan-spec") -public class SplunkScanSpec { +public class SplunkScanSpec implements DrillTableSelection { private final String pluginName; private final String indexName; private final SplunkPluginConfig config; @@ -55,4 +56,9 @@ public class SplunkScanSpec { .field("indexName", indexName) .toString(); } + + @Override + public String digest() { + return toString(); + } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/FileSystemPartitionDescriptor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/FileSystemPartitionDescriptor.java index 5fb84708a7..23e7ef827a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/FileSystemPartitionDescriptor.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/FileSystemPartitionDescriptor.java @@ -41,7 +41,7 @@ import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.Types; import org.apache.drill.exec.physical.base.FileGroupScan; -import org.apache.drill.exec.planner.logical.DirPrunedEnumerableTableScan; +import org.apache.drill.exec.planner.logical.SelectionBasedTableScan; import org.apache.drill.exec.planner.logical.DrillRel; import org.apache.drill.exec.planner.logical.DrillScanRel; import org.apache.drill.exec.planner.logical.DrillTable; @@ -252,8 +252,8 @@ public class FileSystemPartitionDescriptor extends AbstractPartitionDescriptor { RelOptTableImpl newOptTableImpl = RelOptTableImpl.create(relOptTable.getRelOptSchema(), relOptTable.getRowType(), newTable, GuavaUtils.convertToUnshadedImmutableList(relOptTable.getQualifiedName())); - // return an EnumerableTableScan with fileSelection being part of digest of TableScan node. - return DirPrunedEnumerableTableScan.create(scanRel.getCluster(), newOptTableImpl, newFileSelection.toString()); + // return a SelectionBasedTableScan with fileSelection being part of digest of TableScan node. + return SelectionBasedTableScan.create(scanRel.getCluster(), newOptTableImpl, newFileSelection.toString()); } else { throw new UnsupportedOperationException("Only DrillScanRel and EnumerableTableScan is allowed!"); } @@ -271,4 +271,8 @@ public class FileSystemPartitionDescriptor extends AbstractPartitionDescriptor { return selection instanceof FormatSelection && ((FormatSelection)selection).getSelection().getCacheFileRoot() != null; } + + private static boolean supportsScan(TableScan scanRel) { + return scanRel instanceof SelectionBasedTableScan; + } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java index 7a9dc0c081..4857d802d0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java @@ -20,7 +20,6 @@ package org.apache.drill.exec.planner.logical; import java.io.IOException; import java.util.Objects; -import org.apache.calcite.adapter.enumerable.EnumerableTableScan; import org.apache.calcite.config.CalciteConnectionConfig; import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.rel.RelNode; @@ -47,7 +46,7 @@ public abstract class DrillTable implements Table { private final String storageEngineName; private final StoragePluginConfig storageEngineConfig; private final TableType tableType; - private final Object selection; + private final DrillTableSelection selection; private final StoragePlugin plugin; private final String userName; private GroupScan scan; @@ -61,7 +60,7 @@ public abstract class DrillTable implements Table { * @param userName Whom to impersonate while reading the contents of the table. * @param selection Table contents (type and contents depend on type of StoragePlugin). */ - public DrillTable(String storageEngineName, StoragePlugin plugin, String userName, Object selection) { + public DrillTable(String storageEngineName, StoragePlugin plugin, String userName, DrillTableSelection selection) { this(storageEngineName, plugin, TableType.TABLE, userName, selection); } @@ -73,12 +72,12 @@ public abstract class DrillTable implements Table { * @param userName Whom to impersonate while reading the contents of the table. * @param selection Table contents (type and contents depend on type of StoragePlugin). */ - public DrillTable(String storageEngineName, StoragePlugin plugin, TableType tableType, String userName, Object selection) { + public DrillTable(String storageEngineName, StoragePlugin plugin, TableType tableType, String userName, DrillTableSelection selection) { this(storageEngineName, plugin, tableType, userName, selection, null); } public DrillTable(String storageEngineName, StoragePlugin plugin, TableType tableType, - String userName, Object selection, MetadataProviderManager metadataProviderManager) { + String userName, DrillTableSelection selection, MetadataProviderManager metadataProviderManager) { this.selection = selection; this.plugin = plugin; @@ -95,7 +94,7 @@ public abstract class DrillTable implements Table { * process. Once we add impersonation to non-FileSystem storage plugins such as Hive, HBase etc, * we can remove this constructor. */ - public DrillTable(String storageEngineName, StoragePlugin plugin, Object selection) { + public DrillTable(String storageEngineName, StoragePlugin plugin, DrillTableSelection selection) { this(storageEngineName, plugin, ImpersonationUtil.getProcessUserName(), selection); } @@ -166,9 +165,9 @@ public abstract class DrillTable implements Table { } public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable table) { - // returns non-drill table scan to allow directory-based partition pruning - // before table group scan is created - return EnumerableTableScan.create(context.getCluster(), table); + // Returns non-drill table scan to allow directory-based partition pruning + // before table group scan is created. + return SelectionBasedTableScan.create(context.getCluster(), table, selection.digest()); } @Override diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTableSelection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTableSelection.java new file mode 100644 index 0000000000..b514a4995e --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTableSelection.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.logical; + +public interface DrillTableSelection { + + /** + * The digest of the selection represented by the implementation. The + * selections that accompany Tables can modify the contained dataset, e.g. + * a file selection can restrict to a subset of the available data and a + * format selection can include options that affect the behaviour of the + * underlying reader. Two scans will end up being considered identical during + * logical planning if their digests are the same so selection + * implementations should override this method so that exactly those scans + * that really are identical (in terms of the data they produce) have matching + * digests. + * + * @return this selection's digest, normally a string built from its properties. + */ + public String digest(); +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DynamicDrillTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DynamicDrillTable.java index c406d9a644..8164287a02 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DynamicDrillTable.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DynamicDrillTable.java @@ -30,12 +30,12 @@ public class DynamicDrillTable extends DrillTable { private final RelDataTypeHolder holder; - public DynamicDrillTable(StoragePlugin plugin, String storageEngineName, String userName, Object selection) { + public DynamicDrillTable(StoragePlugin plugin, String storageEngineName, String userName, DrillTableSelection selection) { this(plugin, storageEngineName, userName, selection, null); } public DynamicDrillTable(StoragePlugin plugin, String storageEngineName, String userName, - Object selection, MetadataProviderManager metadataProviderManager) { + DrillTableSelection selection, MetadataProviderManager metadataProviderManager) { super(storageEngineName, plugin, Schema.TableType.TABLE, userName, selection, metadataProviderManager); this.holder = new RelDataTypeHolder(); } @@ -46,7 +46,7 @@ public class DynamicDrillTable extends DrillTable { * non-FileSystem storage plugins such as Hive, HBase etc, we can remove this * constructor. */ - public DynamicDrillTable(StoragePlugin plugin, String storageEngineName, Object selection) { + public DynamicDrillTable(StoragePlugin plugin, String storageEngineName, DrillTableSelection selection) { this(plugin, storageEngineName, ImpersonationUtil.getProcessUserName(), selection, null); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DirPrunedEnumerableTableScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/SelectionBasedTableScan.java similarity index 82% rename from exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DirPrunedEnumerableTableScan.java rename to exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/SelectionBasedTableScan.java index 3287a80e53..971d7d1aea 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DirPrunedEnumerableTableScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/SelectionBasedTableScan.java @@ -35,11 +35,11 @@ import java.util.List; * When directory-based partition pruning applied, file selection could be different for the same * table. */ -public class DirPrunedEnumerableTableScan extends EnumerableTableScan { +public class SelectionBasedTableScan extends EnumerableTableScan { private final String digestFromSelection; - public DirPrunedEnumerableTableScan(RelOptCluster cluster, RelTraitSet traitSet, - RelOptTable table, Class elementType, String digestFromSelection) { + public SelectionBasedTableScan(RelOptCluster cluster, RelTraitSet traitSet, + RelOptTable table, Class elementType, String digestFromSelection) { super(cluster, traitSet, table, elementType); this.digestFromSelection = digestFromSelection; } @@ -48,12 +48,11 @@ public class DirPrunedEnumerableTableScan extends EnumerableTableScan { public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) { final Table tbl = this.table.unwrap(Table.class); Class elementType = EnumerableTableScan.deduceElementType(tbl); - - return new DirPrunedEnumerableTableScan(getCluster(), traitSet, table, elementType, digestFromSelection); + return new SelectionBasedTableScan(getCluster(), traitSet, table, elementType, digestFromSelection); } - /** Creates an DirPrunedEnumerableTableScan. */ - public static EnumerableTableScan create(RelOptCluster cluster, + /** Creates an SelectionBasedTableScan. */ + public static SelectionBasedTableScan create(RelOptCluster cluster, RelOptTable relOptTable, String digestFromSelection) { final Table table = relOptTable.unwrap(Table.class); Class elementType = EnumerableTableScan.deduceElementType(table); @@ -66,7 +65,8 @@ public class DirPrunedEnumerableTableScan extends EnumerableTableScan { } return ImmutableList.of(); }); - return new DirPrunedEnumerableTableScan(cluster, traitSet, relOptTable, elementType, digestFromSelection); + + return new SelectionBasedTableScan(cluster, traitSet, relOptTable, elementType, digestFromSelection); } @Override diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java index b72ecee33e..9d8c753056 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java @@ -27,6 +27,7 @@ import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; import org.apache.drill.exec.planner.common.DrillRelOptUtil; +import org.apache.drill.exec.planner.logical.SelectionBasedTableScan; import org.apache.drill.exec.util.DrillFileSystemUtil; import org.apache.drill.shaded.guava.com.google.common.base.Stopwatch; import org.apache.calcite.adapter.enumerable.EnumerableTableScan; @@ -778,4 +779,8 @@ public abstract class PruneScanRule extends StoragePluginOptimizerRule { return false; } } + + private static boolean supportsScan(TableScan scan) { + return scan instanceof SelectionBasedTableScan; + } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java index ebe1a43cea..6563c7803a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java @@ -19,6 +19,7 @@ package org.apache.drill.exec.store.dfs; import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.util.DrillStringUtils; +import org.apache.drill.exec.planner.logical.DrillTableSelection; import org.apache.drill.exec.util.DrillFileSystemUtil; import org.apache.drill.shaded.guava.com.google.common.base.Preconditions; import org.apache.drill.shaded.guava.com.google.common.base.Stopwatch; @@ -40,7 +41,7 @@ import java.util.stream.Collectors; /** * Jackson serializable description of a file selection. */ -public class FileSelection { +public class FileSelection implements DrillTableSelection { private static final Logger logger = LoggerFactory.getLogger(FileSelection.class); private static final String WILD_CARD = "*"; @@ -437,6 +438,11 @@ public class FileSelection { this.emptyDirectory = true; } + @Override + public String digest() { + return toString(); + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatSelection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatSelection.java index d2a55455a1..4210b82f8b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatSelection.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatSelection.java @@ -21,11 +21,12 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.drill.common.logical.FormatPluginConfig; +import org.apache.drill.exec.planner.logical.DrillTableSelection; import org.apache.hadoop.fs.Path; import java.util.List; -public class FormatSelection { +public class FormatSelection implements DrillTableSelection { private FormatPluginConfig format; private FileSelection selection; @@ -62,4 +63,14 @@ public class FormatSelection { public boolean supportsDirPruning() { return selection.supportsDirPruning(); } + + @Override + public String digest() { + return toString(); + } + + @Override + public String toString() { + return String.format("fileSelection=%s,formatConfig=%s", selection, format); + } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java index 3b3ea83cef..d0b1636199 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java @@ -319,6 +319,13 @@ public class EasyGroupScan extends AbstractGroupScanWithMetadata<TableMetadataPr @Override public String toString() { + // Note that the output of this method is incorporated in the digest of + // the corresponding scan node in the query plan. This means that the + // fields included here constitute what the planner will use to decide + // whether two scans are identical or not. E.g. the format config must be + // present here because format config can be overriden using table functions + // Two scans that differ by format config alone may produce different data + // and therefore should not be considered identical. return new PlanStringBuilder(this) .field("selectionRoot", selectionRoot) .field("numFiles", getFiles().size()) @@ -327,6 +334,7 @@ public class EasyGroupScan extends AbstractGroupScanWithMetadata<TableMetadataPr .field("schema", getSchema()) .field("usedMetastore", usedMetastore()) .field("limit", limit) + .field("formatConfig", getFormatConfig()) .toString(); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaTableType.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaTableType.java index 3e38fb2389..f08e380e5b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaTableType.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaTableType.java @@ -21,6 +21,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.schema.SchemaPlus; import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.planner.logical.DrillTableSelection; import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.store.ischema.InfoSchemaTable.Catalogs; import org.apache.drill.exec.store.ischema.InfoSchemaTable.Columns; @@ -40,7 +41,7 @@ import static org.slf4j.LoggerFactory.getLogger; /** * The set of tables / views in INFORMATION_SCHEMA. */ -public enum InfoSchemaTableType { +public enum InfoSchemaTableType implements DrillTableSelection { CATALOGS(new Catalogs()), SCHEMATA(new Schemata()), @@ -91,4 +92,9 @@ public enum InfoSchemaTableType { public RelDataType getRowType(RelDataTypeFactory typeFactory) { return tableDef.getRowType(typeFactory); } + + @Override + public String digest() { + return toString(); + } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java index fb00f836ea..983e150be6 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java @@ -19,7 +19,6 @@ package org.apache.drill.exec.store.mock; import java.io.IOException; import java.net.URL; -import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -39,10 +38,11 @@ import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.store.AbstractSchema; import org.apache.drill.exec.store.AbstractStoragePlugin; import org.apache.drill.exec.store.SchemaConfig; +import org.apache.drill.exec.store.mock.MockTableDef.MockScanEntry; +import org.apache.drill.exec.store.mock.MockTableDef.MockTableSelection; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.drill.shaded.guava.com.google.common.base.Charsets; @@ -64,10 +64,11 @@ public class MockStorageEngine extends AbstractStoragePlugin { public AbstractGroupScan getPhysicalScan(String userName, JSONOptions selection, List<SchemaPath> columns) throws IOException { - List<MockTableDef.MockScanEntry> readEntries = selection.getListWith(new ObjectMapper(), - new TypeReference<ArrayList<MockTableDef.MockScanEntry>>() { - }); - + MockTableSelection tableSelection = selection.getWith( + new ObjectMapper(), + MockTableSelection.class + ); + List<MockScanEntry> readEntries = tableSelection.getEntries(); assert ! readEntries.isEmpty(); return new MockGroupScanPOP(null, readEntries); } @@ -161,6 +162,7 @@ public class MockStorageEngine extends AbstractStoragePlugin { } catch (IOException e) { throw new IllegalArgumentException("Unable to read mock table definition file: " + name, e); } + return new DynamicDrillTable(engine, this.name, mockTableDefn.getEntries()); } @@ -177,10 +179,9 @@ public class MockStorageEngine extends AbstractStoragePlugin { if (unit == null) { } else if (unit.equalsIgnoreCase("K")) { n *= 1000; } else if (unit.equalsIgnoreCase("M")) { n *= 1_000_000; } - MockTableDef.MockScanEntry entry = new MockTableDef.MockScanEntry(n, true, 0, 1, null); - List<MockTableDef.MockScanEntry> list = new ArrayList<>(); - list.add(entry); - return new DynamicDrillTable(engine, this.name, list); + MockScanEntry entry = new MockTableDef.MockScanEntry(n, true, 0, 1, null); + MockTableSelection entries = new MockTableSelection(ImmutableList.<MockScanEntry>of(entry)); + return new DynamicDrillTable(engine, this.name, entries); } @Override diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockTableDef.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockTableDef.java index 1b4af74fa9..5a9a6f59f5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockTableDef.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockTableDef.java @@ -31,6 +31,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; +import org.apache.drill.exec.planner.logical.DrillTableSelection; /** * Structure of a mock table definition file. Yes, using Jackson deserialization to parse @@ -84,6 +85,29 @@ public class MockTableDef { } } + /** + * A tiny wrapper class to add required DrillTableSelection behaviour to + * the entries list. + */ + public static class MockTableSelection implements DrillTableSelection { + private final List<MockScanEntry> entries; + + @JsonCreator + public MockTableSelection(@JsonProperty("entries") List<MockScanEntry> entries) { + this.entries = entries; + } + + @JsonIgnore + @Override + public String digest() { + return entries.toString(); + } + + public List<MockScanEntry> getEntries() { + return entries; + } + } + /** * Meta-data description of the columns we wish to create during a simulated * scan. @@ -189,10 +213,10 @@ public class MockTableDef { } private String descrip; - List<MockTableDef.MockScanEntry> entries; + MockTableSelection entries; public MockTableDef(@JsonProperty("descrip") final String descrip, - @JsonProperty("entries") final List<MockTableDef.MockScanEntry> entries) { + @JsonProperty("entries") final MockTableSelection entries) { this.descrip = descrip; this.entries = entries; } @@ -211,5 +235,5 @@ public class MockTableDef { * @return */ - public List<MockTableDef.MockScanEntry> getEntries() { return entries; } + public MockTableSelection getEntries() { return entries; } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/plan/rel/PluginDrillTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/plan/rel/PluginDrillTable.java index 7528d99b1c..da1a34ecf4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/plan/rel/PluginDrillTable.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/plan/rel/PluginDrillTable.java @@ -23,6 +23,7 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.schema.TranslatableTable; import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.exec.planner.logical.DrillTable; +import org.apache.drill.exec.planner.logical.DrillTableSelection; import org.apache.drill.exec.planner.logical.DynamicDrillTable; import org.apache.drill.exec.store.StoragePlugin; import org.apache.drill.exec.util.Utilities; @@ -36,7 +37,7 @@ public class PluginDrillTable extends DynamicDrillTable implements TranslatableT private final Convention convention; public PluginDrillTable(StoragePlugin plugin, String storageEngineName, - String userName, Object selection, Convention convention) { + String userName, DrillTableSelection selection, Convention convention) { super(plugin, storageEngineName, userName, selection); this.convention = convention; } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/StaticDrillTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/StaticDrillTable.java index dbe58917e0..07df14b56e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/StaticDrillTable.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/StaticDrillTable.java @@ -21,6 +21,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.schema.Schema.TableType; import org.apache.drill.exec.planner.logical.DrillTable; +import org.apache.drill.exec.planner.logical.DrillTableSelection; import org.apache.drill.exec.store.RecordDataType; import org.apache.drill.exec.store.StoragePlugin; import org.apache.drill.exec.util.ImpersonationUtil; @@ -34,7 +35,7 @@ public class StaticDrillTable extends DrillTable { private final RecordDataType dataType; - public StaticDrillTable(String storageEngineName, StoragePlugin plugin, TableType tableType, Object selection, RecordDataType dataType) { + public StaticDrillTable(String storageEngineName, StoragePlugin plugin, TableType tableType, DrillTableSelection selection, RecordDataType dataType) { super(storageEngineName, plugin, tableType, ImpersonationUtil.getProcessUserName(), selection); this.dataType = dataType; } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java index 5bddc8d734..35a4b64cff 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java @@ -21,6 +21,7 @@ import java.util.Iterator; import org.apache.drill.exec.alias.AliasTarget; import org.apache.drill.exec.ops.ExecutorFragmentContext; +import org.apache.drill.exec.planner.logical.DrillTableSelection; import org.apache.drill.exec.store.sys.OptionIterator.OptionValueWrapper; /** @@ -31,7 +32,7 @@ import org.apache.drill.exec.store.sys.OptionIterator.OptionValueWrapper; * PROFILES and PROFILES_JSON are stored in local / distributed storage. * </p> */ -public enum SystemTable { +public enum SystemTable implements DrillTableSelection { OPTIONS_OLD("options_old", false, OptionValueWrapper.class) { @Deprecated @Override @@ -164,4 +165,9 @@ public enum SystemTable { public Class<?> getPojoClass() { return pojoClass; } + + @Override + public String digest() { + return toString(); + } } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/enumerable/plan/EnumMockTable.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/enumerable/plan/EnumMockTable.java index 6628069794..f5a92733c9 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/enumerable/plan/EnumMockTable.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/enumerable/plan/EnumMockTable.java @@ -20,12 +20,13 @@ package org.apache.drill.exec.store.enumerable.plan; import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.rel.RelNode; import org.apache.calcite.schema.TranslatableTable; +import org.apache.drill.exec.planner.logical.DrillTableSelection; import org.apache.drill.exec.planner.logical.DynamicDrillTable; import org.apache.drill.exec.store.StoragePlugin; public class EnumMockTable extends DynamicDrillTable implements TranslatableTable { - public EnumMockTable(StoragePlugin plugin, String storageEngineName, String userName, Object selection) { + public EnumMockTable(StoragePlugin plugin, String storageEngineName, String userName, DrillTableSelection selection) { super(plugin, storageEngineName, userName, selection); } diff --git a/exec/java-exec/src/test/resources/functions/conv/conversionTestWithLogicalPlan.json b/exec/java-exec/src/test/resources/functions/conv/conversionTestWithLogicalPlan.json index acae9e730e..2db5634705 100644 --- a/exec/java-exec/src/test/resources/functions/conv/conversionTestWithLogicalPlan.json +++ b/exec/java-exec/src/test/resources/functions/conv/conversionTestWithLogicalPlan.json @@ -18,30 +18,32 @@ "op" : "scan", "@id" : 1, "storageengine" : "mock", - "selection" : [ { - "records" : 10, - "types" : [ { - "name" : "tinyint_val", - "type" : "TINYINT", - "mode" : "REQUIRED" - }, { - "name" : "smallint_val", - "type" : "SMALLINT", - "mode" : "REQUIRED" - }, { - "name" : "int_val", - "type" : "INT", - "mode" : "REQUIRED" - }, { - "name" : "bigint_val", - "type" : "BIGINT", - "mode" : "REQUIRED" - }, { - "name" : "uint8_val", - "type" : "UINT8", - "mode" : "REQUIRED" + "selection" : { + "entries": [ { + "records" : 10, + "types" : [ { + "name" : "tinyint_val", + "type" : "TINYINT", + "mode" : "REQUIRED" + }, { + "name" : "smallint_val", + "type" : "SMALLINT", + "mode" : "REQUIRED" + }, { + "name" : "int_val", + "type" : "INT", + "mode" : "REQUIRED" + }, { + "name" : "bigint_val", + "type" : "BIGINT", + "mode" : "REQUIRED" + }, { + "name" : "uint8_val", + "type" : "UINT8", + "mode" : "REQUIRED" + } ] } ] - } ] + } }, { "op" : "project", "@id" : 2, @@ -84,4 +86,4 @@ "target" : null, "storageEngine" : "--SCREEN--" } ] -} \ No newline at end of file +} diff --git a/exec/java-exec/src/test/resources/scan_screen_logical.json b/exec/java-exec/src/test/resources/scan_screen_logical.json index 1cf380049c..97f1ab9f95 100644 --- a/exec/java-exec/src/test/resources/scan_screen_logical.json +++ b/exec/java-exec/src/test/resources/scan_screen_logical.json @@ -18,14 +18,16 @@ "op" : "scan", "memo" : "initial_scan", "storageengine" : "mock", - "selection" : [ { - "records" : 100, - "types" : [ { - "name" : "superhero_name", - "type" : "VARCHAR", - "mode" : "REQUIRED" + "selection" : { + "entries": [ { + "records" : 100, + "types" : [ { + "name" : "superhero_name", + "type" : "VARCHAR", + "mode" : "REQUIRED" + } ] } ] - } ] + } }, { "@id" : "2", "input" : 1, @@ -35,4 +37,4 @@ "file" : "console:///stdout" } } ] -} \ No newline at end of file +} diff --git a/pom.xml b/pom.xml index f32c48bb9f..65dd0829f2 100644 --- a/pom.xml +++ b/pom.xml @@ -50,7 +50,7 @@ <slf4j.version>1.7.26</slf4j.version> <shaded.guava.version>28.2-jre</shaded.guava.version> <guava.version>30.1.1-jre</guava.version> - <forkCount>2</forkCount> + <forkCount>1</forkCount> <parquet.version>1.12.2</parquet.version> <parquet.format.version>2.8.0</parquet.format.version> <!--
