This is an automated email from the ASF dual-hosted git repository.
gparai pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git.
from 4627973 DRILL-6950: Row set-based scan framework
new 3233d8a DRILL-1328: Support table statistics
new 469be17 DRILL-1328: Support table statistics - Part 2
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../exec/store/mapr/db/MapRDBFormatPlugin.java | 16 +
.../drill/exec/store/mapr/db/MapRDBGroupScan.java | 1 +
.../store/mapr/streams/StreamsFormatPlugin.java | 18 +
.../exec/store/syslog/SyslogFormatPlugin.java | 19 +
.../native/client/src/protobuf/UserBitShared.pb.cc | 15 +-
.../native/client/src/protobuf/UserBitShared.pb.h | 7 +-
.../drill/exec/store/hbase/HBaseGroupScan.java | 1 +
.../org/apache/drill/exec/store/hive/HiveScan.java | 1 +
.../drill/exec/store/jdbc/JdbcGroupScan.java | 9 +-
.../org/apache/drill/exec/store/jdbc/JdbcPrel.java | 9 +-
.../drill/exec/store/jdbc/JdbcRecordReader.java | 7 +-
.../apache/drill/exec/store/jdbc/JdbcSubScan.java | 9 +-
.../drill/exec/store/kafka/KafkaGroupScan.java | 1 +
.../drill/exec/store/kudu/KuduGroupScan.java | 1 +
.../drill/exec/store/mongo/MongoGroupScan.java | 1 +
.../exec/store/openTSDB/OpenTSDBGroupScan.java | 1 +
exec/java-exec/pom.xml | 5 +
exec/java-exec/src/main/codegen/data/Parser.tdd | 8 +-
.../src/main/codegen/includes/parserImpls.ftl | 48 +-
.../codegen/templates/AbstractRecordWriter.java | 5 +
.../templates/JsonBaseStatisticsRecordWriter.java | 139 +
.../src/main/codegen/templates/RecordWriter.java | 6 +-
...cordWriter.java => StatisticsRecordWriter.java} | 44 +-
.../templates/StatisticsRecordWriterImpl.java | 119 +
.../java/org/apache/drill/exec/ExecConstants.java | 42 +-
.../apache/drill/exec/dotdrill/DotDrillType.java | 6 +-
.../apache/drill/exec/dotdrill/DotDrillUtil.java | 14 +-
.../drill/exec/expr/fn/impl/MathFunctions.java | 28 +
.../exec/expr/fn/impl/StatisticsAggrFunctions.java | 5740 ++++++++++++++++++++
.../apache/drill/exec/ops/BaseFragmentContext.java | 24 +-
.../apache/drill/exec/ops/ContextInformation.java | 1 +
.../org/apache/drill/exec/ops/FragmentContext.java | 29 +-
.../apache/drill/exec/ops/FragmentContextImpl.java | 20 +-
.../org/apache/drill/exec/ops/QueryContext.java | 34 +
.../org/apache/drill/exec/ops/UdfUtilities.java | 8 +-
.../exec/physical/base/AbstractDbGroupScan.java | 2 -
.../exec/physical/base/AbstractGroupScan.java | 5 +
.../physical/base/AbstractPhysicalVisitor.java | 18 +
.../apache/drill/exec/physical/base/GroupScan.java | 12 +-
.../exec/physical/base/PhysicalOperatorUtil.java | 26 +-
.../drill/exec/physical/base/PhysicalVisitor.java | 6 +
.../drill/exec/physical/base/SchemalessScan.java | 6 +
.../config/StatisticsAggregate.java} | 48 +-
.../exec/physical/config/StatisticsMerge.java | 69 +
.../config/UnpivotMaps.java} | 45 +-
...Batch.java => StatisticsWriterRecordBatch.java} | 70 +-
.../exec/physical/impl/WriterRecordBatch.java | 3 +
.../physical/impl/aggregate/InternalBatch.java | 6 +
.../physical/impl/aggregate/StreamingAggBatch.java | 18 +-
.../impl/statistics/AbstractMergedStatistic.java | 48 +
.../impl/statistics/AvgWidthMergedStatistic.java | 150 +
.../impl/statistics/CntDupsMergedStatistic.java | 103 +
.../impl/statistics/ColTypeMergedStatistic.java | 87 +
.../impl/statistics/ColumnMergedStatistic.java | 68 +
.../impl/statistics/HLLMergedStatistic.java | 130 +
.../physical/impl/statistics/MergedStatistic.java | 62 +
.../impl/statistics/MergedStatisticFactory.java | 62 +
.../impl/statistics/NDVMergedStatistic.java | 198 +
.../impl/statistics/NNRowCountMergedStatistic.java | 95 +
.../impl/statistics/RowCountMergedStatistic.java | 96 +
.../exec/physical/impl/statistics/Statistic.java | 44 +
.../impl/statistics/StatisticsAggBatch.java | 249 +
.../impl/statistics/StatisticsAggBatchCreator.java | 37 +
.../impl/statistics/StatisticsMergeBatch.java | 405 ++
.../statistics/StatisticsMergeBatchCreator.java | 38 +
.../impl/unpivot/UnpivotMapsBatchCreator.java | 38 +
.../impl/unpivot/UnpivotMapsRecordBatch.java | 280 +
.../planner/FileSystemPartitionDescriptor.java | 11 +-
.../apache/drill/exec/planner/PlannerPhase.java | 2 +
.../exec/planner/common/DrillAggregateRelBase.java | 9 +
.../exec/planner/common/DrillFilterRelBase.java | 4 +-
.../exec/planner/common/DrillJoinRelBase.java | 53 +-
.../drill/exec/planner/common/DrillRelOptUtil.java | 102 +
.../exec/planner/common/DrillScanRelBase.java | 14 +
.../drill/exec/planner/common/DrillStatsTable.java | 424 ++
.../planner/cost/DrillRelMdDistinctRowCount.java | 179 +-
.../exec/planner/cost/DrillRelMdRowCount.java | 62 +-
.../exec/planner/cost/DrillRelMdSelectivity.java | 201 +-
.../exec/planner/logical/DrillAnalyzeRel.java | 77 +
.../drill/exec/planner/logical/DrillTable.java | 11 +
.../drill/exec/planner/physical/AggPruleBase.java | 2 +-
.../drill/exec/planner/physical/AnalyzePrule.java | 131 +
.../planner/physical/ConvertCountToDirectScan.java | 4 +-
.../exec/planner/physical/DirectScanPrel.java | 33 +-
.../exec/planner/physical/DirectScanPrule.java | 3 -
.../exec/planner/physical/NestedLoopJoinPrule.java | 20 +-
.../exec/planner/physical/PlannerSettings.java | 6 +
.../drill/exec/planner/physical/ScanPrel.java | 29 +-
.../drill/exec/planner/physical/StatsAggPrel.java | 83 +
.../exec/planner/physical/StatsMergePrel.java | 89 +
.../exec/planner/physical/UnpivotMapsPrel.java | 85 +
.../planner/physical/visitor/BasePrelVisitor.java | 6 +
.../exec/planner/physical/visitor/PrelVisitor.java | 2 +
.../physical/visitor/PrelVisualizerVisitor.java | 8 +
.../drill/exec/planner/sql/DrillSqlWorker.java | 20 +
.../drill/exec/planner/sql/SchemaUtilites.java | 19 +
.../planner/sql/handlers/AnalyzeTableHandler.java | 284 +
.../planner/sql/handlers/DefaultSqlHandler.java | 5 +-
.../planner/sql/handlers/FindLimit0Visitor.java | 8 +-
.../sql/parser/CompoundIdentifierConverter.java | 8 +-
.../sql/parser/DrillCompoundIdentifier.java | 27 +-
.../exec/planner/sql/parser/SqlAnalyzeTable.java | 156 +
.../impl/DrillParserWithCompoundIdConverter.java | 9 +-
.../exec/server/options/SystemOptionManager.java | 7 +-
.../drill/exec/server/options/TypeValidators.java | 19 +
.../apache/drill/exec/store/AbstractSchema.java | 34 +
.../org/apache/drill/exec/store/RecordReader.java | 11 +
.../apache/drill/exec/store/SubSchemaWrapper.java | 15 +
.../drill/exec/store/avro/AvroFormatPlugin.java | 18 +
.../exec/store/dfs/FileSystemSchemaFactory.java | 15 +
.../apache/drill/exec/store/dfs/FormatPlugin.java | 11 +-
.../exec/store/dfs/WorkspaceSchemaFactory.java | 123 +-
.../exec/store/dfs/easy/EasyFormatPlugin.java | 17 +-
.../drill/exec/store/dfs/easy/EasyGroupScan.java | 1 +
.../drill/exec/store/direct/DirectSubScan.java | 9 +-
.../exec/store/easy/json/JSONFormatPlugin.java | 100 +-
.../exec/store/easy/json/JsonRecordWriter.java | 1 +
.../easy/json/JsonStatisticsRecordWriter.java | 482 ++
.../sequencefile/SequenceFileFormatPlugin.java | 17 +
.../exec/store/easy/text/TextFormatPlugin.java | 17 +
.../exec/store/httpd/HttpdLogFormatPlugin.java | 17 +
.../drill/exec/store/image/ImageFormatPlugin.java | 18 +
.../drill/exec/store/log/LogFormatPlugin.java | 19 +
.../store/parquet/AbstractParquetGroupScan.java | 1 +
.../exec/store/parquet/ParquetFormatPlugin.java | 46 +-
.../drill/exec/store/pcap/PcapFormatPlugin.java | 18 +
.../exec/store/pcapng/PcapngFormatPlugin.java | 19 +
.../drill/exec/store/pojo/PojoRecordReader.java | 13 +-
.../drill/exec/store/sys/SystemTableScan.java | 1 +
.../java/org/apache/drill/exec/util/Utilities.java | 13 +-
.../java-exec/src/main/resources/drill-module.conf | 9 +-
.../java/org/apache/drill/TestSelectivity.java | 4 +-
.../impersonation/TestImpersonationMetadata.java | 43 +-
.../exec/physical/impl/TestLocalExchange.java | 3 +-
.../impl/partitionsender/TestPartitionSender.java | 3 +-
.../apache/drill/exec/pop/TestFragmentChecker.java | 3 +-
.../org/apache/drill/exec/sql/TestAnalyze.java | 390 ++
.../drill/exec/sql/TestSqlBracketlessSyntax.java | 2 +-
.../org/apache/drill/test/DrillTestWrapper.java | 36 +-
.../java/org/apache/drill/test/TestBuilder.java | 10 +-
exec/jdbc-all/pom.xml | 4 +
.../apache/drill/common/logical/data/Analyze.java | 46 +
.../data/visitors/AbstractLogicalVisitor.java | 6 +
.../logical/data/visitors/LogicalVisitor.java | 2 +
.../org/apache/drill/exec/proto/UserBitShared.java | 39 +-
.../drill/exec/proto/beans/CoreOperatorType.java | 8 +-
protocol/src/main/protobuf/UserBitShared.proto | 3 +
147 files changed, 12483 insertions(+), 336 deletions(-)
create mode 100644
exec/java-exec/src/main/codegen/templates/JsonBaseStatisticsRecordWriter.java
copy exec/java-exec/src/main/codegen/templates/{RecordWriter.java =>
StatisticsRecordWriter.java} (67%)
create mode 100644
exec/java-exec/src/main/codegen/templates/StatisticsRecordWriterImpl.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StatisticsAggrFunctions.java
copy
exec/java-exec/src/main/java/org/apache/drill/exec/{store/direct/DirectSubScan.java
=> physical/config/StatisticsAggregate.java} (50%)
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/StatisticsMerge.java
copy
exec/java-exec/src/main/java/org/apache/drill/exec/{store/direct/DirectSubScan.java
=> physical/config/UnpivotMaps.java} (53%)
copy
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/{WriterRecordBatch.java
=> StatisticsWriterRecordBatch.java} (75%)
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/AbstractMergedStatistic.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/AvgWidthMergedStatistic.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/CntDupsMergedStatistic.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/ColTypeMergedStatistic.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/ColumnMergedStatistic.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/HLLMergedStatistic.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/MergedStatistic.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/MergedStatisticFactory.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/NDVMergedStatistic.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/NNRowCountMergedStatistic.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/RowCountMergedStatistic.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/Statistic.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/StatisticsAggBatch.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/StatisticsAggBatchCreator.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/StatisticsMergeBatch.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/StatisticsMergeBatchCreator.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/unpivot/UnpivotMapsBatchCreator.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/unpivot/UnpivotMapsRecordBatch.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillStatsTable.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAnalyzeRel.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AnalyzePrule.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/StatsAggPrel.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/StatsMergePrel.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnpivotMapsPrel.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/AnalyzeTableHandler.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlAnalyzeTable.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JsonStatisticsRecordWriter.java
create mode 100644
exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestAnalyze.java
create mode 100644
logical/src/main/java/org/apache/drill/common/logical/data/Analyze.java
