This is an automated email from the ASF dual-hosted git repository.
vitalii pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git.
from 2c3e2de DRILL-6642: Update protocol-buffers version
new 7e3b459 DRILL-7074: Scan framework fixes and enhancements
new 3d29faf DRILL-5603: Replace String file paths to Hadoop Path -
replaced all String path representation with org.apache.hadoop.fs.Path - added
PathSerDe.Se JSON serializer - refactoring of DFSPartitionLocation code by
leveraging existing listPartitionValues() functionality
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../exec/store/mapr/db/MapRDBFormatPlugin.java | 4 +-
.../store/mapr/streams/StreamsFormatPlugin.java | 12 +-
.../exec/store/syslog/SyslogRecordReader.java | 3 +-
.../exec/planner/sql/HivePartitionDescriptor.java | 9 +-
.../exec/planner/sql/HivePartitionLocation.java | 8 +-
.../hive/HiveDrillNativeParquetRowGroupScan.java | 2 +-
.../store/hive/HiveDrillNativeParquetScan.java | 6 +-
.../HiveDrillNativeParquetScanBatchCreator.java | 5 +-
.../drill/exec/store/hive/HivePartitionHolder.java | 11 +-
.../apache/drill/exec/physical/PhysicalPlan.java | 3 +-
.../exec/physical/base/AbstractGroupScan.java | 3 +-
.../apache/drill/exec/physical/base/GroupScan.java | 5 +-
.../drill/exec/physical/base/SchemalessScan.java | 9 +-
.../drill/exec/physical/impl/scan/ReaderState.java | 26 +-
.../exec/physical/impl/scan/RowBatchReader.java | 20 +-
.../physical/impl/scan/ScanOperatorEvents.java | 42 +-
.../impl/scan/columns/ColumnsArrayParser.java | 52 +-
.../impl/scan/file/BaseFileScanFramework.java | 2 +-
.../exec/physical/impl/scan/file/FileMetadata.java | 2 +-
.../impl/scan/file/FileMetadataColumnsParser.java | 71 +-
.../impl/scan/file/FileMetadataManager.java | 34 +
.../physical/impl/scan/file/FileScanFramework.java | 37 +-
.../physical/impl/scan/file/MetadataColumn.java | 13 +
.../impl/scan/framework/AbstractScanFramework.java | 76 ++
.../impl/scan/framework/SchemaNegotiator.java | 6 +-
.../impl/scan/framework/SchemaNegotiatorImpl.java | 6 +-
.../impl/scan/framework/ShimBatchReader.java | 15 +-
.../physical/impl/scan/framework/package-info.java | 26 +-
.../exec/physical/impl/scan/package-info.java | 85 +-
.../scan/project/ExplicitSchemaProjection.java | 1 +
.../scan/project/ReaderSchemaOrchestrator.java | 230 +++++
.../impl/scan/project/ScanLevelProjection.java | 66 +-
.../impl/scan/project/ScanSchemaOrchestrator.java | 241 +-----
.../impl/scan/project/SchemaLevelProjection.java | 11 +-
.../physical/impl/scan/project/package-info.java | 50 ++
.../rowSet/project/ImpliedTupleRequest.java | 2 +-
.../rowSet/project/RequestedTupleImpl.java | 44 +-
.../exec/planner/AbstractPartitionDescriptor.java | 3 +-
.../exec/planner/DFSDirPartitionLocation.java | 18 +-
.../exec/planner/DFSFilePartitionLocation.java | 25 +-
.../planner/FileSystemPartitionDescriptor.java | 102 +--
.../exec/planner/ParquetPartitionDescriptor.java | 18 +-
.../exec/planner/ParquetPartitionLocation.java | 8 +-
.../drill/exec/planner/PartitionDescriptor.java | 44 +-
.../drill/exec/planner/PartitionLocation.java | 14 +-
.../drill/exec/planner/PhysicalPlanReader.java | 53 +-
.../exec/planner/SimplePartitionLocation.java | 5 +-
.../planner/logical/partition/PruneScanRule.java | 17 +-
.../planner/sql/handlers/AnalyzeTableHandler.java | 28 +-
.../sql/handlers/RefreshMetadataHandler.java | 4 +-
.../{InstanceSerializer.java => PathSerDe.java} | 20 +-
.../apache/drill/exec/store/ColumnExplorer.java | 46 +-
.../drill/exec/store/avro/AvroDrillTable.java | 4 +-
.../drill/exec/store/avro/AvroRecordReader.java | 12 +-
.../apache/drill/exec/store/dfs/FileSelection.java | 161 ++--
.../drill/exec/store/dfs/FormatSelection.java | 8 +-
.../drill/exec/store/dfs/MetadataContext.java | 7 +-
.../drill/exec/store/dfs/ReadEntryFromHDFS.java | 3 +-
.../drill/exec/store/dfs/ReadEntryWithPath.java | 13 +-
.../exec/store/dfs/easy/EasyFormatPlugin.java | 2 +-
.../drill/exec/store/dfs/easy/EasyGroupScan.java | 33 +-
.../drill/exec/store/dfs/easy/EasySubScan.java | 22 +-
.../apache/drill/exec/store/dfs/easy/FileWork.java | 8 +-
.../exec/store/direct/MetadataDirectGroupScan.java | 10 +-
.../exec/store/easy/json/JSONRecordReader.java | 13 +-
.../sequencefile/SequenceFileFormatPlugin.java | 2 +-
.../exec/store/easy/text/TextFormatPlugin.java | 2 +-
.../exec/store/httpd/HttpdLogFormatPlugin.java | 2 +-
.../drill/exec/store/image/ImageFormatPlugin.java | 9 +-
.../drill/exec/store/image/ImageRecordReader.java | 4 +-
.../drill/exec/store/log/LogRecordReader.java | 5 +-
.../store/parquet/AbstractParquetGroupScan.java | 17 +-
.../parquet/AbstractParquetScanBatchCreator.java | 11 +-
.../drill/exec/store/parquet/ParquetGroupScan.java | 42 +-
.../store/parquet/ParquetGroupScanStatistics.java | 5 +-
.../exec/store/parquet/ParquetRowGroupScan.java | 11 +-
.../store/parquet/ParquetScanBatchCreator.java | 3 +-
.../drill/exec/store/parquet/RowGroupInfo.java | 3 +-
.../exec/store/parquet/RowGroupReadEntry.java | 3 +-
.../parquet/columnreaders/ParquetRecordReader.java | 24 +-
.../exec/store/parquet/metadata/Metadata.java | 37 +-
.../exec/store/parquet/metadata/MetadataBase.java | 5 +-
.../store/parquet/metadata/MetadataPathUtils.java | 29 +-
.../exec/store/parquet/metadata/Metadata_V1.java | 13 +-
.../exec/store/parquet/metadata/Metadata_V2.java | 15 +-
.../exec/store/parquet/metadata/Metadata_V3.java | 15 +-
.../parquet/metadata/ParquetTableMetadataDirs.java | 7 +-
.../exec/store/parquet2/DrillParquetReader.java | 2 +-
.../drill/exec/store/pcap/PcapRecordReader.java | 4 +-
.../exec/store/pcapng/PcapngRecordReader.java | 4 +-
.../drill/exec/store/schedule/BlockMapBuilder.java | 22 +-
.../exec/store/schedule/CompleteFileWork.java | 42 +-
.../drill/exec/util/DrillFileSystemUtil.java | 11 +
.../test/java/org/apache/drill/PlanTestBase.java | 3 +-
.../apache/drill/exec/TestPathSerialization.java | 65 ++
.../exec/physical/impl/scan/TestColumnsArray.java | 172 +++-
.../impl/scan/TestColumnsArrayFramework.java | 19 +-
.../physical/impl/scan/TestColumnsArrayParser.java | 46 +-
.../impl/scan/TestConstantColumnLoader.java | 153 ----
.../impl/scan/TestFileMetadataColumnParser.java | 170 +++-
.../impl/scan/TestFileMetadataProjection.java | 4 +-
.../physical/impl/scan/TestFileScanFramework.java | 13 +-
.../physical/impl/scan/TestNullColumnLoader.java | 281 ------
.../physical/impl/scan/TestRowBatchMerger.java | 459 ----------
.../physical/impl/scan/TestScanBatchWriters.java | 3 +
.../impl/scan/TestScanLevelProjection.java | 223 -----
.../physical/impl/scan/TestScanOperatorExec.java | 11 +-
.../impl/scan/TestScanOrchestratorEarlySchema.java | 255 ++----
.../impl/scan/TestScanOrchestratorLateSchema.java | 6 +-
.../impl/scan/TestScanOrchestratorMetadata.java | 77 +-
.../impl/scan/TestSchemaLevelProjection.java | 557 ------------
.../physical/impl/scan/TestSchemaSmoothing.java | 946 ---------------------
.../scan/project/TestConstantColumnLoader.java | 45 +
.../impl/scan/project/TestScanLevelProjection.java | 113 ++-
.../impl/scan/project/TestSchemaSmoothing.java | 250 +++++-
.../exec/physical/unit/MiniPlanUnitTestBase.java | 13 +-
.../drill/exec/physical/unit/TestMiniPlan.java | 5 +-
.../exec/physical/unit/TestNullInputMiniPlan.java | 8 +-
.../drill/exec/store/CachedSingleFileSystem.java | 8 +-
.../exec/store/dfs/TestDFSPartitionLocation.java | 54 ++
.../drill/exec/store/dfs/TestFileSelection.java | 7 +-
.../store/parquet/ParquetRecordReaderTest.java | 4 +-
.../drill/exec/store/store/TestAssignment.java | 3 +-
.../apache/drill/test/PhysicalOpUnitTestBase.java | 5 +-
.../java/org/apache/drill/test/QueryBuilder.java | 2 +-
125 files changed, 2469 insertions(+), 3799 deletions(-)
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/ReaderSchemaOrchestrator.java
copy
exec/java-exec/src/main/java/org/apache/drill/exec/serialization/{InstanceSerializer.java
=> PathSerDe.java} (60%)
create mode 100644
exec/java-exec/src/test/java/org/apache/drill/exec/TestPathSerialization.java
delete mode 100644
exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/TestConstantColumnLoader.java
delete mode 100644
exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/TestNullColumnLoader.java
delete mode 100644
exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/TestRowBatchMerger.java
delete mode 100644
exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/TestScanLevelProjection.java
delete mode 100644
exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/TestSchemaLevelProjection.java
delete mode 100644
exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/TestSchemaSmoothing.java
create mode 100644
exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestDFSPartitionLocation.java
