This is an automated email from the ASF dual-hosted git repository.
boaz pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git.
from cfaf430 code review comments
new 13e72c1 DRILL-6027: Initial implementation of HashJoin spill, without
memory limits checks yet
new 349ac5a DRILL-6027: - Added memory claculator - Added unit tests
and docs. - Fixed IOB caused by output vector allocation. - Don't double
count records that were spilled in HashJoin
new 89e0fe6 DRILL-6027: - Added fallback option for HashJoin. - No
copy of incoming for single partition, and avoid HT resize. - Fix memory leak
when cancelling while spill file is read - get correct schema when probe side
is empty - Re-create the HashJoinProbe
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../test/java/org/apache/drill/test/DrillTest.java | 2 +-
.../java/org/apache/drill/exec/ExecConstants.java | 22 +
.../apache/drill/exec/cache/VectorSerializer.java | 3 +-
.../drill/exec/physical/base/AbstractBase.java | 4 +-
.../drill/exec/physical/base/PhysicalOperator.java | 4 +-
.../drill/exec/physical/config/ExternalSort.java | 4 +-
.../drill/exec/physical/config/HashAggregate.java | 11 +-
.../drill/exec/physical/config/HashJoinPOP.java | 68 +-
.../drill/exec/physical/impl/BaseRootExec.java | 9 +-
.../apache/drill/exec/physical/impl/ScanBatch.java | 4 +
.../physical/impl/aggregate/HashAggTemplate.java | 18 +-
.../impl/aggregate/SpilledRecordbatch.java | 11 +-
.../physical/impl/common/ChainedHashTable.java | 22 +-
.../exec/physical/impl/common/HashPartition.java | 580 ++++++++++++
.../drill/exec/physical/impl/common/HashTable.java | 88 +-
.../impl/common/HashTableAllocationTracker.java | 73 ++
.../exec/physical/impl/common/HashTableConfig.java | 33 +-
.../physical/impl/common/HashTableTemplate.java | 206 +++--
.../exec/physical/impl/join/HashJoinBatch.java | 967 +++++++++++++++------
.../exec/physical/impl/join/HashJoinHelper.java | 16 +
.../impl/join/HashJoinHelperSizeCalculator.java | 8 +-
.../join/HashJoinHelperSizeCalculatorImpl.java | 51 ++
.../join/HashJoinMechanicalMemoryCalculator.java | 162 ++++
.../impl/join/HashJoinMemoryCalculator.java | 320 +++++++
.../impl/join/HashJoinMemoryCalculatorImpl.java | 832 ++++++++++++++++++
.../exec/physical/impl/join/HashJoinProbe.java | 18 +-
.../physical/impl/join/HashJoinProbeTemplate.java | 378 +++++---
.../exec/physical/impl/join/HashJoinState.java | 42 +
...chCreator.java => HashJoinStateCalculator.java} | 30 +-
.../impl/join/HashTableSizeCalculator.java} | 20 +-
.../HashTableSizeCalculatorConservativeImpl.java | 96 ++
.../impl/join/HashTableSizeCalculatorLeanImpl.java | 109 +++
.../impl/protocol/OperatorRecordBatch.java | 5 +
.../drill/exec/physical/impl/spill/SpillSet.java | 5 +
.../unorderedreceiver/UnorderedReceiverBatch.java | 5 +
.../validate/IteratorValidatorBatchIterator.java | 5 +
.../drill/exec/planner/logical/DrillJoinRel.java | 3 +-
.../drill/exec/record/AbstractRecordBatch.java | 13 +
.../org/apache/drill/exec/record/RecordBatch.java | 7 +
.../apache/drill/exec/record/RecordBatchSizer.java | 42 +
.../apache/drill/exec/record/SchemalessBatch.java | 3 +
.../drill/exec/record/SimpleRecordBatch.java | 5 +
.../apache/drill/exec/record/VectorContainer.java | 65 +-
.../exec/server/options/SystemOptionManager.java | 9 +
.../drill/exec/util/MemoryAllocationUtilities.java | 2 +-
.../work/foreman/rm/ThrottledResourceManager.java | 2 +-
.../java-exec/src/main/resources/drill-module.conf | 26 +
.../drill/exec/cache/TestBatchSerialization.java | 5 +-
.../exec/compile/TestLargeFileCompilation.java | 1 +
.../drill/exec/physical/impl/MockRecordBatch.java | 3 +
.../physical/impl/common/HashPartitionTest.java | 306 +++++++
.../common/HashTableAllocationTrackerTest.java | 101 +++
.../exec/physical/impl/join/PartitionStatImpl.java | 79 ++
.../impl/join/TestBuildSidePartitioningImpl.java | 289 ++++++
.../join/TestHashJoinHelperSizeCalculatorImpl.java | 44 +
.../join/TestHashJoinMemoryCalculatorImpl.java | 82 ++
.../exec/physical/impl/join/TestHashJoinSpill.java | 128 +++
...estHashTableSizeCalculatorConservativeImpl.java | 67 ++
.../join/TestHashTableSizeCalculatorLeanImpl.java | 66 ++
.../exec/physical/impl/join/TestPartitionStat.java | 57 ++
.../impl/join/TestPostBuildCalculationsImpl.java | 434 +++++++++
.../physical/impl/unnest/MockLateralJoinBatch.java | 2 +
.../physical/impl/validate/TestBatchValidator.java | 7 +-
.../xsort/managed/TestExternalSortInternals.java | 14 +-
.../physical/impl/xsort/managed/TestSortImpl.java | 23 +-
.../physical/impl/xsort/managed/TestSorter.java | 7 +-
.../exec/physical/unit/MiniPlanUnitTestBase.java | 2 +-
.../exec/physical/unit/PhysicalOpUnitTestBase.java | 16 +-
.../drill/exec/record/TestVectorContainer.java | 41 +-
.../drill/exec/util/TestQueryMemoryAlloc.java | 13 +-
.../org/apache/drill/test/BaseDirTestWatcher.java | 18 +
.../java/org/apache/drill/test/ConfigBuilder.java | 7 +-
.../org/apache/drill/test/DrillTestWrapper.java | 14 +-
.../org/apache/drill/test/OperatorFixture.java | 28 +-
.../org/apache/drill/test/SubOperatorTest.java | 6 +-
.../org/apache/drill/test/rowSet/RowSetBatch.java | 3 +
.../drill/test/rowSet/test/PerformanceTool.java | 2 +-
.../src/test/resources/empty.json | 0
.../main/codegen/templates/FixedValueVectors.java | 5 +
.../codegen/templates/NullableValueVectors.java | 7 +
.../org/apache/drill/exec/vector/BitVector.java | 5 +
.../apache/drill/exec/vector/FixedWidthVector.java | 6 +
.../drill/exec/vector/UntypedNullVector.java | 5 +
83 files changed, 5717 insertions(+), 584 deletions(-)
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashPartition.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableAllocationTracker.java
copy common/src/main/java/org/apache/drill/common/config/ConfigProvider.java
=>
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinHelperSizeCalculator.java
(79%)
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinHelperSizeCalculatorImpl.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinMechanicalMemoryCalculator.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinMemoryCalculator.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinMemoryCalculatorImpl.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinState.java
copy
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/{NestedLoopJoinBatchCreator.java
=> HashJoinStateCalculator.java} (55%)
copy
exec/java-exec/src/main/java/org/apache/drill/exec/{store/parquet/stat/ColumnStatCollector.java
=> physical/impl/join/HashTableSizeCalculator.java} (67%)
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashTableSizeCalculatorConservativeImpl.java
create mode 100644
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashTableSizeCalculatorLeanImpl.java
create mode 100644
exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/common/HashPartitionTest.java
create mode 100644
exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/common/HashTableAllocationTrackerTest.java
create mode 100644
exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/PartitionStatImpl.java
create mode 100644
exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestBuildSidePartitioningImpl.java
create mode 100644
exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinHelperSizeCalculatorImpl.java
create mode 100644
exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinMemoryCalculatorImpl.java
create mode 100644
exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinSpill.java
create mode 100644
exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashTableSizeCalculatorConservativeImpl.java
create mode 100644
exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashTableSizeCalculatorLeanImpl.java
create mode 100644
exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestPartitionStat.java
create mode 100644
exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestPostBuildCalculationsImpl.java
copy exec/{jdbc => java-exec}/src/test/resources/empty.json (100%)
--
To stop receiving notification emails like this one, please contact
[email protected].
