This is an automated email from the ASF dual-hosted git repository.
jonwei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new e8727dc Fix DeterminePartitionsJob reducer when total rows <
targetPartitionSize * SHARD_COMBINE_THRESHOLD (#8273)
e8727dc is described below
commit e8727dc98b4125dcdd35b44cb12a791d404d926a
Author: Jonathan Wei <[email protected]>
AuthorDate: Fri Aug 9 16:03:30 2019 -0500
Fix DeterminePartitionsJob reducer when total rows < targetPartitionSize *
SHARD_COMBINE_THRESHOLD (#8273)
* Fix DeterminePartitionsJob reducer when rows < targetPartitionSize
* use isEmpty()
---
.../druid/indexer/DeterminePartitionsJob.java | 5 +++--
.../druid/indexer/DeterminePartitionsJobTest.java | 24 ++++++++++++++++++++++
2 files changed, 27 insertions(+), 2 deletions(-)
diff --git
a/indexing-hadoop/src/main/java/org/apache/druid/indexer/DeterminePartitionsJob.java
b/indexing-hadoop/src/main/java/org/apache/druid/indexer/DeterminePartitionsJob.java
index 984d104..f9c4fe8 100644
---
a/indexing-hadoop/src/main/java/org/apache/druid/indexer/DeterminePartitionsJob.java
+++
b/indexing-hadoop/src/main/java/org/apache/druid/indexer/DeterminePartitionsJob.java
@@ -691,8 +691,9 @@ public class DeterminePartitionsJob implements Jobby
// One more shard to go
final ShardSpec shardSpec;
- if (currentDimPartition.rows < config.getTargetPartitionSize() *
SHARD_COMBINE_THRESHOLD) {
- // Combine with previous shard
+ if (currentDimPartition.rows < config.getTargetPartitionSize() *
SHARD_COMBINE_THRESHOLD &&
+ !currentDimPartitions.partitions.isEmpty()) {
+ // Combine with previous shard if it exists and the current
shard is small enough
final DimPartition previousDimPartition =
currentDimPartitions.partitions.remove(
currentDimPartitions.partitions.size() - 1
);
diff --git
a/indexing-hadoop/src/test/java/org/apache/druid/indexer/DeterminePartitionsJobTest.java
b/indexing-hadoop/src/test/java/org/apache/druid/indexer/DeterminePartitionsJobTest.java
index 2c35b75..35174f5 100644
---
a/indexing-hadoop/src/test/java/org/apache/druid/indexer/DeterminePartitionsJobTest.java
+++
b/indexing-hadoop/src/test/java/org/apache/druid/indexer/DeterminePartitionsJobTest.java
@@ -191,6 +191,30 @@ public class DeterminePartitionsJobTest
"2014102200,j.example.com,US,333",
"2014102200,k.example.com,US,555"
)
+ },
+ {
+ true,
+ 1000,
+ "2014-10-22T00:00:00Z/P1D",
+ 1,
+ new int[]{1},
+ new String[][][]{
+ {
+ {null, null}
+ }
+ },
+ ImmutableList.of(
+ "2014102200,a.example.com,CN,100",
+ "2014102200,b.exmaple.com,US,50",
+ "2014102200,c.example.com,US,200",
+ "2014102200,d.example.com,US,250",
+ "2014102200,e.example.com,US,123",
+ "2014102200,f.example.com,US,567",
+ "2014102200,g.example.com,US,11",
+ "2014102200,h.example.com,US,251",
+ "2014102200,i.example.com,US,963",
+ "2014102200,j.example.com,US,333"
+ )
}
}
);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
