Repository: tajo Updated Branches: refs/heads/master ae9a8c9cb -> 2950d8ec6
TAJO-1716: Repartitioner.makeEvenDistributedFetchImpl() does not distribute fetches evenly. Closes #659 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/2950d8ec Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/2950d8ec Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/2950d8ec Branch: refs/heads/master Commit: 2950d8ec60fa769819b60e7e516387259b1491da Parents: ae9a8c9 Author: Jihoon Son <[email protected]> Authored: Wed Jul 29 16:40:42 2015 +0900 Committer: Jihoon Son <[email protected]> Committed: Wed Jul 29 16:40:42 2015 +0900 ---------------------------------------------------------------------- CHANGES | 3 +++ .../main/java/org/apache/tajo/querymaster/Repartitioner.java | 7 ++++--- .../test/java/org/apache/tajo/master/TestRepartitioner.java | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/2950d8ec/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 25a03e6..01116d3 100644 --- a/CHANGES +++ b/CHANGES @@ -198,6 +198,9 @@ Release 0.11.0 - unreleased BUG FIXES + TAJO-1716: Repartitioner.makeEvenDistributedFetchImpl() does not distribute + fetches evenly. (jihoon) + TAJO-1681: Fix TajoDump invalid null check for database name. (Contributed by DaeMyung Kang, Committed by jihoon) http://git-wip-us.apache.org/repos/asf/tajo/blob/2950d8ec/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java index 50b84d4..fdaa0bc 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java @@ -914,8 +914,9 @@ public class Repartitioner { // algorithm. Iterator<FetchGroupMeta> iterator = fetchGroupMetaList.iterator(); - int p = 0; + int p; while(iterator.hasNext()) { + p = 0; while (p < num && iterator.hasNext()) { FetchGroupMeta fetchGroupMeta = iterator.next(); assignedVolumes[p] += fetchGroupMeta.getVolume(); @@ -925,13 +926,13 @@ public class Repartitioner { } p = num - 1; - while (p > 0 && iterator.hasNext()) { + while (p >= 0 && iterator.hasNext()) { FetchGroupMeta fetchGroupMeta = iterator.next(); assignedVolumes[p] += fetchGroupMeta.getVolume(); TUtil.putCollectionToNestedList(fetchesArray[p], tableName, fetchGroupMeta.fetchUrls); // While the current one is smaller than next one, it adds additional fetches to current one. - while(iterator.hasNext() && assignedVolumes[p - 1] > assignedVolumes[p]) { + while(iterator.hasNext() && (p > 0 && assignedVolumes[p - 1] > assignedVolumes[p])) { FetchGroupMeta additionalFetchGroup = iterator.next(); assignedVolumes[p] += additionalFetchGroup.getVolume(); TUtil.putCollectionToNestedList(fetchesArray[p], tableName, additionalFetchGroup.fetchUrls); http://git-wip-us.apache.org/repos/asf/tajo/blob/2950d8ec/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java b/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java index 3c378dd..36942a4 100644 --- a/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java +++ b/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java @@ -136,7 +136,7 @@ public class TestRepartitioner { assertFetchImpl(fetches, results.getSecond()); results = Repartitioner.makeEvenDistributedFetchImpl(fetchGroups, tableName, 2); - long expected0 [] = {130, 165}; + long expected0 [] = {140, 155}; assertFetchVolumes(expected0, results.getFirst()); assertFetchImpl(fetches, results.getSecond());
