Repository: tajo Updated Branches: refs/heads/branch-0.10.0 9ad117f07 -> 02c6c266c
TAJO-1283: ORDER BY with the first descending order causes wrong results. (Keuntae Park) Closes #364 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/02c6c266 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/02c6c266 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/02c6c266 Branch: refs/heads/branch-0.10.0 Commit: 02c6c266c013d8174a287bc57a6d4131da51ba96 Parents: 9ad117f Author: Keuntae Park <[email protected]> Authored: Tue Feb 3 16:57:13 2015 +0900 Committer: Keuntae Park <[email protected]> Committed: Tue Feb 3 16:57:13 2015 +0900 ---------------------------------------------------------------------- CHANGES | 3 ++ .../apache/tajo/querymaster/Repartitioner.java | 9 ++---- .../java/org/apache/tajo/querymaster/Stage.java | 3 +- .../apache/tajo/engine/query/TestSortQuery.java | 34 ++++++++++++++++++++ .../queries/TestSortQuery/testSortFirstDesc.sql | 1 + .../TestSortQuery/testSortFirstDesc.result | 10 ++++++ .../org/apache/tajo/storage/TupleRange.java | 9 ------ 7 files changed, 52 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/02c6c266/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 47eaf9c..5fe46e8 100644 --- a/CHANGES +++ b/CHANGES @@ -176,6 +176,9 @@ Release 0.10.0 - unreleased BUG FIXES + TAJO-1283: ORDER BY with the first descending order causes wrong results. + (Keuntae Park) + TAJO-1324: Remove warehouse directory rewriting in Unit Test. (jinho) TAJO-1321: Cli prints wrong response time. (jihoon) http://git-wip-us.apache.org/repos/asf/tajo/blob/02c6c266/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java index 5a35674..7c2d9f4 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java @@ -725,13 +725,8 @@ public class Repartitioner { } } - boolean ascendingFirstKey = sortSpecs[0].isAscending(); SortedMap<TupleRange, Collection<FetchImpl>> map; - if (ascendingFirstKey) { - map = new TreeMap<TupleRange, Collection<FetchImpl>>(); - } else { - map = new TreeMap<TupleRange, Collection<FetchImpl>>(new TupleRange.DescendingTupleRangeComparator()); - } + map = new TreeMap<TupleRange, Collection<FetchImpl>>(); Set<FetchImpl> fetchSet; try { @@ -740,7 +735,7 @@ public class Repartitioner { fetchSet = new HashSet<FetchImpl>(); for (FetchImpl fetch: fetches) { String rangeParam = - TupleUtil.rangeToQuery(ranges[i], ascendingFirstKey ? i == (ranges.length - 1) : i == 0, encoder); + TupleUtil.rangeToQuery(ranges[i], i == (ranges.length - 1) , encoder); FetchImpl copy = null; try { copy = fetch.clone(); http://git-wip-us.apache.org/repos/asf/tajo/blob/02c6c266/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java index 5673d5b..efadaa7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java @@ -1007,7 +1007,8 @@ public class Stage implements EventHandler<StageEvent> { int mb = (int) Math.ceil((double)volume / 1048576); LOG.info(stage.getId() + ", Table's volume is approximately " + mb + " MB"); // determine the number of task per 64MB - int maxTaskNum = Math.max(1, (int) Math.ceil((double)mb / 64)); + int minTaskNum = Math.max(1, stage.getContext().getQueryMasterContext().getConf().getInt(ConfVars.$TEST_MIN_TASK_NUM.varname, 1)); + int maxTaskNum = Math.max(minTaskNum, (int) Math.ceil((double)mb / 64)); LOG.info(stage.getId() + ", The determined number of non-leaf tasks is " + maxTaskNum); return maxTaskNum; } http://git-wip-us.apache.org/repos/asf/tajo/blob/02c6c266/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java index 9c911f9..ff91177 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java @@ -102,6 +102,40 @@ public class TestSortQuery extends QueryTestCaseBase { cleanupQuery(res); } + + @Test + public final void testSortFirstDesc() throws Exception { + try { + testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_MIN_TASK_NUM.varname, "2"); + KeyValueSet tableOptions = new KeyValueSet(); + tableOptions.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.TEXT_NULL, "\\\\N"); + + Schema schema = new Schema(); + schema.addColumn("col1", Type.INT4); + schema.addColumn("col2", Type.TEXT); + String[] data = new String[]{ + "1|abc", + "3|dfa", + "3|das", + "1|abb", + "1|abc", + "3|dfb", + "3|dat", + "1|abe" + }; + TajoTestingCluster.createTable("sortfirstdesc", schema, tableOptions, data, 2); + + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } finally { + testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_MIN_TASK_NUM.varname, "0"); + executeString("DROP TABLE sortfirstdesc PURGE;").close(); + } + } + + @Test public final void testTopK() throws Exception { ResultSet res = executeQuery(); http://git-wip-us.apache.org/repos/asf/tajo/blob/02c6c266/tajo-core/src/test/resources/queries/TestSortQuery/testSortFirstDesc.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestSortQuery/testSortFirstDesc.sql b/tajo-core/src/test/resources/queries/TestSortQuery/testSortFirstDesc.sql new file mode 100644 index 0000000..644feb0 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSortQuery/testSortFirstDesc.sql @@ -0,0 +1 @@ +select col1, col2 from sortfirstdesc order by col1 desc, col2; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/02c6c266/tajo-core/src/test/resources/results/TestSortQuery/testSortFirstDesc.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestSortQuery/testSortFirstDesc.result b/tajo-core/src/test/resources/results/TestSortQuery/testSortFirstDesc.result new file mode 100644 index 0000000..2ed9235 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSortQuery/testSortFirstDesc.result @@ -0,0 +1,10 @@ +col1,col2 +------------------------------- +3,das +3,dat +3,dfa +3,dfb +1,abb +1,abc +1,abc +1,abe \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/02c6c266/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TupleRange.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TupleRange.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TupleRange.java index e824b99..043409a 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TupleRange.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TupleRange.java @@ -94,15 +94,6 @@ public class TupleRange implements Comparable<TupleRange>, Cloneable { } } - public static class DescendingTupleRangeComparator - implements Comparator<TupleRange> { - - @Override - public int compare(TupleRange left, TupleRange right) { - return right.compareTo(left); - } - } - public TupleRange clone() throws CloneNotSupportedException { TupleRange newRange = (TupleRange) super.clone(); newRange.setStart(start.clone());
