Repository: tajo Updated Branches: refs/heads/master ebfcc43b4 -> 85b38348f
TAJO-713: Missing INET4 in UniformRangePartition. (jihoon) Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/85b38348 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/85b38348 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/85b38348 Branch: refs/heads/master Commit: 85b38348f1da1dec5c187b72ae6ceaf887303b8b Parents: ebfcc43 Author: Jihoon Son <[email protected]> Authored: Sat Mar 29 14:34:13 2014 +0900 Committer: Jihoon Son <[email protected]> Committed: Sat Mar 29 14:34:13 2014 +0900 ---------------------------------------------------------------------- CHANGES.txt | 2 + .../engine/planner/UniformRangePartition.java | 36 +++- .../planner/TestUniformRangePartition.java | 169 +++++++++++++------ 3 files changed, 152 insertions(+), 55 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/85b38348/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 1fdaedb..ac4d325 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -292,6 +292,8 @@ Release 0.8.0 - unreleased BUG FIXES + TAJO-713: Missing INET4 in UniformRangePartition. (jihoon) + TAJO-718: A group-by clause with the same columns but aliased causes planning error. (hyunsik) http://git-wip-us.apache.org/repos/asf/tajo/blob/85b38348/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/UniformRangePartition.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/UniformRangePartition.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/UniformRangePartition.java index a7df3e8..f6922ed 100644 --- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/UniformRangePartition.java +++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/UniformRangePartition.java @@ -29,6 +29,7 @@ import org.apache.tajo.engine.exception.RangeOverflowException; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.TupleRange; import org.apache.tajo.storage.VTuple; +import org.apache.tajo.util.Bytes; import java.math.BigDecimal; import java.math.RoundingMode; @@ -202,6 +203,25 @@ public class UniformRangePartition extends RangePartitionAlgorithm { return candidate.compareTo(new BigDecimal(range.getEnd().get(colId).asChars().charAt(0))) < 0; } } + case INET4: { + int candidateIntVal; + byte[] candidateBytesVal = new byte[4]; + if (sortSpecs[colId].isAscending()) { + candidateIntVal = inc.intValue() + last.asInt4(); + if (candidateIntVal - inc.intValue() != last.asInt4()) { + return true; + } + Bytes.putInt(candidateBytesVal, 0, candidateIntVal); + return Bytes.compareTo(range.getEnd().get(colId).asByteArray(), candidateBytesVal) < 0; + } else { + candidateIntVal = last.asInt4() - inc.intValue(); + if (candidateIntVal + inc.intValue() != last.asInt4()) { + return true; + } + Bytes.putInt(candidateBytesVal, 0, candidateIntVal); + return Bytes.compareTo(candidateBytesVal, range.getEnd().get(colId).asByteArray()) < 0; + } + } } return overflow; } @@ -231,7 +251,8 @@ public class UniformRangePartition extends RangePartitionAlgorithm { } case TIME: case TIMESTAMP: - case INT8: { + case INT8: + case INET4: { long candidate = last.asInt8() + inc; long end = range.getEnd().get(colId).asInt8(); reminder = candidate - end; @@ -407,6 +428,19 @@ public class UniformRangePartition extends RangePartitionAlgorithm { end.put(i, DatumFactory.createTimeStampFromMillis(last.get(i).asInt8() + incs[i].longValue())); } break; + case INET4: + byte[] ipBytes; + if (overflowFlag[i]) { + ipBytes = range.getStart().get(i).asByteArray(); + assert ipBytes.length == 4; + end.put(i, DatumFactory.createInet4(ipBytes)); + } else { + int lastVal = last.get(i).asInt4() + incs[i].intValue(); + ipBytes = new byte[4]; + Bytes.putInt(ipBytes, 0, lastVal); + end.put(i, DatumFactory.createInet4(ipBytes)); + } + break; default: throw new UnsupportedOperationException(column.getDataType() + " is not supported yet"); } http://git-wip-us.apache.org/repos/asf/tajo/blob/85b38348/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestUniformRangePartition.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestUniformRangePartition.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestUniformRangePartition.java index dc5108a..f4c114f 100644 --- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestUniformRangePartition.java +++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestUniformRangePartition.java @@ -18,15 +18,14 @@ package org.apache.tajo.engine.planner; -import org.apache.tajo.catalog.SortSpec; -import org.junit.Test; import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.SortSpec; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.utils.TupleUtil; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.TupleRange; import org.apache.tajo.storage.VTuple; +import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -38,8 +37,8 @@ public class TestUniformRangePartition { @Test public void testIncrement1() { Schema schema = new Schema() - .addColumn("l_returnflag", Type.TEXT) - .addColumn("l_linestatus", Type.TEXT); + .addColumn("l_returnflag", Type.TEXT) + .addColumn("l_linestatus", Type.TEXT); SortSpec[] sortSpecs = PlannerUtil.schemaToSortSpecs(schema); @@ -85,8 +84,8 @@ public class TestUniformRangePartition { @Test public void testIncrement2() { Schema schema = new Schema() - .addColumn("l_returnflag", Type.TEXT) - .addColumn("l_linestatus", Type.TEXT); + .addColumn("l_returnflag", Type.TEXT) + .addColumn("l_linestatus", Type.TEXT); SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema); @@ -130,9 +129,9 @@ public class TestUniformRangePartition { @Test public void testIncrement3() { Schema schema = new Schema() - .addColumn("l_returnflag", Type.TEXT) - .addColumn("l_linestatus", Type.TEXT) - .addColumn("final", Type.TEXT); + .addColumn("l_returnflag", Type.TEXT) + .addColumn("l_linestatus", Type.TEXT) + .addColumn("final", Type.TEXT); SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema); @@ -163,8 +162,8 @@ public class TestUniformRangePartition { @Test public void testIncrement4() { Schema schema = new Schema() - .addColumn("l_orderkey", Type.INT8) - .addColumn("l_linenumber", Type.INT8); + .addColumn("l_orderkey", Type.INT8) + .addColumn("l_linenumber", Type.INT8); SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema); @@ -190,9 +189,9 @@ public class TestUniformRangePartition { @Test public void testIncrement5() { Schema schema = new Schema() - .addColumn("l_orderkey", Type.INT8) - .addColumn("l_linenumber", Type.INT8) - .addColumn("final", Type.INT8); + .addColumn("l_orderkey", Type.INT8) + .addColumn("l_linenumber", Type.INT8) + .addColumn("final", Type.INT8); SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema); @@ -223,9 +222,9 @@ public class TestUniformRangePartition { @Test public void testIncrement6() { Schema schema = new Schema() - .addColumn("l_orderkey", Type.FLOAT8) - .addColumn("l_linenumber", Type.FLOAT8) - .addColumn("final", Type.FLOAT8); + .addColumn("l_orderkey", Type.FLOAT8) + .addColumn("l_linenumber", Type.FLOAT8) + .addColumn("final", Type.FLOAT8); SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema); @@ -254,6 +253,39 @@ public class TestUniformRangePartition { } @Test + public void testIncrement7() { + Schema schema = new Schema() + .addColumn("l_orderkey", Type.INET4) + .addColumn("l_linenumber", Type.INET4) + .addColumn("final", Type.INET4); + + SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema); + + Tuple s = new VTuple(3); + s.put(0, DatumFactory.createInet4("127.0.1.1")); + s.put(1, DatumFactory.createInet4("127.0.0.1")); + s.put(2, DatumFactory.createInet4("128.0.0.253")); + Tuple e = new VTuple(3); + e.put(0, DatumFactory.createInet4("127.0.1.4")); // 4 + e.put(1, DatumFactory.createInet4("127.0.0.2")); // 2 + e.put(2, DatumFactory.createInet4("128.0.0.255")); //x3 = 24 + + TupleRange expected = new TupleRange(sortSpecs, s, e); + + UniformRangePartition partitioner = new UniformRangePartition(expected, sortSpecs); + assertEquals(24, partitioner.getTotalCardinality().longValue()); + + Tuple beforeOverflow = partitioner.increment(s, 5, 2); + assertTrue("127.0.1.1".equals(beforeOverflow.get(0).asChars())); + assertTrue("127.0.0.2".equals(beforeOverflow.get(1).asChars())); + assertTrue("128.0.0.255".equals(beforeOverflow.get(2).asChars())); + Tuple overflow = partitioner.increment(beforeOverflow, 1, 2); + assertTrue("127.0.1.2".equals(overflow.get(0).asChars())); + assertTrue("127.0.0.1".equals(overflow.get(1).asChars())); + assertTrue("128.0.0.253".equals(overflow.get(2).asChars())); + } + + @Test public void testPartition() { Schema schema = new Schema(); schema.addColumn("l_returnflag", Type.TEXT); @@ -286,8 +318,8 @@ public class TestUniformRangePartition { @Test public void testPartitionForOnePartNum() { Schema schema = new Schema() - .addColumn("l_returnflag", Type.TEXT) - .addColumn("l_linestatus", Type.TEXT); + .addColumn("l_returnflag", Type.TEXT) + .addColumn("l_linestatus", Type.TEXT); SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema); @@ -308,8 +340,8 @@ public class TestUniformRangePartition { @Test public void testPartitionForOnePartNumWithOneOfTheValueNull() { Schema schema = new Schema() - .addColumn("l_returnflag", Type.TEXT) - .addColumn("l_linestatus", Type.TEXT); + .addColumn("l_returnflag", Type.TEXT) + .addColumn("l_linestatus", Type.TEXT); SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema); @@ -321,7 +353,7 @@ public class TestUniformRangePartition { e.put(1, DatumFactory.createNullDatum()); TupleRange expected = new TupleRange(sortSpecs, s, e); RangePartitionAlgorithm partitioner = - new UniformRangePartition(expected, sortSpecs, true); + new UniformRangePartition(expected, sortSpecs, true); TupleRange [] ranges = partitioner.partition(1); assertEquals(expected, ranges[0]); @@ -330,8 +362,8 @@ public class TestUniformRangePartition { @Test public void testPartitionForOnePartNumWithBothValueNull() { Schema schema = new Schema() - .addColumn("l_returnflag", Type.TEXT) - .addColumn("l_linestatus", Type.TEXT); + .addColumn("l_returnflag", Type.TEXT) + .addColumn("l_linestatus", Type.TEXT); SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema); @@ -343,39 +375,68 @@ public class TestUniformRangePartition { e.put(1, DatumFactory.createNullDatum()); TupleRange expected = new TupleRange(sortSpecs, s, e); RangePartitionAlgorithm partitioner = - new UniformRangePartition(expected, sortSpecs, true); + new UniformRangePartition(expected, sortSpecs, true); TupleRange [] ranges = partitioner.partition(1); assertEquals(expected, ranges[0]); } - @Test - public void testPartitionWithNull() { - Schema schema = new Schema(); - schema.addColumn("l_returnflag", Type.TEXT); - schema.addColumn("l_linestatus", Type.TEXT); - - SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema); - - Tuple s = new VTuple(2); - s.put(0, DatumFactory.createNullDatum()); - s.put(1, DatumFactory.createText("F")); - Tuple e = new VTuple(2); - e.put(0, DatumFactory.createNullDatum()); - e.put(1, DatumFactory.createText("O")); - TupleRange expected = new TupleRange(sortSpecs, s, e); - RangePartitionAlgorithm partitioner - = new UniformRangePartition(expected, sortSpecs, true); - TupleRange [] ranges = partitioner.partition(10); - - - TupleRange prev = null; - for (TupleRange r : ranges) { - if (prev == null) { - prev = r; - } else { - assertTrue(prev.compareTo(r) > 0); - } - } + @Test + public void testPartitionWithNull() { + Schema schema = new Schema(); + schema.addColumn("l_returnflag", Type.TEXT); + schema.addColumn("l_linestatus", Type.TEXT); + + SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema); + + Tuple s = new VTuple(2); + s.put(0, DatumFactory.createNullDatum()); + s.put(1, DatumFactory.createText("F")); + Tuple e = new VTuple(2); + e.put(0, DatumFactory.createNullDatum()); + e.put(1, DatumFactory.createText("O")); + TupleRange expected = new TupleRange(sortSpecs, s, e); + RangePartitionAlgorithm partitioner + = new UniformRangePartition(expected, sortSpecs, true); + TupleRange [] ranges = partitioner.partition(10); + + + TupleRange prev = null; + for (TupleRange r : ranges) { + if (prev == null) { + prev = r; + } else { + assertTrue(prev.compareTo(r) > 0); + } + } + } + + @Test + public void testPartitionWithINET4() { + Schema schema = new Schema(); + schema.addColumn("l_returnflag", Type.INET4); + schema.addColumn("l_linestatus", Type.INET4); + + SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema); + + Tuple s = new VTuple(2); + s.put(0, DatumFactory.createInet4("127.0.1.10")); + s.put(1, DatumFactory.createInet4("127.0.2.10")); + Tuple e = new VTuple(2); + e.put(0, DatumFactory.createInet4("127.0.1.20")); + e.put(1, DatumFactory.createInet4("127.0.2.20")); + TupleRange expected = new TupleRange(sortSpecs, s, e); + RangePartitionAlgorithm partitioner + = new UniformRangePartition(expected, sortSpecs, true); + TupleRange [] ranges = partitioner.partition(10); + + TupleRange prev = null; + for (TupleRange r : ranges) { + if (prev == null) { + prev = r; + } else { + assertTrue(prev.compareTo(r) < 0); + } } + } }
