Repository: tajo Updated Branches: refs/heads/branch-0.11.2 cfdf4c39d -> 5b33401c8
TAJO-2074: Upgrade hadoop and netty. Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/5b33401c Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/5b33401c Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/5b33401c Branch: refs/heads/branch-0.11.2 Commit: 5b33401c8cc52996624f874cfc2ab273a9530968 Parents: cfdf4c3 Author: Jinho Kim <[email protected]> Authored: Mon Feb 15 17:36:21 2016 +0900 Committer: Jinho Kim <[email protected]> Committed: Mon Feb 15 17:36:21 2016 +0900 ---------------------------------------------------------------------- CHANGES | 2 + .../org/apache/tajo/TajoTestingCluster.java | 1 - .../java/org/apache/tajo/util/BytesUtils.java | 56 ++++++++++++++++++++ .../engine/planner/UniformRangePartition.java | 10 +++- tajo-project/pom.xml | 7 ++- .../tajo/storage/TestByteBufLineReader.java | 2 +- .../apache/tajo/storage/TestFileTablespace.java | 8 +-- .../tajo/storage/raw/TestDirectRawFile.java | 2 +- 8 files changed, 75 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/5b33401c/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 938abda..3e5ae65 100644 --- a/CHANGES +++ b/CHANGES @@ -4,6 +4,8 @@ Release 0.11.2 - unreleased TASKS + TAJO-2074: Upgrade hadoop and netty. (jinho) + TAJO-1939: Implement PgSQLTablespace::getTableVolume() method. (jihoon) TAJO-2017: Replace manual array copy with Collection. http://git-wip-us.apache.org/repos/asf/tajo/blob/5b33401c/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java ---------------------------------------------------------------------- diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java index 5cc5842..3a62e5d 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -264,7 +264,6 @@ public class TajoTestingCluster { builder.hosts(hosts); builder.numDataNodes(servers); builder.format(true); - builder.storagesPerDatanode(1); builder.waitSafeMode(true); this.dfsCluster = builder.build(); http://git-wip-us.apache.org/repos/asf/tajo/blob/5b33401c/tajo-common/src/main/java/org/apache/tajo/util/BytesUtils.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/util/BytesUtils.java b/tajo-common/src/main/java/org/apache/tajo/util/BytesUtils.java index 5df924b..9f72c14 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/BytesUtils.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/BytesUtils.java @@ -18,11 +18,15 @@ package org.apache.tajo.util; +import io.netty.buffer.ByteBuf; + import java.io.ByteArrayOutputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import static io.netty.util.internal.StringUtil.isSurrogate; + /** * Extra utilities for bytes */ @@ -250,4 +254,56 @@ public class BytesUtils { public static byte [] trimBytes(byte [] bytes) { return new String(bytes).trim().getBytes(); } + + /** + * this is an implementation copied from ByteBufUtil in netty4 + */ + public static int writeUtf8(ByteBuf buffer, char[] chars, boolean ignoreSurrogate) { + int oldWriterIndex = buffer.writerIndex(); + int writerIndex = oldWriterIndex; + + // We can use the _set methods as these not need to do any index checks and reference checks. + // This is possible as we called ensureWritable(...) before. + for (int i = 0; i < chars.length; i++) { + char c = chars[i]; + if (c < 0x80) { + buffer.setByte(writerIndex++, (byte) c); + } else if (c < 0x800) { + buffer.setByte(writerIndex++, (byte) (0xc0 | (c >> 6))); + buffer.setByte(writerIndex++, (byte) (0x80 | (c & 0x3f))); + } else if (!ignoreSurrogate && isSurrogate(c)) { + if (!Character.isHighSurrogate(c)) { + throw new IllegalArgumentException("Invalid encoding. " + + "Expected high (leading) surrogate at index " + i + " but got " + c); + } + final char c2; + try { + // Surrogate Pair consumes 2 characters. Optimistically try to get the next character to avoid + // duplicate bounds checking with charAt. If an IndexOutOfBoundsException is thrown we will + // re-throw a more informative exception describing the problem. + c2 = chars[++i]; + } catch (IndexOutOfBoundsException e) { + throw new IllegalArgumentException("Underflow. " + + "Expected low (trailing) surrogate at index " + i + " but no more characters found.", e); + } + if (!Character.isLowSurrogate(c2)) { + throw new IllegalArgumentException("Invalid encoding. " + + "Expected low (trailing) surrogate at index " + i + " but got " + c2); + } + int codePoint = Character.toCodePoint(c, c2); + // See http://www.unicode.org/versions/Unicode7.0.0/ch03.pdf#G2630. + buffer.setByte(writerIndex++, (byte) (0xf0 | (codePoint >> 18))); + buffer.setByte(writerIndex++, (byte) (0x80 | ((codePoint >> 12) & 0x3f))); + buffer.setByte(writerIndex++, (byte) (0x80 | ((codePoint >> 6) & 0x3f))); + buffer.setByte(writerIndex++, (byte) (0x80 | (codePoint & 0x3f))); + } else { + buffer.setByte(writerIndex++, (byte) (0xe0 | (c >> 12))); + buffer.setByte(writerIndex++, (byte) (0x80 | ((c >> 6) & 0x3f))); + buffer.setByte(writerIndex++, (byte) (0x80 | (c & 0x3f))); + } + } + // update the writerIndex without any extra checks for performance reasons + buffer.writerIndex(writerIndex); + return writerIndex - oldWriterIndex; + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/5b33401c/tajo-core/src/main/java/org/apache/tajo/engine/planner/UniformRangePartition.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/UniformRangePartition.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/UniformRangePartition.java index 7c26857..02e397d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/UniformRangePartition.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/UniformRangePartition.java @@ -20,7 +20,8 @@ package org.apache.tajo.engine.planner; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; -import com.sun.tools.javac.util.Convert; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.SortSpec; import org.apache.tajo.common.TajoDataTypes; @@ -622,7 +623,12 @@ public class UniformRangePartition extends RangePartitionAlgorithm { } } - end.put(i, DatumFactory.createText(Convert.chars2utf(lastChars))); + ByteBuf byteBuf = Unpooled.buffer(lastChars.length * 3); + BytesUtils.writeUtf8(byteBuf, lastChars, true); + byte[] bytes = new byte[byteBuf.readableBytes()]; + byteBuf.getBytes(0, bytes); + + end.put(i, DatumFactory.createText(bytes)); } } } http://git-wip-us.apache.org/repos/asf/tajo/blob/5b33401c/tajo-project/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-project/pom.xml b/tajo-project/pom.xml index 76ff220..ac3fbe2 100644 --- a/tajo-project/pom.xml +++ b/tajo-project/pom.xml @@ -33,11 +33,11 @@ <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> - <hadoop.version>2.7.1</hadoop.version> + <hadoop.version>2.7.2</hadoop.version> <protobuf.version>2.5.0</protobuf.version> <hbase.version>1.1.1</hbase.version> <hive.version>1.1.0</hive.version> - <netty.version>4.0.33.Final</netty.version> + <netty.version>4.0.34.Final</netty.version> <jersey.version>2.6</jersey.version> <jetty.version>6.1.26</jetty.version> <tajo.root>${project.parent.relativePath}/..</tajo.root> @@ -398,7 +398,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <version>3.3</version> + <version>3.5</version> <configuration> <source>1.7</source> <target>1.7</target> @@ -632,7 +632,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <version>3.3</version> <configuration> <source>1.7</source> <target>1.7</target> http://git-wip-us.apache.org/repos/asf/tajo/blob/5b33401c/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java index 2cee3bb..18a9b85 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestByteBufLineReader.java @@ -87,7 +87,7 @@ public class TestByteBufLineReader { final Configuration conf = TestFileTablespace.getTestHdfsConfiguration(); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) - .numDataNodes(2).storagesPerDatanode(1).format(true).build(); + .numDataNodes(2).format(true).build(); TajoConf tajoConf = new TajoConf(conf); tajoConf.setVar(TajoConf.ConfVars.ROOT_DIR, cluster.getFileSystem().getUri() + "/tajo"); http://git-wip-us.apache.org/repos/asf/tajo/blob/5b33401c/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java index 0370302..58025c3 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java @@ -122,7 +122,7 @@ public class TestFileTablespace { public void testGetSplit() throws Exception { final Configuration hdfsConf = getTestHdfsConfiguration(); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(hdfsConf) - .numDataNodes(1).storagesPerDatanode(1).format(true).build(); + .numDataNodes(1).format(true).build(); int testCount = 10; Path tablePath = new Path("/testGetSplit"); @@ -174,7 +174,7 @@ public class TestFileTablespace { public void testZeroLengthSplit() throws Exception { final Configuration hdfsConf = getTestHdfsConfiguration(); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(hdfsConf) - .numDataNodes(1).storagesPerDatanode(1).format(true).build(); + .numDataNodes(1).format(true).build(); int testCount = 10; Path tablePath = new Path("/testZeroLengthSplit"); @@ -222,7 +222,7 @@ public class TestFileTablespace { hdfsConf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 2); hdfsConf.setBoolean(DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED, true); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(hdfsConf) - .numDataNodes(2).storagesPerDatanode(1).format(true).build(); + .numDataNodes(2).format(true).build(); int testCount = 10; Path tablePath = new Path("/testGetSplitWithBlockStorageLocationsBatching"); @@ -266,7 +266,7 @@ public class TestFileTablespace { final Configuration hdfsConf = getTestHdfsConfiguration(); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(hdfsConf) - .numDataNodes(1).storagesPerDatanode(1).format(true).build(); + .numDataNodes(1).format(true).build(); URI uri = URI.create(cluster.getFileSystem().getUri() + "/tajo"); try { http://git-wip-us.apache.org/repos/asf/tajo/blob/5b33401c/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java index d027fb8..fb69fa4 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java @@ -101,7 +101,7 @@ public class TestDirectRawFile { cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(1) .format(true) - .storagesPerDatanode(1).build(); + .build(); fs = cluster.getFileSystem(); }
