shaofengshi closed pull request #357: KYLIN-3694 Bug fix and revert guava version URL: https://github.com/apache/kylin/pull/357
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/assembly/pom.xml b/assembly/pom.xml index 6dfe165cbe..1e546cb313 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -34,11 +34,6 @@ </properties> <dependencies> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - <scope>compile</scope> - </dependency> <dependency> <groupId>org.apache.kylin</groupId> <artifactId>kylin-source-hive</artifactId> @@ -228,10 +223,6 @@ <pattern>org.apache.commons.jocl</pattern> <shadedPattern>${shadeBase}.org.apache.commons.jocl</shadedPattern> </relocation> - <relocation> - <pattern>com.google.guava</pattern> - <shadedPattern>${shadeBase}.com.google.guava</shadedPattern> - </relocation> </relocations> <filters> <filter> diff --git a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java index 8f4a1f6919..f78e92b530 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java @@ -87,7 +87,7 @@ Hasher hc = hf.newHasher(); final String cell = row.get(flatDesc.getRowKeyColumnIndexes()[i]); if (cell != null) { - row_hashcodes[i] = hc.putUnencodedChars(cell).hash().asBytes(); + row_hashcodes[i] = hc.putString(cell).hash().asBytes(); } else { row_hashcodes[i] = hc.putInt(0).hash().asBytes(); } diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java index b9fb9841e1..c3642cb22d 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java @@ -147,7 +147,7 @@ public void createDirectChildrenCacheTest() { @Test public void createDirectChildrenCacheStressTest() { - Stopwatch sw = Stopwatch.createStarted(); + Stopwatch sw = new Stopwatch(); Set<Long> cuboidSet = generateMassCuboidSet(); System.out.println("Time elapsed for creating sorted cuboid list: " + sw.elapsed(TimeUnit.MILLISECONDS)); sw.reset(); diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java index 81fe71f9f6..0e15687d7f 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java @@ -167,7 +167,8 @@ public void testEstimateBitmapMemSize() { } } - @Test + // Ignore because of OOM + // @Test public void testEstimateMemSize() throws InterruptedException { int scale = Integer.parseInt(System.getProperty("scale", "1")); scale = Math.max(1, Math.min(10, scale)); diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidMapper.java index 0814281d87..1b32944228 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidMapper.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidMapper.java @@ -146,7 +146,7 @@ public void putRowKeyToHLLOld(String[] row) { Hasher hc = hf.newHasher(); String colValue = row[i]; if (colValue != null) { - rowHashCodes[i] = hc.putUnencodedChars(colValue).hash().asBytes(); + rowHashCodes[i] = hc.putString(colValue).hash().asBytes(); } else { rowHashCodes[i] = hc.putInt(0).hash().asBytes(); } @@ -170,7 +170,7 @@ private void putRowKeyToHLLNew(String[] row) { String colValue = row[i]; if (colValue == null) colValue = "0"; - byte[] bytes = hc.putUnencodedChars(colValue).hash().asBytes(); + byte[] bytes = hc.putString(colValue).hash().asBytes(); rowHashCodesLong[i] = (Bytes.toLong(bytes) + i);//add column ordinal to the hash value to distinguish between (a,b) and (b,a) } diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java index 6c5edda752..7bffce7bcf 100755 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java @@ -350,7 +350,7 @@ private void putRowKeyToHLLOld(String[] row) { Hasher hc = hf.newHasher(); String colValue = row[rowkeyColIndex[i]]; if (colValue != null) { - rowHashCodes[i] = hc.putUnencodedChars(colValue).hash().asBytes(); + rowHashCodes[i] = hc.putString(colValue).hash().asBytes(); } else { rowHashCodes[i] = hc.putInt(0).hash().asBytes(); } @@ -374,7 +374,7 @@ private void putRowKeyToHLLNew(String[] row) { String colValue = row[rowkeyColIndex[i]]; if (colValue == null) colValue = "0"; - byte[] bytes = hc.putUnencodedChars(colValue).hash().asBytes(); + byte[] bytes = hc.putString(colValue).hash().asBytes(); rowHashCodesLong[i] = (Bytes.toLong(bytes) + i);//add column ordinal to the hash value to distinguish between (a,b) and (b,a) } diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java index fcf50c7608..916f9fad8c 100644 --- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java +++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java @@ -105,7 +105,7 @@ private void putRowKeyToHLL(List<String> row) { int x = 0; for (String field : row) { Hasher hc = hf.newHasher(); - row_index[x++] = hc.putUnencodedChars(field).hash().asBytes(); + row_index[x++] = hc.putString(field).hash().asBytes(); } for (int i = 0, n = allCuboidsBitSet.length; i < n; i++) { diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NewCubeSamplingMethodTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NewCubeSamplingMethodTest.java index a82f73d958..97e356d08e 100644 --- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NewCubeSamplingMethodTest.java +++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NewCubeSamplingMethodTest.java @@ -113,7 +113,7 @@ public void run() throws Exception { int x = 0; for (String field : row) { Hasher hc = hf.newHasher(); - colHashValues[x++] = hc.putUnencodedChars(field).hash().asBytes(); + colHashValues[x++] = hc.putString(field).hash().asBytes(); } Hasher hc = hf.newHasher(); @@ -139,7 +139,7 @@ public void run() throws Exception { int x = 0; for (String field : row) { Hasher hc = hf2.newHasher(); - byte[] bytes = hc.putUnencodedChars(x + field).hash().asBytes(); + byte[] bytes = hc.putString(x + field).hash().asBytes(); valueHashLong[x++] = Bytes.toLong(bytes); } @@ -213,7 +213,7 @@ private void putRowKeyToHLL(List<String> row, byte[][] colHashValues, HLLCounter int x = 0; for (String field : row) { Hasher hc = hashFunction.newHasher(); - colHashValues[x++] = hc.putUnencodedChars(field).hash().asBytes(); + colHashValues[x++] = hc.putString(field).hash().asBytes(); } for (int i = 0, n = allCuboidsBitSet.length; i < n; i++) { @@ -230,7 +230,7 @@ private void putRowKeyToHLLNew(List<String> row, long[] hashValuesLong, HLLCount int x = 0; for (String field : row) { Hasher hc = hashFunction.newHasher(); - byte[] bytes = hc.putUnencodedChars(x + field).hash().asBytes(); + byte[] bytes = hc.putString(x + field).hash().asBytes(); hashValuesLong[x++] = Bytes.toLong(bytes); } diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java index 878ae7f5a4..31cadaa3d8 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java @@ -528,7 +528,7 @@ private void putRowKeyToHLLOld(String[] row) { Hasher hc = hf.newHasher(); String colValue = row[rowkeyColIndex[i]]; if (colValue != null) { - rowHashCodes[i] = hc.putUnencodedChars(colValue).hash().asBytes(); + rowHashCodes[i] = hc.putString(colValue).hash().asBytes(); } else { rowHashCodes[i] = hc.putInt(0).hash().asBytes(); } @@ -551,8 +551,8 @@ private void putRowKeyToHLLNew(String[] row) { Hasher hc = hf.newHasher(); String colValue = row[rowkeyColIndex[i]]; if (colValue == null) - colValue = "0"; - byte[] bytes = hc.putUnencodedChars(colValue).hash().asBytes(); + colValue = "0";// + byte[] bytes = hc.putString(colValue).hash().asBytes(); rowHashCodesLong[i] = (Bytes.toLong(bytes) + i);//add column ordinal to the hash value to distinguish between (a,b) and (b,a) } diff --git a/pom.xml b/pom.xml index 7b91a5d8ee..d957f0718d 100644 --- a/pom.xml +++ b/pom.xml @@ -82,7 +82,7 @@ <zookeeper.version>3.4.13</zookeeper.version> <curator.version>2.12.0</curator.version> <jsr305.version>3.0.1</jsr305.version> - <guava.version>16.0.1</guava.version> + <guava.version>14.0</guava.version> <jsch.version>0.1.54</jsch.version> <commons-cli.version>1.2</commons-cli.version> <commons-lang.version>2.6</commons-lang.version> diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java b/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java index 92e25c82b4..dd9cbad0a8 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java +++ b/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java @@ -78,7 +78,7 @@ public KylinAuthenticationProvider(AuthenticationProvider authenticationProvider @Override public Authentication authenticate(Authentication authentication) throws AuthenticationException { - byte[] hashKey = hf.hashUnencodedChars(authentication.getName() + authentication.getCredentials()).asBytes(); + byte[] hashKey = hf.hashString(authentication.getName() + authentication.getCredentials()).asBytes(); String userKey = Arrays.toString(hashKey); if (userService.isEvictCacheFlag()) { diff --git a/storage-druid/src/main/java/org/apache/kylin/storage/druid/DruidSchema.java b/storage-druid/src/main/java/org/apache/kylin/storage/druid/DruidSchema.java index 54c85919d1..8f473c4bdb 100644 --- a/storage-druid/src/main/java/org/apache/kylin/storage/druid/DruidSchema.java +++ b/storage-druid/src/main/java/org/apache/kylin/storage/druid/DruidSchema.java @@ -99,7 +99,7 @@ public static Interval segmentInterval(CubeSegment segment) { if (!segment.getCubeDesc().getModel().getPartitionDesc().isPartitioned()) { return Intervals.utc(0, segment.getCreateTimeUTC()); } - return Intervals.utc(segment.getTSRange().start.v, segment.getTSRange().start.v); + return Intervals.utc(segment.getTSRange().start.v, segment.getTSRange().end.v); } public static String getDataSource(CubeDesc cubeDesc) { diff --git a/storage-druid/src/main/java/org/apache/kylin/storage/druid/read/cursor/HttpDruidClient.java b/storage-druid/src/main/java/org/apache/kylin/storage/druid/read/cursor/HttpDruidClient.java index 9af7c636c4..6bf9d5d5d4 100644 --- a/storage-druid/src/main/java/org/apache/kylin/storage/druid/read/cursor/HttpDruidClient.java +++ b/storage-druid/src/main/java/org/apache/kylin/storage/druid/read/cursor/HttpDruidClient.java @@ -26,6 +26,7 @@ import com.fasterxml.jackson.dataformat.smile.SmileFactory; import com.google.common.base.Throwables; import com.google.common.io.ByteSource; +import com.google.common.io.ByteStreams; import com.google.common.util.concurrent.ListenableFuture; import io.druid.java.util.common.IAE; import io.druid.java.util.common.RE; @@ -162,7 +163,8 @@ public InputStream nextElement() { try { // An empty byte array is put at the end to give the SequenceInputStream.close() as something to close out // after done is set to true, regardless of the rest of the stream's state. - queue.put(ByteSource.empty().openStream()); + ByteSource empty = ByteStreams.asByteSource(new byte[]{}); + queue.put(empty.openStream()); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw Throwables.propagate(e); ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services