shaofengshi closed pull request #357: KYLIN-3694 Bug fix and revert guava
version
URL: https://github.com/apache/kylin/pull/357
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 6dfe165cbe..1e546cb313 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -34,11 +34,6 @@
</properties>
<dependencies>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <scope>compile</scope>
- </dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-source-hive</artifactId>
@@ -228,10 +223,6 @@
<pattern>org.apache.commons.jocl</pattern>
<shadedPattern>${shadeBase}.org.apache.commons.jocl</shadedPattern>
</relocation>
- <relocation>
- <pattern>com.google.guava</pattern>
-
<shadedPattern>${shadeBase}.com.google.guava</shadedPattern>
- </relocation>
</relocations>
<filters>
<filter>
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
index 8f4a1f6919..f78e92b530 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
@@ -87,7 +87,7 @@
Hasher hc = hf.newHasher();
final String cell =
row.get(flatDesc.getRowKeyColumnIndexes()[i]);
if (cell != null) {
- row_hashcodes[i] =
hc.putUnencodedChars(cell).hash().asBytes();
+ row_hashcodes[i] = hc.putString(cell).hash().asBytes();
} else {
row_hashcodes[i] = hc.putInt(0).hash().asBytes();
}
diff --git
a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
index b9fb9841e1..c3642cb22d 100644
---
a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
+++
b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
@@ -147,7 +147,7 @@ public void createDirectChildrenCacheTest() {
@Test
public void createDirectChildrenCacheStressTest() {
- Stopwatch sw = Stopwatch.createStarted();
+ Stopwatch sw = new Stopwatch();
Set<Long> cuboidSet = generateMassCuboidSet();
System.out.println("Time elapsed for creating sorted cuboid list: " +
sw.elapsed(TimeUnit.MILLISECONDS));
sw.reset();
diff --git
a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
index 81fe71f9f6..0e15687d7f 100644
---
a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
+++
b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
@@ -167,7 +167,8 @@ public void testEstimateBitmapMemSize() {
}
}
- @Test
+ // Ignore because of OOM
+ // @Test
public void testEstimateMemSize() throws InterruptedException {
int scale = Integer.parseInt(System.getProperty("scale", "1"));
scale = Math.max(1, Math.min(10, scale));
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidMapper.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidMapper.java
index 0814281d87..1b32944228 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidMapper.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidMapper.java
@@ -146,7 +146,7 @@ public void putRowKeyToHLLOld(String[] row) {
Hasher hc = hf.newHasher();
String colValue = row[i];
if (colValue != null) {
- rowHashCodes[i] =
hc.putUnencodedChars(colValue).hash().asBytes();
+ rowHashCodes[i] = hc.putString(colValue).hash().asBytes();
} else {
rowHashCodes[i] = hc.putInt(0).hash().asBytes();
}
@@ -170,7 +170,7 @@ private void putRowKeyToHLLNew(String[] row) {
String colValue = row[i];
if (colValue == null)
colValue = "0";
- byte[] bytes = hc.putUnencodedChars(colValue).hash().asBytes();
+ byte[] bytes = hc.putString(colValue).hash().asBytes();
rowHashCodesLong[i] = (Bytes.toLong(bytes) + i);//add column
ordinal to the hash value to distinguish between (a,b) and (b,a)
}
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
index 6c5edda752..7bffce7bcf 100755
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
@@ -350,7 +350,7 @@ private void putRowKeyToHLLOld(String[] row) {
Hasher hc = hf.newHasher();
String colValue = row[rowkeyColIndex[i]];
if (colValue != null) {
- rowHashCodes[i] =
hc.putUnencodedChars(colValue).hash().asBytes();
+ rowHashCodes[i] = hc.putString(colValue).hash().asBytes();
} else {
rowHashCodes[i] = hc.putInt(0).hash().asBytes();
}
@@ -374,7 +374,7 @@ private void putRowKeyToHLLNew(String[] row) {
String colValue = row[rowkeyColIndex[i]];
if (colValue == null)
colValue = "0";
- byte[] bytes = hc.putUnencodedChars(colValue).hash().asBytes();
+ byte[] bytes = hc.putString(colValue).hash().asBytes();
rowHashCodesLong[i] = (Bytes.toLong(bytes) + i);//add column
ordinal to the hash value to distinguish between (a,b) and (b,a)
}
diff --git
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
index fcf50c7608..916f9fad8c 100644
---
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
+++
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
@@ -105,7 +105,7 @@ private void putRowKeyToHLL(List<String> row) {
int x = 0;
for (String field : row) {
Hasher hc = hf.newHasher();
- row_index[x++] = hc.putUnencodedChars(field).hash().asBytes();
+ row_index[x++] = hc.putString(field).hash().asBytes();
}
for (int i = 0, n = allCuboidsBitSet.length; i < n; i++) {
diff --git
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NewCubeSamplingMethodTest.java
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NewCubeSamplingMethodTest.java
index a82f73d958..97e356d08e 100644
---
a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NewCubeSamplingMethodTest.java
+++
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NewCubeSamplingMethodTest.java
@@ -113,7 +113,7 @@ public void run() throws Exception {
int x = 0;
for (String field : row) {
Hasher hc = hf.newHasher();
- colHashValues[x++] =
hc.putUnencodedChars(field).hash().asBytes();
+ colHashValues[x++] =
hc.putString(field).hash().asBytes();
}
Hasher hc = hf.newHasher();
@@ -139,7 +139,7 @@ public void run() throws Exception {
int x = 0;
for (String field : row) {
Hasher hc = hf2.newHasher();
- byte[] bytes = hc.putUnencodedChars(x +
field).hash().asBytes();
+ byte[] bytes = hc.putString(x +
field).hash().asBytes();
valueHashLong[x++] = Bytes.toLong(bytes);
}
@@ -213,7 +213,7 @@ private void putRowKeyToHLL(List<String> row, byte[][]
colHashValues, HLLCounter
int x = 0;
for (String field : row) {
Hasher hc = hashFunction.newHasher();
- colHashValues[x++] = hc.putUnencodedChars(field).hash().asBytes();
+ colHashValues[x++] = hc.putString(field).hash().asBytes();
}
for (int i = 0, n = allCuboidsBitSet.length; i < n; i++) {
@@ -230,7 +230,7 @@ private void putRowKeyToHLLNew(List<String> row, long[]
hashValuesLong, HLLCount
int x = 0;
for (String field : row) {
Hasher hc = hashFunction.newHasher();
- byte[] bytes = hc.putUnencodedChars(x + field).hash().asBytes();
+ byte[] bytes = hc.putString(x + field).hash().asBytes();
hashValuesLong[x++] = Bytes.toLong(bytes);
}
diff --git
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java
index 878ae7f5a4..31cadaa3d8 100644
---
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java
+++
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java
@@ -528,7 +528,7 @@ private void putRowKeyToHLLOld(String[] row) {
Hasher hc = hf.newHasher();
String colValue = row[rowkeyColIndex[i]];
if (colValue != null) {
- rowHashCodes[i] =
hc.putUnencodedChars(colValue).hash().asBytes();
+ rowHashCodes[i] = hc.putString(colValue).hash().asBytes();
} else {
rowHashCodes[i] = hc.putInt(0).hash().asBytes();
}
@@ -551,8 +551,8 @@ private void putRowKeyToHLLNew(String[] row) {
Hasher hc = hf.newHasher();
String colValue = row[rowkeyColIndex[i]];
if (colValue == null)
- colValue = "0";
- byte[] bytes = hc.putUnencodedChars(colValue).hash().asBytes();
+ colValue = "0";//
+ byte[] bytes = hc.putString(colValue).hash().asBytes();
rowHashCodesLong[i] = (Bytes.toLong(bytes) + i);//add column
ordinal to the hash value to distinguish between (a,b) and (b,a)
}
diff --git a/pom.xml b/pom.xml
index 7b91a5d8ee..d957f0718d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -82,7 +82,7 @@
<zookeeper.version>3.4.13</zookeeper.version>
<curator.version>2.12.0</curator.version>
<jsr305.version>3.0.1</jsr305.version>
- <guava.version>16.0.1</guava.version>
+ <guava.version>14.0</guava.version>
<jsch.version>0.1.54</jsch.version>
<commons-cli.version>1.2</commons-cli.version>
<commons-lang.version>2.6</commons-lang.version>
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
b/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
index 92e25c82b4..dd9cbad0a8 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
@@ -78,7 +78,7 @@ public KylinAuthenticationProvider(AuthenticationProvider
authenticationProvider
@Override
public Authentication authenticate(Authentication authentication) throws
AuthenticationException {
- byte[] hashKey = hf.hashUnencodedChars(authentication.getName() +
authentication.getCredentials()).asBytes();
+ byte[] hashKey = hf.hashString(authentication.getName() +
authentication.getCredentials()).asBytes();
String userKey = Arrays.toString(hashKey);
if (userService.isEvictCacheFlag()) {
diff --git
a/storage-druid/src/main/java/org/apache/kylin/storage/druid/DruidSchema.java
b/storage-druid/src/main/java/org/apache/kylin/storage/druid/DruidSchema.java
index 54c85919d1..8f473c4bdb 100644
---
a/storage-druid/src/main/java/org/apache/kylin/storage/druid/DruidSchema.java
+++
b/storage-druid/src/main/java/org/apache/kylin/storage/druid/DruidSchema.java
@@ -99,7 +99,7 @@ public static Interval segmentInterval(CubeSegment segment) {
if
(!segment.getCubeDesc().getModel().getPartitionDesc().isPartitioned()) {
return Intervals.utc(0, segment.getCreateTimeUTC());
}
- return Intervals.utc(segment.getTSRange().start.v,
segment.getTSRange().start.v);
+ return Intervals.utc(segment.getTSRange().start.v,
segment.getTSRange().end.v);
}
public static String getDataSource(CubeDesc cubeDesc) {
diff --git
a/storage-druid/src/main/java/org/apache/kylin/storage/druid/read/cursor/HttpDruidClient.java
b/storage-druid/src/main/java/org/apache/kylin/storage/druid/read/cursor/HttpDruidClient.java
index 9af7c636c4..6bf9d5d5d4 100644
---
a/storage-druid/src/main/java/org/apache/kylin/storage/druid/read/cursor/HttpDruidClient.java
+++
b/storage-druid/src/main/java/org/apache/kylin/storage/druid/read/cursor/HttpDruidClient.java
@@ -26,6 +26,7 @@
import com.fasterxml.jackson.dataformat.smile.SmileFactory;
import com.google.common.base.Throwables;
import com.google.common.io.ByteSource;
+import com.google.common.io.ByteStreams;
import com.google.common.util.concurrent.ListenableFuture;
import io.druid.java.util.common.IAE;
import io.druid.java.util.common.RE;
@@ -162,7 +163,8 @@ public InputStream nextElement() {
try {
// An empty byte array is put at the end to give the
SequenceInputStream.close() as something to close out
// after done is set to true, regardless of the rest
of the stream's state.
- queue.put(ByteSource.empty().openStream());
+ ByteSource empty = ByteStreams.asByteSource(new
byte[]{});
+ queue.put(empty.openStream());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw Throwables.propagate(e);
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
